From f51446a38cb0db8dcfeade1e30da43de0f80d266 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 21 Jul 2012 19:40:42 +0000 Subject: [PATCH 001/710] Add an internal interface that allows the code to take advantage of multiple cores by pushing subcomputations off into separate threads. The interface is not currently used. FossilOrigin-Name: 0e4d977a4a07d6de50acbf022c7dd947998b8d96 --- Makefile.in | 6 +- Makefile.msc | 6 +- main.mk | 3 +- manifest | 28 +++++---- manifest.uuid | 2 +- src/sqliteInt.h | 7 +++ src/threads.c | 120 ++++++++++++++++++++++++++++++++++++++ tool/mksqlite3c-noext.tcl | 1 + tool/mksqlite3c.tcl | 1 + 9 files changed, 158 insertions(+), 16 deletions(-) create mode 100644 src/threads.c diff --git a/Makefile.in b/Makefile.in index df424cb190..894b352d09 100644 --- a/Makefile.in +++ b/Makefile.in @@ -176,7 +176,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ - table.lo tokenize.lo trigger.lo \ + table.lo threads.lo tokenize.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo @@ -260,6 +260,7 @@ SRC = \ $(TOP)/src/sqliteInt.h \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/table.c \ + $(TOP)/src/threads.c \ $(TOP)/src/tclsqlite.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/trigger.c \ @@ -708,6 +709,9 @@ status.lo: $(TOP)/src/status.c $(HDR) table.lo: $(TOP)/src/table.c $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/table.c +threads.lo: $(TOP)/src/threads.c $(HDR) + $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/threads.c + tokenize.lo: $(TOP)/src/tokenize.c keywordhash.h $(HDR) $(LTCOMPILE) $(TEMP_STORE) -c $(TOP)/src/tokenize.c diff --git a/Makefile.msc b/Makefile.msc index e16ced35e4..33cde181c5 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -376,7 +376,7 @@ LIBOBJS0 = alter.lo analyze.lo attach.lo auth.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo parse.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ - table.lo tokenize.lo trigger.lo \ + table.lo threads.o tokenize.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbe.lo vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo @@ -463,6 +463,7 @@ SRC = \ $(TOP)\src\sqliteInt.h \ $(TOP)\src\sqliteLimit.h \ $(TOP)\src\table.c \ + $(TOP)\src\threads.c \ $(TOP)\src\tclsqlite.c \ $(TOP)\src\tokenize.c \ $(TOP)\src\trigger.c \ @@ -896,6 +897,9 @@ status.lo: $(TOP)\src\status.c $(HDR) table.lo: $(TOP)\src\table.c $(HDR) $(LTCOMPILE) -c $(TOP)\src\table.c +threads.lo: $(TOP)\src\threads.c $(HDR) + $(LTCOMPILE) -c $(TOP)\src\threads.c + tokenize.lo: $(TOP)\src\tokenize.c keywordhash.h $(HDR) $(LTCOMPILE) -c $(TOP)\src\tokenize.c diff --git a/main.mk b/main.mk index 8682961881..7a7be4b9db 100644 --- a/main.mk +++ b/main.mk @@ -64,7 +64,7 @@ LIBOBJ+= alter.o analyze.o attach.o auth.o \ notify.o opcodes.o os.o os_unix.o os_win.o \ pager.o parse.o pcache.o pcache1.o pragma.o prepare.o printf.o \ random.o resolve.o rowset.o rtree.o select.o status.o \ - table.o tokenize.o trigger.o \ + table.o threads.o tokenize.o trigger.o \ update.o util.o vacuum.o \ vdbe.o vdbeapi.o vdbeaux.o vdbeblob.o vdbemem.o vdbesort.o \ vdbetrace.o wal.o walker.o where.o utf.o vtab.o @@ -142,6 +142,7 @@ SRC = \ $(TOP)/src/sqliteLimit.h \ $(TOP)/src/table.c \ $(TOP)/src/tclsqlite.c \ + $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/trigger.c \ $(TOP)/src/utf.c \ diff --git a/manifest b/manifest index 8576e56420..7e6d5fef2a 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Ensure\sthat\sthere\sis\salways\sat\sleast\sone\saReadMark\sslot\susable\sby\san\sunprivileged\sreader\swhile\sa\scheckpoint\sis\srunning.\sAlso,\sif\sone\sor\smore\stransactions\sare\srecovered\sfrom\sa\slog\sfile,\sinitialize\sone\sof\sthe\saReadMark\sslots\sto\scontain\smxFrame\sas\spart\sof\sthe\srecovery\sprocess. -D 2012-07-17T14:37:12.494 +C Add\san\sinternal\sinterface\sthat\sallows\sthe\scode\sto\stake\sadvantage\sof\smultiple\ncores\sby\spushing\ssubcomputations\soff\sinto\sseparate\sthreads.\s\sThe\sinterface\nis\snot\scurrently\sused. +D 2012-07-21T19:40:42.441 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 8f6d858bf3df9978ba43df19985146a1173025e4 +F Makefile.in 7a89f9692d1369faa4071310164ffba0504c324d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 56ff0fcc3fc3b275aec7f6acb34b3c0526c684bc +F Makefile.msc 332c166048570b65127db33450bca22c53b4f0a2 F Makefile.vxworks 879f034a64062a364b21000266bbd5bc6e0c19b9 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION a71848df48082f1d6585d4b0819d530fc455485d @@ -103,7 +103,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 -F main.mk d109a9342d1fe135d3900aca9f5563f9480a991d +F main.mk dd63e2de89cdcb9be441e8f046a0acce3fbae2a8 F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac @@ -180,7 +180,7 @@ F src/select.c f6c4833c4d8e94714761d99013d74f381e084f1d F src/shell.c 076e1c90d594644f36027c8ecff9a392cf2d3a06 F src/sqlite.h.in 310ae7e538883fa1619ab0638c775ce11ad43015 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h 2bc2ebc2ff1a2b530ee5ed9ffd46c6fce93b244c +F src/sqliteInt.h 273e9d00296d3a76519c0446698c23787d8951f9 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -231,6 +231,7 @@ F src/test_vfs.c da6d0d982b11756c94c1760196355d33d03ff745 F src/test_vfstrace.c 6b28adb2a0e8ecd0f2e3581482e1f658b11b4067 F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 +F src/threads.c df23fc534a54fe7e1ff935628fa14a9318d27ae3 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c ee7e178fb9188f44b532cebd449a7c1df90fb684 F src/update.c d3076782c887c10e882996550345da9c4c9f9dea @@ -978,8 +979,8 @@ F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/mkkeywordhash.c bb52064aa614e1426445e4b2b9b00eeecd23cc79 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 -F tool/mksqlite3c-noext.tcl 8bce31074e4cbe631bb7676526a048335f4c9f02 -F tool/mksqlite3c.tcl 589c7f44e990be1b8443cfe4808dce392b0327fa +F tool/mksqlite3c-noext.tcl 752f1a9d3287f6c0ef5738b1c4add0b96fbe0854 +F tool/mksqlite3c.tcl d4923e8e75b7710ddbe4eb37f83dda5eadef63d8 F tool/mksqlite3h.tcl 78013ad79a5e492e5f764f3c7a8ef834255061f8 F tool/mksqlite3internalh.tcl 3dca7bb5374cee003379b8cbac73714f610ef795 F tool/offsets.c fe4262fdfa378e8f5499a42136d17bf3b98f6091 @@ -1005,7 +1006,10 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 8c9ee1d78f99394eef73a177141ca9e1c67e4e07 -R 21a0c6942de3e9593e3e93f462c443c7 -U dan -Z a98d8c358c03bd601d60af308961371e +P e4163596339c2166f9c4356ab824fff8bda8d0b0 +R e05d278833a84eb0e1d0c443cf6ae5d1 +T *branch * threads +T *sym-threads * +T -sym-trunk * +U drh +Z d49696325d34c649ba05b5124d6547e7 diff --git a/manifest.uuid b/manifest.uuid index dad8b25eaa..250952bf9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4163596339c2166f9c4356ab824fff8bda8d0b0 \ No newline at end of file +0e4d977a4a07d6de50acbf022c7dd947998b8d96 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f7f5fd386e..f0ccc5cd28 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -661,6 +661,7 @@ typedef struct Parse Parse; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; +typedef struct SQLiteThread SQLiteThread; typedef struct SrcList SrcList; typedef struct StrAccum StrAccum; typedef struct Table Table; @@ -3307,4 +3308,10 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); #define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ #define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */ +/* +** Threading interface +*/ +int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); +int sqlite3ThreadJoin(SQLiteThread*, void**); + #endif /* _SQLITEINT_H_ */ diff --git a/src/threads.c b/src/threads.c new file mode 100644 index 0000000000..ed98015510 --- /dev/null +++ b/src/threads.c @@ -0,0 +1,120 @@ +/* +** 2012 July 21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file presents a simple cross-platform threading interface for +** use internally by SQLite. +** +** A "thread" can be created using sqlite3ThreadCreate(). This thread +** runs independently of its creator until it is joined using +** sqlite3ThreadJoin(), at which point it terminates. +** +** Threads do not have to be real. It could be that the work of the +** "thread" is done by the main thread at either the sqlite3ThreadCreate() +** or sqlite3ThreadJoin() call. This is, in fact, what happens in +** single threaded systems. Nothing in SQLite requires multiple threads. +** This interface exists so that applications that want to take advantage +** of multiple cores can do so, while also allowing applications to stay +** single-threaded if desired. +*/ +#include "sqliteInt.h" + +/********************************* Unix Pthreads ****************************/ +#if SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) + +#define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ +#include + +/* A running thread */ +struct SQLiteThread { + pthread_t tid; +}; + +/* Create a new thread */ +int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + int rc; + + *ppThread = p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_OK; + rc = pthread_create(&p->tid, 0, xTask, pIn); + if( rc ){ + sqlite3_free(p); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + +/* Get the results of the thread */ +int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + int rc; + if( p==0 ) return SQLITE_NOMEM; + rc = pthread_join(p->tid, ppOut); + sqlite3_free(p); + return rc ? SQLITE_ERROR : SQLITE_OK; +} + +#endif /* SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) */ +/******************************** End Unix Pthreads *************************/ + + +/********************************* Single-Threaded **************************/ +#ifndef SQLITE_THREADS_IMPLEMENTED +/* +** This implementation does not actually create a new thread. It does the +** work of the thread in the main thread, when either the thread is created +** or when it is joined +*/ + +/* A running thread */ +struct SQLiteThread { + void *(*xTask)(void*); /* The routine to run as a thread */ + void *pIn; /* Argument to xTask */ + void *pResult; /* Result of xTask */ +}; + +/* Create a new thread */ +int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + *ppThread = p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; + if( (SQLITE_PTR_TO_INT(p)/17)&1 ){ + p->xTask = xTask; + p->pIn = pIn; + }else{ + p->xTask = 0; + p->pResult = xTask(pIn); + } + return p; +} + +/* Get the results of the thread */ +int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + if( p==0 ) return SQLITE_NOMEM; + if( p->xTask ){ + *ppOut = = p->xTask(p->pIn); + }else{ + *ppOut = p->pResult; + } + sqlite3_free(p); + return SQLITE_OK; +} + +#endif /* !defined(SQLITE_THREADS_IMPLEMENTED) */ +/****************************** End Single-Threaded *************************/ diff --git a/tool/mksqlite3c-noext.tcl b/tool/mksqlite3c-noext.tcl index 017ad6292f..3a0138b3b2 100644 --- a/tool/mksqlite3c-noext.tcl +++ b/tool/mksqlite3c-noext.tcl @@ -237,6 +237,7 @@ foreach file { malloc.c printf.c random.c + threads.c utf.c util.c hash.c diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 2c569d7a4e..4885728f95 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -242,6 +242,7 @@ foreach file { malloc.c printf.c random.c + threads.c utf.c util.c hash.c From da0e47109e6ba93ab92023a8e7a9ddd7afa5d8b2 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 21 Jul 2012 22:49:08 +0000 Subject: [PATCH 002/710] Add Win32 support to the internal threads interface. Also, add several asserts and fix a few typos. FossilOrigin-Name: 793195d37109c75eba84f7190c8fe0b8722f76f7 --- manifest | 21 +++++------- manifest.uuid | 2 +- src/os_win.c | 4 +++ src/sqliteInt.h | 7 ++++ src/threads.c | 87 ++++++++++++++++++++++++++++++++++++++++++++++--- 5 files changed, 103 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 7e6d5fef2a..dcf166e469 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sinternal\sinterface\sthat\sallows\sthe\scode\sto\stake\sadvantage\sof\smultiple\ncores\sby\spushing\ssubcomputations\soff\sinto\sseparate\sthreads.\s\sThe\sinterface\nis\snot\scurrently\sused. -D 2012-07-21T19:40:42.441 +C Add\sWin32\ssupport\sto\sthe\sinternal\sthreads\sinterface.\s\sAlso,\sadd\sseveral\sasserts\sand\sfix\sa\sfew\stypos. +D 2012-07-21T22:49:08.403 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7a89f9692d1369faa4071310164ffba0504c324d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h c2ebd26a68a4223fe170b003852b97d9e7211498 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d7c96b5d140f550f07345870112fae5d7ef99757 -F src/os_win.c e3d3d3e26b65a35d4293d753137a58510bd3299b +F src/os_win.c 899783012ed47a756cd4358b43ecfa139cf14ace F src/pager.c e381c118b77dc22021a1a59d3fec24815e91df78 F src/pager.h 8b8c9bc065a3c66769df8724dfdf492ee1aab3c5 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 @@ -180,7 +180,7 @@ F src/select.c f6c4833c4d8e94714761d99013d74f381e084f1d F src/shell.c 076e1c90d594644f36027c8ecff9a392cf2d3a06 F src/sqlite.h.in 310ae7e538883fa1619ab0638c775ce11ad43015 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477 -F src/sqliteInt.h 273e9d00296d3a76519c0446698c23787d8951f9 +F src/sqliteInt.h 4bb99c73090cde71ea5534d0f9c3ee299a1c10e7 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 35939e7e03abf1b7577ce311f48f682c40de3208 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -231,7 +231,7 @@ F src/test_vfs.c da6d0d982b11756c94c1760196355d33d03ff745 F src/test_vfstrace.c 6b28adb2a0e8ecd0f2e3581482e1f658b11b4067 F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c df23fc534a54fe7e1ff935628fa14a9318d27ae3 +F src/threads.c 82ea90092f36d02f887e85f7559efc1519f9edd3 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c ee7e178fb9188f44b532cebd449a7c1df90fb684 F src/update.c d3076782c887c10e882996550345da9c4c9f9dea @@ -1006,10 +1006,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P e4163596339c2166f9c4356ab824fff8bda8d0b0 -R e05d278833a84eb0e1d0c443cf6ae5d1 -T *branch * threads -T *sym-threads * -T -sym-trunk * -U drh -Z d49696325d34c649ba05b5124d6547e7 +P 0e4d977a4a07d6de50acbf022c7dd947998b8d96 +R a23a964d0745aaa13c7e5378646293f9 +U mistachkin +Z d6738fe175fa5cec50ba76dfb911d019 diff --git a/manifest.uuid b/manifest.uuid index 250952bf9a..25fa463af5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0e4d977a4a07d6de50acbf022c7dd947998b8d96 \ No newline at end of file +793195d37109c75eba84f7190c8fe0b8722f76f7 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 8509e9272d..d3f9671809 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -945,6 +945,10 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #endif } +DWORD sqlite3Win32Wait(HANDLE hObject){ + return osWaitForSingleObjectEx(hObject, INFINITE, TRUE); +} + /* ** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, ** or WinCE. Return false (zero) for Win95, Win98, or WinME. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f0ccc5cd28..559a552de0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3314,4 +3314,11 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); int sqlite3ThreadJoin(SQLiteThread*, void**); +/* +** Win32 interface +*/ +#if SQLITE_OS_WIN + DWORD sqlite3Win32Wait(HANDLE hObject); +#endif + #endif /* _SQLITEINT_H_ */ diff --git a/src/threads.c b/src/threads.c index ed98015510..dfb44a7c2f 100644 --- a/src/threads.c +++ b/src/threads.c @@ -47,19 +47,25 @@ int sqlite3ThreadCreate( SQLiteThread *p; int rc; - *ppThread = p = sqlite3Malloc(sizeof(*p)); - if( p==0 ) return SQLITE_OK; + assert( ppThread!=0 ); + assert( xTask!=0 ); + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; rc = pthread_create(&p->tid, 0, xTask, pIn); if( rc ){ sqlite3_free(p); return SQLITE_ERROR; } + *ppThread = p; return SQLITE_OK; } /* Get the results of the thread */ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ int rc; + + assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; rc = pthread_join(p->tid, ppOut); sqlite3_free(p); @@ -70,6 +76,71 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ /******************************** End Unix Pthreads *************************/ +/********************************* Win32 Threads ****************************/ +#if SQLITE_OS_WIN && !SQLITE_OS_WINRT + +#define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ +#include + +/* A running thread */ +struct SQLiteThread { + uintptr_t tid; /* The thread handle */ + void *(*xTask)(void*); /* The routine to run as a thread */ + void *pIn; /* Argument to xTask */ + void *pResult; /* Result of xTask */ +}; + +/* Thread procedure Win32 compatibility shim */ +static void sqlite3ThreadProc( + void *pArg /* IN: Pointer to the SQLiteThread structure */ +){ + SQLiteThread *p = (SQLiteThread *)pArg; + + assert( p!=0 ); + assert( p->xTask!=0 ); + p->pResult = p->xTask(p->pIn); + _endthread(); +} + +/* Create a new thread */ +int sqlite3ThreadCreate( + SQLiteThread **ppThread, /* OUT: Write the thread object here */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + SQLiteThread *p; + + assert( ppThread!=0 ); + assert( xTask!=0 ); + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; + p->xTask = xTask; p->pIn = pIn; + p->tid = _beginthread(sqlite3ThreadProc, 0, p); + if( p->tid==(uintptr_t)-1 ){ + sqlite3_free(p); + return SQLITE_ERROR; + } + *ppThread = p; + return SQLITE_OK; +} + +/* Get the results of the thread */ +int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + DWORD rc; + + assert( ppOut!=0 ); + if( p==0 ) return SQLITE_NOMEM; + rc = sqlite3Win32Wait((HANDLE)p->tid); + if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; + sqlite3_free(p); + return (rc==WAIT_OBJECT_0) ? SQLITE_OK : SQLITE_ERROR; +} + +#endif /* SQLITE_OS_WIN && !SQLITE_OS_WINRT */ +/******************************** End Win32 Threads *************************/ + + /********************************* Single-Threaded **************************/ #ifndef SQLITE_THREADS_IMPLEMENTED /* @@ -92,7 +163,11 @@ int sqlite3ThreadCreate( void *pIn /* Argument passed into xTask() */ ){ SQLiteThread *p; - *ppThread = p = sqlite3Malloc(sizeof(*p)); + + assert( ppThread!=0 ); + assert( xTask!=0 ); + *ppThread = 0; + p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM; if( (SQLITE_PTR_TO_INT(p)/17)&1 ){ p->xTask = xTask; @@ -101,14 +176,16 @@ int sqlite3ThreadCreate( p->xTask = 0; p->pResult = xTask(pIn); } - return p; + *ppThread = p; + return SQLITE_OK; } /* Get the results of the thread */ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; if( p->xTask ){ - *ppOut = = p->xTask(p->pIn); + *ppOut = p->xTask(p->pIn); }else{ *ppOut = p->pResult; } From 572df3b1e4a307e2657579658c041f567442f50c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 23 Jul 2012 02:00:38 +0000 Subject: [PATCH 003/710] Enhance implementation of the Win32 thread wait function. FossilOrigin-Name: 049b04117353c3e163ffc87916cbe121403a2821 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index dcf166e469..da25a187c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sWin32\ssupport\sto\sthe\sinternal\sthreads\sinterface.\s\sAlso,\sadd\sseveral\sasserts\sand\sfix\sa\sfew\stypos. -D 2012-07-21T22:49:08.403 +C Enhance\simplementation\sof\sthe\sWin32\sthread\swait\sfunction. +D 2012-07-23T02:00:38.699 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7a89f9692d1369faa4071310164ffba0504c324d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F src/os.c e1acdc09ff3ac2412945cca9766e2dcf4675f31c F src/os.h c2ebd26a68a4223fe170b003852b97d9e7211498 F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c d7c96b5d140f550f07345870112fae5d7ef99757 -F src/os_win.c 899783012ed47a756cd4358b43ecfa139cf14ace +F src/os_win.c f9f2a4569f9a6d2415170261641e6e3ee6ed8121 F src/pager.c e381c118b77dc22021a1a59d3fec24815e91df78 F src/pager.h 8b8c9bc065a3c66769df8724dfdf492ee1aab3c5 F src/parse.y f29df90bd3adc64b33114ab1de9fb7768fcf2099 @@ -1006,7 +1006,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 0e4d977a4a07d6de50acbf022c7dd947998b8d96 -R a23a964d0745aaa13c7e5378646293f9 +P 793195d37109c75eba84f7190c8fe0b8722f76f7 +R 49885bc11b122527dda4cd8bcfe1cfa0 U mistachkin -Z d6738fe175fa5cec50ba76dfb911d019 +Z b0f357746aebd8329bde52b662a0497e diff --git a/manifest.uuid b/manifest.uuid index 25fa463af5..1d07b1a32a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -793195d37109c75eba84f7190c8fe0b8722f76f7 \ No newline at end of file +049b04117353c3e163ffc87916cbe121403a2821 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index d3f9671809..75d0edcddb 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -946,7 +946,10 @@ void sqlite3_win32_sleep(DWORD milliseconds){ } DWORD sqlite3Win32Wait(HANDLE hObject){ - return osWaitForSingleObjectEx(hObject, INFINITE, TRUE); + DWORD rc; + while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, + TRUE))==WAIT_IO_COMPLETION ){} + return rc; } /* From a806475f86220a2bb19033eb8bf304c1601f4994 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 23 Jul 2012 06:47:30 +0000 Subject: [PATCH 004/710] Add an assert() to help verify the return code from the Win32 thread wait function. FossilOrigin-Name: ed3dc7a89f3416622fcd741ae5fba437929d06d6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/threads.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index da25a187c6..e4cf469b13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\simplementation\sof\sthe\sWin32\sthread\swait\sfunction. -D 2012-07-23T02:00:38.699 +C Add\san\sassert()\sto\shelp\sverify\sthe\sreturn\scode\sfrom\sthe\sWin32\sthread\swait\sfunction. +D 2012-07-23T06:47:30.102 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7a89f9692d1369faa4071310164ffba0504c324d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -231,7 +231,7 @@ F src/test_vfs.c da6d0d982b11756c94c1760196355d33d03ff745 F src/test_vfstrace.c 6b28adb2a0e8ecd0f2e3581482e1f658b11b4067 F src/test_wholenumber.c 3d2b9ed1505c40ad5c5ca2ad16ae7a289d6cc251 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 82ea90092f36d02f887e85f7559efc1519f9edd3 +F src/threads.c cde5bd24ab5b33bb694c59f4236ecd56ad8da9b5 F src/tokenize.c 1e86210d3976717a19238ea7b047fac481fe8c12 F src/trigger.c ee7e178fb9188f44b532cebd449a7c1df90fb684 F src/update.c d3076782c887c10e882996550345da9c4c9f9dea @@ -1006,7 +1006,7 @@ F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 -P 793195d37109c75eba84f7190c8fe0b8722f76f7 -R 49885bc11b122527dda4cd8bcfe1cfa0 +P 049b04117353c3e163ffc87916cbe121403a2821 +R 8ad4cfcece99f2543c3fc25ff90ff8da U mistachkin -Z b0f357746aebd8329bde52b662a0497e +Z c7ca4753dded1e68bd82c1792e2c5272 diff --git a/manifest.uuid b/manifest.uuid index 1d07b1a32a..acb6e11d72 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -049b04117353c3e163ffc87916cbe121403a2821 \ No newline at end of file +ed3dc7a89f3416622fcd741ae5fba437929d06d6 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index dfb44a7c2f..61e58f494b 100644 --- a/src/threads.c +++ b/src/threads.c @@ -132,6 +132,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; rc = sqlite3Win32Wait((HANDLE)p->tid); + assert( rc!=WAIT_IO_COMPLETION ); if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; sqlite3_free(p); return (rc==WAIT_OBJECT_0) ? SQLITE_OK : SQLITE_ERROR; From daa3ce4b0805753abb473a216b2057d08326023b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Aug 2012 13:08:15 +0000 Subject: [PATCH 005/710] Update the spellfix virtual table so that all OOM errors are reported out to the application. FossilOrigin-Name: c2cf498513c2633bd2b08372772eaa0f3b3ab25f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_spellfix.c | 32 ++++++++++++++++++++++---------- 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 20486b8f7e..4e50d4dc33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_DISABLE_FTS4_DEFERRED\scompile\stime\soption. -D 2012-08-20T17:24:48.768 +C Update\sthe\sspellfix\svirtual\stable\sso\sthat\sall\sOOM\serrors\sare\sreported\sout\nto\sthe\sapplication. +D 2012-08-21T13:08:15.734 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in abd5c10d21d1395f140d9e50ea999df8fa4d6376 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -221,7 +221,7 @@ F src/test_quota.h 8761e463b25e75ebc078bd67d70e39b9c817a0cb F src/test_rtree.c aba603c949766c4193f1068b91c787f57274e0d9 F src/test_schema.c 8c06ef9ddb240c7a0fcd31bc221a6a2aade58bf0 F src/test_server.c 2f99eb2837dfa06a4aacf24af24c6affdf66a84f -F src/test_spellfix.c deab0f9caf853d2ddbee7c4a680ad27621adf1bf +F src/test_spellfix.c 5cc2bfe3fe4b3e6d7b0b0807d36a485a3a15a1ae F src/test_stat.c d1569c7a4839f13e80187e2c26b2ab4da2d03935 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd F src/test_syscall.c a992d8c80ea91fbf21fb2dd570db40e77dd7e6ae @@ -1011,7 +1011,7 @@ F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/win/sqlite.vsix 67d8a99aceb56384a81b3f30d6c71743146d2cc9 -P be1faadebd9464f1c7d4cc26104f219ed35384b8 -R 9d3902f63cfac6bb8ee3fe0483a0e736 -U dan -Z a16ab6913bdfea9d4fad9944795f8c23 +P e799222f3b8246e65657a758437914ece7069ba9 +R 41231569c08717226d41316ed2c512be +U drh +Z 09392d5c35b740ca7443d8f9f40e84b9 diff --git a/manifest.uuid b/manifest.uuid index 61af4bab2b..51c86e8c1f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e799222f3b8246e65657a758437914ece7069ba9 \ No newline at end of file +c2cf498513c2633bd2b08372772eaa0f3b3ab25f \ No newline at end of file diff --git a/src/test_spellfix.c b/src/test_spellfix.c index ec73b0b3cf..fe6992e535 100644 --- a/src/test_spellfix.c +++ b/src/test_spellfix.c @@ -2508,15 +2508,17 @@ static int spellfix1Filter( */ static int spellfix1Next(sqlite3_vtab_cursor *cur){ spellfix1_cursor *pCur = (spellfix1_cursor *)cur; + rc = SQLITE_OK; if( pCur->iRow < pCur->nRow ){ if( pCur->pFullScan ){ - int rc = sqlite3_step(pCur->pFullScan); + rc = sqlite3_step(pCur->pFullScan); if( rc!=SQLITE_ROW ) pCur->iRow = pCur->nRow; + if( rc==SQLITE_ROW || rc==SQLITE_DONE ) rc = SQLITE_OK; }else{ pCur->iRow++; } } - return SQLITE_OK; + return rc; } /* @@ -2773,25 +2775,35 @@ static sqlite3_module spellfix1Module = { ** Register the various functions and the virtual table. */ static int spellfix1Register(sqlite3 *db){ - int nErr = 0; + int rc = SQLITE_OK; int i; - nErr += sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0, + rc = sqlite3_create_function(db, "spellfix1_translit", 1, SQLITE_UTF8, 0, transliterateSqlFunc, 0, 0); - nErr += sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0, + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "spellfix1_editdist", 2, SQLITE_UTF8, 0, editdistSqlFunc, 0, 0); - nErr += sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0, + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "spellfix1_phonehash", 1, SQLITE_UTF8, 0, phoneticHashSqlFunc, 0, 0); - nErr += sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0, + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "spellfix1_scriptcode", 1, SQLITE_UTF8, 0, scriptCodeSqlFunc, 0, 0); - nErr += sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0); - nErr += editDist3Install(db); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "spellfix1", &spellfix1Module, 0); + } + if( rc==SQLITE_OK ){ + rc = editDist3Install(db); + } /* Verify sanity of the translit[] table */ for(i=0; i Date: Mon, 17 Mar 2014 15:43:05 +0000 Subject: [PATCH 006/710] Add an experimental multi-threaded capability to vdbesorter.c. FossilOrigin-Name: ff0b5c851ba7d04d1836d7c6a3222713e7d8d357 --- manifest | 22 +- manifest.uuid | 2 +- src/threads.c | 1 - src/vdbesort.c | 812 +++++++++++++++++++++++++++++++++---------------- 4 files changed, 567 insertions(+), 270 deletions(-) diff --git a/manifest b/manifest index 4f0c8a0be4..2de90048c0 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Merge\slatest\strunk\schanges\sinto\sthis\sbranch. -D 2014-03-13T15:41:09.146 +C Add\san\sexperimental\smulti-threaded\scapability\sto\svdbesorter.c. +D 2014-03-17T15:43:05.543 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc 153eb9b9725bc7b8e4dbe963219298e0c4a644b0 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 -F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 w README +F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION c3b0d47c3c5cf25c5bd4ff9e6f3af2f9d7934ea6 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 @@ -108,17 +108,17 @@ F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 678056a4bfcd83c4e82dea81d37543cd1d6dbee1 F ext/misc/closure.c 636024302cde41b2bf0c542f81c40c624cfb7012 -F ext/misc/fuzzer.c 136533c53cfce0957f0b48fa11dba27e21c5c01d w src/test_fuzzer.c +F ext/misc/fuzzer.c 136533c53cfce0957f0b48fa11dba27e21c5c01d F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a -F ext/misc/spellfix.c 93f3961074cebe63c31fcefe62ca2a032ee8dfed w src/test_spellfix.c +F ext/misc/spellfix.c 93f3961074cebe63c31fcefe62ca2a032ee8dfed F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e -F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 w src/test_wholenumber.c +F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 F ext/rtree/rtree.c 2d9f95da404d850474e628c720c5ce15d29b47de F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e @@ -272,7 +272,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 2b918d1f4f0b0831e8f41c49bcaa097f01490120 +F src/threads.c cde9d885fd562b5427f89a42a8829085f88b17df F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e45e3f9daf38c5be3fd39e9aacc1c9066af57a06 F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 +F src/vdbesort.c 1d973fcd0e00c77836e9d505e7c45df02601ffc2 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c92b0fe1371e7c20a5fbdf5fa96e30da14c40880 6504aa47a8ebb13827be017c4cb4b2dc3c4c55f4 -R c6806fc7f1eff740ed4af1293ffc55d0 +P d17231b63d48c1f9c4dee109c90cec112e2f0fd4 +R 3c79acd4eecb6926efd889b756a27898 U dan -Z b3b09b0f0acb184b570781ded35d947c +Z 226b72b7faaae8e6d7b6586f32d5c5a0 diff --git a/manifest.uuid b/manifest.uuid index c9d158cd10..bf397f3d56 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d17231b63d48c1f9c4dee109c90cec112e2f0fd4 \ No newline at end of file +ff0b5c851ba7d04d1836d7c6a3222713e7d8d357 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 33781a7aac..7cc964250d 100644 --- a/src/threads.c +++ b/src/threads.c @@ -47,7 +47,6 @@ int sqlite3ThreadCreate( void *pIn /* Argument passed into xTask() */ ){ SQLiteThread *p; - int rc; assert( ppThread!=0 ); assert( xTask!=0 ); diff --git a/src/vdbesort.c b/src/vdbesort.c index b9ed97e8b3..e84a33fccc 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -20,9 +20,83 @@ typedef struct VdbeSorterIter VdbeSorterIter; +typedef struct SorterThread SorterThread; typedef struct SorterRecord SorterRecord; +typedef struct SorterMerger SorterMerger; typedef struct FileWriter FileWriter; + +/* +** Maximum number of threads to use. Setting this value to 1 forces all +** operations to be single-threaded. +*/ +#ifndef SQLITE_MAX_SORTER_THREAD +# define SQLITE_MAX_SORTER_THREAD 1 +#endif + +/* +** Candidate values for SorterThread.eWork +*/ +#define SORTER_THREAD_SORT 1 +#define SORTER_THREAD_TO_PMA 2 +#define SORTER_THREAD_CONS 3 + +/* +** Much of the work performed in this module to sort the list of records is +** broken down into smaller units that may be peformed in parallel. In order +** to perform such a unit of work, an instance of the following structure +** is configured and passed to vdbeSorterThreadMain() - either directly by +** the main thread or via a background thread. +** +** Exactly SQLITE_MAX_SORTER_THREAD instances of this structure are allocated +** as part of each VdbeSorter object. Instances are never allocated any other +** way. +** +** When a background thread is launched to perform work, SorterThread.bDone +** is set to 0 and the SorterThread.pThread variable set to point to the +** thread handle. SorterThread.bDone is set to 1 (to indicate to the main +** thread that joining SorterThread.pThread will not block) before the thread +** exits. SorterThread.pThread and bDone are always cleared after the +** background thread has been joined. +** +** One object (specifically, VdbeSorter.aThread[SQLITE_MAX_SORTER_THREAD-1]) +** is reserved for the foreground thread. +** +** The nature of the work performed is determined by SorterThread.eWork, +** as follows: +** +** SORTER_THREAD_SORT: +** Sort the linked list of records at SorterThread.pList. +** +** SORTER_THREAD_TO_PMA: +** Sort the linked list of records at SorterThread.pList, and write +** the results to a new PMA in temp file SorterThread.pTemp1. Open +** the temp file if it is not already open. +** +** SORTER_THREAD_CONS: +** Merge existing PMAs until SorterThread.nConsolidate or fewer +** remain in temp file SorterThread.pTemp1. +*/ +struct SorterThread { + SQLiteThread *pThread; /* Thread handle, or NULL */ + int bDone; /* Set to true by pThread when finished */ + + sqlite3_vfs *pVfs; /* VFS used to open temporary files */ + KeyInfo *pKeyInfo; /* How to compare records */ + UnpackedRecord *pUnpacked; /* Space to unpack a record */ + int pgsz; /* Main database page size */ + + u8 eWork; /* One of the SORTER_THREAD_* constants */ + int nConsolidate; /* For THREAD_CONS, max final PMAs */ + SorterRecord *pList; /* List of records for pThread to sort */ + int nInMemory; /* Expected size of PMA based on pList */ + + int nPMA; /* Number of PMAs currently in pTemp1 */ + i64 iTemp1Off; /* Offset to write to in pTemp1 */ + sqlite3_file *pTemp1; /* File to write PMAs to, or NULL */ +}; + + /* ** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES: ** @@ -92,19 +166,24 @@ typedef struct FileWriter FileWriter; ** key comparison operations are required, where N is the number of segments ** being merged (rounded up to the next power of 2). */ -struct VdbeSorter { - i64 iWriteOff; /* Current write offset within file pTemp1 */ - i64 iReadOff; /* Current read offset within file pTemp1 */ - int nInMemory; /* Current size of pRecord list as PMA */ +struct SorterMerger { int nTree; /* Used size of aTree/aIter (power of 2) */ - int nPMA; /* Number of PMAs stored in pTemp1 */ + int *aTree; /* Current state of incremental merge */ + VdbeSorterIter *aIter; /* Array of iterators to merge data from */ +}; + +/* +** Main sorter structure. A single instance of this is allocated for each +** sorter cursor created by the VDBE. +*/ +struct VdbeSorter { + int nInMemory; /* Current size of pRecord list as PMA */ int mnPmaSize; /* Minimum PMA size, in bytes */ int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ - VdbeSorterIter *aIter; /* Array of iterators to merge */ - int *aTree; /* Current state of incremental merge */ - sqlite3_file *pTemp1; /* PMA file 1 */ + int bUsePMA; /* True if one or more PMAs created */ SorterRecord *pRecord; /* Head of in-memory record list */ - UnpackedRecord *pUnpacked; /* Used to unpack keys */ + SorterMerger *pMerger; /* For final merge of PMAs (by caller) */ + SorterThread aThread[SQLITE_MAX_SORTER_THREAD]; }; /* @@ -150,7 +229,8 @@ struct SorterRecord { SorterRecord *pNext; }; -/* Minimum allowable value for the VdbeSorter.nWorking variable */ +/* The minimum PMA size is set to this value multiplied by the database +** page size in bytes. */ #define SORTER_MIN_WORKING 10 /* Maximum number of segments to merge in a single pass. */ @@ -160,9 +240,9 @@ struct SorterRecord { ** Free all memory belonging to the VdbeSorterIter object passed as the second ** argument. All structure fields are set to zero before returning. */ -static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){ - sqlite3DbFree(db, pIter->aAlloc); - sqlite3DbFree(db, pIter->aBuffer); +static void vdbeSorterIterZero(VdbeSorterIter *pIter){ + sqlite3_free(pIter->aAlloc); + sqlite3_free(pIter->aBuffer); memset(pIter, 0, sizeof(VdbeSorterIter)); } @@ -176,7 +256,6 @@ static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){ ** next call to this function. */ static int vdbeSorterIterRead( - sqlite3 *db, /* Database handle (for malloc) */ VdbeSorterIter *p, /* Iterator */ int nByte, /* Bytes of data to read */ u8 **ppOut /* OUT: Pointer to buffer containing data */ @@ -222,11 +301,13 @@ static int vdbeSorterIterRead( /* Extend the p->aAlloc[] allocation if required. */ if( p->nAllocnAlloc*2; while( nByte>nNew ) nNew = nNew*2; - p->aAlloc = sqlite3DbReallocOrFree(db, p->aAlloc, nNew); - if( !p->aAlloc ) return SQLITE_NOMEM; + aNew = sqlite3Realloc(p->aAlloc, nNew); + if( !aNew ) return SQLITE_NOMEM; p->nAlloc = nNew; + p->aAlloc = aNew; } /* Copy as much data as is available in the buffer into the start of @@ -244,7 +325,7 @@ static int vdbeSorterIterRead( nCopy = nRem; if( nRem>p->nBuffer ) nCopy = p->nBuffer; - rc = vdbeSorterIterRead(db, p, nCopy, &aNext); + rc = vdbeSorterIterRead(p, nCopy, &aNext); if( rc!=SQLITE_OK ) return rc; assert( aNext!=p->aAlloc ); memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); @@ -261,7 +342,7 @@ static int vdbeSorterIterRead( ** Read a varint from the stream of data accessed by p. Set *pnOut to ** the value read. */ -static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ +static int vdbeSorterIterVarint(VdbeSorterIter *p, u64 *pnOut){ int iBuf; iBuf = p->iReadOff % p->nBuffer; @@ -271,7 +352,7 @@ static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ u8 aVarint[16], *a; int i = 0, rc; do{ - rc = vdbeSorterIterRead(db, p, 1, &a); + rc = vdbeSorterIterRead(p, 1, &a); if( rc ) return rc; aVarint[(i++)&0xf] = a[0]; }while( (a[0]&0x80)!=0 ); @@ -286,23 +367,20 @@ static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ ** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if ** no error occurs, or an SQLite error code if one does. */ -static int vdbeSorterIterNext( - sqlite3 *db, /* Database handle (for sqlite3DbMalloc() ) */ - VdbeSorterIter *pIter /* Iterator to advance */ -){ +static int vdbeSorterIterNext(VdbeSorterIter *pIter){ int rc; /* Return Code */ u64 nRec = 0; /* Size of record in bytes */ if( pIter->iReadOff>=pIter->iEof ){ /* This is an EOF condition */ - vdbeSorterIterZero(db, pIter); + vdbeSorterIterZero(pIter); return SQLITE_OK; } - rc = vdbeSorterIterVarint(db, pIter, &nRec); + rc = vdbeSorterIterVarint(pIter, &nRec); if( rc==SQLITE_OK ){ pIter->nKey = (int)nRec; - rc = vdbeSorterIterRead(db, pIter, (int)nRec, &pIter->aKey); + rc = vdbeSorterIterRead(pIter, (int)nRec, &pIter->aKey); } return rc; @@ -315,26 +393,23 @@ static int vdbeSorterIterNext( ** PMA is empty). */ static int vdbeSorterIterInit( - sqlite3 *db, /* Database handle */ - const VdbeSorter *pSorter, /* Sorter object */ - i64 iStart, /* Start offset in pFile */ + SorterThread *pThread, /* Thread context */ + i64 iStart, /* Start offset in pThread->pTemp1 */ VdbeSorterIter *pIter, /* Iterator to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ ){ int rc = SQLITE_OK; - int nBuf; + int nBuf = pThread->pgsz; - nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - - assert( pSorter->iWriteOff>iStart ); + assert( pThread->iTemp1Off>iStart ); assert( pIter->aAlloc==0 ); assert( pIter->aBuffer==0 ); - pIter->pFile = pSorter->pTemp1; + pIter->pFile = pThread->pTemp1; pIter->iReadOff = iStart; pIter->nAlloc = 128; - pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc); + pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc); pIter->nBuffer = nBuf; - pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); + pIter->aBuffer = (u8*)sqlite3Malloc(nBuf); if( !pIter->aBuffer ){ rc = SQLITE_NOMEM; @@ -344,26 +419,26 @@ static int vdbeSorterIterInit( iBuf = iStart % nBuf; if( iBuf ){ int nRead = nBuf - iBuf; - if( (iStart + nRead) > pSorter->iWriteOff ){ - nRead = (int)(pSorter->iWriteOff - iStart); + if( (iStart + nRead) > pThread->iTemp1Off ){ + nRead = (int)(pThread->iTemp1Off - iStart); } rc = sqlite3OsRead( - pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart + pThread->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart ); assert( rc!=SQLITE_IOERR_SHORT_READ ); } if( rc==SQLITE_OK ){ - u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pSorter->iWriteOff; - rc = vdbeSorterIterVarint(db, pIter, &nByte); + u64 nByte; + pIter->iEof = pThread->iTemp1Off; + rc = vdbeSorterIterVarint(pIter, &nByte); pIter->iEof = pIter->iReadOff + nByte; *pnByte += nByte; } } if( rc==SQLITE_OK ){ - rc = vdbeSorterIterNext(db, pIter); + rc = vdbeSorterIterNext(pIter); } return rc; } @@ -385,15 +460,14 @@ static int vdbeSorterIterInit( ** has been allocated and contains an unpacked record that is used as key2. */ static void vdbeSorterCompare( - const VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ + SorterThread *pThread, /* Thread context (for pKeyInfo) */ int nIgnore, /* Ignore the last nIgnore fields */ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2, /* Right side of comparison */ int *pRes /* OUT: Result of comparison */ ){ - KeyInfo *pKeyInfo = pCsr->pKeyInfo; - VdbeSorter *pSorter = pCsr->pSorter; - UnpackedRecord *r2 = pSorter->pUnpacked; + KeyInfo *pKeyInfo = pThread->pKeyInfo; + UnpackedRecord *r2 = pThread->pUnpacked; int i; if( pKey2 ){ @@ -420,26 +494,29 @@ static void vdbeSorterCompare( ** multiple b-tree segments. Parameter iOut is the index of the aTree[] ** value to recalculate. */ -static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ - VdbeSorter *pSorter = pCsr->pSorter; +static int vdbeSorterDoCompare( + SorterThread *pThread, + SorterMerger *pMerger, + int iOut +){ int i1; int i2; int iRes; VdbeSorterIter *p1; VdbeSorterIter *p2; - assert( iOutnTree && iOut>0 ); + assert( iOutnTree && iOut>0 ); - if( iOut>=(pSorter->nTree/2) ){ - i1 = (iOut - pSorter->nTree/2) * 2; + if( iOut>=(pMerger->nTree/2) ){ + i1 = (iOut - pMerger->nTree/2) * 2; i2 = i1 + 1; }else{ - i1 = pSorter->aTree[iOut*2]; - i2 = pSorter->aTree[iOut*2+1]; + i1 = pMerger->aTree[iOut*2]; + i2 = pMerger->aTree[iOut*2+1]; } - p1 = &pSorter->aIter[i1]; - p2 = &pSorter->aIter[i2]; + p1 = &pMerger->aIter[i1]; + p2 = &pMerger->aIter[i2]; if( p1->pFile==0 ){ iRes = i2; @@ -447,9 +524,9 @@ static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ iRes = i1; }else{ int res; - assert( pCsr->pSorter->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ + assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ vdbeSorterCompare( - pCsr, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res + pThread, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res ); if( res<=0 ){ iRes = i1; @@ -458,7 +535,7 @@ static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ } } - pSorter->aTree[iOut] = iRes; + pMerger->aTree[iOut] = iRes; return SQLITE_OK; } @@ -467,22 +544,32 @@ static int vdbeSorterDoCompare(const VdbeCursor *pCsr, int iOut){ */ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ int pgsz; /* Page size of main database */ + int i; /* Used to iterate through aThread[] */ int mxCache; /* Cache size */ VdbeSorter *pSorter; /* The new sorter */ - char *d; /* Dummy */ + KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ + int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ assert( pCsr->pKeyInfo && pCsr->pBt==0 ); - pCsr->pSorter = pSorter = sqlite3DbMallocZero(db, sizeof(VdbeSorter)); + szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); + pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sizeof(VdbeSorter)+szKeyInfo); + pCsr->pSorter = pSorter; if( pSorter==0 ){ return SQLITE_NOMEM; } - - pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, 0, 0, &d); - if( pSorter->pUnpacked==0 ) return SQLITE_NOMEM; - assert( pSorter->pUnpacked==(UnpackedRecord *)d ); + pKeyInfo = (KeyInfo*)&pSorter[1]; + memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); + pKeyInfo->db = 0; + pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + + for(i=0; iaThread[i]; + pThread->pKeyInfo = pKeyInfo; + pThread->pVfs = db->pVfs; + pThread->pgsz = pgsz; + } if( !sqlite3TempInMemory(db) ){ - pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; mxCache = db->aDb[0].pSchema->cache_size; if( mxCachepUnpacked); + vdbeSorterRecordFree(0, pThread->pList); + if( pThread->pTemp1 ){ + sqlite3OsCloseFree(pThread->pTemp1); + } + memset(pThread, 0, sizeof(SorterThread)); +} + +/* +** Join all threads. +*/ +static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ + int rc = rcin; + int i; + for(i=0; iaThread[i]; + if( pThread->pThread ){ + void *pRet; + int rc2 = sqlite3ThreadJoin(pThread->pThread, &pRet); + pThread->pThread = 0; + pThread->bDone = 0; + if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); + } + } + return rc; +} + +/* +** Allocate a new SorterMerger object with space for nIter iterators. +*/ +static SorterMerger *vdbeSorterMergerNew(int nIter){ + int N = 2; /* Smallest power of two >= nIter */ + int nByte; /* Total bytes of space to allocate */ + SorterMerger *pNew; /* Pointer to allocated object to return */ + + assert( nIter<=SORTER_MAX_MERGE_COUNT ); + while( NnTree = N; + pNew->aIter = (VdbeSorterIter*)&pNew[1]; + pNew->aTree = (int*)&pNew->aIter[N]; + } + + return pNew; +} + +/* +** Free the SorterMerger object passed as the only argument. +*/ +static void vdbeSorterMergerFree(SorterMerger *pMerger){ + if( pMerger ){ + int i; + for(i=0; inTree; i++){ + vdbeSorterIterZero(&pMerger->aIter[i]); + } + sqlite3_free(pMerger); + } +} + /* ** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. */ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ - if( pSorter->aIter ){ - int i; - for(i=0; inTree; i++){ - vdbeSorterIterZero(db, &pSorter->aIter[i]); - } - sqlite3DbFree(db, pSorter->aIter); + int i; + vdbeSorterJoinAll(pSorter, SQLITE_OK); + for(i=0; iaThread[i]; + vdbeSorterThreadCleanup(db, pThread); } - if( pSorter->pTemp1 ){ - sqlite3OsCloseFree(pSorter->pTemp1); - } - vdbeSorterRecordFree(db, pSorter->pRecord); - sqlite3DbFree(db, pSorter->pUnpacked); + + vdbeSorterRecordFree(0, pSorter->pRecord); + vdbeSorterMergerFree(pSorter->pMerger); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } @@ -532,9 +684,9 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ ** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK. ** Otherwise, set *ppFile to 0 and return an SQLite error code. */ -static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ +static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){ int dummy; - return sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, + return sqlite3OsOpenMalloc(pVfs, 0, ppFile, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &dummy @@ -546,7 +698,7 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ ** Set *ppOut to the head of the new list. */ static void vdbeSorterMerge( - const VdbeCursor *pCsr, /* For pKeyInfo */ + SorterThread *pThread, /* Calling thread context */ SorterRecord *p1, /* First list to merge */ SorterRecord *p2, /* Second list to merge */ SorterRecord **ppOut /* OUT: Head of merged list */ @@ -557,7 +709,7 @@ static void vdbeSorterMerge( while( p1 && p2 ){ int res; - vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + vdbeSorterCompare(pThread, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); if( res<=0 ){ *pp = p1; pp = &p1->pNext; @@ -576,27 +728,26 @@ static void vdbeSorterMerge( } /* -** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK -** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error -** occurs. +** Sort the linked list of records headed at pThread->pList. Return +** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if +** an error occurs. */ -static int vdbeSorterSort(const VdbeCursor *pCsr){ +static int vdbeSorterSort(SorterThread *pThread){ int i; SorterRecord **aSlot; SorterRecord *p; - VdbeSorter *pSorter = pCsr->pSorter; aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ return SQLITE_NOMEM; } - p = pSorter->pRecord; + p = pThread->pList; while( p ){ SorterRecord *pNext = p->pNext; p->pNext = 0; for(i=0; aSlot[i]; i++){ - vdbeSorterMerge(pCsr, p, aSlot[i], &p); + vdbeSorterMerge(pThread, p, aSlot[i], &p); aSlot[i] = 0; } aSlot[i] = p; @@ -605,9 +756,9 @@ static int vdbeSorterSort(const VdbeCursor *pCsr){ p = 0; for(i=0; i<64; i++){ - vdbeSorterMerge(pCsr, p, aSlot[i], &p); + vdbeSorterMerge(pThread, p, aSlot[i], &p); } - pSorter->pRecord = p; + pThread->pList = p; sqlite3_free(aSlot); return SQLITE_OK; @@ -617,15 +768,13 @@ static int vdbeSorterSort(const VdbeCursor *pCsr){ ** Initialize a file-writer object. */ static void fileWriterInit( - sqlite3 *db, /* Database (for malloc) */ sqlite3_file *pFile, /* File to write to */ FileWriter *p, /* Object to populate */ + int nBuf, /* Buffer size */ i64 iStart /* Offset of pFile to begin writing at */ ){ - int nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - memset(p, 0, sizeof(FileWriter)); - p->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); + p->aBuffer = (u8*)sqlite3Malloc(nBuf); if( !p->aBuffer ){ p->eFWErr = SQLITE_NOMEM; }else{ @@ -673,7 +822,7 @@ static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ ** Before returning, set *piEof to the offset immediately following the ** last byte written to the file. */ -static int fileWriterFinish(sqlite3 *db, FileWriter *p, i64 *piEof){ +static int fileWriterFinish(FileWriter *p, i64 *piEof){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ p->eFWErr = sqlite3OsWrite(p->pFile, @@ -682,7 +831,7 @@ static int fileWriterFinish(sqlite3 *db, FileWriter *p, i64 *piEof){ ); } *piEof = (p->iWriteOff + p->iBufEnd); - sqlite3DbFree(db, p->aBuffer); + sqlite3_free(p->aBuffer); rc = p->eFWErr; memset(p, 0, sizeof(FileWriter)); return rc; @@ -712,43 +861,234 @@ static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ +static int vdbeSorterListToPMA(SorterThread *pThread){ int rc = SQLITE_OK; /* Return code */ - VdbeSorter *pSorter = pCsr->pSorter; - FileWriter writer; + FileWriter writer; /* Object used to write to the file */ memset(&writer, 0, sizeof(FileWriter)); - - if( pSorter->nInMemory==0 ){ - assert( pSorter->pRecord==0 ); - return rc; - } - - rc = vdbeSorterSort(pCsr); + assert( pThread->nInMemory>0 ); /* If the first temporary PMA file has not been opened, open it now. */ - if( rc==SQLITE_OK && pSorter->pTemp1==0 ){ - rc = vdbeSorterOpenTempFile(db, &pSorter->pTemp1); - assert( rc!=SQLITE_OK || pSorter->pTemp1 ); - assert( pSorter->iWriteOff==0 ); - assert( pSorter->nPMA==0 ); + if( pThread->pTemp1==0 ){ + rc = vdbeSorterOpenTempFile(pThread->pVfs, &pThread->pTemp1); + assert( rc!=SQLITE_OK || pThread->pTemp1 ); + assert( pThread->iTemp1Off==0 ); + assert( pThread->nPMA==0 ); } if( rc==SQLITE_OK ){ SorterRecord *p; SorterRecord *pNext = 0; - fileWriterInit(db, pSorter->pTemp1, &writer, pSorter->iWriteOff); - pSorter->nPMA++; - fileWriterWriteVarint(&writer, pSorter->nInMemory); - for(p=pSorter->pRecord; p; p=pNext){ + fileWriterInit(pThread->pTemp1, &writer, pThread->pgsz, pThread->iTemp1Off); + pThread->nPMA++; + fileWriterWriteVarint(&writer, pThread->nInMemory); + for(p=pThread->pList; p; p=pNext){ pNext = p->pNext; fileWriterWriteVarint(&writer, p->nVal); fileWriterWrite(&writer, p->pVal, p->nVal); - sqlite3DbFree(db, p); + sqlite3_free(p); + } + pThread->pList = p; + rc = fileWriterFinish(&writer, &pThread->iTemp1Off); + } + + return rc; +} + +/* +** Advance the SorterMerger iterator passed as the second argument to +** the next entry. Set *pbEof to true if this means the iterator has +** reached EOF. +** +** Return SQLITE_OK if successful or an error code if an error occurs. +*/ +static int vdbeSorterNext( + SorterThread *pThread, + SorterMerger *pMerger, + int *pbEof +){ + int rc; + int iPrev = pMerger->aTree[1];/* Index of iterator to advance */ + int i; /* Index of aTree[] to recalculate */ + + /* Advance the current iterator */ + rc = vdbeSorterIterNext(&pMerger->aIter[iPrev]); + + /* Update contents of aTree[] */ + for(i=(pMerger->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ + rc = vdbeSorterDoCompare(pThread, pMerger, i); + } + + *pbEof = (pMerger->aIter[pMerger->aTree[1]].pFile==0); + return rc; +} + +/* +** The main routine for sorter-thread operations. +*/ +static void *vdbeSorterThreadMain(void *pCtx){ + int rc = SQLITE_OK; + SorterThread *pThread = (SorterThread*)pCtx; + + assert( pThread->eWork==SORTER_THREAD_SORT + || pThread->eWork==SORTER_THREAD_TO_PMA + || pThread->eWork==SORTER_THREAD_CONS + ); + assert( pThread->bDone==0 ); + + if( pThread->pUnpacked==0 ){ + char *pFree; + pThread->pUnpacked = sqlite3VdbeAllocUnpackedRecord( + pThread->pKeyInfo, 0, 0, &pFree + ); + assert( pThread->pUnpacked==(UnpackedRecord*)pFree ); + if( pFree==0 ){ + rc = SQLITE_NOMEM; + goto thread_out; + } + } + + if( pThread->eWork==SORTER_THREAD_CONS ){ + assert( pThread->pList==0 ); + while( pThread->nPMA>pThread->nConsolidate && rc==SQLITE_OK ){ + int nIter = MIN(pThread->nPMA, SORTER_MAX_MERGE_COUNT); + sqlite3_file *pTemp2 = 0; /* Second temp file to use */ + SorterMerger *pMerger; /* Object for reading/merging PMA data */ + i64 iReadOff = 0; /* Offset in pTemp1 to read from */ + i64 iWriteOff = 0; /* Offset in pTemp2 to write to */ + int i; + + /* Allocate a merger object to merge PMAs together. */ + pMerger = vdbeSorterMergerNew(nIter); + if( pMerger==0 ){ + rc = SQLITE_NOMEM; + break; + } + + /* Open a second temp file to write merged data to */ + rc = vdbeSorterOpenTempFile(pThread->pVfs, &pTemp2); + if( rc!=SQLITE_OK ){ + vdbeSorterMergerFree(pMerger); + break; + } + + /* This loop runs once for each output PMA. Each output PMA is made + ** of data merged from up to SORTER_MAX_MERGE_COUNT input PMAs. */ + for(i=0; inPMA; i+=SORTER_MAX_MERGE_COUNT){ + FileWriter writer; /* Object for writing data to pTemp2 */ + i64 nOut = 0; /* Bytes of data in output PMA */ + int bEof = 0; + int rc2; + + /* Configure the merger object to read and merge data from the next + ** SORTER_MAX_MERGE_COUNT PMAs in pTemp1 (or from all remaining PMAs, + ** if that is fewer). */ + int iIter; + for(iIter=0; iIteraIter[iIter]; + rc = vdbeSorterIterInit(pThread, iReadOff, pIter, &nOut); + iReadOff = pIter->iEof; + if( iReadOff>=pThread->iTemp1Off || rc!=SQLITE_OK ) break; + } + for(iIter=pMerger->nTree-1; rc==SQLITE_OK && iIter>0; iIter--){ + rc = vdbeSorterDoCompare(pThread, pMerger, iIter); + } + + fileWriterInit(pTemp2, &writer, pThread->pgsz, iWriteOff); + fileWriterWriteVarint(&writer, nOut); + while( rc==SQLITE_OK && bEof==0 ){ + VdbeSorterIter *pIter = &pMerger->aIter[ pMerger->aTree[1] ]; + assert( pIter->pFile!=0 ); /* pIter is not at EOF */ + fileWriterWriteVarint(&writer, pIter->nKey); + fileWriterWrite(&writer, pIter->aKey, pIter->nKey); + rc = vdbeSorterNext(pThread, pMerger, &bEof); + } + rc2 = fileWriterFinish(&writer, &iWriteOff); + if( rc==SQLITE_OK ) rc = rc2; + } + + vdbeSorterMergerFree(pMerger); + sqlite3OsCloseFree(pThread->pTemp1); + pThread->pTemp1 = pTemp2; + pThread->nPMA = (i / SORTER_MAX_MERGE_COUNT); + pThread->iTemp1Off = iWriteOff; + } + }else{ + /* Sort the pThread->pList list */ + rc = vdbeSorterSort(pThread); + + /* If required, write the list out to a PMA. */ + if( rc==SQLITE_OK && pThread->eWork==SORTER_THREAD_TO_PMA ){ +#ifdef SQLITE_DEBUG + i64 nExpect = pThread->nInMemory + + sqlite3VarintLen(pThread->nInMemory) + + pThread->iTemp1Off; +#endif + rc = vdbeSorterListToPMA(pThread); + assert( rc!=SQLITE_OK || (nExpect==pThread->iTemp1Off) ); + } + } + + thread_out: + pThread->bDone = 1; + return SQLITE_INT_TO_PTR(rc); +} + +/* +** Run the activity scheduled by the object passed as the only argument +** in the current thread. +*/ +static int vdbeSorterRunThread(SorterThread *pThread){ + int rc = SQLITE_PTR_TO_INT( vdbeSorterThreadMain((void*)pThread) ); + assert( pThread->bDone ); + pThread->bDone = 0; + return rc; +} + +/* +** Flush the current contents of VdbeSorter.pRecord to a new PMA, possibly +** using a background thread. +** +** If argument bFg is non-zero, the operation always uses the calling thread. +*/ +static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ + VdbeSorter *pSorter = pCsr->pSorter; + int rc = SQLITE_OK; + int i; + SorterThread *pThread; /* Thread context used to create new PMA */ + + pSorter->bUsePMA = 1; + for(i=0; ALWAYS( iaThread[i]; + if( pThread->bDone ){ + void *pRet; + assert( pThread->pThread ); + rc = sqlite3ThreadJoin(pThread->pThread, &pRet); + pThread->pThread = 0; + pThread->bDone = 0; + if( rc==SQLITE_OK ){ + rc = SQLITE_PTR_TO_INT(pRet); + } + } + if( pThread->pThread==0 ) break; + } + + if( rc==SQLITE_OK ){ + assert( pThread->pThread==0 && pThread->bDone==0 ); + pThread->eWork = SORTER_THREAD_TO_PMA; + pThread->pList = pSorter->pRecord; + pThread->nInMemory = pSorter->nInMemory; + pSorter->nInMemory = 0; + pSorter->pRecord = 0; + + if( bFg || i<(SQLITE_MAX_SORTER_THREAD-1) ){ + void *pCtx = (void*)pThread; + rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSorterThreadMain, pCtx); + }else{ + /* Use the foreground thread for this operation */ + rc = vdbeSorterRunThread(pThread); } - pSorter->pRecord = p; - rc = fileWriterFinish(db, &writer, &pSorter->iWriteOff); } return rc; @@ -769,7 +1109,7 @@ int sqlite3VdbeSorterWrite( assert( pSorter ); pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n + sizeof(SorterRecord)); + pNew = (SorterRecord *)sqlite3Malloc(pVal->n + sizeof(SorterRecord)); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ @@ -793,48 +1133,22 @@ int sqlite3VdbeSorterWrite( (pSorter->nInMemory>pSorter->mxPmaSize) || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) )){ -#ifdef SQLITE_DEBUG - i64 nExpect = pSorter->iWriteOff - + sqlite3VarintLen(pSorter->nInMemory) - + pSorter->nInMemory; -#endif - rc = vdbeSorterListToPMA(db, pCsr); - pSorter->nInMemory = 0; - assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + rc = vdbeSorterFlushPMA(db, pCsr, 0); } return rc; } /* -** Helper function for sqlite3VdbeSorterRewind(). +** Return the total number of PMAs in all temporary files. */ -static int vdbeSorterInitMerge( - sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Cursor handle for this sorter */ - i64 *pnByte /* Sum of bytes in all opened PMAs */ -){ - VdbeSorter *pSorter = pCsr->pSorter; - int rc = SQLITE_OK; /* Return code */ - int i; /* Used to iterator through aIter[] */ - i64 nByte = 0; /* Total bytes in all opened PMAs */ - - /* Initialize the iterators. */ - for(i=0; iaIter[i]; - rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte); - pSorter->iReadOff = pIter->iEof; - assert( rc!=SQLITE_OK || pSorter->iReadOff<=pSorter->iWriteOff ); - if( rc!=SQLITE_OK || pSorter->iReadOff>=pSorter->iWriteOff ) break; +static int vdbeSorterCountPMA(VdbeSorter *pSorter){ + int nPMA = 0; + int i; + for(i=0; iaThread[i].nPMA; } - - /* Initialize the aTree[] array. */ - for(i=pSorter->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pCsr, i); - } - - *pnByte = nByte; - return rc; + return nPMA; } /* @@ -843,107 +1157,98 @@ static int vdbeSorterInitMerge( */ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; - int rc; /* Return code */ - sqlite3_file *pTemp2 = 0; /* Second temp file to use */ - i64 iWrite2 = 0; /* Write offset for pTemp2 */ - int nIter; /* Number of iterators used */ - int nByte; /* Bytes of space required for aIter/aTree */ - int N = 2; /* Power of 2 >= nIter */ + int rc = SQLITE_OK; /* Return code */ assert( pSorter ); /* If no data has been written to disk, then do not do so now. Instead, ** sort the VdbeSorter.pRecord list. The vdbe layer will read data directly ** from the in-memory list. */ - if( pSorter->nPMA==0 ){ - *pbEof = !pSorter->pRecord; - assert( pSorter->aTree==0 ); - return vdbeSorterSort(pCsr); + if( pSorter->bUsePMA==0 ){ + if( pSorter->pRecord ){ + SorterThread *pThread = &pSorter->aThread[0]; + *pbEof = 0; + pThread->pList = pSorter->pRecord; + pThread->eWork = SORTER_THREAD_SORT; + rc = vdbeSorterRunThread(pThread); + pSorter->pRecord = pThread->pList; + pThread->pList = 0; + }else{ + *pbEof = 1; + } + return rc; } /* Write the current in-memory list to a PMA. */ - rc = vdbeSorterListToPMA(db, pCsr); - if( rc!=SQLITE_OK ) return rc; - - /* Allocate space for aIter[] and aTree[]. */ - nIter = pSorter->nPMA; - if( nIter>SORTER_MAX_MERGE_COUNT ) nIter = SORTER_MAX_MERGE_COUNT; - assert( nIter>0 ); - while( NaIter = (VdbeSorterIter *)sqlite3DbMallocZero(db, nByte); - if( !pSorter->aIter ) return SQLITE_NOMEM; - pSorter->aTree = (int *)&pSorter->aIter[N]; - pSorter->nTree = N; - - do { - int iNew; /* Index of new, merged, PMA */ - - for(iNew=0; - rc==SQLITE_OK && iNew*SORTER_MAX_MERGE_COUNTnPMA; - iNew++ - ){ - int rc2; /* Return code from fileWriterFinish() */ - FileWriter writer; /* Object used to write to disk */ - i64 nWrite; /* Number of bytes in new PMA */ - - memset(&writer, 0, sizeof(FileWriter)); - - /* If there are SORTER_MAX_MERGE_COUNT or less PMAs in file pTemp1, - ** initialize an iterator for each of them and break out of the loop. - ** These iterators will be incrementally merged as the VDBE layer calls - ** sqlite3VdbeSorterNext(). - ** - ** Otherwise, if pTemp1 contains more than SORTER_MAX_MERGE_COUNT PMAs, - ** initialize interators for SORTER_MAX_MERGE_COUNT of them. These PMAs - ** are merged into a single PMA that is written to file pTemp2. - */ - rc = vdbeSorterInitMerge(db, pCsr, &nWrite); - assert( rc!=SQLITE_OK || pSorter->aIter[ pSorter->aTree[1] ].pFile ); - if( rc!=SQLITE_OK || pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ - break; - } - - /* Open the second temp file, if it is not already open. */ - if( pTemp2==0 ){ - assert( iWrite2==0 ); - rc = vdbeSorterOpenTempFile(db, &pTemp2); - } - - if( rc==SQLITE_OK ){ - int bEof = 0; - fileWriterInit(db, pTemp2, &writer, iWrite2); - fileWriterWriteVarint(&writer, nWrite); - while( rc==SQLITE_OK && bEof==0 ){ - VdbeSorterIter *pIter = &pSorter->aIter[ pSorter->aTree[1] ]; - assert( pIter->pFile ); - - fileWriterWriteVarint(&writer, pIter->nKey); - fileWriterWrite(&writer, pIter->aKey, pIter->nKey); - rc = sqlite3VdbeSorterNext(db, pCsr, &bEof); - } - rc2 = fileWriterFinish(db, &writer, &iWrite2); - if( rc==SQLITE_OK ) rc = rc2; - } - } - - if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ - break; - }else{ - sqlite3_file *pTmp = pSorter->pTemp1; - pSorter->nPMA = iNew; - pSorter->pTemp1 = pTemp2; - pTemp2 = pTmp; - pSorter->iWriteOff = iWrite2; - pSorter->iReadOff = 0; - iWrite2 = 0; - } - }while( rc==SQLITE_OK ); - - if( pTemp2 ){ - sqlite3OsCloseFree(pTemp2); + if( pSorter->pRecord ){ + rc = vdbeSorterFlushPMA(db, pCsr, 1); + } + + /* Join all threads */ + rc = vdbeSorterJoinAll(pSorter, rc); + + /* If there are more than SORTER_MAX_MERGE_COUNT PMAs on disk, merge + ** some of them together so that this is no longer the case. */ + assert( SORTER_MAX_MERGE_COUNT>=SQLITE_MAX_SORTER_THREAD ); + if( vdbeSorterCountPMA(pSorter)>SORTER_MAX_MERGE_COUNT ){ + int i; + for(i=0; rc==SQLITE_OK && iaThread[i]; + if( pThread->pTemp1 ){ + pThread->nConsolidate = SORTER_MAX_MERGE_COUNT/SQLITE_MAX_SORTER_THREAD; + pThread->eWork = SORTER_THREAD_CONS; + + if( i<(SQLITE_MAX_SORTER_THREAD-1) ){ + void *pCtx = (void*)pThread; + rc = sqlite3ThreadCreate(&pThread->pThread,vdbeSorterThreadMain,pCtx); + }else{ + rc = vdbeSorterRunThread(pThread); + } + } + } + } + + /* Join all threads */ + rc = vdbeSorterJoinAll(pSorter, rc); + + /* Assuming no errors have occurred, set up a merger structure to read + ** and merge all remaining PMAs. */ + assert( pSorter->pMerger==0 ); + if( rc==SQLITE_OK ){ + int nIter = 0; /* Number of iterators used */ + int i; + SorterMerger *pMerger; + for(i=0; iaThread[i].nPMA; + } + + pSorter->pMerger = pMerger = vdbeSorterMergerNew(nIter); + if( pMerger==0 ){ + rc = SQLITE_NOMEM; + }else{ + int iIter = 0; + int iThread = 0; + for(iThread=0; iThreadaThread[iThread]; + for(iPMA=0; iPMAnPMA && rc==SQLITE_OK; iPMA++){ + i64 nDummy = 0; + VdbeSorterIter *pIter = &pMerger->aIter[iIter++]; + rc = vdbeSorterIterInit(pThread, iReadOff, pIter, &nDummy); + iReadOff = pIter->iEof; + } + } + + for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ + rc = vdbeSorterDoCompare(&pSorter->aThread[0], pMerger, i); + } + } + } + + if( rc==SQLITE_OK ){ + *pbEof = (pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]].pFile==0); } - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); return rc; } @@ -954,16 +1259,8 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; int rc; /* Return code */ - if( pSorter->aTree ){ - int iPrev = pSorter->aTree[1];/* Index of iterator to advance */ - int i; /* Index of aTree[] to recalculate */ - - rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); - for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ - rc = vdbeSorterDoCompare(pCsr, i); - } - - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + if( pSorter->pMerger ){ + rc = vdbeSorterNext(&pSorter->aThread[0], pSorter->pMerger, pbEof); }else{ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->pNext; @@ -984,9 +1281,9 @@ static void *vdbeSorterRowkey( int *pnKey /* OUT: Size of current key in bytes */ ){ void *pKey; - if( pSorter->aTree ){ + if( pSorter->pMerger ){ VdbeSorterIter *pIter; - pIter = &pSorter->aIter[ pSorter->aTree[1] ]; + pIter = &pSorter->pMerger->aIter[ pSorter->pMerger->aTree[1] ]; *pnKey = pIter->nKey; pKey = pIter->aKey; }else{ @@ -1031,9 +1328,10 @@ int sqlite3VdbeSorterCompare( int *pRes /* OUT: Result of comparison */ ){ VdbeSorter *pSorter = pCsr->pSorter; + SorterThread *pMain = &pSorter->aThread[0]; void *pKey; int nKey; /* Sorter key to compare pVal with */ pKey = vdbeSorterRowkey(pSorter, &nKey); - vdbeSorterCompare(pCsr, nIgnore, pVal->z, pVal->n, pKey, nKey, pRes); + vdbeSorterCompare(pMain, nIgnore, pVal->z, pVal->n, pKey, nKey, pRes); return SQLITE_OK; } From f45f2326a28870f9533d5e75f8daa7562708b96a Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 23 Mar 2014 17:45:03 +0000 Subject: [PATCH 007/710] Use only a single OP_MakeRecord instead of two when constructing entries to go onto a sorter. FossilOrigin-Name: d696cdedacd39075aa7fc407ab7c7e50f01d9f39 --- manifest | 14 ++++----- manifest.uuid | 2 +- src/select.c | 82 ++++++++++++++++++++++++++----------------------- test/tester.tcl | 2 ++ 4 files changed, 53 insertions(+), 47 deletions(-) diff --git a/manifest b/manifest index d2ff62a057..1724c9322b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sOFFSET-on-query-without-FROM\sfix\sfrom\strunk. -D 2014-03-21T18:45:19.215 +C Use\sonly\sa\ssingle\sOP_MakeRecord\sinstead\sof\stwo\swhen\sconstructing\sentries\nto\sgo\sonto\sa\ssorter. +D 2014-03-23T17:45:03.365 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 596b8098be41e6256968f167d1d8ece2be08d082 +F src/select.c b1e0ac15d846c1d584c26f95bd92e887313b55df F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -845,7 +845,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 -F test/tester.tcl f31bea1483ea1d39620f982130026e76f872d744 +F test/tester.tcl bc0889a2f86d9c17307992ca1e70391794780265 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e70cfa28aa393661ccc742ecd5e672d807bdd0a9 179ef81648b0ad557df78b7712f216b876b6fb65 -R 654a6218c0c034c3e6462d9f9d0a06b2 +P 71e9ae72c272dc86720b2bfe719f57de437c400b +R d93a4c6f8aefee44f30ed73f4e05ae09 U drh -Z 7a7ab7eaa2946af0174662f0ca0e6009 +Z 5ef6faddb00682067ed8bb051e6b9892 diff --git a/manifest.uuid b/manifest.uuid index d1f85a0601..b62e4b84bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71e9ae72c272dc86720b2bfe719f57de437c400b \ No newline at end of file +d696cdedacd39075aa7fc407ab7c7e50f01d9f39 \ No newline at end of file diff --git a/src/select.c b/src/select.c index f81a5ef3e7..6a3dff28a9 100644 --- a/src/select.c +++ b/src/select.c @@ -455,26 +455,29 @@ static KeyInfo *keyInfoFromExprList( ); /* -** Insert code into "v" that will push the record in register regData -** into the sorter. +** Generate code that will push the record in registers regData +** through regData+nData-1 onto the sorter. */ static void pushOntoSorter( Parse *pParse, /* Parser context */ SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ - int regData /* Register holding data to be sorted */ + int regData, /* First register holding data to be sorted */ + int nData /* Number of elements in the data array */ ){ - Vdbe *v = pParse->pVdbe; - int nExpr = pSort->pOrderBy->nExpr; - int regBase = sqlite3GetTempRange(pParse, nExpr+2); - int regRecord = sqlite3GetTempReg(pParse); - int nOBSat = pSort->nOBSat; - int op; + Vdbe *v = pParse->pVdbe; /* Stmt under construction */ + int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ + int nBase = nExpr + 1 + nData; /* Fields in sorter record */ + int regBase = sqlite3GetTempRange(pParse, nBase); /* Regs for sorter record */ + int regRecord = sqlite3GetTempReg(pParse); /* Assemblied sorter record */ + int nOBSat = pSort->nOBSat; /* No. ORDER BY terms to skip */ + int op; /* Opcode to add sorter record to sorter */ + sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); - sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat, regRecord); + sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ int addrFirst; /* Address of the OP_IfNot opcode */ @@ -509,7 +512,7 @@ static void pushOntoSorter( sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); if( nOBSat==0 ){ sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); + sqlite3ReleaseTempRange(pParse, regBase, nBase); } if( pSelect->iLimit ){ int addr1, addr2; @@ -773,7 +776,7 @@ static void selectInnerLoop( } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1); + pushOntoSorter(pParse, pSort, p, r1, 1); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -799,7 +802,7 @@ static void selectInnerLoop( ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult); + pushOntoSorter(pParse, pSort, p, regResult, 1); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -825,7 +828,7 @@ static void selectInnerLoop( case SRT_Mem: { assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult); + pushOntoSorter(pParse, pSort, p, regResult, 1); }else{ sqlite3ExprCodeMove(pParse, regResult, iParm, 1); /* The LIMIT clause will jump out of the loop for us */ @@ -839,10 +842,7 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - pushOntoSorter(pParse, pSort, p, r1); - sqlite3ReleaseTempReg(pParse, r1); + pushOntoSorter(pParse, pSort, p, regResult, nResultCol); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ @@ -1129,6 +1129,10 @@ static void generateSortTail( int regRow; int regRowid; int nKey; + int iSortTab; /* Sorter cursor to read from */ + int nSortData; /* Trailing values to read from sorter */ + u8 p5; /* p5 parameter for 1st OP_Column */ + int i; if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); @@ -1142,26 +1146,35 @@ static void generateSortTail( pseudoTab = pParse->nTab++; sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn); regRowid = 0; + regRow = pDest->iSdst; + nSortData = nColumn; }else{ regRowid = sqlite3GetTempReg(pParse); + regRow = sqlite3GetTempReg(pParse); + nSortData = 1; } nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; - int ptab2 = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, nKey+2); + iSortTab = pParse->nTab++; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); - sqlite3VdbeAddOp3(v, OP_Column, ptab2, nKey+1, regRow); - sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); + p5 = OPFLAG_CLEARCACHE; }else{ if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow); + iSortTab = iTab; + p5 = 0; + } + for(i=0; iiSdst+i ); - sqlite3VdbeAddOp3(v, OP_Column, pseudoTab, i, pDest->iSdst+i); - if( i==0 ){ - sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); - } - } if( eDest==SRT_Output ){ sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); sqlite3ExprCacheAffinityChange(pParse, pDest->iSdst, nColumn); @@ -1210,9 +1215,10 @@ static void generateSortTail( break; } } - sqlite3ReleaseTempReg(pParse, regRow); - sqlite3ReleaseTempReg(pParse, regRowid); - + if( regRowid ){ + sqlite3ReleaseTempReg(pParse, regRow); + sqlite3ReleaseTempReg(pParse, regRowid); + } /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); @@ -1223,9 +1229,6 @@ static void generateSortTail( } if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); sqlite3VdbeResolveLabel(v, addrBreak); - if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); - } } /* @@ -4760,8 +4763,9 @@ int sqlite3Select( sSort.iECursor = pParse->nTab++; sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sSort.iECursor, sSort.pOrderBy->nExpr+2, 0, - (char*)pKeyInfo, P4_KEYINFO); + sSort.iECursor, sSort.pOrderBy->nExpr+1+pEList->nExpr, 0, + (char*)pKeyInfo, P4_KEYINFO + ); }else{ sSort.addrSortIndex = -1; } diff --git a/test/tester.tcl b/test/tester.tcl index 1c4e93937c..4d55042b43 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -1076,6 +1076,7 @@ proc explain_i {sql {db db}} { foreach opcode { Seek SeekGe SeekGt SeekLe SeekLt NotFound Last Rewind NoConflict Next Prev VNext VPrev VFilter + SorterSort SorterNext } { set color($opcode) $B } @@ -1098,6 +1099,7 @@ proc explain_i {sql {db db}} { if {$opcode=="Next" || $opcode=="Prev" || $opcode=="VNext" || $opcode=="VPrev" + || $opcode=="SorterNext" } { for {set i $p2} {$i<$addr} {incr i} { incr x($i) 2 From 70f624c3a9ed4a9d1efc61ddd3a6744d741be886 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Mar 2014 01:43:50 +0000 Subject: [PATCH 008/710] Further enhancements to geneverated VDBE code for ORDER BY. FossilOrigin-Name: e7188fad87ec82d36a39b80ccaf8006bf45a9bcd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 140d5c124c..979df5c0f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C merge\sfixes\sfrom\strunk -D 2014-03-23T18:47:00.384 +C Further\senhancements\sto\sgeneverated\sVDBE\scode\sfor\sORDER\sBY. +D 2014-03-24T01:43:50.641 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 09fef04ec0746d168ddcff37031ee804ac19dd0e +F src/select.c bf5446f892259f2bb59b0fd9f123b37862b6282b F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d696cdedacd39075aa7fc407ab7c7e50f01d9f39 641408a1395bfc911ca619ef9e5f073b913d856b -R e64080ab6e1cc6573ca05b14626c10fb +P faf7f9caf526ab33a6fdb9c89b45a0483510db21 +R 55d45b1a91856051dc771a7feb58050e U drh -Z 0df19c139e23d451d51cd0e8cf9cddd3 +Z 450407adee455925374bbdaef08e9b36 diff --git a/manifest.uuid b/manifest.uuid index 49e295d769..61b2c1f687 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -faf7f9caf526ab33a6fdb9c89b45a0483510db21 \ No newline at end of file +e7188fad87ec82d36a39b80ccaf8006bf45a9bcd \ No newline at end of file diff --git a/src/select.c b/src/select.c index 1acc6d1020..5ececf1540 100644 --- a/src/select.c +++ b/src/select.c @@ -473,8 +473,7 @@ static void pushOntoSorter( int nOBSat = pSort->nOBSat; /* No. ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ - sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); @@ -1127,7 +1126,6 @@ static void generateSortTail( int addr; int addrOnce = 0; int iTab; - int pseudoTab = 0; ExprList *pOrderBy = pSort->pOrderBy; int eDest = pDest->eDest; int iParm = pDest->iSDParm; @@ -1138,6 +1136,9 @@ static void generateSortTail( int nSortData; /* Trailing values to read from sorter */ u8 p5; /* p5 parameter for 1st OP_Column */ int i; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + struct ExprList_item *aOutEx = p->pEList->a; +#endif if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); @@ -1148,8 +1149,6 @@ static void generateSortTail( iTab = pSort->iECursor; regRow = sqlite3GetTempReg(pParse); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ - pseudoTab = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, pseudoTab, regRow, nColumn); regRowid = 0; regRow = pDest->iSdst; nSortData = nColumn; @@ -1180,6 +1179,7 @@ static void generateSortTail( for(i=0; i Date: Mon, 24 Mar 2014 02:20:53 +0000 Subject: [PATCH 009/710] Remove a pointless OP_Once operation in ORDER BY clauses with LIMIT. FossilOrigin-Name: e6c59d23316c83b318b1a94d9b28a5d321737fa5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 979df5c0f1..e046917d18 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\senhancements\sto\sgeneverated\sVDBE\scode\sfor\sORDER\sBY. -D 2014-03-24T01:43:50.641 +C Remove\sa\spointless\sOP_Once\soperation\sin\sORDER\sBY\sclauses\swith\sLIMIT. +D 2014-03-24T02:20:53.278 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c bf5446f892259f2bb59b0fd9f123b37862b6282b +F src/select.c ece2324b5505317477cbb30cde1ddf67f6304a73 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P faf7f9caf526ab33a6fdb9c89b45a0483510db21 -R 55d45b1a91856051dc771a7feb58050e +P e7188fad87ec82d36a39b80ccaf8006bf45a9bcd +R 7cf4844b34fb28597f11c358a9ceae37 U drh -Z 450407adee455925374bbdaef08e9b36 +Z 5144b0c0a9c74c80111921622c3f3ee8 diff --git a/manifest.uuid b/manifest.uuid index 61b2c1f687..0c4d7e566f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e7188fad87ec82d36a39b80ccaf8006bf45a9bcd \ No newline at end of file +e6c59d23316c83b318b1a94d9b28a5d321737fa5 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 5ececf1540..013feb019f 100644 --- a/src/select.c +++ b/src/select.c @@ -1144,7 +1144,6 @@ static void generateSortTail( sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); - addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); } iTab = pSort->iECursor; regRow = sqlite3GetTempReg(pParse); @@ -1161,6 +1160,9 @@ static void generateSortTail( if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; + if( pSort->labelBkOut ){ + addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); @@ -1169,10 +1171,8 @@ static void generateSortTail( sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); p5 = OPFLAG_CLEARCACHE; }else{ - if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); - sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow); iSortTab = iTab; p5 = 0; } From 3f802ebce2d497fce3ba9330fb8bc4c0ab1cf539 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Mar 2014 09:34:58 +0000 Subject: [PATCH 010/710] Remove an unnecessary temporary register allocation. FossilOrigin-Name: 5d506743f541b022cde04a9606baa4680cdfd70b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e046917d18..feb6340a70 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\spointless\sOP_Once\soperation\sin\sORDER\sBY\sclauses\swith\sLIMIT. -D 2014-03-24T02:20:53.278 +C Remove\san\sunnecessary\stemporary\sregister\sallocation. +D 2014-03-24T09:34:58.396 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c ece2324b5505317477cbb30cde1ddf67f6304a73 +F src/select.c a088183774c4efae4105076355ac4010c62390a8 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e7188fad87ec82d36a39b80ccaf8006bf45a9bcd -R 7cf4844b34fb28597f11c358a9ceae37 -U drh -Z 5144b0c0a9c74c80111921622c3f3ee8 +P e6c59d23316c83b318b1a94d9b28a5d321737fa5 +R ab9aa4f9f7d5ea8c370ac77061403538 +U dan +Z 631d6a70f4aeb5d6a455a81761c8aefd diff --git a/manifest.uuid b/manifest.uuid index 0c4d7e566f..36b51c0a51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6c59d23316c83b318b1a94d9b28a5d321737fa5 \ No newline at end of file +5d506743f541b022cde04a9606baa4680cdfd70b \ No newline at end of file diff --git a/src/select.c b/src/select.c index 013feb019f..641a54faa2 100644 --- a/src/select.c +++ b/src/select.c @@ -1146,7 +1146,6 @@ static void generateSortTail( sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } iTab = pSort->iECursor; - regRow = sqlite3GetTempReg(pParse); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ regRowid = 0; regRow = pDest->iSdst; From fd0a2f9756745401f68fc0e76b868c682194d023 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Mar 2014 18:08:15 +0000 Subject: [PATCH 011/710] Avoid unnecessary moving of content between registers during an ORDER BY. FossilOrigin-Name: 4f472accf072d9cb64f209923924b26f21b13d27 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 38 +++++++++++++++++++++++++++----------- 3 files changed, 35 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index feb6340a70..a1b753f05e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\stemporary\sregister\sallocation. -D 2014-03-24T09:34:58.396 +C Avoid\sunnecessary\smoving\sof\scontent\sbetween\sregisters\sduring\san\sORDER\sBY. +D 2014-03-24T18:08:15.960 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c a088183774c4efae4105076355ac4010c62390a8 +F src/select.c 7f4a1ef9c9e893ee6da160441cd773c951f3d44e F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e6c59d23316c83b318b1a94d9b28a5d321737fa5 -R ab9aa4f9f7d5ea8c370ac77061403538 -U dan -Z 631d6a70f4aeb5d6a455a81761c8aefd +P 5d506743f541b022cde04a9606baa4680cdfd70b +R edfc832bbac864fb9d0e2cb38ae7f098 +U drh +Z a04be0daeb9646c039a07b2f019c65a9 diff --git a/manifest.uuid b/manifest.uuid index 36b51c0a51..1c7384bd0a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d506743f541b022cde04a9606baa4680cdfd70b \ No newline at end of file +4f472accf072d9cb64f209923924b26f21b13d27 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 641a54faa2..40fd59daab 100644 --- a/src/select.c +++ b/src/select.c @@ -463,19 +463,28 @@ static void pushOntoSorter( SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ int regData, /* First register holding data to be sorted */ - int nData /* Number of elements in the data array */ + int nData, /* Number of elements in the data array */ + int nPrefixReg /* No. of reg prior to regData available for use */ ){ Vdbe *v = pParse->pVdbe; /* Stmt under construction */ int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ int nBase = nExpr + 1 + nData; /* Fields in sorter record */ - int regBase = sqlite3GetTempRange(pParse, nBase); /* Regs for sorter record */ + int regBase; /* Regs for sorter record */ int regRecord = sqlite3GetTempReg(pParse); /* Assemblied sorter record */ int nOBSat = pSort->nOBSat; /* No. ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ + if( nPrefixReg ){ + assert( nPrefixReg==nExpr+1 ); + regBase = regData - nExpr - 1; + }else{ + regBase = sqlite3GetTempRange(pParse, nBase); + } sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); - sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); + if( nPrefixReg==0 ){ + sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); + } sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ @@ -515,7 +524,9 @@ static void pushOntoSorter( sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); if( nOBSat==0 ){ sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nBase); + if( nPrefixReg==0 ){ + sqlite3ReleaseTempRange(pParse, regBase, nBase); + } } if( pSelect->iLimit ){ int addr1, addr2; @@ -631,6 +642,7 @@ static void selectInnerLoop( int eDest = pDest->eDest; /* How to dispose of results */ int iParm = pDest->iSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ + int nPrefixReg = 0; /* Number of extra registers before regResult */ assert( v ); assert( pEList!=0 ); @@ -646,6 +658,10 @@ static void selectInnerLoop( nResultCol = pEList->nExpr; if( pDest->iSdst==0 ){ + if( pSort ){ + nPrefixReg = pSort->pOrderBy->nExpr + 1; + pParse->nMem += nPrefixReg; + } pDest->iSdst = pParse->nMem+1; pParse->nMem += nResultCol; }else if( pDest->iSdst+nResultCol > pParse->nMem ){ @@ -762,10 +778,10 @@ static void selectInnerLoop( case SRT_DistFifo: case SRT_Table: case SRT_EphemTab: { - int r1 = sqlite3GetTempReg(pParse); + int r1 = sqlite3GetTempRange(pParse, nPrefixReg+1); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ /* If the destination is DistFifo, then cursor (iParm+1) is open @@ -780,7 +796,7 @@ static void selectInnerLoop( } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1, 1); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, 1, nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -788,7 +804,7 @@ static void selectInnerLoop( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3ReleaseTempReg(pParse, r2); } - sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r1, nPrefixReg+1); break; } @@ -806,7 +822,7 @@ static void selectInnerLoop( ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult, 1); + pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -832,7 +848,7 @@ static void selectInnerLoop( case SRT_Mem: { assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, 1); + pushOntoSorter(pParse, pSort, p, regResult, 1, nPrefixReg); }else{ sqlite3ExprCodeMove(pParse, regResult, iParm, 1); /* The LIMIT clause will jump out of the loop for us */ @@ -846,7 +862,7 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, nResultCol); + pushOntoSorter(pParse, pSort, p, regResult, nResultCol, nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ From dd23c6bfb42cc87dc5919657a972d84afff2162a Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Mar 2014 20:19:07 +0000 Subject: [PATCH 012/710] Omit the sequence value from sorter records used by GROUP BY queries that cannot use an index. FossilOrigin-Name: 3f90abddc31ac20739778c235a834c33f7057997 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 9 ++++----- src/vdbesort.c | 1 + 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index a1b753f05e..63d2130c2b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sunnecessary\smoving\sof\scontent\sbetween\sregisters\sduring\san\sORDER\sBY. -D 2014-03-24T18:08:15.960 +C Omit\sthe\ssequence\svalue\sfrom\ssorter\srecords\sused\sby\sGROUP\sBY\squeries\sthat\scannot\suse\san\sindex. +D 2014-03-24T20:19:07.793 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 7f4a1ef9c9e893ee6da160441cd773c951f3d44e +F src/select.c 2b8722c9888be5e2b358dcd1369a652b38d7ccc4 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 5078ca7de4fd5ba4535bd17fe44d5b56c2d3294c F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 +F src/vdbesort.c 46e50c6bc9300625cff144f8948381a2c53116bf F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5d506743f541b022cde04a9606baa4680cdfd70b -R edfc832bbac864fb9d0e2cb38ae7f098 -U drh -Z a04be0daeb9646c039a07b2f019c65a9 +P 4f472accf072d9cb64f209923924b26f21b13d27 +R e1309a603202b4d83715c783fe89f3da +U dan +Z 8e3a9a2ddbbe8575a70cbc6b0dba434a diff --git a/manifest.uuid b/manifest.uuid index 1c7384bd0a..0ff8edcdd3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4f472accf072d9cb64f209923924b26f21b13d27 \ No newline at end of file +3f90abddc31ac20739778c235a834c33f7057997 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 40fd59daab..7e0085dd36 100644 --- a/src/select.c +++ b/src/select.c @@ -4910,7 +4910,7 @@ int sqlite3Select( sNC.pSrcList = pTabList; sNC.pAggInfo = &sAggInfo; sAggInfo.mnReg = pParse->nMem+1; - sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; + sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; sAggInfo.pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); @@ -5002,8 +5002,8 @@ int sqlite3Select( groupBySort = 1; nGroupBy = pGroupBy->nExpr; - nCol = nGroupBy + 1; - j = nGroupBy+1; + nCol = nGroupBy; + j = nGroupBy; for(i=0; i=j ){ nCol++; @@ -5013,8 +5013,7 @@ int sqlite3Select( regBase = sqlite3GetTempRange(pParse, nCol); sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, sAggInfo.sortingIdx,regBase+nGroupBy); - j = nGroupBy+1; + j = nGroupBy; for(i=0; iiSorterColumn>=j ){ diff --git a/src/vdbesort.c b/src/vdbesort.c index d1b726b727..d55156a2c1 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -480,6 +480,7 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pCsr->pKeyInfo, 0, 0, &d); if( pSorter->pUnpacked==0 ) return SQLITE_NOMEM; assert( pSorter->pUnpacked==(UnpackedRecord *)d ); + pSorter->pUnpacked->nField = pCsr->pKeyInfo->nField; if( !sqlite3TempInMemory(db) ){ pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); From 3c863634ef70229b61573d4e98165ac564e44658 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Mar 2014 14:12:16 +0000 Subject: [PATCH 013/710] Enable four sorting threads by default in the command-line shell. FossilOrigin-Name: 1cab83577c814feb35b4fb91af0d52a9751d99bc --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/shell.c | 1 - src/threads.c | 4 ++-- src/vdbesort.c | 2 +- 5 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 7ccdd5b9a5..d29d622a2f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\sfixes\sand\senhancements\sfrom\strunk. -D 2014-03-25T13:17:41.050 +C Enable\sfour\ssorting\sthreads\sby\sdefault\sin\sthe\scommand-line\sshell. +D 2014-03-25T14:12:16.989 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -218,7 +218,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 269c3e31a450fce642a10569221a49180348c88e -F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf +F src/shell.c f48b63f8e582e7998ecefd051d697f91fb1453df F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -272,7 +272,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c cde9d885fd562b5427f89a42a8829085f88b17df +F src/threads.c b96d62f88c06d4fa980a4a92685d1b130c4c84d3 F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 68dbdc77cdc008eeabc088b8b8a60aa743ba8d2a F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 0fbaf5b3ec3e779d81c4db4eb2f0ae5f44fbb02c +F src/vdbesort.c c3e427de848b78e9e9feaa25f68fb64686bab6cd F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ff0b5c851ba7d04d1836d7c6a3222713e7d8d357 e6798871ce94961135762669af418cd78540c121 -R 5887d7b59f581a1221f99e7d95aac2b8 +P b415dfb6cb0df0c69992ca2bb700c15664f158e6 +R a4b6dbfcec045a7fea4740daec225662 U drh -Z 5e51466328d6bc63d3b26ee7aec599ce +Z a12cd4438cfae19c4441f02ed27b5761 diff --git a/manifest.uuid b/manifest.uuid index 628a8f1b85..e0e5369892 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b415dfb6cb0df0c69992ca2bb700c15664f158e6 \ No newline at end of file +1cab83577c814feb35b4fb91af0d52a9751d99bc \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 1313112709..c9abd9ec7d 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3532,7 +3532,6 @@ static void main_init(struct callback_data *data) { sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); - sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); } /* diff --git a/src/threads.c b/src/threads.c index 7cc964250d..cb148b6c6f 100644 --- a/src/threads.c +++ b/src/threads.c @@ -28,7 +28,7 @@ #include "sqliteInt.h" /********************************* Unix Pthreads ****************************/ -#if SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) +#if SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) && SQLITE_THREADSAFE>0 #define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ #include @@ -85,7 +85,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ /********************************* Win32 Threads ****************************/ -#if SQLITE_OS_WIN && !SQLITE_OS_WINRT +#if SQLITE_OS_WIN && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0 #define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ #include diff --git a/src/vdbesort.c b/src/vdbesort.c index 3c8e91d987..a3f7aba10e 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -31,7 +31,7 @@ typedef struct FileWriter FileWriter; ** operations to be single-threaded. */ #ifndef SQLITE_MAX_SORTER_THREAD -# define SQLITE_MAX_SORTER_THREAD 1 +# define SQLITE_MAX_SORTER_THREAD 4 #endif /* From 78d5843245b2ef9c1fb92213d0c0e4bb49e6c70d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Mar 2014 15:04:07 +0000 Subject: [PATCH 014/710] Remove the sequence values from sorter records used by ORDER BY as well. FossilOrigin-Name: c3ae3697832a00d4d5758988a8766bdbb691e6b8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 40 +++++++++++++++++++++++++++------------- src/vdbe.c | 18 ++++++++++++++++++ 4 files changed, 53 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 63d2130c2b..2690ecef90 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sthe\ssequence\svalue\sfrom\ssorter\srecords\sused\sby\sGROUP\sBY\squeries\sthat\scannot\suse\san\sindex. -D 2014-03-24T20:19:07.793 +C Remove\sthe\ssequence\svalues\sfrom\ssorter\srecords\sused\sby\sORDER\sBY\sas\swell. +D 2014-03-25T15:04:07.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 2b8722c9888be5e2b358dcd1369a652b38d7ccc4 +F src/select.c 20055cf917222e660c4222fea306bd13a0623caa F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -278,7 +278,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 5c0feeb6c9e6a0e0cc2a9715aa6045830643809d +F src/vdbe.c 42177064bd02fc55984aabc3a241368fda01e8b4 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4f472accf072d9cb64f209923924b26f21b13d27 -R e1309a603202b4d83715c783fe89f3da +P 3f90abddc31ac20739778c235a834c33f7057997 +R 743d88a284437e5e0babc8e33ec37e61 U dan -Z 8e3a9a2ddbbe8575a70cbc6b0dba434a +Z 5024d9c780855d9abccbd4c6c23f3054 diff --git a/manifest.uuid b/manifest.uuid index 0ff8edcdd3..b187241ae4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3f90abddc31ac20739778c235a834c33f7057997 \ No newline at end of file +c3ae3697832a00d4d5758988a8766bdbb691e6b8 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 7e0085dd36..7480eeb94b 100644 --- a/src/select.c +++ b/src/select.c @@ -467,24 +467,29 @@ static void pushOntoSorter( int nPrefixReg /* No. of reg prior to regData available for use */ ){ Vdbe *v = pParse->pVdbe; /* Stmt under construction */ + int bSeq = ((pSort->sortFlags & SORTFLAG_UseSorter)==0); int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ - int nBase = nExpr + 1 + nData; /* Fields in sorter record */ + int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ int regBase; /* Regs for sorter record */ - int regRecord = sqlite3GetTempReg(pParse); /* Assemblied sorter record */ - int nOBSat = pSort->nOBSat; /* No. ORDER BY terms to skip */ - int op; /* Opcode to add sorter record to sorter */ + int regRecord = sqlite3GetTempReg(pParse); /* Assembled sorter record */ + int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ + int op; /* Opcode to add sorter record to sorter */ + assert( bSeq==0 || bSeq==1 ); if( nPrefixReg ){ - assert( nPrefixReg==nExpr+1 ); - regBase = regData - nExpr - 1; + assert( nPrefixReg==nExpr+bSeq ); + regBase = regData - nExpr - bSeq; }else{ regBase = sqlite3GetTempRange(pParse, nBase); } sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, SQLITE_ECEL_DUP); - sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); - if( nPrefixReg==0 ){ - sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+1, nData); + if( bSeq ){ + sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } + if( nPrefixReg==0 ){ + sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+bSeq, nData); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ @@ -496,8 +501,13 @@ static void pushOntoSorter( regPrevKey = pParse->nMem+1; pParse->nMem += pSort->nOBSat; - nKey = nExpr - pSort->nOBSat + 1; - addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); + nKey = nExpr - pSort->nOBSat + bSeq; + if( bSeq ){ + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); + }else{ + addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor); + } + VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); if( pParse->db->mallocFailed ) return; @@ -659,7 +669,8 @@ static void selectInnerLoop( if( pDest->iSdst==0 ){ if( pSort ){ - nPrefixReg = pSort->pOrderBy->nExpr + 1; + nPrefixReg = pSort->pOrderBy->nExpr; + if( !(pSort->sortFlags & SORTFLAG_UseSorter) ) nPrefixReg++; pParse->nMem += nPrefixReg; } pDest->iSdst = pParse->nMem+1; @@ -1152,6 +1163,7 @@ static void generateSortTail( int nSortData; /* Trailing values to read from sorter */ u8 p5; /* p5 parameter for 1st OP_Column */ int i; + int bSeq; /* True if sorter record includes seq. no. */ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS struct ExprList_item *aOutEx = p->pEList->a; #endif @@ -1185,14 +1197,16 @@ static void generateSortTail( codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); p5 = OPFLAG_CLEARCACHE; + bSeq = 0; }else{ addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); iSortTab = iTab; p5 = 0; + bSeq = 1; } for(i=0; ip1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC->pSorter ); + if( (pC->seqCount++)==0 ){ + pc = pOp->p2 - 1; + } + break; +} + /* Opcode: OpenPseudo P1 P2 P3 * * ** Synopsis: P3 columns in r[P2] ** From ab1dcc1a4be2d1636bb43685525ab60c2e2e6ee8 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Mar 2014 17:07:48 +0000 Subject: [PATCH 015/710] Fix a problem in the code added by [707ea170b3] causing vdbesort.c to sort unstably. FossilOrigin-Name: d3e640afe611b6ae0b7f2cff5b00900d7e4d5ee3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 20 ++++++++++++++++---- test/sort.test | 23 +++++++++++++++++++++++ 4 files changed, 47 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 2690ecef90..eb635d79a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\ssequence\svalues\sfrom\ssorter\srecords\sused\sby\sORDER\sBY\sas\swell. -D 2014-03-25T15:04:07.777 +C Fix\sa\sproblem\sin\sthe\scode\sadded\sby\s[707ea170b3]\scausing\svdbesort.c\sto\ssort\sunstably. +D 2014-03-25T17:07:48.821 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 5078ca7de4fd5ba4535bd17fe44d5b56c2d3294c F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 46e50c6bc9300625cff144f8948381a2c53116bf +F src/vdbesort.c 691f2186ae0943cd746ea7f5498cc9abebb7a7cc F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -816,7 +816,7 @@ F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 +F test/sort.test cb76a6e9db897b6871ef4dbc206ebc6dbc033bf4 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 3f90abddc31ac20739778c235a834c33f7057997 -R 743d88a284437e5e0babc8e33ec37e61 +P c3ae3697832a00d4d5758988a8766bdbb691e6b8 +R eb749164f2115a0b3ef6dbb32fd74b6a U dan -Z 5024d9c780855d9abccbd4c6c23f3054 +Z ecf1eb93a5e3757595a49ddb23f9c671 diff --git a/manifest.uuid b/manifest.uuid index b187241ae4..bb3b1adce9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3ae3697832a00d4d5758988a8766bdbb691e6b8 \ No newline at end of file +d3e640afe611b6ae0b7f2cff5b00900d7e4d5ee3 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index d55156a2c1..7b43554282 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -562,6 +562,9 @@ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ /* ** Merge the two sorted lists p1 and p2 into a single list. ** Set *ppOut to the head of the new list. +** +** In cases where key values are equal, keys from list p1 are considered +** to be smaller than list p2. */ static void vdbeSorterMerge( const VdbeCursor *pCsr, /* For pKeyInfo */ @@ -597,6 +600,11 @@ static void vdbeSorterMerge( ** Sort the linked list of records headed at pCsr->pRecord. Return SQLITE_OK ** if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if an error ** occurs. +** +** The sort is required to be stable - if two elements compare as equal +** then the one added to the sorter first is considered the smaller. +** Currently, the list is sorted from newest to oldest - pSorter->pRecord +** points to the most recently added sort key. */ static int vdbeSorterSort(const VdbeCursor *pCsr){ int i; @@ -837,7 +845,7 @@ static int vdbeSorterInitMerge( int i; /* Used to iterator through aIter[] */ i64 nByte = 0; /* Total bytes in all opened PMAs */ - /* Initialize the iterators. */ + /* Initialize the iterators. Iterator 0 contains the oldest data. */ for(i=0; iaIter[i]; rc = vdbeSorterIterInit(db, pSorter, pSorter->iReadOff, pIter, &nByte); @@ -1009,8 +1017,13 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() ** was actually called above, then pSorter->pUnpacked now contains ** a value equivalent to pIter2. So set pKey2 to NULL to prevent - ** vdbeSorterCompare() from decoding pIter2 again. */ - if( iRes<=0 ){ + ** vdbeSorterCompare() from decoding pIter2 again. + ** + ** If the two values were equal, then the value from the oldest + ** PMA should be considered smaller. The VdbeSorter.aIter[] array + ** is sorted from oldest to newest, so pIter1 contains older values + ** than pIter2 iff (pIter1aTree[i] = (int)(pIter1 - pSorter->aIter); pIter2 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; pKey2 = pIter2->aKey; @@ -1019,7 +1032,6 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ pSorter->aTree[i] = (int)(pIter2 - pSorter->aIter); pIter1 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; } - } *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); } diff --git a/test/sort.test b/test/sort.test index 08d496b259..b543ffad24 100644 --- a/test/sort.test +++ b/test/sort.test @@ -464,4 +464,27 @@ do_test sort-12.1 { } } {1 2 xxx 1 3 yyy 1 1 zzz} + +#------------------------------------------------------------------------- +# Check that the sorter in vdbesort.c sorts in a stable fashion. +# +do_execsql_test sort-13.0 { + CREATE TABLE t10(a, b); +} +do_test sort-13.1 { + db transaction { + for {set i 0} {$i < 100000} {incr i} { + execsql { INSERT INTO t10 VALUES( $i/10, $i%10 ) } + } + } +} {} +do_execsql_test sort-13.2 { + SELECT a, b FROM t10 ORDER BY a; +} [db eval {SELECT a, b FROM t10 ORDER BY a, b}] +do_execsql_test sort-13.3 { + PRAGMA cache_size = 5; + SELECT a, b FROM t10 ORDER BY a; +} [db eval {SELECT a, b FROM t10 ORDER BY a, b}] + + finish_test From face0872125725b1d104b5a5d8e0f04079c9e7a6 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Mar 2014 17:23:41 +0000 Subject: [PATCH 016/710] Use xFetch() to access temporary files in vdbesort.c. Use a single large allocation instead of many small allocations when accumulating records in vdbesort.c. This is an interim commit - it allocates a buffer the size of the page-cache every time data is sorted. FossilOrigin-Name: f4ac1bf28c4ba395ccab8f1c9df72614a61095a7 --- manifest | 14 ++-- manifest.uuid | 2 +- src/vdbesort.c | 204 +++++++++++++++++++++++++++++++++++-------------- 3 files changed, 156 insertions(+), 64 deletions(-) diff --git a/manifest b/manifest index 50b1f52a8e..dc6ca295c7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sand\sthe\sfix\sfor\sthe\scrash\son\sa\scorrupt\ndatabase. -D 2014-03-27T00:09:00.185 +C Use\sxFetch()\sto\saccess\stemporary\sfiles\sin\svdbesort.c.\sUse\sa\ssingle\slarge\sallocation\sinstead\sof\smany\ssmall\sallocations\swhen\saccumulating\srecords\sin\svdbesort.c.\sThis\sis\san\sinterim\scommit\s-\sit\sallocates\sa\sbuffer\sthe\ssize\sof\sthe\spage-cache\severy\stime\sdata\sis\ssorted. +D 2014-03-27T17:23:41.403 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c f81ef920dcf76aceaa1ce77081e9fc5d7a0993dd F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 691f2186ae0943cd746ea7f5498cc9abebb7a7cc +F src/vdbesort.c d46f384af1997a4441ef9c65759181954efc89cf F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e005f2d6dd9faf38cc8fdb9428b5aa6192a6adae f585f5d7a0f9bf8c590388654a3638231eba8892 -R 9012cf758152dd3775ad665fff626f11 -U drh -Z ffbdc2f043e4deddac6c4b509aff14ac +P 0b35346c32dba14963c85ec178f2b46aa2bbf6dc +R 8f417577dbf4ab5b5ce5a802872c70f0 +U dan +Z 2c1a0a402e8f93dd4df9cfe6e1bca5e6 diff --git a/manifest.uuid b/manifest.uuid index f8b9980e3d..853b60d3d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0b35346c32dba14963c85ec178f2b46aa2bbf6dc \ No newline at end of file +f4ac1bf28c4ba395ccab8f1c9df72614a61095a7 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 7b43554282..2a15168fd2 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -105,6 +105,8 @@ struct VdbeSorter { sqlite3_file *pTemp1; /* PMA file 1 */ SorterRecord *pRecord; /* Head of in-memory record list */ UnpackedRecord *pUnpacked; /* Used to unpack keys */ + u8* aMemory; /* Block to allocate records from */ + int iMemory; /* Offset of free space in aMemory */ }; /* @@ -121,6 +123,7 @@ struct VdbeSorterIter { u8 *aKey; /* Pointer to current key */ u8 *aBuffer; /* Current read buffer */ int nBuffer; /* Size of read buffer in bytes */ + u8 *aMap; /* Pointer to mapping of pFile */ }; /* @@ -163,6 +166,7 @@ struct SorterRecord { static void vdbeSorterIterZero(sqlite3 *db, VdbeSorterIter *pIter){ sqlite3DbFree(db, pIter->aAlloc); sqlite3DbFree(db, pIter->aBuffer); + if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); memset(pIter, 0, sizeof(VdbeSorterIter)); } @@ -183,6 +187,13 @@ static int vdbeSorterIterRead( ){ int iBuf; /* Offset within buffer to read from */ int nAvail; /* Bytes of data available in buffer */ + + if( p->aMap ){ + *ppOut = &p->aMap[p->iReadOff]; + p->iReadOff += nByte; + return SQLITE_OK; + } + assert( p->aBuffer ); /* If there is no more data to be read from the buffer, read the next @@ -264,18 +275,22 @@ static int vdbeSorterIterRead( static int vdbeSorterIterVarint(sqlite3 *db, VdbeSorterIter *p, u64 *pnOut){ int iBuf; - iBuf = p->iReadOff % p->nBuffer; - if( iBuf && (p->nBuffer-iBuf)>=9 ){ - p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + if( p->aMap ){ + p->iReadOff += sqlite3GetVarint(&p->aMap[p->iReadOff], pnOut); }else{ - u8 aVarint[16], *a; - int i = 0, rc; - do{ - rc = vdbeSorterIterRead(db, p, 1, &a); - if( rc ) return rc; - aVarint[(i++)&0xf] = a[0]; - }while( (a[0]&0x80)!=0 ); - sqlite3GetVarint(aVarint, pnOut); + iBuf = p->iReadOff % p->nBuffer; + if( iBuf && (p->nBuffer-iBuf)>=9 ){ + p->iReadOff += sqlite3GetVarint(&p->aBuffer[iBuf], pnOut); + }else{ + u8 aVarint[16], *a; + int i = 0, rc; + do{ + rc = vdbeSorterIterRead(db, p, 1, &a); + if( rc ) return rc; + aVarint[(i++)&0xf] = a[0]; + }while( (a[0]&0x80)!=0 ); + sqlite3GetVarint(aVarint, pnOut); + } } return SQLITE_OK; @@ -323,6 +338,7 @@ static int vdbeSorterIterInit( ){ int rc = SQLITE_OK; int nBuf; + void *pMap; nBuf = sqlite3BtreeGetPageSize(db->aDb[0].pBt); @@ -333,33 +349,41 @@ static int vdbeSorterIterInit( pIter->iReadOff = iStart; pIter->nAlloc = 128; pIter->aAlloc = (u8 *)sqlite3DbMallocRaw(db, pIter->nAlloc); - pIter->nBuffer = nBuf; - pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); - if( !pIter->aBuffer ){ - rc = SQLITE_NOMEM; + /* See if this PMA can be read using xFetch. */ + rc = sqlite3OsFetch(pIter->pFile, 0, pSorter->iWriteOff, &pMap); + if( rc!=SQLITE_OK ) return rc; + if( pMap ){ + pIter->aMap = (u8*)pMap; }else{ - int iBuf; + pIter->nBuffer = nBuf; + pIter->aBuffer = (u8 *)sqlite3DbMallocRaw(db, nBuf); - iBuf = iStart % nBuf; - if( iBuf ){ - int nRead = nBuf - iBuf; - if( (iStart + nRead) > pSorter->iWriteOff ){ - nRead = (int)(pSorter->iWriteOff - iStart); + if( !pIter->aBuffer ){ + rc = SQLITE_NOMEM; + }else{ + int iBuf; + + iBuf = iStart % nBuf; + if( iBuf ){ + int nRead = nBuf - iBuf; + if( (iStart + nRead) > pSorter->iWriteOff ){ + nRead = (int)(pSorter->iWriteOff - iStart); + } + rc = sqlite3OsRead( + pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart + ); + assert( rc!=SQLITE_IOERR_SHORT_READ ); } - rc = sqlite3OsRead( - pSorter->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart - ); - assert( rc!=SQLITE_IOERR_SHORT_READ ); } + } - if( rc==SQLITE_OK ){ - u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pSorter->iWriteOff; - rc = vdbeSorterIterVarint(db, pIter, &nByte); - pIter->iEof = pIter->iReadOff + nByte; - *pnByte += nByte; - } + if( rc==SQLITE_OK ){ + u64 nByte; /* Size of PMA in bytes */ + pIter->iEof = pSorter->iWriteOff; + rc = vdbeSorterIterVarint(db, pIter, &nByte); + pIter->iEof = pIter->iReadOff + nByte; + *pnByte += nByte; } if( rc==SQLITE_OK ){ @@ -488,6 +512,10 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ mxCache = db->aDb[0].pSchema->cache_size; if( mxCachemxPmaSize = mxCache * pgsz; + + pSorter->aMemory = (u8*)sqlite3DbMallocRaw(db, pSorter->mxPmaSize); + assert( pSorter->iMemory==0 ); + if( !pSorter->aMemory ) return SQLITE_NOMEM; } return SQLITE_OK; @@ -521,7 +549,9 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ sqlite3OsCloseFree(pSorter->pTemp1); pSorter->pTemp1 = 0; } - vdbeSorterRecordFree(db, pSorter->pRecord); + if( pSorter->aMemory==0 ){ + vdbeSorterRecordFree(db, pSorter->pRecord); + } pSorter->pRecord = 0; pSorter->iWriteOff = 0; pSorter->iReadOff = 0; @@ -529,6 +559,7 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ pSorter->nTree = 0; pSorter->nPMA = 0; pSorter->aTree = 0; + pSorter->iMemory = 0; } @@ -540,6 +571,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); sqlite3DbFree(db, pSorter->pUnpacked); + sqlite3DbFree(db, pSorter->aMemory); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } @@ -551,12 +583,17 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ ** Otherwise, set *ppFile to 0 and return an SQLite error code. */ static int vdbeSorterOpenTempFile(sqlite3 *db, sqlite3_file **ppFile){ - int dummy; - return sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, + int rc; + rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | - SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &dummy + SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &rc ); + if( rc==SQLITE_OK ){ + i64 max = SQLITE_MAX_MMAP_SIZE; + sqlite3OsFileControlHint( *ppFile, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); + } + return rc; } /* @@ -725,6 +762,29 @@ static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ fileWriterWrite(p, aByte, nByte); } +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** The first argument is a file-handle open on a temporary file. The file +** is guaranteed to be nByte bytes or smaller in size. This function +** attempts to extend the file to nByte bytes in size and to ensure that +** the VFS has memory mapped it. +** +** Whether or not the file does end up memory mapped of course depends on +** the specific VFS implementation. +*/ +static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ + int rc = sqlite3OsTruncate(pFile, nByte); + if( rc==SQLITE_OK ){ + void *p = 0; + sqlite3OsFetch(pFile, 0, nByte, &p); + sqlite3OsUnfetch(pFile, 0, p); + } + return rc; +} +#else +# define vdbeSorterExtendFile(x,y) SQLITE_OK +#endif + /* ** Write the current contents of the in-memory linked-list to a PMA. Return ** SQLITE_OK if successful, or an SQLite error code otherwise. @@ -760,6 +820,13 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ assert( pSorter->nPMA==0 ); } + /* Try to get the file to memory map */ + if( rc==SQLITE_OK ){ + rc = vdbeSorterExtendFile( + pSorter->pTemp1, pSorter->iWriteOff + pSorter->nInMemory + 9 + ); + } + if( rc==SQLITE_OK ){ SorterRecord *p; SorterRecord *pNext = 0; @@ -771,12 +838,14 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ pNext = p->pNext; fileWriterWriteVarint(&writer, p->nVal); fileWriterWrite(&writer, p->pVal, p->nVal); - sqlite3DbFree(db, p); + if( pSorter->aMemory==0 ) sqlite3DbFree(db, p); } pSorter->pRecord = p; rc = fileWriterFinish(db, &writer, &pSorter->iWriteOff); } + if( pSorter->aMemory ) pSorter->pRecord = 0; + assert( pSorter->pRecord==0 || rc!=SQLITE_OK ); return rc; } @@ -789,23 +858,37 @@ int sqlite3VdbeSorterWrite( Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; + SorterRecord sRecord; /* Used for aMemory overflow record */ int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ assert( pSorter ); pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n + sizeof(SorterRecord)); - if( pNew==0 ){ - rc = SQLITE_NOMEM; + if( pSorter->aMemory ){ + int nReq = sizeof(SorterRecord) + pVal->n; + if( (pSorter->iMemory+nReq) > pSorter->mxPmaSize ){ + pNew = &sRecord; + pNew->pVal = pVal->z; + }else{ + pNew = &pSorter->aMemory[pSorter->iMemory]; + pSorter->iMemory += ROUND8(nReq); + } }else{ - pNew->pVal = (void *)&pNew[1]; - memcpy(pNew->pVal, pVal->z, pVal->n); - pNew->nVal = pVal->n; - pNew->pNext = pSorter->pRecord; - pSorter->pRecord = pNew; + pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n+sizeof(SorterRecord)); + if( pNew==0 ){ + return SQLITE_NOMEM; + } } + if( pNew!=&sRecord ){ + pNew->pVal = (void*)&pNew[1]; + memcpy(pNew->pVal, pVal->z, pVal->n); + } + pNew->nVal = pVal->n; + pNew->pNext = pSorter->pRecord; + pSorter->pRecord = pNew; + /* See if the contents of the sorter should now be written out. They ** are written out when either of the following are true: ** @@ -815,18 +898,22 @@ int sqlite3VdbeSorterWrite( ** * The total memory allocated for the in-memory list is greater ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ - if( rc==SQLITE_OK && pSorter->mxPmaSize>0 && ( - (pSorter->nInMemory>pSorter->mxPmaSize) - || (pSorter->nInMemory>pSorter->mnPmaSize && sqlite3HeapNearlyFull()) - )){ + if( pSorter->mxPmaSize>0 ){ + if( (pNew==&sRecord) || (pSorter->aMemory==0 && ( + (pSorter->nInMemory > pSorter->mxPmaSize) + || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) + ))){ #ifdef SQLITE_DEBUG - i64 nExpect = pSorter->iWriteOff - + sqlite3VarintLen(pSorter->nInMemory) - + pSorter->nInMemory; + i64 nExpect = pSorter->iWriteOff + + sqlite3VarintLen(pSorter->nInMemory) + + pSorter->nInMemory; #endif - rc = vdbeSorterListToPMA(db, pCsr); - pSorter->nInMemory = 0; - assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + rc = vdbeSorterListToPMA(db, pCsr); + pSorter->nInMemory = 0; + pSorter->iMemory = 0; + assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); + } } return rc; @@ -934,6 +1021,9 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ if( pTemp2==0 ){ assert( iWrite2==0 ); rc = vdbeSorterOpenTempFile(db, &pTemp2); + if( rc==SQLITE_OK ){ + rc = vdbeSorterExtendFile(pTemp2, pSorter->iWriteOff); + } } if( rc==SQLITE_OK ){ @@ -1039,7 +1129,9 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->pNext; pFree->pNext = 0; - vdbeSorterRecordFree(db, pFree); + if( pSorter->aMemory==0 ){ + vdbeSorterRecordFree(db, pFree); + } *pbEof = !pSorter->pRecord; rc = SQLITE_OK; } From 6971952c650cdf1d4e76e63fdb89165bac5c1af0 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Mar 2014 19:25:02 +0000 Subject: [PATCH 017/710] Instead of allocating a single large buffer at the beginning of each sort operation, start with a small buffer and extend it using realloc() as required. FossilOrigin-Name: 81987c8ceb64f051528a6ca42673821d9ab7c0ff --- manifest | 12 ++-- manifest.uuid | 2 +- src/vdbesort.c | 191 ++++++++++++++++++++++++++++++++----------------- 3 files changed, 134 insertions(+), 71 deletions(-) diff --git a/manifest b/manifest index dc6ca295c7..986addf1fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sxFetch()\sto\saccess\stemporary\sfiles\sin\svdbesort.c.\sUse\sa\ssingle\slarge\sallocation\sinstead\sof\smany\ssmall\sallocations\swhen\saccumulating\srecords\sin\svdbesort.c.\sThis\sis\san\sinterim\scommit\s-\sit\sallocates\sa\sbuffer\sthe\ssize\sof\sthe\spage-cache\severy\stime\sdata\sis\ssorted. -D 2014-03-27T17:23:41.403 +C Instead\sof\sallocating\sa\ssingle\slarge\sbuffer\sat\sthe\sbeginning\sof\seach\ssort\soperation,\sstart\swith\sa\ssmall\sbuffer\sand\sextend\sit\susing\srealloc()\sas\srequired. +D 2014-03-27T19:25:02.222 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c f81ef920dcf76aceaa1ce77081e9fc5d7a0993dd F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c d46f384af1997a4441ef9c65759181954efc89cf +F src/vdbesort.c 08d5e1ee199599d9571942f0560f84963c7a1a9b F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 0b35346c32dba14963c85ec178f2b46aa2bbf6dc -R 8f417577dbf4ab5b5ce5a802872c70f0 +P f4ac1bf28c4ba395ccab8f1c9df72614a61095a7 +R 5fefa7763a5f0d580046ba5219de21a5 U dan -Z 2c1a0a402e8f93dd4df9cfe6e1bca5e6 +Z 8f7301353335dd4530b24cac1ad9c37a diff --git a/manifest.uuid b/manifest.uuid index 853b60d3d3..cf2496b0b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4ac1bf28c4ba395ccab8f1c9df72614a61095a7 \ No newline at end of file +81987c8ceb64f051528a6ca42673821d9ab7c0ff \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 2a15168fd2..f229b1f18b 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -107,6 +107,7 @@ struct VdbeSorter { UnpackedRecord *pUnpacked; /* Used to unpack keys */ u8* aMemory; /* Block to allocate records from */ int iMemory; /* Offset of free space in aMemory */ + int nMemory; /* Current size of allocation at aMemory */ }; /* @@ -144,15 +145,37 @@ struct FileWriter { /* ** A structure to store a single record. All in-memory records are connected -** together into a linked list headed at VdbeSorter.pRecord using the -** SorterRecord.pNext pointer. +** together into a linked list headed at VdbeSorter.pRecord. +** +** How the linked list is connected depends on how memory is being managed +** by this module. If using a separate allocation for each in-memory record +** (VdbeSorter.aMemory==0), then the list is always connected using the +** SorterRecord.u.pNext pointers. +** +** Or, if using the single large allocation method (VdbeSorter.aMemory!=0), +** then while records are being accumulated the list is linked using the +** SorterRecord.u.iNext offset. This is because the aMemory[] array may +** be sqlite3Realloc()ed while records are being accumulated. Once the VM +** has finished passing records to the sorter, or when the in-memory buffer +** is full, the list is sorted. As part of the sorting process, it is +** converted to use the SorterRecord.u.pNext pointers. See function +** vdbeSorterSort() for details. */ struct SorterRecord { - void *pVal; int nVal; - SorterRecord *pNext; + union { + SorterRecord *pNext; /* Pointer to next record in list */ + int iNext; /* Offset within aMemory of next record */ + } u; }; +/* Return a pointer to the buffer containing the record data for SorterRecord +** object p. Should be used as if: +** +** void *SRVAL(SorterRecord *p) { return (void*)&p[1]; } +*/ +#define SRVAL(p) ((void*)((SorterRecord*)(p) + 1)) + /* Minimum allowable value for the VdbeSorter.nWorking variable */ #define SORTER_MIN_WORKING 10 @@ -513,9 +536,15 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ if( mxCachemxPmaSize = mxCache * pgsz; - pSorter->aMemory = (u8*)sqlite3DbMallocRaw(db, pSorter->mxPmaSize); - assert( pSorter->iMemory==0 ); - if( !pSorter->aMemory ) return SQLITE_NOMEM; + /* If the application is using memsys3 or memsys5, use a separate + ** allocation for each sort-key in memory. Otherwise, use a single big + ** allocation at pSorter->aMemory for all sort-keys. */ + if( sqlite3GlobalConfig.pHeap==0 ){ + assert( pSorter->iMemory==0 ); + pSorter->nMemory = pgsz; + pSorter->aMemory = (u8*)sqlite3Malloc(pSorter->nMemory); + if( !pSorter->aMemory ) return SQLITE_NOMEM; + } } return SQLITE_OK; @@ -528,7 +557,7 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ SorterRecord *p; SorterRecord *pNext; for(p=pRecord; p; p=pNext){ - pNext = p->pNext; + pNext = p->u.pNext; sqlite3DbFree(db, p); } } @@ -611,22 +640,22 @@ static void vdbeSorterMerge( ){ SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; - void *pVal2 = p2 ? p2->pVal : 0; + void *pVal2 = p2 ? SRVAL(p2) : 0; while( p1 && p2 ){ int res; - vdbeSorterCompare(pCsr, 0, p1->pVal, p1->nVal, pVal2, p2->nVal, &res); + vdbeSorterCompare(pCsr, 0, SRVAL(p1), p1->nVal, pVal2, p2->nVal, &res); if( res<=0 ){ *pp = p1; - pp = &p1->pNext; - p1 = p1->pNext; + pp = &p1->u.pNext; + p1 = p1->u.pNext; pVal2 = 0; }else{ *pp = p2; - pp = &p2->pNext; - p2 = p2->pNext; + pp = &p2->u.pNext; + p2 = p2->u.pNext; if( p2==0 ) break; - pVal2 = p2->pVal; + pVal2 = SRVAL(p2); } } *pp = p1 ? p1 : p2; @@ -656,8 +685,18 @@ static int vdbeSorterSort(const VdbeCursor *pCsr){ p = pSorter->pRecord; while( p ){ - SorterRecord *pNext = p->pNext; - p->pNext = 0; + SorterRecord *pNext; + if( pSorter->aMemory ){ + assert( p->u.iNextnMemory ); + if( (u8*)p==pSorter->aMemory ){ + pNext = 0; + }else{ + pNext = (SorterRecord*)&pSorter->aMemory[p->u.iNext]; + } + }else{ + pNext = p->u.pNext; + } + p->u.pNext = 0; for(i=0; aSlot[i]; i++){ vdbeSorterMerge(pCsr, p, aSlot[i], &p); aSlot[i] = 0; @@ -835,9 +874,9 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ pSorter->nPMA++; fileWriterWriteVarint(&writer, pSorter->nInMemory); for(p=pSorter->pRecord; p; p=pNext){ - pNext = p->pNext; + pNext = p->u.pNext; fileWriterWriteVarint(&writer, p->nVal); - fileWriterWrite(&writer, p->pVal, p->nVal); + fileWriterWrite(&writer, SRVAL(p), p->nVal); if( pSorter->aMemory==0 ) sqlite3DbFree(db, p); } pSorter->pRecord = p; @@ -858,39 +897,24 @@ int sqlite3VdbeSorterWrite( Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; - SorterRecord sRecord; /* Used for aMemory overflow record */ int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ + int bFlush; /* True to flush contents of memory to PMA */ + int nReq; /* Bytes of memory required */ + int nPMA; /* Bytes of PMA space required */ + assert( pSorter ); - pSorter->nInMemory += sqlite3VarintLen(pVal->n) + pVal->n; - if( pSorter->aMemory ){ - int nReq = sizeof(SorterRecord) + pVal->n; - if( (pSorter->iMemory+nReq) > pSorter->mxPmaSize ){ - pNew = &sRecord; - pNew->pVal = pVal->z; - }else{ - pNew = &pSorter->aMemory[pSorter->iMemory]; - pSorter->iMemory += ROUND8(nReq); - } - }else{ - pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n+sizeof(SorterRecord)); - if( pNew==0 ){ - return SQLITE_NOMEM; - } - } - - if( pNew!=&sRecord ){ - pNew->pVal = (void*)&pNew[1]; - memcpy(pNew->pVal, pVal->z, pVal->n); - } - pNew->nVal = pVal->n; - pNew->pNext = pSorter->pRecord; - pSorter->pRecord = pNew; - - /* See if the contents of the sorter should now be written out. They - ** are written out when either of the following are true: + /* Figure out whether or not the current contents of memory should be + ** flushed to a PMA before continuing. If so, do so. + ** + ** If using the single large allocation mode (pSorter->aMemory!=0), then + ** flush the contents of memory to a new PMA if (a) at least one value is + ** already in memory and (b) the new value will not fit in memory. + ** + ** Or, if using separate allocations for each record, flush the contents + ** of memory to a PMA if either of the following are true: ** ** * The total memory allocated for the in-memory list is greater ** than (page-size * cache-size), or @@ -898,23 +922,62 @@ int sqlite3VdbeSorterWrite( ** * The total memory allocated for the in-memory list is greater ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ - if( pSorter->mxPmaSize>0 ){ - if( (pNew==&sRecord) || (pSorter->aMemory==0 && ( + nReq = pVal->n + sizeof(SorterRecord); + nPMA = pVal->n + sqlite3VarintLen(pVal->n); + if( pSorter->aMemory ){ + bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize; + }else{ + bFlush = ( (pSorter->nInMemory > pSorter->mxPmaSize) || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) - ))){ -#ifdef SQLITE_DEBUG - i64 nExpect = pSorter->iWriteOff - + sqlite3VarintLen(pSorter->nInMemory) - + pSorter->nInMemory; -#endif - rc = vdbeSorterListToPMA(db, pCsr); - pSorter->nInMemory = 0; - pSorter->iMemory = 0; - assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); - assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); - } + ); } + if( bFlush ){ +#ifdef SQLITE_DEBUG + i64 nExpect = pSorter->iWriteOff + + sqlite3VarintLen(pSorter->nInMemory) + + pSorter->nInMemory; +#endif + rc = vdbeSorterListToPMA(db, pCsr); + pSorter->nInMemory = 0; + pSorter->iMemory = 0; + assert( rc!=SQLITE_OK || (nExpect==pSorter->iWriteOff) ); + assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); + } + + pSorter->nInMemory += nPMA; + + if( pSorter->aMemory ){ + int nMin = pSorter->iMemory + nReq; + + if( nMin>pSorter->nMemory ){ + u8 *aNew; + int nNew = pSorter->nMemory * 2; + while( nNew < nMin ) nNew = nNew*2; + if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; + if( nNew < nMin ) nNew = nMin; + + aNew = sqlite3Realloc(pSorter->aMemory, nNew); + if( !aNew ) return SQLITE_NOMEM; + pSorter->pRecord = aNew + ((u8*)pSorter->pRecord - pSorter->aMemory); + pSorter->aMemory = aNew; + pSorter->nMemory = nNew; + } + + pNew = (SorterRecord*)&pSorter->aMemory[pSorter->iMemory]; + pSorter->iMemory += ROUND8(nReq); + pNew->u.iNext = (u8*)(pSorter->pRecord) - pSorter->aMemory; + }else{ + pNew = (SorterRecord *)sqlite3DbMallocRaw(db, pVal->n+sizeof(SorterRecord)); + if( pNew==0 ){ + return SQLITE_NOMEM; + } + pNew->u.pNext = pSorter->pRecord; + } + + memcpy(SRVAL(pNew), pVal->z, pVal->n); + pNew->nVal = pVal->n; + pSorter->pRecord = pNew; return rc; } @@ -1127,8 +1190,8 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ } }else{ SorterRecord *pFree = pSorter->pRecord; - pSorter->pRecord = pFree->pNext; - pFree->pNext = 0; + pSorter->pRecord = pFree->u.pNext; + pFree->u.pNext = 0; if( pSorter->aMemory==0 ){ vdbeSorterRecordFree(db, pFree); } @@ -1154,7 +1217,7 @@ static void *vdbeSorterRowkey( pKey = pIter->aKey; }else{ *pnKey = pSorter->pRecord->nVal; - pKey = pSorter->pRecord->pVal; + pKey = SRVAL(pSorter->pRecord); } return pKey; } From 73e7d8b2bbccfc462e57973939c2acd887e1067b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Mar 2014 19:47:19 +0000 Subject: [PATCH 018/710] Fix a compiler warning and an after-OOM memory leak. FossilOrigin-Name: 58f7ca29303c280229f86d01f418e1de5f5aebba --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 8 +++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a3dc17de19..e6d729925d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\schanges\sfrom\strunk. -D 2014-03-28T18:35:39.779 +C Fix\sa\scompiler\swarning\sand\san\safter-OOM\smemory\sleak. +D 2014-03-28T19:47:19.372 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 08d5e1ee199599d9571942f0560f84963c7a1a9b +F src/vdbesort.c ad0f9f717a73b870c12c0a0f47781b8b042a5348 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 81987c8ceb64f051528a6ca42673821d9ab7c0ff 27deb6e49bcc76714dbdc61b34748603155ac770 -R 79fcad4572db3d584834fdf0de4170a6 +P 3047a25f1c41e83f0b4772f7c36fbfec0f12dc7e +R 911f6d7d99d9350a186d106d9b5cf527 U drh -Z 871d90dec169e4d494b7ff32475fff97 +Z 55f6b91a865b632e99503eb67df5d777 diff --git a/manifest.uuid b/manifest.uuid index 518ac58c2a..cf96f01707 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3047a25f1c41e83f0b4772f7c36fbfec0f12dc7e \ No newline at end of file +58f7ca29303c280229f86d01f418e1de5f5aebba \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index f229b1f18b..791a2465ca 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -959,7 +959,8 @@ int sqlite3VdbeSorterWrite( aNew = sqlite3Realloc(pSorter->aMemory, nNew); if( !aNew ) return SQLITE_NOMEM; - pSorter->pRecord = aNew + ((u8*)pSorter->pRecord - pSorter->aMemory); + pSorter->pRecord = (SorterRecord*) + (aNew + ((u8*)pSorter->pRecord - pSorter->aMemory)); pSorter->aMemory = aNew; pSorter->nMemory = nNew; } @@ -1052,7 +1053,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ pSorter->aTree = (int *)&pSorter->aIter[N]; pSorter->nTree = N; - do { + while(1){ int iNew; /* Index of new, merged, PMA */ for(iNew=0; @@ -1105,6 +1106,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ if( rc==SQLITE_OK ) rc = rc2; } } + if( rc ) break; if( pSorter->nPMA<=SORTER_MAX_MERGE_COUNT ){ break; @@ -1117,7 +1119,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ pSorter->iReadOff = 0; iWrite2 = 0; } - }while( rc==SQLITE_OK ); + } if( pTemp2 ){ sqlite3OsCloseFree(pTemp2); From ff9fce4d607a92715636898af6bb96ea53a3f84d Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 29 Mar 2014 06:27:35 +0000 Subject: [PATCH 019/710] Add the optimization to avoid some unnecessary calls to sqlite3VdbeRecordUnpack() added to the trunk by [707ea170b3]. FossilOrigin-Name: fc4d04e6b039ea5aeb47739e38c5926e63a4b01b --- manifest | 12 +++++------ manifest.uuid | 2 +- src/vdbesort.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 58 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 52c9a03b24..a8a84cabb7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\schanges\sfrom\sorderby-planning\sbranch. -D 2014-03-28T19:18:16.969 +C Add\sthe\soptimization\sto\savoid\ssome\sunnecessary\scalls\sto\ssqlite3VdbeRecordUnpack()\sadded\sto\sthe\strunk\sby\s[707ea170b3]. +D 2014-03-29T06:27:35.162 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 01068b89364fa2bffeba9b929367ed04661e97f7 +F src/vdbesort.c d7ef3c431d7996907815a13579952506bf2bdf8b F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8cb2b02baa7ef9aa96319e977f0315328f944237 3047a25f1c41e83f0b4772f7c36fbfec0f12dc7e -R 132477cf9a18d083088d044c20454723 +P 4c7fb5423430f3b936befaa7c309f8e1968ee7d8 +R c555f8c440b397229a1b1d80867cea18 U dan -Z b189ca5a1e8d41a36f80415af9011f68 +Z 5b7820939497d4f0728d4d29380b4189 diff --git a/manifest.uuid b/manifest.uuid index a600160612..d7c0207d3a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c7fb5423430f3b936befaa7c309f8e1968ee7d8 \ No newline at end of file +fc4d04e6b039ea5aeb47739e38c5926e63a4b01b \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index e01790d600..e6fbbdd2df 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1052,11 +1052,59 @@ static int vdbeSorterNext( rc = vdbeSorterIterNext(&pMerger->aIter[iPrev]); /* Update contents of aTree[] */ - for(i=(pMerger->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ - rc = vdbeSorterDoCompare(pThread, pMerger, i); + if( rc==SQLITE_OK ){ + int i; /* Index of aTree[] to recalculate */ + VdbeSorterIter *pIter1; /* First iterator to compare */ + VdbeSorterIter *pIter2; /* Second iterator to compare */ + u8 *pKey2; /* To pIter2->aKey, or 0 if record cached */ + + /* Find the first two iterators to compare. The one that was just + ** advanced (iPrev) and the one next to it in the array. */ + pIter1 = &pMerger->aIter[(iPrev & 0xFFFE)]; + pIter2 = &pMerger->aIter[(iPrev | 0x0001)]; + pKey2 = pIter2->aKey; + + for(i=(pMerger->nTree+iPrev)/2; i>0; i=i/2){ + /* Compare pIter1 and pIter2. Store the result in variable iRes. */ + int iRes; + if( pIter1->pFile==0 ){ + iRes = +1; + }else if( pIter2->pFile==0 ){ + iRes = -1; + }else{ + vdbeSorterCompare(pThread, 0, + pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey, &iRes + ); + } + + /* If pIter1 contained the smaller value, set aTree[i] to its index. + ** Then set pIter2 to the next iterator to compare to pIter1. In this + ** case there is no cache of pIter2 in pThread->pUnpacked, so set + ** pKey2 to point to the record belonging to pIter2. + ** + ** Alternatively, if pIter2 contains the smaller of the two values, + ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() + ** was actually called above, then pThread->pUnpacked now contains + ** a value equivalent to pIter2. So set pKey2 to NULL to prevent + ** vdbeSorterCompare() from decoding pIter2 again. + ** + ** If the two values were equal, then the value from the oldest + ** PMA should be considered smaller. The VdbeSorter.aIter[] array + ** is sorted from oldest to newest, so pIter1 contains older values + ** than pIter2 iff (pIter1aTree[i] = (int)(pIter1 - pMerger->aIter); + pIter2 = &pMerger->aIter[ pMerger->aTree[i ^ 0x0001] ]; + pKey2 = pIter2->aKey; + }else{ + if( pIter1->pFile ) pKey2 = 0; + pMerger->aTree[i] = (int)(pIter2 - pMerger->aIter); + pIter1 = &pMerger->aIter[ pMerger->aTree[i ^ 0x0001] ]; + } + } + *pbEof = (pMerger->aIter[pMerger->aTree[1]].pFile==0); } - *pbEof = (pMerger->aIter[pMerger->aTree[1]].pFile==0); return rc; } From e7c84cc7e30a90fdefc8508b5ef6fc2bed48e115 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 29 Mar 2014 09:34:45 +0000 Subject: [PATCH 020/710] Fix a problem in vdbesort.c causing spurious SQLITE_NOMEM errors when using memsys3 or memsys5. FossilOrigin-Name: a683c05f6250389e84b980b16559e162ba1a27c2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 44 ++++++++++++++++++++++++-------------------- 3 files changed, 31 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index a8a84cabb7..5859cf8e2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\soptimization\sto\savoid\ssome\sunnecessary\scalls\sto\ssqlite3VdbeRecordUnpack()\sadded\sto\sthe\strunk\sby\s[707ea170b3]. -D 2014-03-29T06:27:35.162 +C Fix\sa\sproblem\sin\svdbesort.c\scausing\sspurious\sSQLITE_NOMEM\serrors\swhen\susing\smemsys3\sor\smemsys5. +D 2014-03-29T09:34:45.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c d7ef3c431d7996907815a13579952506bf2bdf8b +F src/vdbesort.c 80812ceb596febe778d506587d24bef6fabc00d4 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4c7fb5423430f3b936befaa7c309f8e1968ee7d8 -R c555f8c440b397229a1b1d80867cea18 +P fc4d04e6b039ea5aeb47739e38c5926e63a4b01b +R 58d9f269ef794b7845b0943176283ea0 U dan -Z 5b7820939497d4f0728d4d29380b4189 +Z 849635fb9198ef253e10fe8fcdddee0e diff --git a/manifest.uuid b/manifest.uuid index d7c0207d3a..0e4e18ad35 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc4d04e6b039ea5aeb47739e38c5926e63a4b01b \ No newline at end of file +a683c05f6250389e84b980b16559e162ba1a27c2 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index e6fbbdd2df..d9c26f209e 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1046,7 +1046,6 @@ static int vdbeSorterNext( ){ int rc; int iPrev = pMerger->aTree[1];/* Index of iterator to advance */ - int i; /* Index of aTree[] to recalculate */ /* Advance the current iterator */ rc = vdbeSorterIterNext(&pMerger->aIter[iPrev]); @@ -1281,11 +1280,14 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ if( bUseFg==0 ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; - if( pSorter->aMemory==0 ){ - pSorter->aMemory = sqlite3Malloc(pSorter->nMemory); - if( pSorter->aMemory==0 ) return SQLITE_NOMEM; - }else{ - pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory); + assert( pSorter->aMemory==0 || pThread->aListMemory==0 ); + if( pThread->aListMemory ){ + if( pSorter->aMemory==0 ){ + pSorter->aMemory = sqlite3Malloc(pSorter->nMemory); + if( pSorter->aMemory==0 ) return SQLITE_NOMEM; + }else{ + pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory); + } } rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSorterThreadMain, pCtx); }else{ @@ -1337,19 +1339,21 @@ int sqlite3VdbeSorterWrite( */ nReq = pVal->n + sizeof(SorterRecord); nPMA = pVal->n + sqlite3VarintLen(pVal->n); - if( pSorter->aMemory ){ - bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize; - }else{ - bFlush = ( - (pSorter->nInMemory > pSorter->mxPmaSize) - || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) - ); - } - if( bFlush ){ - rc = vdbeSorterFlushPMA(db, pCsr, 0); - pSorter->nInMemory = 0; - pSorter->iMemory = 0; - assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); + if( pSorter->mxPmaSize ){ + if( pSorter->aMemory ){ + bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize; + }else{ + bFlush = ( + (pSorter->nInMemory > pSorter->mxPmaSize) + || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) + ); + } + if( bFlush ){ + rc = vdbeSorterFlushPMA(db, pCsr, 0); + pSorter->nInMemory = 0; + pSorter->iMemory = 0; + assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); + } } pSorter->nInMemory += nPMA; @@ -1375,7 +1379,7 @@ int sqlite3VdbeSorterWrite( pSorter->iMemory += ROUND8(nReq); pNew->u.iNext = (u8*)(pSorter->pRecord) - pSorter->aMemory; }else{ - pNew = (SorterRecord *)sqlite3Malloc(pVal->n+sizeof(SorterRecord)); + pNew = (SorterRecord *)sqlite3Malloc(nReq); if( pNew==0 ){ return SQLITE_NOMEM; } From 853c4a7621b37ecb3c4cab8ca1fb7fc06fb1a533 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 29 Mar 2014 10:01:58 +0000 Subject: [PATCH 021/710] Fix a broken assert() in vdbesort.c. FossilOrigin-Name: 18d1b402f2dbe78f1a1113bb356b710e348365ef --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5859cf8e2e..0b2bd63f50 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\svdbesort.c\scausing\sspurious\sSQLITE_NOMEM\serrors\swhen\susing\smemsys3\sor\smemsys5. -D 2014-03-29T09:34:45.457 +C Fix\sa\sbroken\sassert()\sin\svdbesort.c. +D 2014-03-29T10:01:58.802 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 80812ceb596febe778d506587d24bef6fabc00d4 +F src/vdbesort.c 2881297f4acdba5908078c5d7f00635288a1ca08 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P fc4d04e6b039ea5aeb47739e38c5926e63a4b01b -R 58d9f269ef794b7845b0943176283ea0 +P a683c05f6250389e84b980b16559e162ba1a27c2 +R d8e63408790a442d33eec2a57272c54d U dan -Z 849635fb9198ef253e10fe8fcdddee0e +Z 6c5e71d99f167a1ded5f09353b3e0513 diff --git a/manifest.uuid b/manifest.uuid index 0e4e18ad35..1bf7c269ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a683c05f6250389e84b980b16559e162ba1a27c2 \ No newline at end of file +18d1b402f2dbe78f1a1113bb356b710e348365ef \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index d9c26f209e..c6927f87a7 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1280,7 +1280,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ if( bUseFg==0 ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; - assert( pSorter->aMemory==0 || pThread->aListMemory==0 ); + assert( pSorter->aMemory==0 || pThread->aListMemory!=0 ); if( pThread->aListMemory ){ if( pSorter->aMemory==0 ){ pSorter->aMemory = sqlite3Malloc(pSorter->nMemory); From b3f56fdb69201329c96ef9bb904a5df46fc62c0d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 31 Mar 2014 19:57:34 +0000 Subject: [PATCH 022/710] Add the SQLITE_MAX_WORKER_THREADS compile time option. And the SQLITE_CONFIG_WORKER_THREADS sqlite3_config() switch. FossilOrigin-Name: 2774710df8cd2bfaca49888c69f1b01c0ddadf9a --- manifest | 22 +++++++-------- manifest.uuid | 2 +- src/global.c | 1 + src/main.c | 7 +++++ src/sqlite.h.in | 11 ++++++++ src/sqliteInt.h | 14 ++++++++++ src/threads.c | 3 +++ src/vdbesort.c | 72 ++++++++++++++++++++++++++++--------------------- 8 files changed, 90 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index 0b2bd63f50..4a4f37cc87 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbroken\sassert()\sin\svdbesort.c. -D 2014-03-29T10:01:58.802 +C Add\sthe\sSQLITE_MAX_WORKER_THREADS\scompile\stime\soption.\sAnd\sthe\sSQLITE_CONFIG_WORKER_THREADS\ssqlite3_config()\sswitch. +D 2014-03-31T19:57:34.075 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/expr.c da2b3cb41081af6b56e95e7c9e95949564ce2e21 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 -F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 +F src/global.c 57d9dd92f4e2469cf4046847f456d45bdda0f202 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -186,7 +186,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 691b25754bef596108fe60ff1bcbe8445369c9db +F src/main.c d3655832585baef4c2356529a5c6ca5ca3bd7c1f F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -219,10 +219,10 @@ F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 20055cf917222e660c4222fea306bd13a0623caa F src/shell.c f48b63f8e582e7998ecefd051d697f91fb1453df -F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 +F src/sqlite.h.in 0249af5d9d3bbeab0dc1f58e1f9fee878807732a F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 3f5190a4e07ca227035334da8d66ebe227071528 +F src/sqliteInt.h 7f42c2792b951db22fa189bbed828a5e3b38789c F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -272,7 +272,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c b96d62f88c06d4fa980a4a92685d1b130c4c84d3 +F src/threads.c 6992f70cab8d5d8451a6b5641a9256d1749af87b F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 2881297f4acdba5908078c5d7f00635288a1ca08 +F src/vdbesort.c b4d6133bada297e118492420346f83cd76c6da31 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a683c05f6250389e84b980b16559e162ba1a27c2 -R d8e63408790a442d33eec2a57272c54d +P 18d1b402f2dbe78f1a1113bb356b710e348365ef +R 8b3347b8372cc17a3226330524ab6da6 U dan -Z 6c5e71d99f167a1ded5f09353b3e0513 +Z 11cb5db8cbfdddf6047ddaf9c26850df diff --git a/manifest.uuid b/manifest.uuid index 1bf7c269ad..096bf0eab2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18d1b402f2dbe78f1a1113bb356b710e348365ef \ No newline at end of file +2774710df8cd2bfaca49888c69f1b01c0ddadf9a \ No newline at end of file diff --git a/src/global.c b/src/global.c index 1ee3f6436f..4ed597d292 100644 --- a/src/global.c +++ b/src/global.c @@ -167,6 +167,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ + SQLITE_MAX_WORKER_THREADS, /* nWorker */ /* All the rest should always be initialized to zero */ 0, /* isInit */ 0, /* inProgress */ diff --git a/src/main.c b/src/main.c index 9e83d4963a..c1eaa6849a 100644 --- a/src/main.c +++ b/src/main.c @@ -515,6 +515,13 @@ int sqlite3_config(int op, ...){ } #endif + case SQLITE_CONFIG_WORKER_THREADS: { + int n = va_arg(ap, int); + if( n>SQLITE_MAX_WORKER_THREADS ) n = SQLITE_MAX_WORKER_THREADS; + if( n>=0 ) sqlite3GlobalConfig.nWorker = n; + break; + } + default: { rc = SQLITE_ERROR; break; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 5d2c87552c..78aa9c36e1 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1715,6 +1715,16 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** +** +** [[SQLITE_CONFIG_WORKER_THREADS]] +**
SQLITE_CONFIG_WORKER_THREADS +**
^SQLITE_CONFIG_WORKER_THREADS takes a single argument of type int. +** It is used to set the number of background worker threads that may be +** launched when sorting large amounts of data. A value of 0 means launch +** no background threads at all. The maximum number of background threads +** allowed is configured at build-time by the SQLITE_MAX_WORKER_THREADS +** pre-processor option. +** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -1739,6 +1749,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ #define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ +#define SQLITE_CONFIG_WORKER_THREADS 24 /* int nWorker */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 41219c710a..5f85e80d82 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -422,6 +422,19 @@ # define SQLITE_TEMP_STORE_xc 1 /* Exclude from ctime.c */ #endif +/* +** If no value has been provided for SQLITE_MAX_WORKER_THREADS, or if +** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it +** to zero. +*/ +#if SQLITE_TEMP_STORE==3 +# undef SQLITE_MAX_WORKER_THREADS +#endif +#ifndef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 0 +#endif + + /* ** GCC does not define the offsetof() macro so we'll have to do it ** ourselves. @@ -2685,6 +2698,7 @@ struct Sqlite3Config { int nPage; /* Number of pages in pPage[] */ int mxParserStack; /* maximum depth of the parser stack */ int sharedCacheEnabled; /* true if shared-cache mode enabled */ + int nWorker; /* Number of worker threads to use */ /* The above might be initialized to non-zero. The following need to always ** initially be zero, however. */ int isInit; /* True after initialization has finished */ diff --git a/src/threads.c b/src/threads.c index cb148b6c6f..96cb17cef7 100644 --- a/src/threads.c +++ b/src/threads.c @@ -27,6 +27,8 @@ */ #include "sqliteInt.h" +#if SQLITE_MAX_WORKER_THREADS>0 + /********************************* Unix Pthreads ****************************/ #if SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) && SQLITE_THREADSAFE>0 @@ -215,3 +217,4 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ #endif /* !defined(SQLITE_THREADS_IMPLEMENTED) */ /****************************** End Single-Threaded *************************/ +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ diff --git a/src/vdbesort.c b/src/vdbesort.c index c6927f87a7..84c6fadc94 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -26,14 +26,6 @@ typedef struct SorterMerger SorterMerger; typedef struct FileWriter FileWriter; -/* -** Maximum number of threads to use. Setting this value to 1 forces all -** operations to be single-threaded. -*/ -#ifndef SQLITE_MAX_SORTER_THREAD -# define SQLITE_MAX_SORTER_THREAD 4 -#endif - /* ** Candidate values for SorterThread.eWork */ @@ -48,9 +40,10 @@ typedef struct FileWriter FileWriter; ** is configured and passed to vdbeSorterThreadMain() - either directly by ** the main thread or via a background thread. ** -** Exactly SQLITE_MAX_SORTER_THREAD instances of this structure are allocated +** Exactly SorterThread.nThread instances of this structure are allocated ** as part of each VdbeSorter object. Instances are never allocated any other -** way. +** way. SorterThread.nThread is set to the number of worker threads allowed +** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). ** ** When a background thread is launched to perform work, SorterThread.bDone ** is set to 0 and the SorterThread.pThread variable set to point to the @@ -59,7 +52,7 @@ typedef struct FileWriter FileWriter; ** exits. SorterThread.pThread and bDone are always cleared after the ** background thread has been joined. ** -** One object (specifically, VdbeSorter.aThread[SQLITE_MAX_SORTER_THREAD-1]) +** One object (specifically, VdbeSorter.aThread[SorterThread.nThread-1]) ** is reserved for the foreground thread. ** ** The nature of the work performed is determined by SorterThread.eWork, @@ -187,7 +180,8 @@ struct VdbeSorter { u8 *aMemory; /* Block of memory to alloc records from */ int iMemory; /* Offset of first free byte in aMemory */ int nMemory; /* Size of aMemory allocation in bytes */ - SorterThread aThread[SQLITE_MAX_SORTER_THREAD]; + int nThread; /* Size of aThread[] array */ + SorterThread aThread[1]; }; /* @@ -572,7 +566,7 @@ static int vdbeSorterDoCompare( iRes = i1; }else{ int res; - assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterMerge() */ + assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterThreadMain() */ vdbeSorterCompare( pThread, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res ); @@ -597,21 +591,26 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter; /* The new sorter */ KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ + int sz; /* Size of pSorter in bytes */ int rc = SQLITE_OK; + int nWorker = (sqlite3GlobalConfig.bCoreMutex?sqlite3GlobalConfig.nWorker:0); assert( pCsr->pKeyInfo && pCsr->pBt==0 ); szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); - pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sizeof(VdbeSorter)+szKeyInfo); + sz = sizeof(VdbeSorter) + nWorker * sizeof(SorterThread); + + pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); pCsr->pSorter = pSorter; if( pSorter==0 ){ rc = SQLITE_NOMEM; }else{ - pKeyInfo = (KeyInfo*)&pSorter[1]; + pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - for(i=0; inThread = nWorker + 1; + for(i=0; inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; pThread->pKeyInfo = pKeyInfo; pThread->pVfs = db->pVfs; @@ -674,10 +673,11 @@ static void vdbeSorterThreadCleanup(sqlite3 *db, SorterThread *pThread){ /* ** Join all threads. */ +#if SQLITE_MAX_WORKER_THREADS>0 static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int rc = rcin; int i; - for(i=0; inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; if( pThread->pThread ){ void *pRet; @@ -690,6 +690,9 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ } return rc; } +#else +# define vdbeSorterJoinAll(x,rcin) (rcin) +#endif /* ** Allocate a new SorterMerger object with space for nIter iterators. @@ -739,7 +742,7 @@ static void vdbeSorterMergerFree(SorterMerger *pMerger){ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ int i; vdbeSorterJoinAll(pSorter, SQLITE_OK); - for(i=0; inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; vdbeSorterThreadCleanup(db, pThread); } @@ -1246,8 +1249,9 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ SorterThread *pThread; /* Thread context used to create new PMA */ pSorter->bUsePMA = 1; - for(i=0; ALWAYS( inThread ); i++){ pThread = &pSorter->aThread[i]; +#if SQLITE_MAX_WORKER_THREADS>0 if( pThread->bDone ){ void *pRet; assert( pThread->pThread ); @@ -1258,11 +1262,12 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ rc = SQLITE_PTR_TO_INT(pRet); } } +#endif if( pThread->pThread==0 ) break; } if( rc==SQLITE_OK ){ - int bUseFg = (bFg || i==(SQLITE_MAX_SORTER_THREAD-1)); + int bUseFg = (bFg || i==(pSorter->nThread-1)); assert( pThread->pThread==0 && pThread->bDone==0 ); pThread->eWork = SORTER_THREAD_TO_PMA; @@ -1277,6 +1282,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ pSorter->aMemory = aMem; } +#if SQLITE_MAX_WORKER_THREADS>0 if( bUseFg==0 ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; @@ -1290,7 +1296,9 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } } rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSorterThreadMain, pCtx); - }else{ + }else +#endif + { /* Use the foreground thread for this operation */ u8 *aMem; rc = vdbeSorterRunThread(pThread); @@ -1370,7 +1378,9 @@ int sqlite3VdbeSorterWrite( aNew = sqlite3Realloc(pSorter->aMemory, nNew); if( !aNew ) return SQLITE_NOMEM; - pSorter->pRecord = aNew + ((u8*)pSorter->pRecord - pSorter->aMemory); + pSorter->pRecord = (SorterRecord*)( + aNew + ((u8*)pSorter->pRecord - pSorter->aMemory) + ); pSorter->aMemory = aNew; pSorter->nMemory = nNew; } @@ -1399,7 +1409,7 @@ int sqlite3VdbeSorterWrite( static int vdbeSorterCountPMA(VdbeSorter *pSorter){ int nPMA = 0; int i; - for(i=0; inThread; i++){ nPMA += pSorter->aThread[i].nPMA; } return nPMA; @@ -1446,19 +1456,21 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ /* If there are more than SORTER_MAX_MERGE_COUNT PMAs on disk, merge ** some of them together so that this is no longer the case. */ - assert( SORTER_MAX_MERGE_COUNT>=SQLITE_MAX_SORTER_THREAD ); if( vdbeSorterCountPMA(pSorter)>SORTER_MAX_MERGE_COUNT ){ int i; - for(i=0; rc==SQLITE_OK && inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; if( pThread->pTemp1 ){ - pThread->nConsolidate = SORTER_MAX_MERGE_COUNT/SQLITE_MAX_SORTER_THREAD; + pThread->nConsolidate = SORTER_MAX_MERGE_COUNT/pSorter->nThread; pThread->eWork = SORTER_THREAD_CONS; - if( i<(SQLITE_MAX_SORTER_THREAD-1) ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( i<(pSorter->nThread-1) ){ void *pCtx = (void*)pThread; rc = sqlite3ThreadCreate(&pThread->pThread,vdbeSorterThreadMain,pCtx); - }else{ + }else +#endif + { rc = vdbeSorterRunThread(pThread); } } @@ -1475,7 +1487,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ int nIter = 0; /* Number of iterators used */ int i; SorterMerger *pMerger; - for(i=0; inThread; i++){ nIter += pSorter->aThread[i].nPMA; } @@ -1485,7 +1497,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ }else{ int iIter = 0; int iThread = 0; - for(iThread=0; iThreadnThread; iThread++){ int iPMA; i64 iReadOff = 0; SorterThread *pThread = &pSorter->aThread[iThread]; From 38fdead890bc6006ee4d550b9694aa495ddc9798 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 1 Apr 2014 10:19:02 +0000 Subject: [PATCH 023/710] Fix a problem with OOM handling in the sorter code. FossilOrigin-Name: 59cd5229e2b5be5272cf57c7e7d09e97d16a5425 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/btree.c | 9 ++++++--- src/sqliteInt.h | 2 +- src/test1.c | 41 +++++++++++++++++++++++++++++++++++++++++ src/vdbeaux.c | 26 ++++++++++++++++---------- src/vdbesort.c | 9 ++++++--- test/malloc.test | 22 ++++++++++++++++++++++ 8 files changed, 104 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 4a4f37cc87..b819b3428f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_MAX_WORKER_THREADS\scompile\stime\soption.\sAnd\sthe\sSQLITE_CONFIG_WORKER_THREADS\ssqlite3_config()\sswitch. -D 2014-03-31T19:57:34.075 +C Fix\sa\sproblem\swith\sOOM\shandling\sin\sthe\ssorter\scode. +D 2014-04-01T10:19:02.635 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c a59a199f21338ae1847d69f5db87c3e8ef1b1578 +F src/btree.c 0d1be67448c45eccc40114556186397eb9da7f7d F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 @@ -222,12 +222,12 @@ F src/shell.c f48b63f8e582e7998ecefd051d697f91fb1453df F src/sqlite.h.in 0249af5d9d3bbeab0dc1f58e1f9fee878807732a F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 7f42c2792b951db22fa189bbed828a5e3b38789c +F src/sqliteInt.h 3ed0fedb5b64ece395a2114b7c73417678f3e420 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd -F src/test1.c 2401eee14a4309a7cfe2aeb2f30ad517a1d9c299 +F src/test1.c 31596bf8a9c0629f88e514a4ec864847c8946c4e F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -283,10 +283,10 @@ F src/vdbe.c 02f2de0b2f3b198438e3e64a2ceba9407bb8348b F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d +F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c b4d6133bada297e118492420346f83cd76c6da31 +F src/vdbesort.c 35c270630fa5af14791fc6abc70024d1aeeaac0e F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -663,7 +663,7 @@ F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 F test/lookaside.test 93f07bac140c5bb1d49f3892d2684decafdc7af2 F test/main.test 39c4bb8a157f57298ed1659d6df89d9f35aaf2c8 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 -F test/malloc.test fd368e31fe98d4779ed80442f311ed9f03bcd1f7 +F test/malloc.test 26ae08a09cc15a98d147ee63925e3a66048e71c9 F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 F test/malloc5.test fafce0aa9157060445cd1a56ad50fc79d82f28c3 @@ -1160,7 +1160,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 18d1b402f2dbe78f1a1113bb356b710e348365ef -R 8b3347b8372cc17a3226330524ab6da6 +P 2774710df8cd2bfaca49888c69f1b01c0ddadf9a +R a6fe6d7d652069432584e1a691602cb5 U dan -Z 11cb5db8cbfdddf6047ddaf9c26850df +Z ac080311c2ad2c3d7bf733448ffc0f03 diff --git a/manifest.uuid b/manifest.uuid index 096bf0eab2..7a6826cad1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2774710df8cd2bfaca49888c69f1b01c0ddadf9a \ No newline at end of file +59cd5229e2b5be5272cf57c7e7d09e97d16a5425 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c3055836c9..fb49746567 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4588,7 +4588,7 @@ int sqlite3BtreeMovetoUnpacked( if( pIdxKey ){ xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); - pIdxKey->isCorrupt = 0; + pIdxKey->errCode = 0; assert( pIdxKey->default_rc==1 || pIdxKey->default_rc==0 || pIdxKey->default_rc==-1 @@ -4712,7 +4712,10 @@ int sqlite3BtreeMovetoUnpacked( c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); sqlite3_free(pCellKey); } - assert( pIdxKey->isCorrupt==0 || c==0 ); + assert( + (pIdxKey->errCode!=SQLITE_CORRUPT || c==0) + && (pIdxKey->errCode!=SQLITE_NOMEM || !pCur->pBtree->db->mallocFailed) + ); if( c<0 ){ lwr = idx+1; }else if( c>0 ){ @@ -4722,7 +4725,7 @@ int sqlite3BtreeMovetoUnpacked( *pRes = 0; rc = SQLITE_OK; pCur->aiIdx[pCur->iPage] = (u16)idx; - if( pIdxKey->isCorrupt ) rc = SQLITE_CORRUPT; + if( pIdxKey->errCode ) rc = SQLITE_CORRUPT; goto moveto_finish; } if( lwr>upr ) break; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5f85e80d82..0763f085ab 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1641,7 +1641,7 @@ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ - u8 isCorrupt; /* Corruption detected by xRecordCompare() */ + u8 errCode; /* Error detected by xRecordCompare (CORRUPT or NOMEM) */ Mem *aMem; /* Values */ int r1; /* Value to return if (lhs > rhs) */ int r2; /* Value to return if (rhs < lhs) */ diff --git a/src/test1.c b/src/test1.c index 3000288c7d..4b485ce741 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2703,6 +2703,46 @@ bad_args: return TCL_ERROR; } +/* +** Usage: add_test_utf16bin_collate +** +** Add a utf-16 collation sequence named "utf16bin" to the database +** handle. This collation sequence compares arguments in the same way as the +** built-in collation "binary". +*/ +static int test_utf16bin_collate_func( + void *pCtx, + int nA, const void *zA, + int nB, const void *zB +){ + int nCmp = (nA>nB ? nB : nA); + int res = memcmp(zA, zB, nCmp); + if( res==0 ) res = nA - nB; + return res; +} +static int test_utf16bin_collate( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3 *db; + int rc; + + if( objc!=2 ) goto bad_args; + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + + rc = sqlite3_create_collation(db, "utf16bin", SQLITE_UTF16, 0, + test_utf16bin_collate_func + ); + if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; + return TCL_OK; + +bad_args: + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; +} + /* ** When the collation needed callback is invoked, record the name of ** the requested collating function here. The recorded name is linked @@ -6481,6 +6521,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "add_test_collate", test_collate, 0 }, { "add_test_collate_needed", test_collate_needed, 0 }, { "add_test_function", test_function, 0 }, + { "add_test_utf16bin_collate", test_utf16bin_collate, 0 }, #endif { "sqlite3_test_errstr", test_errstr, 0 }, { "tcl_variable_type", tcl_variable_type, 0 }, diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0ce21378d5..7194dea36a 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3229,7 +3229,8 @@ static int vdbeRecordCompareDebug( static int vdbeCompareMemString( const Mem *pMem1, const Mem *pMem2, - const CollSeq *pColl + const CollSeq *pColl, + u8 *prcErr /* If an OOM occurs, set to SQLITE_NOMEM */ ){ if( pMem1->enc==pColl->enc ){ /* The strings are already in the correct encoding. Call the @@ -3252,6 +3253,7 @@ static int vdbeCompareMemString( rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); sqlite3VdbeMemRelease(&c1); sqlite3VdbeMemRelease(&c2); + if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM; return rc; } } @@ -3334,7 +3336,7 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ assert( !pColl || pColl->xCmp ); if( pColl ){ - return vdbeCompareMemString(pMem1, pMem2, pColl); + return vdbeCompareMemString(pMem1, pMem2, pColl, 0); } /* If a NULL pointer was passed as the collate function, fall through ** to the blob case and use memcmp(). */ @@ -3406,8 +3408,10 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** fields that appear in both keys are equal, then pPKey2->default_rc is ** returned. ** -** If database corruption is discovered, set pPKey2->isCorrupt to non-zero -** and return 0. +** If database corruption is discovered, set pPKey2->errCode to +** SQLITE_CORRUPT and return 0. If an OOM error is encountered, +** pPKey2->errCode is set to SQLITE_NOMEM and, if it is not NULL, the +** malloc-failed flag set on database handle (pPKey2->pKeyInfo->db). */ int sqlite3VdbeRecordCompare( int nKey1, const void *pKey1, /* Left key */ @@ -3438,7 +3442,7 @@ int sqlite3VdbeRecordCompare( idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; if( d1>(unsigned)nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ } i = 0; @@ -3517,14 +3521,16 @@ int sqlite3VdbeRecordCompare( testcase( (d1+mem1.n)==(unsigned)nKey1 ); testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); if( (d1+mem1.n) > (unsigned)nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ }else if( pKeyInfo->aColl[i] ){ mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; mem1.flags = MEM_Str; mem1.z = (char*)&aKey1[d1]; - rc = vdbeCompareMemString(&mem1, pRhs, pKeyInfo->aColl[i]); + rc = vdbeCompareMemString( + &mem1, pRhs, pKeyInfo->aColl[i], &pPKey2->errCode + ); }else{ int nCmp = MIN(mem1.n, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); @@ -3544,7 +3550,7 @@ int sqlite3VdbeRecordCompare( testcase( (d1+nStr)==(unsigned)nKey1 ); testcase( (d1+nStr+1)==(unsigned)nKey1 ); if( (d1+nStr) > (unsigned)nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ }else{ int nCmp = MIN(nStr, pRhs->n); @@ -3564,7 +3570,7 @@ int sqlite3VdbeRecordCompare( if( pKeyInfo->aSortOrder[i] ){ rc = -rc; } - assert( CORRUPT_DB + assert( CORRUPT_DB || pKeyInfo->db==0 || (rc<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) || (rc>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) || pKeyInfo->db->mallocFailed @@ -3724,7 +3730,7 @@ static int vdbeRecordCompareString( nStr = (serial_type-12) / 2; if( (szHdr + nStr) > nKey1 ){ - pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ } nCmp = MIN( pPKey2->aMem[0].n, nStr ); diff --git a/src/vdbesort.c b/src/vdbesort.c index 84c6fadc94..60a023c8f5 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1134,6 +1134,7 @@ static void *vdbeSorterThreadMain(void *pCtx){ goto thread_out; } pThread->pUnpacked->nField = pThread->pKeyInfo->nField; + pThread->pUnpacked->errCode = 0; } if( pThread->eWork==SORTER_THREAD_CONS ){ @@ -1222,6 +1223,10 @@ static void *vdbeSorterThreadMain(void *pCtx){ thread_out: pThread->bDone = 1; + if( rc==SQLITE_OK && pThread->pUnpacked->errCode ){ + assert( pThread->pUnpacked->errCode==SQLITE_NOMEM ); + rc = SQLITE_NOMEM; + } return SQLITE_INT_TO_PTR(rc); } @@ -1267,8 +1272,6 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } if( rc==SQLITE_OK ){ - int bUseFg = (bFg || i==(pSorter->nThread-1)); - assert( pThread->pThread==0 && pThread->bDone==0 ); pThread->eWork = SORTER_THREAD_TO_PMA; pThread->pList = pSorter->pRecord; @@ -1283,7 +1286,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } #if SQLITE_MAX_WORKER_THREADS>0 - if( bUseFg==0 ){ + if( bFg || i==(pSorter->nThread-1) ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; assert( pSorter->aMemory==0 || pThread->aListMemory!=0 ); diff --git a/test/malloc.test b/test/malloc.test index 5d03aa8fe8..10d2a18c96 100644 --- a/test/malloc.test +++ b/test/malloc.test @@ -880,6 +880,28 @@ do_malloc_test 39 -tclprep { db close } +reset_db +add_test_utf16bin_collate db +do_execsql_test 40.1 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES('fghij'); + INSERT INTO t1 VALUES('pqrst'); + INSERT INTO t1 VALUES('abcde'); + INSERT INTO t1 VALUES('uvwxy'); + INSERT INTO t1 VALUES('klmno'); +} +do_execsql_test 40.2 { + SELECT * FROM t1 ORDER BY 1 COLLATE utf16bin; +} {abcde fghij klmno pqrst uvwxy} +do_faultsim_test 40.3 -faults oom-trans* -body { + execsql { + SELECT * FROM t1 ORDER BY 1 COLLATE utf16bin; + } +} -test { + faultsim_test_result {0 {abcde fghij klmno pqrst uvwxy}} + faultsim_integrity_check +} + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} From 578e1ca8d7d00805fb4410210ba317d2b8f9e846 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 1 Apr 2014 15:38:44 +0000 Subject: [PATCH 024/710] Even if compile time option SQLITE_MAX_WORKER_THREADS is set to one or greater, set the default number of worker threads to zero. Distribute data more evenly between threads in sqlite3VdbeSorterWrite() to improve performance when sorting large amounts of data. Add new test file sort2.test. FossilOrigin-Name: 643c86a056168e39fcb7f39b8a72731f1eb246db --- manifest | 21 +++++++++++---------- manifest.uuid | 2 +- src/global.c | 2 +- src/shell.c | 2 ++ src/test_config.c | 6 ++++++ src/test_malloc.c | 27 +++++++++++++++++++++++++++ src/vdbesort.c | 18 +++++++++++++----- test/sort2.test | 41 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 102 insertions(+), 17 deletions(-) create mode 100644 test/sort2.test diff --git a/manifest b/manifest index b819b3428f..73c52a7c23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sOOM\shandling\sin\sthe\ssorter\scode. -D 2014-04-01T10:19:02.635 +C Even\sif\scompile\stime\soption\sSQLITE_MAX_WORKER_THREADS\sis\sset\sto\sone\sor\sgreater,\sset\sthe\sdefault\snumber\sof\sworker\sthreads\sto\szero.\sDistribute\sdata\smore\sevenly\sbetween\sthreads\sin\ssqlite3VdbeSorterWrite()\sto\simprove\sperformance\swhen\ssorting\slarge\samounts\sof\sdata.\sAdd\snew\stest\sfile\ssort2.test. +D 2014-04-01T15:38:44.990 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/expr.c da2b3cb41081af6b56e95e7c9e95949564ce2e21 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 -F src/global.c 57d9dd92f4e2469cf4046847f456d45bdda0f202 +F src/global.c deadd872189b92aca4ee2566332a86315839f811 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -218,7 +218,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 20055cf917222e660c4222fea306bd13a0623caa -F src/shell.c f48b63f8e582e7998ecefd051d697f91fb1453df +F src/shell.c a08060750f92461fc462b4f767e3b0d19d6b832e F src/sqlite.h.in 0249af5d9d3bbeab0dc1f58e1f9fee878807732a F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -240,7 +240,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c 0336e0bdbe541b4af89d7e3dd0656e8e6b51e585 +F src/test_config.c ebd0a42983b696f2b121515d577753cf2afdc9b0 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -251,7 +251,7 @@ F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61 F src/test_intarray.h 2ece66438cfd177b78d1bfda7a4180cd3a10844d F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 -F src/test_malloc.c 1ff5b1243d96124c9a180f3b89424820a1f337f3 +F src/test_malloc.c 27047a841f5bff1cb638123806a2c30714771307 F src/test_multiplex.c 9f304bf04170c91c0318238d512df2da039eb1c8 F src/test_multiplex.h 110a8c4d356e0aa464ca8730375608a9a0b61ae1 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 35c270630fa5af14791fc6abc70024d1aeeaac0e +F src/vdbesort.c 375919a7165647cccf9f7218422540fa61e1dcb1 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -818,6 +818,7 @@ F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a +F test/sort2.test 65c1e8b49f13f5973d1b03e3ef5b1c864eadd574 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1160,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2774710df8cd2bfaca49888c69f1b01c0ddadf9a -R a6fe6d7d652069432584e1a691602cb5 +P 59cd5229e2b5be5272cf57c7e7d09e97d16a5425 +R 574aa31e9a2482d16cbf0b6e42831c35 U dan -Z ac080311c2ad2c3d7bf733448ffc0f03 +Z 53e6ec40073ce4b7a6ae1cb9efd4c01b diff --git a/manifest.uuid b/manifest.uuid index 7a6826cad1..957ecbe9a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59cd5229e2b5be5272cf57c7e7d09e97d16a5425 \ No newline at end of file +643c86a056168e39fcb7f39b8a72731f1eb246db \ No newline at end of file diff --git a/src/global.c b/src/global.c index 4ed597d292..091b750803 100644 --- a/src/global.c +++ b/src/global.c @@ -167,7 +167,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ - SQLITE_MAX_WORKER_THREADS, /* nWorker */ + 0, /* nWorker */ /* All the rest should always be initialized to zero */ 0, /* isInit */ 0, /* inProgress */ diff --git a/src/shell.c b/src/shell.c index c9abd9ec7d..6fc85ae394 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3532,6 +3532,8 @@ static void main_init(struct callback_data *data) { sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); + sqlite3_config(SQLITE_CONFIG_MULTITHREAD); + sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 3); } /* diff --git a/src/test_config.c b/src/test_config.c index 1db8198641..2201fba007 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -99,6 +99,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); #endif +#if SQLITE_MAX_WORKER_THREADS>0 + Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "0", TCL_GLOBAL_ONLY); +#endif + #if 1 /* def SQLITE_MEMDEBUG */ Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY); #else diff --git a/src/test_malloc.c b/src/test_malloc.c index e3cfcaa9f0..6ac030f23c 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -1253,6 +1253,32 @@ static int test_config_cis( return TCL_OK; } +/* +** Usage: sqlite3_config_worker_threads N +*/ +static int test_config_worker_threads( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int rc; + int nThread; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "N"); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[1], &nThread) ){ + return TCL_ERROR; + } + + rc = sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, nThread); + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); + + return TCL_OK; +} + /* ** Usage: sqlite3_dump_memsys3 FILENAME ** sqlite3_dump_memsys5 FILENAME @@ -1506,6 +1532,7 @@ int Sqlitetest_malloc_Init(Tcl_Interp *interp){ { "sqlite3_config_error", test_config_error ,0 }, { "sqlite3_config_uri", test_config_uri ,0 }, { "sqlite3_config_cis", test_config_cis ,0 }, + { "sqlite3_config_worker_threads", test_config_worker_threads ,0 }, { "sqlite3_db_config_lookaside",test_db_config_lookaside ,0 }, { "sqlite3_dump_memsys3", test_dump_memsys3 ,3 }, { "sqlite3_dump_memsys5", test_dump_memsys3 ,5 }, diff --git a/src/vdbesort.c b/src/vdbesort.c index 60a023c8f5..746618cc25 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -180,6 +180,7 @@ struct VdbeSorter { u8 *aMemory; /* Block of memory to alloc records from */ int iMemory; /* Offset of first free byte in aMemory */ int nMemory; /* Size of aMemory allocation in bytes */ + int iPrev; /* Previous thread used to flush PMA */ int nThread; /* Size of aThread[] array */ SorterThread aThread[1]; }; @@ -1251,11 +1252,13 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; int i; - SorterThread *pThread; /* Thread context used to create new PMA */ + SorterThread *pThread = 0; /* Thread context used to create new PMA */ + int nWorker = (pSorter->nThread-1); pSorter->bUsePMA = 1; - for(i=0; ALWAYS( inThread ); i++){ - pThread = &pSorter->aThread[i]; + for(i=0; iiPrev + i + 1) % nWorker; + pThread = &pSorter->aThread[iTest]; #if SQLITE_MAX_WORKER_THREADS>0 if( pThread->bDone ){ void *pRet; @@ -1269,7 +1272,12 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } #endif if( pThread->pThread==0 ) break; + pThread = 0; } + if( pThread==0 ){ + pThread = &pSorter->aThread[nWorker]; + } + pSorter->iPrev = (pThread - pSorter->aThread); if( rc==SQLITE_OK ){ assert( pThread->pThread==0 && pThread->bDone==0 ); @@ -1286,7 +1294,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ } #if SQLITE_MAX_WORKER_THREADS>0 - if( bFg || i==(pSorter->nThread-1) ){ + if( !bFg && pThread!=&pSorter->aThread[nWorker] ){ /* Launch a background thread for this operation */ void *pCtx = (void*)pThread; assert( pSorter->aMemory==0 || pThread->aListMemory!=0 ); @@ -1464,7 +1472,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ for(i=0; rc==SQLITE_OK && inThread; i++){ SorterThread *pThread = &pSorter->aThread[i]; if( pThread->pTemp1 ){ - pThread->nConsolidate = SORTER_MAX_MERGE_COUNT/pSorter->nThread; + pThread->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nThread; pThread->eWork = SORTER_THREAD_CONS; #if SQLITE_MAX_WORKER_THREADS>0 diff --git a/test/sort2.test b/test/sort2.test new file mode 100644 index 0000000000..2a77b61ed1 --- /dev/null +++ b/test/sort2.test @@ -0,0 +1,41 @@ +# 2014 March 25. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sort2 + +db close +sqlite3_shutdown +sqlite3_config_worker_threads 7 +reset_db + +do_execsql_test 1 { + PRAGMA cache_size = 5; + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 100000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) +} { + 20000 100 20000 100 20000 100 20000 100 20000 100 +} + +db close +sqlite3_shutdown +sqlite3_config_worker_threads 0 +sqlite3_initialize +finish_test + From fad9f9a8a6920c795223d84cc3923920fe2269a9 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 1 Apr 2014 18:41:51 +0000 Subject: [PATCH 025/710] When sorting data for a CREATE INDEX statement in single-threaded mode, assume that keys are delivered to the sorter in primary key order. Also fix various comments that had fallen out of date. FossilOrigin-Name: 821d1ac4504243fa13b9e3c0d56361ad9fb80d78 --- manifest | 20 ++++++------- manifest.uuid | 2 +- src/build.c | 2 +- src/vdbe.c | 8 +++-- src/vdbeInt.h | 2 +- src/vdbesort.c | 77 ++++++++++++++++++++++++------------------------- test/sort2.test | 14 +++++++++ 7 files changed, 70 insertions(+), 55 deletions(-) diff --git a/manifest b/manifest index 73c52a7c23..d0ef35d91d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Even\sif\scompile\stime\soption\sSQLITE_MAX_WORKER_THREADS\sis\sset\sto\sone\sor\sgreater,\sset\sthe\sdefault\snumber\sof\sworker\sthreads\sto\szero.\sDistribute\sdata\smore\sevenly\sbetween\sthreads\sin\ssqlite3VdbeSorterWrite()\sto\simprove\sperformance\swhen\ssorting\slarge\samounts\sof\sdata.\sAdd\snew\stest\sfile\ssort2.test. -D 2014-04-01T15:38:44.990 +C When\ssorting\sdata\sfor\sa\sCREATE\sINDEX\sstatement\sin\ssingle-threaded\smode,\sassume\sthat\skeys\sare\sdelivered\sto\sthe\ssorter\sin\sprimary\skey\sorder.\sAlso\sfix\svarious\scomments\sthat\shad\sfallen\sout\sof\sdate. +D 2014-04-01T18:41:51.759 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 0d1be67448c45eccc40114556186397eb9da7f7d F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 -F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 +F src/build.c b507fb9b4ce943139401d5c9a2daa94a568a8cf1 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -279,14 +279,14 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 02f2de0b2f3b198438e3e64a2ceba9407bb8348b +F src/vdbe.c eed2230017ca8ac5ca8fc221b38be036bba48499 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 -F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 +F src/vdbeInt.h ba1069627d0ab75e9ddb8f9c10958b86cdbd333d F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 375919a7165647cccf9f7218422540fa61e1dcb1 +F src/vdbesort.c be494ad4a1cb5845c51c9bc25b8ec151f504d49c F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -818,7 +818,7 @@ F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a -F test/sort2.test 65c1e8b49f13f5973d1b03e3ef5b1c864eadd574 +F test/sort2.test 21cd865e31adecdc8fc81c8d95431e629676a8d8 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 59cd5229e2b5be5272cf57c7e7d09e97d16a5425 -R 574aa31e9a2482d16cbf0b6e42831c35 +P 643c86a056168e39fcb7f39b8a72731f1eb246db +R c2ac1c0b48f44ef7c812322485a83f86 U dan -Z 53e6ec40073ce4b7a6ae1cb9efd4c01b +Z 23ba0058a55a91ed976e2b1f4e943278 diff --git a/manifest.uuid b/manifest.uuid index 957ecbe9a9..823d08f629 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -643c86a056168e39fcb7f39b8a72731f1eb246db \ No newline at end of file +821d1ac4504243fa13b9e3c0d56361ad9fb80d78 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 10077e5018..26c2922ebd 100644 --- a/src/build.c +++ b/src/build.c @@ -2669,7 +2669,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; - sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, 0, (char*) + sqlite3VdbeAddOp4(v, OP_SorterOpen, iSorter, 0, pIndex->nKeyCol, (char*) sqlite3KeyInfoRef(pKey), P4_KEYINFO); /* Open the table. Loop through all rows of the table, inserting index diff --git a/src/vdbe.c b/src/vdbe.c index de02d1f638..21da5007ab 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3390,11 +3390,15 @@ case OP_OpenEphemeral: { break; } -/* Opcode: SorterOpen P1 P2 * P4 * +/* Opcode: SorterOpen P1 P2 P3 P4 * ** ** This opcode works like OP_OpenEphemeral except that it opens ** a transient index that is specifically designed to sort large ** tables using an external merge-sort algorithm. +** +** If argument P3 is non-zero, then it indicates that the sorter may +** assume that a stable sort considering the first P3 fields of each +** key is sufficient to produce the required results. */ case OP_SorterOpen: { VdbeCursor *pCx; @@ -3406,7 +3410,7 @@ case OP_SorterOpen: { pCx->pKeyInfo = pOp->p4.pKeyInfo; assert( pCx->pKeyInfo->db==db ); assert( pCx->pKeyInfo->enc==ENC(db) ); - rc = sqlite3VdbeSorterInit(db, pCx); + rc = sqlite3VdbeSorterInit(db, pOp->p3, pCx); break; } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 043b56da59..03a303ca1b 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -437,7 +437,7 @@ void sqlite3VdbeFrameDelete(VdbeFrame*); int sqlite3VdbeFrameRestore(VdbeFrame *); int sqlite3VdbeTransferError(Vdbe *p); -int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); +int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *); void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); diff --git a/src/vdbesort.c b/src/vdbesort.c index 746618cc25..81b2ff5c2b 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -489,47 +489,27 @@ static int vdbeSorterIterInit( /* ** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, -** size nKey2 bytes). Argument pKeyInfo supplies the collation functions -** used by the comparison. If an error occurs, return an SQLite error code. -** Otherwise, return SQLITE_OK and set *pRes to a negative, zero or positive -** value, depending on whether key1 is smaller, equal to or larger than key2. +** size nKey2 bytes). Use (pThread->pKeyInfo) for the collation sequences +** used by the comparison. Return the result of the comparison. ** -** If the bOmitRowid argument is non-zero, assume both keys end in a rowid -** field. For the purposes of the comparison, ignore it. Also, if bOmitRowid -** is true and key1 contains even a single NULL value, it is considered to -** be less than key2. Even if key2 also contains NULL values. +** Before returning, object (pThread->pUnpacked) is populated with the +** unpacked version of key2. Or, if pKey2 is passed a NULL pointer, then it +** is assumed that the (pThread->pUnpacked) structure already contains the +** unpacked key to use as key2. ** -** If pKey2 is passed a NULL pointer, then it is assumed that the pCsr->aSpace -** has been allocated and contains an unpacked record that is used as key2. +** If an OOM error is encountered, (pThread->pUnpacked->error_rc) is set +** to SQLITE_NOMEM. */ -static void vdbeSorterCompare( +static int vdbeSorterCompare( SorterThread *pThread, /* Thread context (for pKeyInfo) */ - int nIgnore, /* Ignore the last nIgnore fields */ const void *pKey1, int nKey1, /* Left side of comparison */ - const void *pKey2, int nKey2, /* Right side of comparison */ - int *pRes /* OUT: Result of comparison */ + const void *pKey2, int nKey2 /* Right side of comparison */ ){ - KeyInfo *pKeyInfo = pThread->pKeyInfo; UnpackedRecord *r2 = pThread->pUnpacked; - int i; - if( pKey2 ){ - sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(pThread->pKeyInfo, nKey2, pKey2, r2); } - - if( nIgnore ){ - r2->nField = pKeyInfo->nField - nIgnore; - assert( r2->nField>0 ); - for(i=0; inField; i++){ - if( r2->aMem[i].flags & MEM_Null ){ - *pRes = -1; - return; - } - } - assert( r2->default_rc==0 ); - } - - *pRes = sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); + return sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); } /* @@ -568,8 +548,8 @@ static int vdbeSorterDoCompare( }else{ int res; assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterThreadMain() */ - vdbeSorterCompare( - pThread, 0, p1->aKey, p1->nKey, p2->aKey, p2->nKey, &res + res = vdbeSorterCompare( + pThread, p1->aKey, p1->nKey, p2->aKey, p2->nKey ); if( res<=0 ){ iRes = i1; @@ -585,7 +565,7 @@ static int vdbeSorterDoCompare( /* ** Initialize the temporary index cursor just opened as a sorter cursor. */ -int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ +int sqlite3VdbeSorterInit(sqlite3 *db, int nField, VdbeCursor *pCsr){ int pgsz; /* Page size of main database */ int i; /* Used to iterate through aThread[] */ int mxCache; /* Cache size */ @@ -608,6 +588,7 @@ int sqlite3VdbeSorterInit(sqlite3 *db, VdbeCursor *pCsr){ pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; + if( nField && nWorker==0 ) pKeyInfo->nField = nField; pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nThread = nWorker + 1; @@ -806,7 +787,7 @@ static void vdbeSorterMerge( while( p1 && p2 ){ int res; - vdbeSorterCompare(pThread, 0, SRVAL(p1), p1->nVal, pVal2, p2->nVal, &res); + res = vdbeSorterCompare(pThread, SRVAL(p1), p1->nVal, pVal2, p2->nVal); if( res<=0 ){ *pp = p1; pp = &p1->u.pNext; @@ -1075,8 +1056,8 @@ static int vdbeSorterNext( }else if( pIter2->pFile==0 ){ iRes = -1; }else{ - vdbeSorterCompare(pThread, 0, - pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey, &iRes + iRes = vdbeSorterCompare(pThread, + pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey ); } @@ -1596,6 +1577,9 @@ int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ ** passed as the first argument currently points to. For the purposes of ** the comparison, ignore the rowid field at the end of each record. ** +** If the sorter cursor key contains any NULL values, consider it to be +** less than pVal. Evn if pVal also contains NULL values. +** ** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM). ** Otherwise, set *pRes to a negative, zero or positive value if the ** key in pVal is smaller than, equal to or larger than the current sorter @@ -1608,10 +1592,23 @@ int sqlite3VdbeSorterCompare( int *pRes /* OUT: Result of comparison */ ){ VdbeSorter *pSorter = pCsr->pSorter; - SorterThread *pMain = &pSorter->aThread[0]; + UnpackedRecord *r2 = pSorter->aThread[0].pUnpacked; + KeyInfo *pKeyInfo = pCsr->pKeyInfo; + int i; void *pKey; int nKey; /* Sorter key to compare pVal with */ + assert( r2->nField>=pKeyInfo->nField-nIgnore ); + r2->nField = pKeyInfo->nField-nIgnore; + pKey = vdbeSorterRowkey(pSorter, &nKey); - vdbeSorterCompare(pMain, nIgnore, pVal->z, pVal->n, pKey, nKey, pRes); + sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2); + for(i=0; inField; i++){ + if( r2->aMem[i].flags & MEM_Null ){ + *pRes = -1; + return SQLITE_OK; + } + } + + *pRes = sqlite3VdbeRecordCompare(pVal->n, pVal->z, r2, 0); return SQLITE_OK; } diff --git a/test/sort2.test b/test/sort2.test index 2a77b61ed1..f8bfb0fe51 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -33,6 +33,20 @@ do_execsql_test 1 { 20000 100 20000 100 20000 100 20000 100 20000 100 } +do_execsql_test 2.1 { + CREATE TABLE t1(a, b); + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 10000 + ) INSERT INTO t1 SELECT * FROM r; +} + +do_execsql_test 2.2 { + CREATE UNIQUE INDEX i1 ON t1(b, a); +} + db close sqlite3_shutdown sqlite3_config_worker_threads 0 From 8d8f56296b1101784d2223f18808d50a245c8fec Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 2 Apr 2014 14:38:14 +0000 Subject: [PATCH 026/710] Change the name of the SorterThread object to "SortSubtask" to avoid confusion with the SQLiteThread object. FossilOrigin-Name: 4ee2d910fbbed8d4def15e4e99ee225839f3a739 --- manifest | 14 ++++---- manifest.uuid | 2 +- src/vdbesort.c | 92 +++++++++++++++++++++++++------------------------- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/manifest b/manifest index d0ef35d91d..8bc41fb375 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\ssorting\sdata\sfor\sa\sCREATE\sINDEX\sstatement\sin\ssingle-threaded\smode,\sassume\sthat\skeys\sare\sdelivered\sto\sthe\ssorter\sin\sprimary\skey\sorder.\sAlso\sfix\svarious\scomments\sthat\shad\sfallen\sout\sof\sdate. -D 2014-04-01T18:41:51.759 +C Change\sthe\sname\sof\sthe\sSorterThread\sobject\sto\s"SortSubtask"\sto\savoid\sconfusion\nwith\sthe\sSQLiteThread\sobject. +D 2014-04-02T14:38:14.734 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c be494ad4a1cb5845c51c9bc25b8ec151f504d49c +F src/vdbesort.c 0cb83fc36f8a56cc6e4ecdefc3c7d4155169cef2 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 643c86a056168e39fcb7f39b8a72731f1eb246db -R c2ac1c0b48f44ef7c812322485a83f86 -U dan -Z 23ba0058a55a91ed976e2b1f4e943278 +P 821d1ac4504243fa13b9e3c0d56361ad9fb80d78 +R 2283517ddbfd0881db16b2410c51a15c +U drh +Z e6ea0be46d70fe73f5ea59fa8b577076 diff --git a/manifest.uuid b/manifest.uuid index 823d08f629..0fe5750cf3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -821d1ac4504243fa13b9e3c0d56361ad9fb80d78 \ No newline at end of file +4ee2d910fbbed8d4def15e4e99ee225839f3a739 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 81b2ff5c2b..ff315494e6 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -20,57 +20,57 @@ typedef struct VdbeSorterIter VdbeSorterIter; -typedef struct SorterThread SorterThread; +typedef struct SortSubtask SortSubtask; typedef struct SorterRecord SorterRecord; typedef struct SorterMerger SorterMerger; typedef struct FileWriter FileWriter; /* -** Candidate values for SorterThread.eWork +** Candidate values for SortSubtask.eWork */ -#define SORTER_THREAD_SORT 1 -#define SORTER_THREAD_TO_PMA 2 -#define SORTER_THREAD_CONS 3 +#define SORTER_THREAD_SORT 1 /* Sort records on pList */ +#define SORTER_THREAD_TO_PMA 2 /* Xfer pList to Packed-Memory-Array pFile */ +#define SORTER_THREAD_CONS 3 /* Consolidate multiple PMAs */ /* ** Much of the work performed in this module to sort the list of records is ** broken down into smaller units that may be peformed in parallel. In order ** to perform such a unit of work, an instance of the following structure -** is configured and passed to vdbeSorterThreadMain() - either directly by +** is configured and passed to vdbeSortSubtaskMain() - either directly by ** the main thread or via a background thread. ** -** Exactly SorterThread.nThread instances of this structure are allocated +** Exactly SortSubtask.nThread instances of this structure are allocated ** as part of each VdbeSorter object. Instances are never allocated any other -** way. SorterThread.nThread is set to the number of worker threads allowed +** way. SortSubtask.nThread is set to the number of worker threads allowed ** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). ** -** When a background thread is launched to perform work, SorterThread.bDone -** is set to 0 and the SorterThread.pThread variable set to point to the -** thread handle. SorterThread.bDone is set to 1 (to indicate to the main -** thread that joining SorterThread.pThread will not block) before the thread -** exits. SorterThread.pThread and bDone are always cleared after the +** When a background thread is launched to perform work, SortSubtask.bDone +** is set to 0 and the SortSubtask.pThread variable set to point to the +** thread handle. SortSubtask.bDone is set to 1 (to indicate to the main +** thread that joining SortSubtask.pThread will not block) before the thread +** exits. SortSubtask.pThread and bDone are always cleared after the ** background thread has been joined. ** -** One object (specifically, VdbeSorter.aThread[SorterThread.nThread-1]) +** One object (specifically, VdbeSorter.aThread[SortSubtask.nThread-1]) ** is reserved for the foreground thread. ** -** The nature of the work performed is determined by SorterThread.eWork, +** The nature of the work performed is determined by SortSubtask.eWork, ** as follows: ** ** SORTER_THREAD_SORT: -** Sort the linked list of records at SorterThread.pList. +** Sort the linked list of records at SortSubtask.pList. ** ** SORTER_THREAD_TO_PMA: -** Sort the linked list of records at SorterThread.pList, and write -** the results to a new PMA in temp file SorterThread.pTemp1. Open +** Sort the linked list of records at SortSubtask.pList, and write +** the results to a new PMA in temp file SortSubtask.pTemp1. Open ** the temp file if it is not already open. ** ** SORTER_THREAD_CONS: -** Merge existing PMAs until SorterThread.nConsolidate or fewer -** remain in temp file SorterThread.pTemp1. +** Merge existing PMAs until SortSubtask.nConsolidate or fewer +** remain in temp file SortSubtask.pTemp1. */ -struct SorterThread { +struct SortSubtask { SQLiteThread *pThread; /* Thread handle, or NULL */ int bDone; /* Set to true by pThread when finished */ @@ -182,7 +182,7 @@ struct VdbeSorter { int nMemory; /* Size of aMemory allocation in bytes */ int iPrev; /* Previous thread used to flush PMA */ int nThread; /* Size of aThread[] array */ - SorterThread aThread[1]; + SortSubtask aThread[1]; }; /* @@ -427,7 +427,7 @@ static int vdbeSorterIterNext(VdbeSorterIter *pIter){ ** PMA is empty). */ static int vdbeSorterIterInit( - SorterThread *pThread, /* Thread context */ + SortSubtask *pThread, /* Thread context */ i64 iStart, /* Start offset in pThread->pTemp1 */ VdbeSorterIter *pIter, /* Iterator to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ @@ -501,7 +501,7 @@ static int vdbeSorterIterInit( ** to SQLITE_NOMEM. */ static int vdbeSorterCompare( - SorterThread *pThread, /* Thread context (for pKeyInfo) */ + SortSubtask *pThread, /* Thread context (for pKeyInfo) */ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2 /* Right side of comparison */ ){ @@ -518,7 +518,7 @@ static int vdbeSorterCompare( ** value to recalculate. */ static int vdbeSorterDoCompare( - SorterThread *pThread, + SortSubtask *pThread, SorterMerger *pMerger, int iOut ){ @@ -547,7 +547,7 @@ static int vdbeSorterDoCompare( iRes = i1; }else{ int res; - assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSorterThreadMain() */ + assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSortSubtaskMain() */ res = vdbeSorterCompare( pThread, p1->aKey, p1->nKey, p2->aKey, p2->nKey ); @@ -578,7 +578,7 @@ int sqlite3VdbeSorterInit(sqlite3 *db, int nField, VdbeCursor *pCsr){ assert( pCsr->pKeyInfo && pCsr->pBt==0 ); szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); - sz = sizeof(VdbeSorter) + nWorker * sizeof(SorterThread); + sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); pCsr->pSorter = pSorter; @@ -593,7 +593,7 @@ int sqlite3VdbeSorterInit(sqlite3 *db, int nField, VdbeCursor *pCsr){ pSorter->nThread = nWorker + 1; for(i=0; inThread; i++){ - SorterThread *pThread = &pSorter->aThread[i]; + SortSubtask *pThread = &pSorter->aThread[i]; pThread->pKeyInfo = pKeyInfo; pThread->pVfs = db->pVfs; pThread->pgsz = pgsz; @@ -636,7 +636,7 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ ** Free all resources owned by the object indicated by argument pThread. All ** fields of *pThread are zeroed before returning. */ -static void vdbeSorterThreadCleanup(sqlite3 *db, SorterThread *pThread){ +static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pThread){ sqlite3DbFree(db, pThread->pUnpacked); pThread->pUnpacked = 0; if( pThread->aListMemory==0 ){ @@ -660,7 +660,7 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int rc = rcin; int i; for(i=0; inThread; i++){ - SorterThread *pThread = &pSorter->aThread[i]; + SortSubtask *pThread = &pSorter->aThread[i]; if( pThread->pThread ){ void *pRet; int rc2 = sqlite3ThreadJoin(pThread->pThread, &pRet); @@ -725,8 +725,8 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ int i; vdbeSorterJoinAll(pSorter, SQLITE_OK); for(i=0; inThread; i++){ - SorterThread *pThread = &pSorter->aThread[i]; - vdbeSorterThreadCleanup(db, pThread); + SortSubtask *pThread = &pSorter->aThread[i]; + vdbeSortSubtaskCleanup(db, pThread); } if( pSorter->aMemory==0 ){ vdbeSorterRecordFree(0, pSorter->pRecord); @@ -776,7 +776,7 @@ static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){ ** Set *ppOut to the head of the new list. */ static void vdbeSorterMerge( - SorterThread *pThread, /* Calling thread context */ + SortSubtask *pThread, /* Calling thread context */ SorterRecord *p1, /* First list to merge */ SorterRecord *p2, /* Second list to merge */ SorterRecord **ppOut /* OUT: Head of merged list */ @@ -810,7 +810,7 @@ static void vdbeSorterMerge( ** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if ** an error occurs. */ -static int vdbeSorterSort(SorterThread *pThread){ +static int vdbeSorterSort(SortSubtask *pThread){ int i; SorterRecord **aSlot; SorterRecord *p; @@ -974,7 +974,7 @@ static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterListToPMA(SorterThread *pThread){ +static int vdbeSorterListToPMA(SortSubtask *pThread){ int rc = SQLITE_OK; /* Return code */ FileWriter writer; /* Object used to write to the file */ @@ -1025,7 +1025,7 @@ static int vdbeSorterListToPMA(SorterThread *pThread){ ** Return SQLITE_OK if successful or an error code if an error occurs. */ static int vdbeSorterNext( - SorterThread *pThread, + SortSubtask *pThread, SorterMerger *pMerger, int *pbEof ){ @@ -1095,9 +1095,9 @@ static int vdbeSorterNext( /* ** The main routine for sorter-thread operations. */ -static void *vdbeSorterThreadMain(void *pCtx){ +static void *vdbeSortSubtaskMain(void *pCtx){ int rc = SQLITE_OK; - SorterThread *pThread = (SorterThread*)pCtx; + SortSubtask *pThread = (SortSubtask*)pCtx; assert( pThread->eWork==SORTER_THREAD_SORT || pThread->eWork==SORTER_THREAD_TO_PMA @@ -1216,8 +1216,8 @@ static void *vdbeSorterThreadMain(void *pCtx){ ** Run the activity scheduled by the object passed as the only argument ** in the current thread. */ -static int vdbeSorterRunThread(SorterThread *pThread){ - int rc = SQLITE_PTR_TO_INT( vdbeSorterThreadMain((void*)pThread) ); +static int vdbeSorterRunThread(SortSubtask *pThread){ + int rc = SQLITE_PTR_TO_INT( vdbeSortSubtaskMain((void*)pThread) ); assert( pThread->bDone ); pThread->bDone = 0; return rc; @@ -1233,7 +1233,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; int i; - SorterThread *pThread = 0; /* Thread context used to create new PMA */ + SortSubtask *pThread = 0; /* Thread context used to create new PMA */ int nWorker = (pSorter->nThread-1); pSorter->bUsePMA = 1; @@ -1287,7 +1287,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory); } } - rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSorterThreadMain, pCtx); + rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSortSubtaskMain, pCtx); }else #endif { @@ -1422,7 +1422,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** from the in-memory list. */ if( pSorter->bUsePMA==0 ){ if( pSorter->pRecord ){ - SorterThread *pThread = &pSorter->aThread[0]; + SortSubtask *pThread = &pSorter->aThread[0]; *pbEof = 0; pThread->pList = pSorter->pRecord; pThread->eWork = SORTER_THREAD_SORT; @@ -1451,7 +1451,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ if( vdbeSorterCountPMA(pSorter)>SORTER_MAX_MERGE_COUNT ){ int i; for(i=0; rc==SQLITE_OK && inThread; i++){ - SorterThread *pThread = &pSorter->aThread[i]; + SortSubtask *pThread = &pSorter->aThread[i]; if( pThread->pTemp1 ){ pThread->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nThread; pThread->eWork = SORTER_THREAD_CONS; @@ -1459,7 +1459,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ #if SQLITE_MAX_WORKER_THREADS>0 if( i<(pSorter->nThread-1) ){ void *pCtx = (void*)pThread; - rc = sqlite3ThreadCreate(&pThread->pThread,vdbeSorterThreadMain,pCtx); + rc = sqlite3ThreadCreate(&pThread->pThread,vdbeSortSubtaskMain,pCtx); }else #endif { @@ -1492,7 +1492,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ for(iThread=0; iThreadnThread; iThread++){ int iPMA; i64 iReadOff = 0; - SorterThread *pThread = &pSorter->aThread[iThread]; + SortSubtask *pThread = &pSorter->aThread[iThread]; for(iPMA=0; iPMAnPMA && rc==SQLITE_OK; iPMA++){ i64 nDummy = 0; VdbeSorterIter *pIter = &pMerger->aIter[iIter++]; From dd95d30f82ff926c7b79e0410788575c0e66cdfb Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 2 Apr 2014 15:15:25 +0000 Subject: [PATCH 027/710] Fix some problems with OOM handling in vdbesort.c. FossilOrigin-Name: 47e702bd8392bc50c4edaf6a2c8c499af87b520e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbesort.c | 36 +++++++++++++++++------------------- test/mallocA.test | 23 ++++++++++++++++++++++- 4 files changed, 48 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 8bc41fb375..2cae57fb97 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\sSorterThread\sobject\sto\s"SortSubtask"\sto\savoid\sconfusion\nwith\sthe\sSQLiteThread\sobject. -D 2014-04-02T14:38:14.734 +C Fix\ssome\sproblems\swith\sOOM\shandling\sin\svdbesort.c. +D 2014-04-02T15:15:25.762 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 0cb83fc36f8a56cc6e4ecdefc3c7d4155169cef2 +F src/vdbesort.c e830ea4a7333ff07177fc367918ede2b33fcfe10 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -671,7 +671,7 @@ F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151 F test/malloc7.test 7c68a32942858bc715284856c5507446bba88c3a F test/malloc8.test 9b7a3f8cb9cf0b12fff566e80a980b1767bd961d F test/malloc9.test 2307c6ee3703b0a21391f3ea92388b4b73f9105e -F test/mallocA.test 1ba0367fb5434e7bc2fa4afcb30b14174d91b160 +F test/mallocA.test c049224adeb0244b8f6eb770c1fa6ac40f9b3518 F test/mallocAll.test 98f1be74bc9f49a858bc4f361fc58e26486798be F test/mallocB.test bc475ab850cda896142ab935bbfbc74c24e51ed6 F test/mallocC.test 3dffe16532f109293ce1ccecd0c31dca55ef08c4 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 821d1ac4504243fa13b9e3c0d56361ad9fb80d78 -R 2283517ddbfd0881db16b2410c51a15c -U drh -Z e6ea0be46d70fe73f5ea59fa8b577076 +P 4ee2d910fbbed8d4def15e4e99ee225839f3a739 +R 18420994a2ccade11be299f196096342 +U dan +Z 3f3c411cd945b7957dc8b5eeb10e94fa diff --git a/manifest.uuid b/manifest.uuid index 0fe5750cf3..4eba9e128c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4ee2d910fbbed8d4def15e4e99ee225839f3a739 \ No newline at end of file +47e702bd8392bc50c4edaf6a2c8c499af87b520e \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index ff315494e6..50680007b6 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -443,10 +443,13 @@ static int vdbeSorterIterInit( pIter->iReadOff = iStart; pIter->nAlloc = 128; pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc); - - /* Try to xFetch() a mapping of the entire temp file. If this is possible, - ** the PMA will be read via the mapping. Otherwise, use xRead(). */ - rc = sqlite3OsFetch(pIter->pFile, 0, pThread->iTemp1Off, &pMap); + if( pIter->aAlloc ){ + /* Try to xFetch() a mapping of the entire temp file. If this is possible, + ** the PMA will be read via the mapping. Otherwise, use xRead(). */ + rc = sqlite3OsFetch(pIter->pFile, 0, pThread->iTemp1Off, &pMap); + }else{ + rc = SQLITE_NOMEM; + } if( rc==SQLITE_OK ){ if( pMap ){ @@ -698,23 +701,15 @@ static SorterMerger *vdbeSorterMergerNew(int nIter){ } /* -** Reset a merger +** Free the SorterMerger object passed as the only argument. */ -static void vdbeSorterMergerReset(SorterMerger *pMerger){ +static void vdbeSorterMergerFree(SorterMerger *pMerger){ int i; if( pMerger ){ for(i=0; inTree; i++){ vdbeSorterIterZero(&pMerger->aIter[i]); } } -} - - -/* -** Free the SorterMerger object passed as the only argument. -*/ -static void vdbeSorterMergerFree(SorterMerger *pMerger){ - vdbeSorterMergerReset(pMerger); sqlite3_free(pMerger); } @@ -724,6 +719,8 @@ static void vdbeSorterMergerFree(SorterMerger *pMerger){ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ int i; vdbeSorterJoinAll(pSorter, SQLITE_OK); + vdbeSorterMergerFree(pSorter->pMerger); + pSorter->pMerger = 0; for(i=0; inThread; i++){ SortSubtask *pThread = &pSorter->aThread[i]; vdbeSortSubtaskCleanup(db, pThread); @@ -731,7 +728,6 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ if( pSorter->aMemory==0 ){ vdbeSorterRecordFree(0, pSorter->pRecord); } - vdbeSorterMergerReset(pSorter->pMerger); pSorter->pRecord = 0; pSorter->nInMemory = 0; pSorter->bUsePMA = 0; @@ -1292,11 +1288,13 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ #endif { /* Use the foreground thread for this operation */ - u8 *aMem; rc = vdbeSorterRunThread(pThread); - aMem = pThread->aListMemory; - pThread->aListMemory = pSorter->aMemory; - pSorter->aMemory = aMem; + if( rc==SQLITE_OK ){ + u8 *aMem = pThread->aListMemory; + pThread->aListMemory = pSorter->aMemory; + pSorter->aMemory = aMem; + assert( pThread->pList==0 ); + } } } diff --git a/test/mallocA.test b/test/mallocA.test index 61e88a61e7..d6d6de8222 100644 --- a/test/mallocA.test +++ b/test/mallocA.test @@ -25,7 +25,6 @@ if {!$MEMDEBUG} { return } - # Construct a test database # forcedelete test.db.bu @@ -116,6 +115,28 @@ ifcapable stat3 { } } +do_execsql_test 7.0 { + PRAGMA cache_size = 5; +} +do_faultsim_test 7 -faults oom-trans* -prep { + if {$iFail < 500} { set iFail 2000 } + if {$iFail > 1215} { set iFail 2000 } +} -body { + execsql { + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 1000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } +} -test { + set res [list 200 100 200 100 200 100 200 100 200 100] + faultsim_test_result [list 0 $res] +} + + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} From ac4f0039c0aeabb76f63085697371d9673cd7a70 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 2 Apr 2014 18:58:49 +0000 Subject: [PATCH 028/710] Add a big introductory comment to vdbesort.c explaining its operation at a high level. Also adjust some symbolic names and fix other comment issues in that file. FossilOrigin-Name: eef60f1bf54fcdc7b32f96ebb87a9a0bf0776e8b --- manifest | 14 ++--- manifest.uuid | 2 +- src/vdbesort.c | 137 +++++++++++++++++++++++++++++++++++++------------ 3 files changed, 113 insertions(+), 40 deletions(-) diff --git a/manifest b/manifest index 2cae57fb97..e1d3814d6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\swith\sOOM\shandling\sin\svdbesort.c. -D 2014-04-02T15:15:25.762 +C Add\sa\sbig\sintroductory\scomment\sto\svdbesort.c\sexplaining\sits\soperation\sat\sa\nhigh\slevel.\s\sAlso\sadjust\ssome\ssymbolic\snames\sand\sfix\sother\scomment\sissues\sin\nthat\sfile. +D 2014-04-02T18:58:49.259 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c e830ea4a7333ff07177fc367918ede2b33fcfe10 +F src/vdbesort.c 523283d7c3f499444df97d700503f3c9ddd746b7 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4ee2d910fbbed8d4def15e4e99ee225839f3a739 -R 18420994a2ccade11be299f196096342 -U dan -Z 3f3c411cd945b7957dc8b5eeb10e94fa +P 47e702bd8392bc50c4edaf6a2c8c499af87b520e +R a364509f4308441b3af07ec467078f09 +U drh +Z a6f0e24eb45caefc4a14c210e684429c diff --git a/manifest.uuid b/manifest.uuid index 4eba9e128c..30ed24d4f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -47e702bd8392bc50c4edaf6a2c8c499af87b520e \ No newline at end of file +eef60f1bf54fcdc7b32f96ebb87a9a0bf0776e8b \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 50680007b6..c5d13c685c 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1,5 +1,5 @@ /* -** 2011 July 9 +** 2011-07-09 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -10,15 +10,87 @@ ** ************************************************************************* ** This file contains code for the VdbeSorter object, used in concert with -** a VdbeCursor to sort large numbers of keys (as may be required, for -** example, by CREATE INDEX statements on tables too large to fit in main -** memory). +** a VdbeCursor to sort large numbers of keys for CREATE TABLE statements +** or by SELECT statements with ORDER BY clauses that cannot be satisfied +** using indexes and without LIMIT clauses. +** +** The VdbeSorter object implements a multi-threaded external merge sort +** algorithm that is efficient even if the number of element being sorted +** exceeds the available memory. +** +** Here is the (internal, non-API) interface between this module and the +** rest of the SQLite system: +** +** sqlite3VdbeSorterInit() Create a new VdbeSorter object. +** +** sqlite3VdbeSorterWrite() Add a single new row to the VdbeSorter +** object. The row is a binary blob in the +** OP_MakeRecord format that contains both +** the ORDER BY key columns and result columns +** in the case of a SELECT w/ ORDER BY, or +** the complete record for an index entry +** in the case of a CREATE INDEX. +** +** sqlite3VdbeSorterRewind() Sort all content previously added. +** Position the read cursor on the +** first sorted element. +** +** sqlite3VdbeSorterNext() Advance the read cursor to the next sorted +** element. +** +** sqlite3VdbeSorterRowkey() Return the complete binary blob for the +** row currently under the read cursor. +** +** sqlite3VdbeSorterCompare() Compare the binary blob for the row +** currently under the read cursor against +** another binary blob X and report if +** X is strictly less than the read cursor. +** Used to enforce uniqueness in a +** CREATE UNIQUE INDEX statement. +** +** sqlite3VdbeSorterClose() Close the VdbeSorter object and reclaim +** all resources. +** +** sqlite3VdbeSorterReset() Refurbish the VdbeSorter for reuse. This +** is like Close() followed by Init() only +** much faster. +** +** The interfaces above must be called in a particular order. Write() can +** only occur in between Init()/Reset() and Rewind(). Next(), Rowkey(), and +** Compare() can only occur in between Rewind() and Close()/Reset(). +** +** Algorithm: +** +** Records to be sorted are initially held in memory, in the order in +** which they arrive from Write(). When the amount of memory needed exceeds +** a threshold, all in-memory records are sorted and then appended to +** a temporary file as a "Packed-Memory-Array" or "PMA" and the memory is +** reset. There is a single temporary file used for all PMAs. The PMAs +** are packed one after another in the file. The VdbeSorter object keeps +** track of the number of PMAs written. +** +** When the Rewind() is seen, any records still held in memory are sorted. +** If no PMAs have been written (if all records are still held in memory) +** then subsequent Rowkey(), Next(), and Compare() operations work directly +** from memory. But if PMAs have been written things get a little more +** complicated. +** +** When Rewind() is seen after PMAs have been written, any records still +** in memory are sorted and written as a final PMA. Then all the PMAs +** are merged together into a single massive PMA that Next(), Rowkey(), +** and Compare() walk to extract the records in sorted order. +** +** If SQLITE_MAX_WORKER_THREADS is non-zero, various steps of the above +** algorithm might be performed in parallel by separate threads. Threads +** are only used when one or more PMA spill to disk. If the sort is small +** enough to fit entirely in memory, everything happens on the main thread. */ - #include "sqliteInt.h" #include "vdbeInt.h" - +/* +** Private objects used by the sorter +*/ typedef struct VdbeSorterIter VdbeSorterIter; typedef struct SortSubtask SortSubtask; typedef struct SorterRecord SorterRecord; @@ -29,20 +101,18 @@ typedef struct FileWriter FileWriter; /* ** Candidate values for SortSubtask.eWork */ -#define SORTER_THREAD_SORT 1 /* Sort records on pList */ -#define SORTER_THREAD_TO_PMA 2 /* Xfer pList to Packed-Memory-Array pFile */ -#define SORTER_THREAD_CONS 3 /* Consolidate multiple PMAs */ +#define SORT_SUBTASK_SORT 1 /* Sort records on pList */ +#define SORT_SUBTASK_TO_PMA 2 /* Xfer pList to Packed-Memory-Array pTemp1 */ +#define SORT_SUBTASK_CONS 3 /* Consolidate multiple PMAs */ /* -** Much of the work performed in this module to sort the list of records is -** broken down into smaller units that may be peformed in parallel. In order -** to perform such a unit of work, an instance of the following structure -** is configured and passed to vdbeSortSubtaskMain() - either directly by -** the main thread or via a background thread. +** Sorting is divided up into smaller subtasks. Each subtask is controlled +** by an instance of this object. Subtask might run in either the main thread +** or in a background thread. ** -** Exactly SortSubtask.nThread instances of this structure are allocated +** Exactly VdbeSorter.nThread instances of this object are allocated ** as part of each VdbeSorter object. Instances are never allocated any other -** way. SortSubtask.nThread is set to the number of worker threads allowed +** way. VdbeSorter.nThread is set to the number of worker threads allowed ** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). ** ** When a background thread is launched to perform work, SortSubtask.bDone @@ -52,21 +122,21 @@ typedef struct FileWriter FileWriter; ** exits. SortSubtask.pThread and bDone are always cleared after the ** background thread has been joined. ** -** One object (specifically, VdbeSorter.aThread[SortSubtask.nThread-1]) +** One object (specifically, VdbeSorter.aThread[VdbeSorter.nThread-1]) ** is reserved for the foreground thread. ** ** The nature of the work performed is determined by SortSubtask.eWork, ** as follows: ** -** SORTER_THREAD_SORT: +** SORT_SUBTASK_SORT: ** Sort the linked list of records at SortSubtask.pList. ** -** SORTER_THREAD_TO_PMA: +** SORT_SUBTASK_TO_PMA: ** Sort the linked list of records at SortSubtask.pList, and write ** the results to a new PMA in temp file SortSubtask.pTemp1. Open ** the temp file if it is not already open. ** -** SORTER_THREAD_CONS: +** SORT_SUBTASK_CONS: ** Merge existing PMAs until SortSubtask.nConsolidate or fewer ** remain in temp file SortSubtask.pTemp1. */ @@ -79,8 +149,8 @@ struct SortSubtask { UnpackedRecord *pUnpacked; /* Space to unpack a record */ int pgsz; /* Main database page size */ - u8 eWork; /* One of the SORTER_THREAD_* constants */ - int nConsolidate; /* For THREAD_CONS, max final PMAs */ + u8 eWork; /* One of the SORT_SUBTASK_* constants */ + int nConsolidate; /* For SORT_SUBTASK_CONS, max final PMAs */ SorterRecord *pList; /* List of records for pThread to sort */ int nInMemory; /* Expected size of PMA based on pList */ u8 *aListMemory; /* Records memory (or NULL) */ @@ -1095,9 +1165,9 @@ static void *vdbeSortSubtaskMain(void *pCtx){ int rc = SQLITE_OK; SortSubtask *pThread = (SortSubtask*)pCtx; - assert( pThread->eWork==SORTER_THREAD_SORT - || pThread->eWork==SORTER_THREAD_TO_PMA - || pThread->eWork==SORTER_THREAD_CONS + assert( pThread->eWork==SORT_SUBTASK_SORT + || pThread->eWork==SORT_SUBTASK_TO_PMA + || pThread->eWork==SORT_SUBTASK_CONS ); assert( pThread->bDone==0 ); @@ -1115,7 +1185,7 @@ static void *vdbeSortSubtaskMain(void *pCtx){ pThread->pUnpacked->errCode = 0; } - if( pThread->eWork==SORTER_THREAD_CONS ){ + if( pThread->eWork==SORT_SUBTASK_CONS ){ assert( pThread->pList==0 ); while( pThread->nPMA>pThread->nConsolidate && rc==SQLITE_OK ){ int nIter = MIN(pThread->nPMA, SORTER_MAX_MERGE_COUNT); @@ -1188,7 +1258,7 @@ static void *vdbeSortSubtaskMain(void *pCtx){ rc = vdbeSorterSort(pThread); /* If required, write the list out to a PMA. */ - if( rc==SQLITE_OK && pThread->eWork==SORTER_THREAD_TO_PMA ){ + if( rc==SQLITE_OK && pThread->eWork==SORT_SUBTASK_TO_PMA ){ #ifdef SQLITE_DEBUG i64 nExpect = pThread->nInMemory + sqlite3VarintLen(pThread->nInMemory) @@ -1258,7 +1328,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ if( rc==SQLITE_OK ){ assert( pThread->pThread==0 && pThread->bDone==0 ); - pThread->eWork = SORTER_THREAD_TO_PMA; + pThread->eWork = SORT_SUBTASK_TO_PMA; pThread->pList = pSorter->pRecord; pThread->nInMemory = pSorter->nInMemory; pSorter->nInMemory = 0; @@ -1306,7 +1376,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ */ int sqlite3VdbeSorterWrite( sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Sorter cursor */ + const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; @@ -1423,7 +1493,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ SortSubtask *pThread = &pSorter->aThread[0]; *pbEof = 0; pThread->pList = pSorter->pRecord; - pThread->eWork = SORTER_THREAD_SORT; + pThread->eWork = SORT_SUBTASK_SORT; assert( pThread->aListMemory==0 ); pThread->aListMemory = pSorter->aMemory; rc = vdbeSorterRunThread(pThread); @@ -1452,7 +1522,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ SortSubtask *pThread = &pSorter->aThread[i]; if( pThread->pTemp1 ){ pThread->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nThread; - pThread->eWork = SORTER_THREAD_CONS; + pThread->eWork = SORT_SUBTASK_CONS; #if SQLITE_MAX_WORKER_THREADS>0 if( i<(pSorter->nThread-1) ){ @@ -1576,12 +1646,15 @@ int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ ** the comparison, ignore the rowid field at the end of each record. ** ** If the sorter cursor key contains any NULL values, consider it to be -** less than pVal. Evn if pVal also contains NULL values. +** less than pVal. Even if pVal also contains NULL values. ** ** If an error occurs, return an SQLite error code (i.e. SQLITE_NOMEM). ** Otherwise, set *pRes to a negative, zero or positive value if the ** key in pVal is smaller than, equal to or larger than the current sorter ** key. +** +** This routine forms the core of the OP_SorterCompare opcode, which in +** turn is used to verify uniqueness when constructing a UNIQUE INDEX. */ int sqlite3VdbeSorterCompare( const VdbeCursor *pCsr, /* Sorter cursor */ From a634fb15778e176b4c63ca5d50fac06fe6e16947 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 3 Apr 2014 02:54:27 +0000 Subject: [PATCH 029/710] Refactor local object and method names in vdbesort.c so that their names more closely reflect their actual use. FossilOrigin-Name: d284e30eb1db144965fa85566e4234e30464350b --- manifest | 12 +- manifest.uuid | 2 +- src/vdbesort.c | 610 +++++++++++++++++++++++++------------------------ 3 files changed, 314 insertions(+), 310 deletions(-) diff --git a/manifest b/manifest index e1d3814d6f..a1d8fa4b08 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sbig\sintroductory\scomment\sto\svdbesort.c\sexplaining\sits\soperation\sat\sa\nhigh\slevel.\s\sAlso\sadjust\ssome\ssymbolic\snames\sand\sfix\sother\scomment\sissues\sin\nthat\sfile. -D 2014-04-02T18:58:49.259 +C Refactor\slocal\sobject\sand\smethod\snames\sin\svdbesort.c\sso\sthat\stheir\snames\nmore\sclosely\sreflect\stheir\sactual\suse. +D 2014-04-03T02:54:27.677 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 523283d7c3f499444df97d700503f3c9ddd746b7 +F src/vdbesort.c a4e349ff62196dc3d98eacb0f9281e2468c44cab F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 47e702bd8392bc50c4edaf6a2c8c499af87b520e -R a364509f4308441b3af07ec467078f09 +P eef60f1bf54fcdc7b32f96ebb87a9a0bf0776e8b +R 0d1e8a8cee97e3fb0928896d3dcac011 U drh -Z a6f0e24eb45caefc4a14c210e684429c +Z 523d07a773fda305bad8a184629c5049 diff --git a/manifest.uuid b/manifest.uuid index 30ed24d4f7..0af0f90890 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eef60f1bf54fcdc7b32f96ebb87a9a0bf0776e8b \ No newline at end of file +d284e30eb1db144965fa85566e4234e30464350b \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index c5d13c685c..6b94ddfebf 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -91,11 +91,11 @@ /* ** Private objects used by the sorter */ -typedef struct VdbeSorterIter VdbeSorterIter; -typedef struct SortSubtask SortSubtask; -typedef struct SorterRecord SorterRecord; -typedef struct SorterMerger SorterMerger; -typedef struct FileWriter FileWriter; +typedef struct MergeEngine MergeEngine; /* Merge PMAs together */ +typedef struct PmaReader PmaReader; /* Incrementally read one PMA */ +typedef struct PmaWriter PmaWriter; /* Incrementally write on PMA */ +typedef struct SorterRecord SorterRecord; /* A record being sorted */ +typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ /* @@ -107,22 +107,22 @@ typedef struct FileWriter FileWriter; /* ** Sorting is divided up into smaller subtasks. Each subtask is controlled -** by an instance of this object. Subtask might run in either the main thread +** by an instance of this object. A Subtask might run in either the main thread ** or in a background thread. ** -** Exactly VdbeSorter.nThread instances of this object are allocated +** Exactly VdbeSorter.nTask instances of this object are allocated ** as part of each VdbeSorter object. Instances are never allocated any other -** way. VdbeSorter.nThread is set to the number of worker threads allowed +** way. VdbeSorter.nTask is set to the number of worker threads allowed ** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). ** ** When a background thread is launched to perform work, SortSubtask.bDone -** is set to 0 and the SortSubtask.pThread variable set to point to the +** is set to 0 and the SortSubtask.pTask variable set to point to the ** thread handle. SortSubtask.bDone is set to 1 (to indicate to the main -** thread that joining SortSubtask.pThread will not block) before the thread -** exits. SortSubtask.pThread and bDone are always cleared after the +** thread that joining SortSubtask.pTask will not block) before the thread +** exits. SortSubtask.pTask and bDone are always cleared after the ** background thread has been joined. ** -** One object (specifically, VdbeSorter.aThread[VdbeSorter.nThread-1]) +** One object (specifically, VdbeSorter.aTask[VdbeSorter.nTask-1]) ** is reserved for the foreground thread. ** ** The nature of the work performed is determined by SortSubtask.eWork, @@ -142,7 +142,7 @@ typedef struct FileWriter FileWriter; */ struct SortSubtask { SQLiteThread *pThread; /* Thread handle, or NULL */ - int bDone; /* Set to true by pThread when finished */ + int bDone; /* Set to true by pTask when finished */ sqlite3_vfs *pVfs; /* VFS used to open temporary files */ KeyInfo *pKeyInfo; /* How to compare records */ @@ -151,7 +151,7 @@ struct SortSubtask { u8 eWork; /* One of the SORT_SUBTASK_* constants */ int nConsolidate; /* For SORT_SUBTASK_CONS, max final PMAs */ - SorterRecord *pList; /* List of records for pThread to sort */ + SorterRecord *pList; /* List of records for pTask to sort */ int nInMemory; /* Expected size of PMA based on pList */ u8 *aListMemory; /* Records memory (or NULL) */ @@ -162,29 +162,23 @@ struct SortSubtask { /* -** NOTES ON DATA STRUCTURE USED FOR N-WAY MERGES: +** The MergeEngine object is used to combine two or more smaller PMAs into +** one big PMA using a merge operation. Separate PMAs all need to be +** combined into one big PMA in order to be able to step through the sorted +** records in order. ** -** As keys are added to the sorter, they are written to disk in a series -** of sorted packed-memory-arrays (PMAs). The size of each PMA is roughly -** the same as the cache-size allowed for temporary databases. In order -** to allow the caller to extract keys from the sorter in sorted order, -** all PMAs currently stored on disk must be merged together. This comment -** describes the data structure used to do so. The structure supports -** merging any number of arrays in a single pass with no redundant comparison -** operations. -** -** The aIter[] array contains an iterator for each of the PMAs being merged. -** An aIter[] iterator either points to a valid key or else is at EOF. For -** the purposes of the paragraphs below, we assume that the array is actually -** N elements in size, where N is the smallest power of 2 greater to or equal -** to the number of iterators being merged. The extra aIter[] elements are -** treated as if they are empty (always at EOF). +** The aIter[] array contains a PmaReader object for each of the PMAs being +** merged. An aIter[] object either points to a valid key or else is at EOF. +** For the purposes of the paragraphs below, we assume that the array is +** actually N elements in size, where N is the smallest power of 2 greater +** to or equal to the number of PMAs being merged. The extra aIter[] elements +** are treated as if they are empty (always at EOF). ** ** The aTree[] array is also N elements in size. The value of N is stored in -** the VdbeSorter.nTree variable. +** the MergeEngine.nTree variable. ** ** The final (N/2) elements of aTree[] contain the results of comparing -** pairs of iterator keys together. Element i contains the result of +** pairs of PMA keys together. Element i contains the result of ** comparing aIter[2*i-N] and aIter[2*i-N+1]. Whichever key is smaller, the ** aTree element is set to the index of it. ** @@ -230,10 +224,10 @@ struct SortSubtask { ** key comparison operations are required, where N is the number of segments ** being merged (rounded up to the next power of 2). */ -struct SorterMerger { - int nTree; /* Used size of aTree/aIter (power of 2) */ - int *aTree; /* Current state of incremental merge */ - VdbeSorterIter *aIter; /* Array of iterators to merge data from */ +struct MergeEngine { + int nTree; /* Used size of aTree/aIter (power of 2) */ + int *aTree; /* Current state of incremental merge */ + PmaReader *aIter; /* Array of iterators to merge data from */ }; /* @@ -246,20 +240,21 @@ struct VdbeSorter { int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ int bUsePMA; /* True if one or more PMAs created */ SorterRecord *pRecord; /* Head of in-memory record list */ - SorterMerger *pMerger; /* For final merge of PMAs (by caller) */ + MergeEngine *pMerger; /* For final merge of PMAs (by caller) */ u8 *aMemory; /* Block of memory to alloc records from */ int iMemory; /* Offset of first free byte in aMemory */ int nMemory; /* Size of aMemory allocation in bytes */ int iPrev; /* Previous thread used to flush PMA */ - int nThread; /* Size of aThread[] array */ - SortSubtask aThread[1]; + int nTask; /* Size of aTask[] array */ + SortSubtask aTask[1]; /* One or more subtasks */ }; /* -** The following type is an iterator for a PMA. It caches the current key in -** variables nKey/aKey. If the iterator is at EOF, pFile==0. +** An instance of the following object is used to read records out of a +** PMA, in sorted order. The next key to be read is cached in nKey/aKey. +** pFile==0 at EOF. */ -struct VdbeSorterIter { +struct PmaReader { i64 iReadOff; /* Current read offset */ i64 iEof; /* 1 byte past EOF for this iterator */ int nAlloc; /* Bytes of space at aAlloc */ @@ -273,12 +268,14 @@ struct VdbeSorterIter { }; /* -** An instance of this structure is used to organize the stream of records -** being written to files by the merge-sort code into aligned, page-sized -** blocks. Doing all I/O in aligned page-sized blocks helps I/O to go -** faster on many operating systems. +** An instance of this object is used for writing a PMA. +** +** The PMA is written one record at a time. Each record is of an arbitrary +** size. But I/O is more efficient if it occurs in page-sized blocks where +** each block is aligned on a page boundary. This object caches writes to +** the PMA so that aligned, page-size blocks are written. */ -struct FileWriter { +struct PmaWriter { int eFWErr; /* Non-zero if in an error state */ u8 *aBuffer; /* Pointer to write buffer */ int nBuffer; /* Size of write buffer in bytes */ @@ -289,8 +286,8 @@ struct FileWriter { }; /* -** A structure to store a single record. All in-memory records are connected -** together into a linked list headed at VdbeSorter.pRecord. +** This object is the header on a single record while that record is being +** held in memory and prior to being written out as part of a PMA. ** ** How the linked list is connected depends on how memory is being managed ** by this module. If using a separate allocation for each in-memory record @@ -307,11 +304,12 @@ struct FileWriter { ** vdbeSorterSort() for details. */ struct SorterRecord { - int nVal; + int nVal; /* Size of the record in bytes */ union { SorterRecord *pNext; /* Pointer to next record in list */ int iNext; /* Offset within aMemory of next record */ } u; + /* The data for the record immediately follows this header */ }; /* Return a pointer to the buffer containing the record data for SorterRecord @@ -325,18 +323,18 @@ struct SorterRecord { ** page size in bytes. */ #define SORTER_MIN_WORKING 10 -/* Maximum number of segments to merge in a single pass. */ +/* Maximum number of PMAs that a single MergeEngine can merge */ #define SORTER_MAX_MERGE_COUNT 16 /* -** Free all memory belonging to the VdbeSorterIter object passed as the second +** Free all memory belonging to the PmaReader object passed as the second ** argument. All structure fields are set to zero before returning. */ -static void vdbeSorterIterZero(VdbeSorterIter *pIter){ +static void vdbePmaReaderClear(PmaReader *pIter){ sqlite3_free(pIter->aAlloc); sqlite3_free(pIter->aBuffer); if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); - memset(pIter, 0, sizeof(VdbeSorterIter)); + memset(pIter, 0, sizeof(PmaReader)); } /* @@ -348,8 +346,8 @@ static void vdbeSorterIterZero(VdbeSorterIter *pIter){ ** The buffer indicated by *ppOut may only be considered valid until the ** next call to this function. */ -static int vdbeSorterIterRead( - VdbeSorterIter *p, /* Iterator */ +static int vdbePmaReadBlob( + PmaReader *p, /* Iterator */ int nByte, /* Bytes of data to read */ u8 **ppOut /* OUT: Pointer to buffer containing data */ ){ @@ -419,13 +417,13 @@ static int vdbeSorterIterRead( /* The following loop copies up to p->nBuffer bytes per iteration into ** the p->aAlloc[] buffer. */ while( nRem>0 ){ - int rc; /* vdbeSorterIterRead() return code */ + int rc; /* vdbePmaReadBlob() return code */ int nCopy; /* Number of bytes to copy */ u8 *aNext; /* Pointer to buffer to copy data from */ nCopy = nRem; if( nRem>p->nBuffer ) nCopy = p->nBuffer; - rc = vdbeSorterIterRead(p, nCopy, &aNext); + rc = vdbePmaReadBlob(p, nCopy, &aNext); if( rc!=SQLITE_OK ) return rc; assert( aNext!=p->aAlloc ); memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); @@ -442,7 +440,7 @@ static int vdbeSorterIterRead( ** Read a varint from the stream of data accessed by p. Set *pnOut to ** the value read. */ -static int vdbeSorterIterVarint(VdbeSorterIter *p, u64 *pnOut){ +static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ int iBuf; if( p->aMap ){ @@ -455,7 +453,7 @@ static int vdbeSorterIterVarint(VdbeSorterIter *p, u64 *pnOut){ u8 aVarint[16], *a; int i = 0, rc; do{ - rc = vdbeSorterIterRead(p, 1, &a); + rc = vdbePmaReadBlob(p, 1, &a); if( rc ) return rc; aVarint[(i++)&0xf] = a[0]; }while( (a[0]&0x80)!=0 ); @@ -471,20 +469,20 @@ static int vdbeSorterIterVarint(VdbeSorterIter *p, u64 *pnOut){ ** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if ** no error occurs, or an SQLite error code if one does. */ -static int vdbeSorterIterNext(VdbeSorterIter *pIter){ +static int vdbePmaReaderNext(PmaReader *pIter){ int rc; /* Return Code */ u64 nRec = 0; /* Size of record in bytes */ if( pIter->iReadOff>=pIter->iEof ){ /* This is an EOF condition */ - vdbeSorterIterZero(pIter); + vdbePmaReaderClear(pIter); return SQLITE_OK; } - rc = vdbeSorterIterVarint(pIter, &nRec); + rc = vdbePmaReadVarint(pIter, &nRec); if( rc==SQLITE_OK ){ pIter->nKey = (int)nRec; - rc = vdbeSorterIterRead(pIter, (int)nRec, &pIter->aKey); + rc = vdbePmaReadBlob(pIter, (int)nRec, &pIter->aKey); } return rc; @@ -496,27 +494,27 @@ static int vdbeSorterIterNext(VdbeSorterIter *pIter){ ** leaves the iterator pointing to the first key in the PMA (or EOF if the ** PMA is empty). */ -static int vdbeSorterIterInit( - SortSubtask *pThread, /* Thread context */ - i64 iStart, /* Start offset in pThread->pTemp1 */ - VdbeSorterIter *pIter, /* Iterator to populate */ +static int vdbePmaReaderInit( + SortSubtask *pTask, /* Thread context */ + i64 iStart, /* Start offset in pTask->pTemp1 */ + PmaReader *pIter, /* Iterator to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ ){ int rc = SQLITE_OK; - int nBuf = pThread->pgsz; + int nBuf = pTask->pgsz; void *pMap = 0; /* Mapping of temp file */ - assert( pThread->iTemp1Off>iStart ); + assert( pTask->iTemp1Off>iStart ); assert( pIter->aAlloc==0 ); assert( pIter->aBuffer==0 ); - pIter->pFile = pThread->pTemp1; + pIter->pFile = pTask->pTemp1; pIter->iReadOff = iStart; pIter->nAlloc = 128; pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc); if( pIter->aAlloc ){ /* Try to xFetch() a mapping of the entire temp file. If this is possible, ** the PMA will be read via the mapping. Otherwise, use xRead(). */ - rc = sqlite3OsFetch(pIter->pFile, 0, pThread->iTemp1Off, &pMap); + rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap); }else{ rc = SQLITE_NOMEM; } @@ -533,11 +531,11 @@ static int vdbeSorterIterInit( int iBuf = iStart % nBuf; if( iBuf ){ int nRead = nBuf - iBuf; - if( (iStart + nRead) > pThread->iTemp1Off ){ - nRead = (int)(pThread->iTemp1Off - iStart); + if( (iStart + nRead) > pTask->iTemp1Off ){ + nRead = (int)(pTask->iTemp1Off - iStart); } rc = sqlite3OsRead( - pThread->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart + pTask->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart ); assert( rc!=SQLITE_IOERR_SHORT_READ ); } @@ -547,14 +545,14 @@ static int vdbeSorterIterInit( if( rc==SQLITE_OK ){ u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pThread->iTemp1Off; - rc = vdbeSorterIterVarint(pIter, &nByte); + pIter->iEof = pTask->iTemp1Off; + rc = vdbePmaReadVarint(pIter, &nByte); pIter->iEof = pIter->iReadOff + nByte; *pnByte += nByte; } if( rc==SQLITE_OK ){ - rc = vdbeSorterIterNext(pIter); + rc = vdbePmaReaderNext(pIter); } return rc; } @@ -562,25 +560,25 @@ static int vdbeSorterIterInit( /* ** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, -** size nKey2 bytes). Use (pThread->pKeyInfo) for the collation sequences +** size nKey2 bytes). Use (pTask->pKeyInfo) for the collation sequences ** used by the comparison. Return the result of the comparison. ** -** Before returning, object (pThread->pUnpacked) is populated with the +** Before returning, object (pTask->pUnpacked) is populated with the ** unpacked version of key2. Or, if pKey2 is passed a NULL pointer, then it -** is assumed that the (pThread->pUnpacked) structure already contains the +** is assumed that the (pTask->pUnpacked) structure already contains the ** unpacked key to use as key2. ** -** If an OOM error is encountered, (pThread->pUnpacked->error_rc) is set +** If an OOM error is encountered, (pTask->pUnpacked->error_rc) is set ** to SQLITE_NOMEM. */ static int vdbeSorterCompare( - SortSubtask *pThread, /* Thread context (for pKeyInfo) */ + SortSubtask *pTask, /* Subtask context (for pKeyInfo) */ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2 /* Right side of comparison */ ){ - UnpackedRecord *r2 = pThread->pUnpacked; + UnpackedRecord *r2 = pTask->pUnpacked; if( pKey2 ){ - sqlite3VdbeRecordUnpack(pThread->pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(pTask->pKeyInfo, nKey2, pKey2, r2); } return sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); } @@ -591,15 +589,15 @@ static int vdbeSorterCompare( ** value to recalculate. */ static int vdbeSorterDoCompare( - SortSubtask *pThread, - SorterMerger *pMerger, + SortSubtask *pTask, + MergeEngine *pMerger, int iOut ){ int i1; int i2; int iRes; - VdbeSorterIter *p1; - VdbeSorterIter *p2; + PmaReader *p1; + PmaReader *p2; assert( iOutnTree && iOut>0 ); @@ -620,9 +618,9 @@ static int vdbeSorterDoCompare( iRes = i1; }else{ int res; - assert( pThread->pUnpacked!=0 ); /* allocated in vdbeSortSubtaskMain() */ + assert( pTask->pUnpacked!=0 ); /* allocated in vdbeSortSubtaskMain() */ res = vdbeSorterCompare( - pThread, p1->aKey, p1->nKey, p2->aKey, p2->nKey + pTask, p1->aKey, p1->nKey, p2->aKey, p2->nKey ); if( res<=0 ){ iRes = i1; @@ -638,9 +636,13 @@ static int vdbeSorterDoCompare( /* ** Initialize the temporary index cursor just opened as a sorter cursor. */ -int sqlite3VdbeSorterInit(sqlite3 *db, int nField, VdbeCursor *pCsr){ +int sqlite3VdbeSorterInit( + sqlite3 *db, /* Database connection (for malloc()) */ + int nField, /* Number of key fields in each record */ + VdbeCursor *pCsr /* Cursor that holds the new sorter */ +){ int pgsz; /* Page size of main database */ - int i; /* Used to iterate through aThread[] */ + int i; /* Used to iterate through aTask[] */ int mxCache; /* Cache size */ VdbeSorter *pSorter; /* The new sorter */ KeyInfo *pKeyInfo; /* Copy of pCsr->pKeyInfo with db==0 */ @@ -664,12 +666,12 @@ int sqlite3VdbeSorterInit(sqlite3 *db, int nField, VdbeCursor *pCsr){ if( nField && nWorker==0 ) pKeyInfo->nField = nField; pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - pSorter->nThread = nWorker + 1; - for(i=0; inThread; i++){ - SortSubtask *pThread = &pSorter->aThread[i]; - pThread->pKeyInfo = pKeyInfo; - pThread->pVfs = db->pVfs; - pThread->pgsz = pgsz; + pSorter->nTask = nWorker + 1; + for(i=0; inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + pTask->pKeyInfo = pKeyInfo; + pTask->pVfs = db->pVfs; + pTask->pgsz = pgsz; } if( !sqlite3TempInMemory(db) ){ @@ -706,22 +708,22 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ } /* -** Free all resources owned by the object indicated by argument pThread. All -** fields of *pThread are zeroed before returning. +** Free all resources owned by the object indicated by argument pTask. All +** fields of *pTask are zeroed before returning. */ -static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pThread){ - sqlite3DbFree(db, pThread->pUnpacked); - pThread->pUnpacked = 0; - if( pThread->aListMemory==0 ){ - vdbeSorterRecordFree(0, pThread->pList); +static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ + sqlite3DbFree(db, pTask->pUnpacked); + pTask->pUnpacked = 0; + if( pTask->aListMemory==0 ){ + vdbeSorterRecordFree(0, pTask->pList); }else{ - sqlite3_free(pThread->aListMemory); - pThread->aListMemory = 0; + sqlite3_free(pTask->aListMemory); + pTask->aListMemory = 0; } - pThread->pList = 0; - if( pThread->pTemp1 ){ - sqlite3OsCloseFree(pThread->pTemp1); - pThread->pTemp1 = 0; + pTask->pList = 0; + if( pTask->pTemp1 ){ + sqlite3OsCloseFree(pTask->pTemp1); + pTask->pTemp1 = 0; } } @@ -732,13 +734,13 @@ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pThread){ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int rc = rcin; int i; - for(i=0; inThread; i++){ - SortSubtask *pThread = &pSorter->aThread[i]; - if( pThread->pThread ){ + for(i=0; inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + if( pTask->pTask ){ void *pRet; - int rc2 = sqlite3ThreadJoin(pThread->pThread, &pRet); - pThread->pThread = 0; - pThread->bDone = 0; + int rc2 = sqlite3ThreadJoin(pTask->pTask, &pRet); + pTask->pTask = 0; + pTask->bDone = 0; if( rc==SQLITE_OK ) rc = rc2; if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); } @@ -750,34 +752,34 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ #endif /* -** Allocate a new SorterMerger object with space for nIter iterators. +** Allocate a new MergeEngine object with space for nIter iterators. */ -static SorterMerger *vdbeSorterMergerNew(int nIter){ +static MergeEngine *vdbeMergeEngineNew(int nIter){ int N = 2; /* Smallest power of two >= nIter */ int nByte; /* Total bytes of space to allocate */ - SorterMerger *pNew; /* Pointer to allocated object to return */ + MergeEngine *pNew; /* Pointer to allocated object to return */ assert( nIter<=SORTER_MAX_MERGE_COUNT ); while( NnTree = N; - pNew->aIter = (VdbeSorterIter*)&pNew[1]; + pNew->aIter = (PmaReader*)&pNew[1]; pNew->aTree = (int*)&pNew->aIter[N]; } return pNew; } /* -** Free the SorterMerger object passed as the only argument. +** Free the MergeEngine object passed as the only argument. */ -static void vdbeSorterMergerFree(SorterMerger *pMerger){ +static void vdbeMergeEngineFree(MergeEngine *pMerger){ int i; if( pMerger ){ for(i=0; inTree; i++){ - vdbeSorterIterZero(&pMerger->aIter[i]); + vdbePmaReaderClear(&pMerger->aIter[i]); } } sqlite3_free(pMerger); @@ -788,12 +790,12 @@ static void vdbeSorterMergerFree(SorterMerger *pMerger){ */ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ int i; - vdbeSorterJoinAll(pSorter, SQLITE_OK); - vdbeSorterMergerFree(pSorter->pMerger); + (void)vdbeSorterJoinAll(pSorter, SQLITE_OK); + vdbeMergeEngineFree(pSorter->pMerger); pSorter->pMerger = 0; - for(i=0; inThread; i++){ - SortSubtask *pThread = &pSorter->aThread[i]; - vdbeSortSubtaskCleanup(db, pThread); + for(i=0; inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + vdbeSortSubtaskCleanup(db, pTask); } if( pSorter->aMemory==0 ){ vdbeSorterRecordFree(0, pSorter->pRecord); @@ -811,7 +813,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); - vdbeSorterMergerFree(pSorter->pMerger); + vdbeMergeEngineFree(pSorter->pMerger); sqlite3_free(pSorter->aMemory); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; @@ -842,7 +844,7 @@ static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){ ** Set *ppOut to the head of the new list. */ static void vdbeSorterMerge( - SortSubtask *pThread, /* Calling thread context */ + SortSubtask *pTask, /* Calling thread context */ SorterRecord *p1, /* First list to merge */ SorterRecord *p2, /* Second list to merge */ SorterRecord **ppOut /* OUT: Head of merged list */ @@ -853,7 +855,7 @@ static void vdbeSorterMerge( while( p1 && p2 ){ int res; - res = vdbeSorterCompare(pThread, SRVAL(p1), p1->nVal, pVal2, p2->nVal); + res = vdbeSorterCompare(pTask, SRVAL(p1), p1->nVal, pVal2, p2->nVal); if( res<=0 ){ *pp = p1; pp = &p1->u.pNext; @@ -872,11 +874,11 @@ static void vdbeSorterMerge( } /* -** Sort the linked list of records headed at pThread->pList. Return +** Sort the linked list of records headed at pTask->pList. Return ** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if ** an error occurs. */ -static int vdbeSorterSort(SortSubtask *pThread){ +static int vdbeSorterSort(SortSubtask *pTask){ int i; SorterRecord **aSlot; SorterRecord *p; @@ -886,15 +888,15 @@ static int vdbeSorterSort(SortSubtask *pThread){ return SQLITE_NOMEM; } - p = pThread->pList; + p = pTask->pList; while( p ){ SorterRecord *pNext; - if( pThread->aListMemory ){ - if( (u8*)p==pThread->aListMemory ){ + if( pTask->aListMemory ){ + if( (u8*)p==pTask->aListMemory ){ pNext = 0; }else{ - assert( p->u.iNextaListMemory) ); - pNext = (SorterRecord*)&pThread->aListMemory[p->u.iNext]; + assert( p->u.iNextaListMemory) ); + pNext = (SorterRecord*)&pTask->aListMemory[p->u.iNext]; } }else{ pNext = p->u.pNext; @@ -902,7 +904,7 @@ static int vdbeSorterSort(SortSubtask *pThread){ p->u.pNext = 0; for(i=0; aSlot[i]; i++){ - vdbeSorterMerge(pThread, p, aSlot[i], &p); + vdbeSorterMerge(pTask, p, aSlot[i], &p); aSlot[i] = 0; } aSlot[i] = p; @@ -911,24 +913,24 @@ static int vdbeSorterSort(SortSubtask *pThread){ p = 0; for(i=0; i<64; i++){ - vdbeSorterMerge(pThread, p, aSlot[i], &p); + vdbeSorterMerge(pTask, p, aSlot[i], &p); } - pThread->pList = p; + pTask->pList = p; sqlite3_free(aSlot); return SQLITE_OK; } /* -** Initialize a file-writer object. +** Initialize a PMA-writer object. */ -static void fileWriterInit( +static void vdbePmaWriterInit( sqlite3_file *pFile, /* File to write to */ - FileWriter *p, /* Object to populate */ + PmaWriter *p, /* Object to populate */ int nBuf, /* Buffer size */ i64 iStart /* Offset of pFile to begin writing at */ ){ - memset(p, 0, sizeof(FileWriter)); + memset(p, 0, sizeof(PmaWriter)); p->aBuffer = (u8*)sqlite3Malloc(nBuf); if( !p->aBuffer ){ p->eFWErr = SQLITE_NOMEM; @@ -941,10 +943,10 @@ static void fileWriterInit( } /* -** Write nData bytes of data to the file-write object. Return SQLITE_OK +** Write nData bytes of data to the PMA. Return SQLITE_OK ** if successful, or an SQLite error code if an error occurs. */ -static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ +static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ int nRem = nData; while( nRem>0 && p->eFWErr==0 ){ int nCopy = nRem; @@ -969,15 +971,15 @@ static void fileWriterWrite(FileWriter *p, u8 *pData, int nData){ } /* -** Flush any buffered data to disk and clean up the file-writer object. -** The results of using the file-writer after this call are undefined. +** Flush any buffered data to disk and clean up the PMA-writer object. +** The results of using the PMA-writer after this call are undefined. ** Return SQLITE_OK if flushing the buffered data succeeds or is not ** required. Otherwise, return an SQLite error code. ** ** Before returning, set *piEof to the offset immediately following the ** last byte written to the file. */ -static int fileWriterFinish(FileWriter *p, i64 *piEof){ +static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ p->eFWErr = sqlite3OsWrite(p->pFile, @@ -988,19 +990,19 @@ static int fileWriterFinish(FileWriter *p, i64 *piEof){ *piEof = (p->iWriteOff + p->iBufEnd); sqlite3_free(p->aBuffer); rc = p->eFWErr; - memset(p, 0, sizeof(FileWriter)); + memset(p, 0, sizeof(PmaWriter)); return rc; } /* -** Write value iVal encoded as a varint to the file-write object. Return +** Write value iVal encoded as a varint to the PMA. Return ** SQLITE_OK if successful, or an SQLite error code if an error occurs. */ -static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ +static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ int nByte; u8 aByte[10]; nByte = sqlite3PutVarint(aByte, iVal); - fileWriterWrite(p, aByte, nByte); + vdbePmaWriteBlob(p, aByte, nByte); } #if SQLITE_MAX_MMAP_SIZE>0 @@ -1040,25 +1042,25 @@ static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterListToPMA(SortSubtask *pThread){ +static int vdbeSorterListToPMA(SortSubtask *pTask){ int rc = SQLITE_OK; /* Return code */ - FileWriter writer; /* Object used to write to the file */ + PmaWriter writer; /* Object used to write to the file */ - memset(&writer, 0, sizeof(FileWriter)); - assert( pThread->nInMemory>0 ); + memset(&writer, 0, sizeof(PmaWriter)); + assert( pTask->nInMemory>0 ); /* If the first temporary PMA file has not been opened, open it now. */ - if( pThread->pTemp1==0 ){ - rc = vdbeSorterOpenTempFile(pThread->pVfs, &pThread->pTemp1); - assert( rc!=SQLITE_OK || pThread->pTemp1 ); - assert( pThread->iTemp1Off==0 ); - assert( pThread->nPMA==0 ); + if( pTask->pTemp1==0 ){ + rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTask->pTemp1); + assert( rc!=SQLITE_OK || pTask->pTemp1 ); + assert( pTask->iTemp1Off==0 ); + assert( pTask->nPMA==0 ); } /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ rc = vdbeSorterExtendFile( - pThread->pTemp1, pThread->iTemp1Off + pThread->nInMemory + 9 + pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9 ); } @@ -1066,46 +1068,47 @@ static int vdbeSorterListToPMA(SortSubtask *pThread){ SorterRecord *p; SorterRecord *pNext = 0; - fileWriterInit(pThread->pTemp1, &writer, pThread->pgsz, pThread->iTemp1Off); - pThread->nPMA++; - fileWriterWriteVarint(&writer, pThread->nInMemory); - for(p=pThread->pList; p; p=pNext){ + vdbePmaWriterInit(pTask->pTemp1, &writer, pTask->pgsz, + pTask->iTemp1Off); + pTask->nPMA++; + vdbePmaWriteVarint(&writer, pTask->nInMemory); + for(p=pTask->pList; p; p=pNext){ pNext = p->u.pNext; - fileWriterWriteVarint(&writer, p->nVal); - fileWriterWrite(&writer, SRVAL(p), p->nVal); - if( pThread->aListMemory==0 ) sqlite3_free(p); + vdbePmaWriteVarint(&writer, p->nVal); + vdbePmaWriteBlob(&writer, SRVAL(p), p->nVal); + if( pTask->aListMemory==0 ) sqlite3_free(p); } - pThread->pList = p; - rc = fileWriterFinish(&writer, &pThread->iTemp1Off); + pTask->pList = p; + rc = vdbePmaWriterFinish(&writer, &pTask->iTemp1Off); } - assert( pThread->pList==0 || rc!=SQLITE_OK ); + assert( pTask->pList==0 || rc!=SQLITE_OK ); return rc; } /* -** Advance the SorterMerger iterator passed as the second argument to +** Advance the MergeEngine iterator passed as the second argument to ** the next entry. Set *pbEof to true if this means the iterator has ** reached EOF. ** ** Return SQLITE_OK if successful or an error code if an error occurs. */ static int vdbeSorterNext( - SortSubtask *pThread, - SorterMerger *pMerger, + SortSubtask *pTask, + MergeEngine *pMerger, int *pbEof ){ int rc; int iPrev = pMerger->aTree[1];/* Index of iterator to advance */ /* Advance the current iterator */ - rc = vdbeSorterIterNext(&pMerger->aIter[iPrev]); + rc = vdbePmaReaderNext(&pMerger->aIter[iPrev]); /* Update contents of aTree[] */ if( rc==SQLITE_OK ){ int i; /* Index of aTree[] to recalculate */ - VdbeSorterIter *pIter1; /* First iterator to compare */ - VdbeSorterIter *pIter2; /* Second iterator to compare */ + PmaReader *pIter1; /* First iterator to compare */ + PmaReader *pIter2; /* Second iterator to compare */ u8 *pKey2; /* To pIter2->aKey, or 0 if record cached */ /* Find the first two iterators to compare. The one that was just @@ -1122,19 +1125,19 @@ static int vdbeSorterNext( }else if( pIter2->pFile==0 ){ iRes = -1; }else{ - iRes = vdbeSorterCompare(pThread, + iRes = vdbeSorterCompare(pTask, pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey ); } /* If pIter1 contained the smaller value, set aTree[i] to its index. ** Then set pIter2 to the next iterator to compare to pIter1. In this - ** case there is no cache of pIter2 in pThread->pUnpacked, so set + ** case there is no cache of pIter2 in pTask->pUnpacked, so set ** pKey2 to point to the record belonging to pIter2. ** ** Alternatively, if pIter2 contains the smaller of the two values, ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() - ** was actually called above, then pThread->pUnpacked now contains + ** was actually called above, then pTask->pUnpacked now contains ** a value equivalent to pIter2. So set pKey2 to NULL to prevent ** vdbeSorterCompare() from decoding pIter2 again. ** @@ -1163,59 +1166,59 @@ static int vdbeSorterNext( */ static void *vdbeSortSubtaskMain(void *pCtx){ int rc = SQLITE_OK; - SortSubtask *pThread = (SortSubtask*)pCtx; + SortSubtask *pTask = (SortSubtask*)pCtx; - assert( pThread->eWork==SORT_SUBTASK_SORT - || pThread->eWork==SORT_SUBTASK_TO_PMA - || pThread->eWork==SORT_SUBTASK_CONS + assert( pTask->eWork==SORT_SUBTASK_SORT + || pTask->eWork==SORT_SUBTASK_TO_PMA + || pTask->eWork==SORT_SUBTASK_CONS ); - assert( pThread->bDone==0 ); + assert( pTask->bDone==0 ); - if( pThread->pUnpacked==0 ){ + if( pTask->pUnpacked==0 ){ char *pFree; - pThread->pUnpacked = sqlite3VdbeAllocUnpackedRecord( - pThread->pKeyInfo, 0, 0, &pFree + pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( + pTask->pKeyInfo, 0, 0, &pFree ); - assert( pThread->pUnpacked==(UnpackedRecord*)pFree ); + assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); if( pFree==0 ){ rc = SQLITE_NOMEM; goto thread_out; } - pThread->pUnpacked->nField = pThread->pKeyInfo->nField; - pThread->pUnpacked->errCode = 0; + pTask->pUnpacked->nField = pTask->pKeyInfo->nField; + pTask->pUnpacked->errCode = 0; } - if( pThread->eWork==SORT_SUBTASK_CONS ){ - assert( pThread->pList==0 ); - while( pThread->nPMA>pThread->nConsolidate && rc==SQLITE_OK ){ - int nIter = MIN(pThread->nPMA, SORTER_MAX_MERGE_COUNT); + if( pTask->eWork==SORT_SUBTASK_CONS ){ + assert( pTask->pList==0 ); + while( pTask->nPMA>pTask->nConsolidate && rc==SQLITE_OK ){ + int nIter = MIN(pTask->nPMA, SORTER_MAX_MERGE_COUNT); sqlite3_file *pTemp2 = 0; /* Second temp file to use */ - SorterMerger *pMerger; /* Object for reading/merging PMA data */ + MergeEngine *pMerger; /* Object for reading/merging PMA data */ i64 iReadOff = 0; /* Offset in pTemp1 to read from */ i64 iWriteOff = 0; /* Offset in pTemp2 to write to */ int i; /* Allocate a merger object to merge PMAs together. */ - pMerger = vdbeSorterMergerNew(nIter); + pMerger = vdbeMergeEngineNew(nIter); if( pMerger==0 ){ rc = SQLITE_NOMEM; break; } /* Open a second temp file to write merged data to */ - rc = vdbeSorterOpenTempFile(pThread->pVfs, &pTemp2); + rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTemp2); if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile(pTemp2, pThread->iTemp1Off); + rc = vdbeSorterExtendFile(pTemp2, pTask->iTemp1Off); } if( rc!=SQLITE_OK ){ - vdbeSorterMergerFree(pMerger); + vdbeMergeEngineFree(pMerger); break; } /* This loop runs once for each output PMA. Each output PMA is made ** of data merged from up to SORTER_MAX_MERGE_COUNT input PMAs. */ - for(i=0; inPMA; i+=SORTER_MAX_MERGE_COUNT){ - FileWriter writer; /* Object for writing data to pTemp2 */ + for(i=0; inPMA; i+=SORTER_MAX_MERGE_COUNT){ + PmaWriter writer; /* Object for writing data to pTemp2 */ i64 nOut = 0; /* Bytes of data in output PMA */ int bEof = 0; int rc2; @@ -1225,54 +1228,54 @@ static void *vdbeSortSubtaskMain(void *pCtx){ ** if that is fewer). */ int iIter; for(iIter=0; iIteraIter[iIter]; - rc = vdbeSorterIterInit(pThread, iReadOff, pIter, &nOut); + PmaReader *pIter = &pMerger->aIter[iIter]; + rc = vdbePmaReaderInit(pTask, iReadOff, pIter, &nOut); iReadOff = pIter->iEof; - if( iReadOff>=pThread->iTemp1Off || rc!=SQLITE_OK ) break; + if( iReadOff>=pTask->iTemp1Off || rc!=SQLITE_OK ) break; } for(iIter=pMerger->nTree-1; rc==SQLITE_OK && iIter>0; iIter--){ - rc = vdbeSorterDoCompare(pThread, pMerger, iIter); + rc = vdbeSorterDoCompare(pTask, pMerger, iIter); } - fileWriterInit(pTemp2, &writer, pThread->pgsz, iWriteOff); - fileWriterWriteVarint(&writer, nOut); + vdbePmaWriterInit(pTemp2, &writer, pTask->pgsz, iWriteOff); + vdbePmaWriteVarint(&writer, nOut); while( rc==SQLITE_OK && bEof==0 ){ - VdbeSorterIter *pIter = &pMerger->aIter[ pMerger->aTree[1] ]; + PmaReader *pIter = &pMerger->aIter[ pMerger->aTree[1] ]; assert( pIter->pFile!=0 ); /* pIter is not at EOF */ - fileWriterWriteVarint(&writer, pIter->nKey); - fileWriterWrite(&writer, pIter->aKey, pIter->nKey); - rc = vdbeSorterNext(pThread, pMerger, &bEof); + vdbePmaWriteVarint(&writer, pIter->nKey); + vdbePmaWriteBlob(&writer, pIter->aKey, pIter->nKey); + rc = vdbeSorterNext(pTask, pMerger, &bEof); } - rc2 = fileWriterFinish(&writer, &iWriteOff); + rc2 = vdbePmaWriterFinish(&writer, &iWriteOff); if( rc==SQLITE_OK ) rc = rc2; } - vdbeSorterMergerFree(pMerger); - sqlite3OsCloseFree(pThread->pTemp1); - pThread->pTemp1 = pTemp2; - pThread->nPMA = (i / SORTER_MAX_MERGE_COUNT); - pThread->iTemp1Off = iWriteOff; + vdbeMergeEngineFree(pMerger); + sqlite3OsCloseFree(pTask->pTemp1); + pTask->pTemp1 = pTemp2; + pTask->nPMA = (i / SORTER_MAX_MERGE_COUNT); + pTask->iTemp1Off = iWriteOff; } }else{ - /* Sort the pThread->pList list */ - rc = vdbeSorterSort(pThread); + /* Sort the pTask->pList list */ + rc = vdbeSorterSort(pTask); /* If required, write the list out to a PMA. */ - if( rc==SQLITE_OK && pThread->eWork==SORT_SUBTASK_TO_PMA ){ + if( rc==SQLITE_OK && pTask->eWork==SORT_SUBTASK_TO_PMA ){ #ifdef SQLITE_DEBUG - i64 nExpect = pThread->nInMemory - + sqlite3VarintLen(pThread->nInMemory) - + pThread->iTemp1Off; + i64 nExpect = pTask->nInMemory + + sqlite3VarintLen(pTask->nInMemory) + + pTask->iTemp1Off; #endif - rc = vdbeSorterListToPMA(pThread); - assert( rc!=SQLITE_OK || (nExpect==pThread->iTemp1Off) ); + rc = vdbeSorterListToPMA(pTask); + assert( rc!=SQLITE_OK || (nExpect==pTask->iTemp1Off) ); } } thread_out: - pThread->bDone = 1; - if( rc==SQLITE_OK && pThread->pUnpacked->errCode ){ - assert( pThread->pUnpacked->errCode==SQLITE_NOMEM ); + pTask->bDone = 1; + if( rc==SQLITE_OK && pTask->pUnpacked->errCode ){ + assert( pTask->pUnpacked->errCode==SQLITE_NOMEM ); rc = SQLITE_NOMEM; } return SQLITE_INT_TO_PTR(rc); @@ -1282,10 +1285,10 @@ static void *vdbeSortSubtaskMain(void *pCtx){ ** Run the activity scheduled by the object passed as the only argument ** in the current thread. */ -static int vdbeSorterRunThread(SortSubtask *pThread){ - int rc = SQLITE_PTR_TO_INT( vdbeSortSubtaskMain((void*)pThread) ); - assert( pThread->bDone ); - pThread->bDone = 0; +static int vdbeSorterRunTask(SortSubtask *pTask){ + int rc = SQLITE_PTR_TO_INT( vdbeSortSubtaskMain((void*)pTask) ); + assert( pTask->bDone ); + pTask->bDone = 0; return rc; } @@ -1299,53 +1302,53 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; int i; - SortSubtask *pThread = 0; /* Thread context used to create new PMA */ - int nWorker = (pSorter->nThread-1); + SortSubtask *pTask = 0; /* Thread context used to create new PMA */ + int nWorker = (pSorter->nTask-1); pSorter->bUsePMA = 1; for(i=0; iiPrev + i + 1) % nWorker; - pThread = &pSorter->aThread[iTest]; + pTask = &pSorter->aTask[iTest]; #if SQLITE_MAX_WORKER_THREADS>0 - if( pThread->bDone ){ + if( pTask->bDone ){ void *pRet; - assert( pThread->pThread ); - rc = sqlite3ThreadJoin(pThread->pThread, &pRet); - pThread->pThread = 0; - pThread->bDone = 0; + assert( pTask->pTask ); + rc = sqlite3ThreadJoin(pTask->pTask, &pRet); + pTask->pTask = 0; + pTask->bDone = 0; if( rc==SQLITE_OK ){ rc = SQLITE_PTR_TO_INT(pRet); } } #endif - if( pThread->pThread==0 ) break; - pThread = 0; + if( pTask->pThread==0 ) break; + pTask = 0; } - if( pThread==0 ){ - pThread = &pSorter->aThread[nWorker]; + if( pTask==0 ){ + pTask = &pSorter->aTask[nWorker]; } - pSorter->iPrev = (pThread - pSorter->aThread); + pSorter->iPrev = (pTask - pSorter->aTask); if( rc==SQLITE_OK ){ - assert( pThread->pThread==0 && pThread->bDone==0 ); - pThread->eWork = SORT_SUBTASK_TO_PMA; - pThread->pList = pSorter->pRecord; - pThread->nInMemory = pSorter->nInMemory; + assert( pTask->pThread==0 && pTask->bDone==0 ); + pTask->eWork = SORT_SUBTASK_TO_PMA; + pTask->pList = pSorter->pRecord; + pTask->nInMemory = pSorter->nInMemory; pSorter->nInMemory = 0; pSorter->pRecord = 0; if( pSorter->aMemory ){ - u8 *aMem = pThread->aListMemory; - pThread->aListMemory = pSorter->aMemory; + u8 *aMem = pTask->aListMemory; + pTask->aListMemory = pSorter->aMemory; pSorter->aMemory = aMem; } #if SQLITE_MAX_WORKER_THREADS>0 - if( !bFg && pThread!=&pSorter->aThread[nWorker] ){ + if( !bFg && pTask!=&pSorter->aTask[nWorker] ){ /* Launch a background thread for this operation */ - void *pCtx = (void*)pThread; - assert( pSorter->aMemory==0 || pThread->aListMemory!=0 ); - if( pThread->aListMemory ){ + void *pCtx = (void*)pTask; + assert( pSorter->aMemory==0 || pTask->aListMemory!=0 ); + if( pTask->aListMemory ){ if( pSorter->aMemory==0 ){ pSorter->aMemory = sqlite3Malloc(pSorter->nMemory); if( pSorter->aMemory==0 ) return SQLITE_NOMEM; @@ -1353,17 +1356,17 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory); } } - rc = sqlite3ThreadCreate(&pThread->pThread, vdbeSortSubtaskMain, pCtx); + rc = sqlite3ThreadCreate(&pTask->pTask, vdbeSortSubtaskMain, pCtx); }else #endif { /* Use the foreground thread for this operation */ - rc = vdbeSorterRunThread(pThread); + rc = vdbeSorterRunTask(pTask); if( rc==SQLITE_OK ){ - u8 *aMem = pThread->aListMemory; - pThread->aListMemory = pSorter->aMemory; + u8 *aMem = pTask->aListMemory; + pTask->aListMemory = pSorter->aMemory; pSorter->aMemory = aMem; - assert( pThread->pList==0 ); + assert( pTask->pList==0 ); } } } @@ -1469,15 +1472,16 @@ int sqlite3VdbeSorterWrite( static int vdbeSorterCountPMA(VdbeSorter *pSorter){ int nPMA = 0; int i; - for(i=0; inThread; i++){ - nPMA += pSorter->aThread[i].nPMA; + for(i=0; inTask; i++){ + nPMA += pSorter->aTask[i].nPMA; } return nPMA; } /* -** Once the sorter has been populated, this function is called to prepare -** for iterating through its contents in sorted order. +** Once the sorter has been populated by calls to sqlite3VdbeSorterWrite, +** this function is called to prepare for iterating through the records +** in sorted order. */ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; @@ -1490,16 +1494,16 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** from the in-memory list. */ if( pSorter->bUsePMA==0 ){ if( pSorter->pRecord ){ - SortSubtask *pThread = &pSorter->aThread[0]; + SortSubtask *pTask = &pSorter->aTask[0]; *pbEof = 0; - pThread->pList = pSorter->pRecord; - pThread->eWork = SORT_SUBTASK_SORT; - assert( pThread->aListMemory==0 ); - pThread->aListMemory = pSorter->aMemory; - rc = vdbeSorterRunThread(pThread); - pThread->aListMemory = 0; - pSorter->pRecord = pThread->pList; - pThread->pList = 0; + pTask->pList = pSorter->pRecord; + pTask->eWork = SORT_SUBTASK_SORT; + assert( pTask->aListMemory==0 ); + pTask->aListMemory = pSorter->aMemory; + rc = vdbeSorterRunTask(pTask); + pTask->aListMemory = 0; + pSorter->pRecord = pTask->pList; + pTask->pList = 0; }else{ *pbEof = 1; } @@ -1518,20 +1522,20 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** some of them together so that this is no longer the case. */ if( vdbeSorterCountPMA(pSorter)>SORTER_MAX_MERGE_COUNT ){ int i; - for(i=0; rc==SQLITE_OK && inThread; i++){ - SortSubtask *pThread = &pSorter->aThread[i]; - if( pThread->pTemp1 ){ - pThread->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nThread; - pThread->eWork = SORT_SUBTASK_CONS; + for(i=0; rc==SQLITE_OK && inTask; i++){ + SortSubtask *pTask = &pSorter->aTask[i]; + if( pTask->pTemp1 ){ + pTask->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nTask; + pTask->eWork = SORT_SUBTASK_CONS; #if SQLITE_MAX_WORKER_THREADS>0 - if( i<(pSorter->nThread-1) ){ - void *pCtx = (void*)pThread; - rc = sqlite3ThreadCreate(&pThread->pThread,vdbeSortSubtaskMain,pCtx); + if( i<(pSorter->nTask-1) ){ + void *pCtx = (void*)pTask; + rc = sqlite3ThreadCreate(&pTask->pTask,vdbeSortSubtaskMain,pCtx); }else #endif { - rc = vdbeSorterRunThread(pThread); + rc = vdbeSorterRunTask(pTask); } } } @@ -1546,31 +1550,31 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ if( rc==SQLITE_OK ){ int nIter = 0; /* Number of iterators used */ int i; - SorterMerger *pMerger; - for(i=0; inThread; i++){ - nIter += pSorter->aThread[i].nPMA; + MergeEngine *pMerger; + for(i=0; inTask; i++){ + nIter += pSorter->aTask[i].nPMA; } - pSorter->pMerger = pMerger = vdbeSorterMergerNew(nIter); + pSorter->pMerger = pMerger = vdbeMergeEngineNew(nIter); if( pMerger==0 ){ rc = SQLITE_NOMEM; }else{ int iIter = 0; int iThread = 0; - for(iThread=0; iThreadnThread; iThread++){ + for(iThread=0; iThreadnTask; iThread++){ int iPMA; i64 iReadOff = 0; - SortSubtask *pThread = &pSorter->aThread[iThread]; - for(iPMA=0; iPMAnPMA && rc==SQLITE_OK; iPMA++){ + SortSubtask *pTask = &pSorter->aTask[iThread]; + for(iPMA=0; iPMAnPMA && rc==SQLITE_OK; iPMA++){ i64 nDummy = 0; - VdbeSorterIter *pIter = &pMerger->aIter[iIter++]; - rc = vdbeSorterIterInit(pThread, iReadOff, pIter, &nDummy); + PmaReader *pIter = &pMerger->aIter[iIter++]; + rc = vdbePmaReaderInit(pTask, iReadOff, pIter, &nDummy); iReadOff = pIter->iEof; } } for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(&pSorter->aThread[0], pMerger, i); + rc = vdbeSorterDoCompare(&pSorter->aTask[0], pMerger, i); } } } @@ -1589,7 +1593,7 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ int rc; /* Return code */ if( pSorter->pMerger ){ - rc = vdbeSorterNext(&pSorter->aThread[0], pSorter->pMerger, pbEof); + rc = vdbeSorterNext(&pSorter->aTask[0], pSorter->pMerger, pbEof); }else{ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->u.pNext; @@ -1611,7 +1615,7 @@ static void *vdbeSorterRowkey( ){ void *pKey; if( pSorter->pMerger ){ - VdbeSorterIter *pIter; + PmaReader *pIter; pIter = &pSorter->pMerger->aIter[ pSorter->pMerger->aTree[1] ]; *pnKey = pIter->nKey; pKey = pIter->aKey; @@ -1663,7 +1667,7 @@ int sqlite3VdbeSorterCompare( int *pRes /* OUT: Result of comparison */ ){ VdbeSorter *pSorter = pCsr->pSorter; - UnpackedRecord *r2 = pSorter->aThread[0].pUnpacked; + UnpackedRecord *r2 = pSorter->aTask[0].pUnpacked; KeyInfo *pKeyInfo = pCsr->pKeyInfo; int i; void *pKey; int nKey; /* Sorter key to compare pVal with */ From 6e4cc55e1f8e5cce7d7be6dd4fd5fd576d787a33 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Apr 2014 14:29:08 +0000 Subject: [PATCH 030/710] Fix minor errors causing compilation to fail with SQLITE_MAX_WORKER_THREADS set to a value greater than zero. FossilOrigin-Name: 0561272abf357a2f4709f6c02866e570d19cd344 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 16 ++++++++-------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index a1d8fa4b08..2e20875ba9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\slocal\sobject\sand\smethod\snames\sin\svdbesort.c\sso\sthat\stheir\snames\nmore\sclosely\sreflect\stheir\sactual\suse. -D 2014-04-03T02:54:27.677 +C Fix\sminor\serrors\scausing\scompilation\sto\sfail\swith\sSQLITE_MAX_WORKER_THREADS\sset\sto\sa\svalue\sgreater\sthan\szero. +D 2014-04-03T14:29:08.251 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c a4e349ff62196dc3d98eacb0f9281e2468c44cab +F src/vdbesort.c 5e7ed44bb4f2af809b6d229ae00f97825efab89a F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eef60f1bf54fcdc7b32f96ebb87a9a0bf0776e8b -R 0d1e8a8cee97e3fb0928896d3dcac011 -U drh -Z 523d07a773fda305bad8a184629c5049 +P d284e30eb1db144965fa85566e4234e30464350b +R 45d899d78ea7a6cd4a92080d8bb33ecf +U dan +Z 013157fb51930f7eb005a94358375580 diff --git a/manifest.uuid b/manifest.uuid index 0af0f90890..9cf01bf6ac 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d284e30eb1db144965fa85566e4234e30464350b \ No newline at end of file +0561272abf357a2f4709f6c02866e570d19cd344 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 6b94ddfebf..78f5183b5d 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -736,10 +736,10 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int i; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; - if( pTask->pTask ){ + if( pTask->pThread ){ void *pRet; - int rc2 = sqlite3ThreadJoin(pTask->pTask, &pRet); - pTask->pTask = 0; + int rc2 = sqlite3ThreadJoin(pTask->pThread, &pRet); + pTask->pThread = 0; pTask->bDone = 0; if( rc==SQLITE_OK ) rc = rc2; if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); @@ -1312,9 +1312,9 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ #if SQLITE_MAX_WORKER_THREADS>0 if( pTask->bDone ){ void *pRet; - assert( pTask->pTask ); - rc = sqlite3ThreadJoin(pTask->pTask, &pRet); - pTask->pTask = 0; + assert( pTask->pThread ); + rc = sqlite3ThreadJoin(pTask->pThread, &pRet); + pTask->pThread = 0; pTask->bDone = 0; if( rc==SQLITE_OK ){ rc = SQLITE_PTR_TO_INT(pRet); @@ -1356,7 +1356,7 @@ static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory); } } - rc = sqlite3ThreadCreate(&pTask->pTask, vdbeSortSubtaskMain, pCtx); + rc = sqlite3ThreadCreate(&pTask->pThread, vdbeSortSubtaskMain, pCtx); }else #endif { @@ -1531,7 +1531,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ #if SQLITE_MAX_WORKER_THREADS>0 if( i<(pSorter->nTask-1) ){ void *pCtx = (void*)pTask; - rc = sqlite3ThreadCreate(&pTask->pTask,vdbeSortSubtaskMain,pCtx); + rc = sqlite3ThreadCreate(&pTask->pThread, vdbeSortSubtaskMain, pCtx); }else #endif { From 8930c2ab0cf4f38a7aef3db0cf5965c4c36583e1 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 3 Apr 2014 16:25:29 +0000 Subject: [PATCH 031/710] Fix an integer overflow problem in the sorter. FossilOrigin-Name: 9d3351b8d713232133dad149c73fb2a27c72abb1 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/main.c | 8 ++++++++ src/sqlite.h.in | 3 ++- src/sqliteInt.h | 1 + src/test1.c | 14 ++++++++++++++ src/vdbesort.c | 33 +++++++++++++++++++-------------- test/permutations.test | 1 + 8 files changed, 57 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 2e20875ba9..1e7716c432 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\serrors\scausing\scompilation\sto\sfail\swith\sSQLITE_MAX_WORKER_THREADS\sset\sto\sa\svalue\sgreater\sthan\szero. -D 2014-04-03T14:29:08.251 +C Fix\san\sinteger\soverflow\sproblem\sin\sthe\ssorter. +D 2014-04-03T16:25:29.778 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -186,7 +186,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c d3655832585baef4c2356529a5c6ca5ca3bd7c1f +F src/main.c fcceb01d74a79c2d7984f33545b35b06da3bb1e8 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -219,15 +219,15 @@ F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 20055cf917222e660c4222fea306bd13a0623caa F src/shell.c a08060750f92461fc462b4f767e3b0d19d6b832e -F src/sqlite.h.in 0249af5d9d3bbeab0dc1f58e1f9fee878807732a +F src/sqlite.h.in 81221c50addbf698c3247154d92efd1095bfd885 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 3ed0fedb5b64ece395a2114b7c73417678f3e420 +F src/sqliteInt.h 78c89401120b062660427c7b642de4de7673bc46 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd -F src/test1.c 31596bf8a9c0629f88e514a4ec864847c8946c4e +F src/test1.c 0cd73ae82fdf7add76ca603e3575380ae7539ae2 F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 5e7ed44bb4f2af809b6d229ae00f97825efab89a +F src/vdbesort.c 252d7ab7620649945b53289510a172bc73133f17 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -738,7 +738,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 40add071ba71aefe1c04f5845308cf46f7de8d04 +F test/permutations.test a214a42b4767bbbc7cd0fd965ea6198044ab414d F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d284e30eb1db144965fa85566e4234e30464350b -R 45d899d78ea7a6cd4a92080d8bb33ecf +P 0561272abf357a2f4709f6c02866e570d19cd344 +R 8288f2959bddd3667c4349a94ca23e0f U dan -Z 013157fb51930f7eb005a94358375580 +Z 4eb0e7377049f06d09d1ea7ce591ab92 diff --git a/manifest.uuid b/manifest.uuid index 9cf01bf6ac..cdc8213641 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0561272abf357a2f4709f6c02866e570d19cd344 \ No newline at end of file +9d3351b8d713232133dad149c73fb2a27c72abb1 \ No newline at end of file diff --git a/src/main.c b/src/main.c index c1eaa6849a..f18d1a6507 100644 --- a/src/main.c +++ b/src/main.c @@ -2504,6 +2504,7 @@ static int openDatabase( db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; + db->nMaxSorterMmap = 0x7FFFFFFF; db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill #if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX | SQLITE_AutoIndex @@ -3330,6 +3331,13 @@ int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, nMax); */ + case SQLITE_TESTCTRL_SORTER_MMAP: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->nMaxSorterMmap = va_arg(ap, int); + break; + } + } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 78aa9c36e1..469504b149 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6129,7 +6129,8 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_EXPLAIN_STMT 19 #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 #define SQLITE_TESTCTRL_VDBE_COVERAGE 21 -#define SQLITE_TESTCTRL_LAST 21 +#define SQLITE_TESTCTRL_SORTER_MMAP 22 +#define SQLITE_TESTCTRL_LAST 22 /* ** CAPI3REF: SQLite Runtime Status diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0763f085ab..b802d7aab7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -981,6 +981,7 @@ struct sqlite3 { int nChange; /* Value returned by sqlite3_changes() */ int nTotalChange; /* Value returned by sqlite3_total_changes() */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ + int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */ struct sqlite3InitInfo { /* Information used during initialization */ int newTnum; /* Rootpage of table being initialized */ u8 iDb; /* Which db file is being initialized */ diff --git a/src/test1.c b/src/test1.c index 4b485ce741..44e96c2c1f 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5884,6 +5884,7 @@ static int test_test_control( int i; } aVerb[] = { { "SQLITE_TESTCTRL_LOCALTIME_FAULT", SQLITE_TESTCTRL_LOCALTIME_FAULT }, + { "SQLITE_TESTCTRL_SORTER_MMAP", SQLITE_TESTCTRL_SORTER_MMAP }, }; int iVerb; int iFlag; @@ -5911,6 +5912,19 @@ static int test_test_control( sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, val); break; } + + case SQLITE_TESTCTRL_SORTER_MMAP: { + int val; + sqlite3 *db; + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 2, objv, "DB LIMIT"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[2]), &db) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[3], &val) ) return TCL_ERROR; + sqlite3_test_control(SQLITE_TESTCTRL_SORTER_MMAP, db, val); + break; + } } Tcl_ResetResult(interp); diff --git a/src/vdbesort.c b/src/vdbesort.c index 78f5183b5d..ece66d4c11 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -144,7 +144,7 @@ struct SortSubtask { SQLiteThread *pThread; /* Thread handle, or NULL */ int bDone; /* Set to true by pTask when finished */ - sqlite3_vfs *pVfs; /* VFS used to open temporary files */ + sqlite3 *db; /* Database connection */ KeyInfo *pKeyInfo; /* How to compare records */ UnpackedRecord *pUnpacked; /* Space to unpack a record */ int pgsz; /* Main database page size */ @@ -514,7 +514,9 @@ static int vdbePmaReaderInit( if( pIter->aAlloc ){ /* Try to xFetch() a mapping of the entire temp file. If this is possible, ** the PMA will be read via the mapping. Otherwise, use xRead(). */ - rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap); + if( pTask->iTemp1Off<=(i64)(pTask->db->nMaxSorterMmap) ){ + rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap); + } }else{ rc = SQLITE_NOMEM; } @@ -670,8 +672,8 @@ int sqlite3VdbeSorterInit( for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; pTask->pKeyInfo = pKeyInfo; - pTask->pVfs = db->pVfs; pTask->pgsz = pgsz; + pTask->db = db; } if( !sqlite3TempInMemory(db) ){ @@ -1015,17 +1017,20 @@ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ ** Whether or not the file does end up memory mapped of course depends on ** the specific VFS implementation. */ -static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ - int rc = sqlite3OsTruncate(pFile, nByte); - if( rc==SQLITE_OK ){ - void *p = 0; - sqlite3OsFetch(pFile, 0, nByte, &p); - sqlite3OsUnfetch(pFile, 0, p); +static int vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ + int rc = SQLITE_OK; + if( nByte<=(i64)(db->nMaxSorterMmap) ){ + rc = sqlite3OsTruncate(pFile, nByte); + if( rc==SQLITE_OK ){ + void *p = 0; + sqlite3OsFetch(pFile, 0, nByte, &p); + sqlite3OsUnfetch(pFile, 0, p); + } } return rc; } #else -# define vdbeSorterExtendFile(x,y) SQLITE_OK +# define vdbeSorterExtendFile(x,y,z) SQLITE_OK #endif @@ -1051,7 +1056,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ /* If the first temporary PMA file has not been opened, open it now. */ if( pTask->pTemp1==0 ){ - rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTask->pTemp1); + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->pTemp1); assert( rc!=SQLITE_OK || pTask->pTemp1 ); assert( pTask->iTemp1Off==0 ); assert( pTask->nPMA==0 ); @@ -1059,7 +1064,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile( + rc = vdbeSorterExtendFile(pTask->db, pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9 ); } @@ -1206,9 +1211,9 @@ static void *vdbeSortSubtaskMain(void *pCtx){ } /* Open a second temp file to write merged data to */ - rc = vdbeSorterOpenTempFile(pTask->pVfs, &pTemp2); + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTemp2); if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile(pTemp2, pTask->iTemp1Off); + rc = vdbeSorterExtendFile(pTask->db, pTemp2, pTask->iTemp1Off); } if( rc!=SQLITE_OK ){ vdbeMergeEngineFree(pMerger); diff --git a/test/permutations.test b/test/permutations.test index 7f1485f831..4487af055b 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -112,6 +112,7 @@ set allquicktests [test_set $alltests -exclude { incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test + sort3.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] From 6b5d4759cea2278aacd2f06bdd150bbe40c8fbea Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 4 Apr 2014 02:13:26 +0000 Subject: [PATCH 032/710] Change vdbeSorterExtendFile() so that it makes a best effort to create the PMA file of the desired size, but does not return an error if unable. FossilOrigin-Name: 217814bc4b53fab7bdad433e24e8aef8998c38fe --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 15 ++++++--------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index b7536a4c10..51e948c44c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\schanges\sfrom\strunk. -D 2014-04-03T16:42:21.478 +C Change\svdbeSorterExtendFile()\sso\sthat\sit\smakes\sa\sbest\seffort\sto\screate\sthe\nPMA\sfile\sof\sthe\sdesired\ssize,\sbut\sdoes\snot\sreturn\san\serror\sif\sunable. +D 2014-04-04T02:13:26.507 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 252d7ab7620649945b53289510a172bc73133f17 +F src/vdbesort.c 8da916fc74e78edd5bc95653206942e01710ac09 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9d3351b8d713232133dad149c73fb2a27c72abb1 d5513dfa23baa0b0a095aaf17d19aacd30dcef61 -R 6137d1586a278eeecb535398f89aa582 +P a0910079adde95245680dee59b43613b60903f10 +R ae0d06309ee79fa9e2b8e2b495c941f2 U drh -Z 41afe239279763eb22b85c89d966d688 +Z 2dc7a665c4d0ee5a145d17c4c35dd575 diff --git a/manifest.uuid b/manifest.uuid index 701e5c70e7..c76da35986 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0910079adde95245680dee59b43613b60903f10 \ No newline at end of file +217814bc4b53fab7bdad433e24e8aef8998c38fe \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index ece66d4c11..f59e8f51f5 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1017,17 +1017,15 @@ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ ** Whether or not the file does end up memory mapped of course depends on ** the specific VFS implementation. */ -static int vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ - int rc = SQLITE_OK; +static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ if( nByte<=(i64)(db->nMaxSorterMmap) ){ - rc = sqlite3OsTruncate(pFile, nByte); + int rc = sqlite3OsTruncate(pFile, nByte); if( rc==SQLITE_OK ){ void *p = 0; sqlite3OsFetch(pFile, 0, nByte, &p); sqlite3OsUnfetch(pFile, 0, p); } } - return rc; } #else # define vdbeSorterExtendFile(x,y,z) SQLITE_OK @@ -1064,7 +1062,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile(pTask->db, + vdbeSorterExtendFile(pTask->db, pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9 ); } @@ -1213,16 +1211,15 @@ static void *vdbeSortSubtaskMain(void *pCtx){ /* Open a second temp file to write merged data to */ rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTemp2); if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile(pTask->db, pTemp2, pTask->iTemp1Off); - } - if( rc!=SQLITE_OK ){ + vdbeSorterExtendFile(pTask->db, pTemp2, pTask->iTemp1Off); + }else{ vdbeMergeEngineFree(pMerger); break; } /* This loop runs once for each output PMA. Each output PMA is made ** of data merged from up to SORTER_MAX_MERGE_COUNT input PMAs. */ - for(i=0; inPMA; i+=SORTER_MAX_MERGE_COUNT){ + for(i=0; rc==SQLITE_OK && inPMA; i+=SORTER_MAX_MERGE_COUNT){ PmaWriter writer; /* Object for writing data to pTemp2 */ i64 nOut = 0; /* Bytes of data in output PMA */ int bEof = 0; From 8daefc2af05742f0c36ca7a992981283408a0c36 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 4 Apr 2014 07:52:44 +0000 Subject: [PATCH 033/710] Add test file sort3.test, which should have been part of commit [9d3351b8d7]. FossilOrigin-Name: dceed2c803fca23c83c02c448d5ae7c4698efee1 --- manifest | 13 +++++----- manifest.uuid | 2 +- test/sort3.test | 63 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 test/sort3.test diff --git a/manifest b/manifest index 51e948c44c..ba8d195a88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\svdbeSorterExtendFile()\sso\sthat\sit\smakes\sa\sbest\seffort\sto\screate\sthe\nPMA\sfile\sof\sthe\sdesired\ssize,\sbut\sdoes\snot\sreturn\san\serror\sif\sunable. -D 2014-04-04T02:13:26.507 +C Add\stest\sfile\ssort3.test,\swhich\sshould\shave\sbeen\spart\sof\scommit\s[9d3351b8d7]. +D 2014-04-04T07:52:44.563 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -819,6 +819,7 @@ F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a F test/sort2.test 21cd865e31adecdc8fc81c8d95431e629676a8d8 +F test/sort3.test c3f88d233452a129de519de311d109a0ad0da0af F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1161,7 +1162,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a0910079adde95245680dee59b43613b60903f10 -R ae0d06309ee79fa9e2b8e2b495c941f2 -U drh -Z 2dc7a665c4d0ee5a145d17c4c35dd575 +P 217814bc4b53fab7bdad433e24e8aef8998c38fe +R 3decea650c26960f7b93e4d30e9eb4e6 +U dan +Z 1dd1e90c2d62e06b6853f5bd55d6e7b2 diff --git a/manifest.uuid b/manifest.uuid index c76da35986..b284b74278 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -217814bc4b53fab7bdad433e24e8aef8998c38fe \ No newline at end of file +dceed2c803fca23c83c02c448d5ae7c4698efee1 \ No newline at end of file diff --git a/test/sort3.test b/test/sort3.test new file mode 100644 index 0000000000..9963dcccd3 --- /dev/null +++ b/test/sort3.test @@ -0,0 +1,63 @@ +# 2014 March 25. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sort3 + +# Sort roughly 20MB of data. Once with a mmap limit of 5MB and once without. +# +foreach {itest limit} { + 1 5000000 + 2 0x7FFFFFFF +} { + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $limit + do_execsql_test 1.$itest { + WITH r(x,y) AS ( + SELECT 1, randomblob(1000) + UNION ALL + SELECT x+1, randomblob(1000) FROM r + LIMIT 20000 + ) + SELECT count(*), sum(length(y)) FROM r GROUP BY (x%5); + } { + 4000 4000000 + 4000 4000000 + 4000 4000000 + 4000 4000000 + 4000 4000000 + } +} + +# Sort more than 2GB of data. At one point this was causing a problem. +# This test might take one minute or more to run. +# +do_execsql_test 2 { + PRAGMA cache_size = 20000; + WITH r(x,y) AS ( + SELECT 1, randomblob(1000) + UNION ALL + SELECT x+1, randomblob(1000) FROM r + LIMIT 2200000 + ) + SELECT count(*), sum(length(y)) FROM r GROUP BY (x%5); +} { + 440000 440000000 + 440000 440000000 + 440000 440000000 + 440000 440000000 + 440000 440000000 +} + +finish_test + From f7da5555da88a8b4660909bf350725bb25fedae5 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 4 Apr 2014 21:40:38 +0000 Subject: [PATCH 034/710] Fix typo in a Windows threading support routine. FossilOrigin-Name: 5e3dfa27c71a666e122e3cf64897038ff8424800 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/threads.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4728b7994c..b330e04b4c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sinto\sthe\sthreads\sbranch. -D 2014-04-04T18:37:36.101 +C Fix\stypo\sin\sa\sWindows\sthreading\ssupport\sroutine. +D 2014-04-04T21:40:38.243 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -272,7 +272,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 6992f70cab8d5d8451a6b5641a9256d1749af87b +F src/threads.c b8e7232f2b9c9d148d6886117160de394d172f85 F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P dceed2c803fca23c83c02c448d5ae7c4698efee1 683dd379a293b2f330e1e4cd746f190527fe48ee -R ded894407cec029874d1f7230b628469 -U drh -Z 4f065338769b0de2d9e8b418eaae9ff0 +P 39ac79cffe716f88af0871bdd206231b6a6511ff +R 0cd4553a2bbfab9bd06e3f865048f05f +U mistachkin +Z cb7697a0807ce9ae67c5a22136134623 diff --git a/manifest.uuid b/manifest.uuid index d9a62a86f8..89d1ae46cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -39ac79cffe716f88af0871bdd206231b6a6511ff \ No newline at end of file +5e3dfa27c71a666e122e3cf64897038ff8424800 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 96cb17cef7..eeab5379ce 100644 --- a/src/threads.c +++ b/src/threads.c @@ -149,7 +149,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; if( p->xTask==0 ){ - rc = WAIT_OBJECT_O; + rc = WAIT_OBJECT_0; }else{ rc = sqlite3Win32Wait((HANDLE)p->tid); assert( rc!=WAIT_IO_COMPLETION ); From bf20a35d5f692d0c5e850d043f155797094dd99e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 4 Apr 2014 22:44:59 +0000 Subject: [PATCH 035/710] Fix harmless compiler warnings. FossilOrigin-Name: e54dded2012f0ab486ee138e9bd57c528af33980 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/build.c | 2 +- src/sqliteInt.h | 2 ++ 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b330e04b4c..2eb2b1da22 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\sa\sWindows\sthreading\ssupport\sroutine. -D 2014-04-04T21:40:38.243 +C Fix\sharmless\scompiler\swarnings. +D 2014-04-04T22:44:59.018 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 0d1be67448c45eccc40114556186397eb9da7f7d F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 -F src/build.c b507fb9b4ce943139401d5c9a2daa94a568a8cf1 +F src/build.c e120a3693ac78ab3e74048205f07a3a522d9b982 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -222,7 +222,7 @@ F src/shell.c afc0b1a5a646d287142ef0c9a2a6e3139d57cba2 F src/sqlite.h.in 81221c50addbf698c3247154d92efd1095bfd885 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 78c89401120b062660427c7b642de4de7673bc46 +F src/sqliteInt.h 533154bf9d8401ff3e6f8030f4373774c0a6ad41 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 39ac79cffe716f88af0871bdd206231b6a6511ff -R 0cd4553a2bbfab9bd06e3f865048f05f -U mistachkin -Z cb7697a0807ce9ae67c5a22136134623 +P 5e3dfa27c71a666e122e3cf64897038ff8424800 +R bbcde0d30a2bb025a3c15f4a10b2b404 +U drh +Z 9376e61a8443d421f4f6f69d3d5500ac diff --git a/manifest.uuid b/manifest.uuid index 89d1ae46cd..44109c7358 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5e3dfa27c71a666e122e3cf64897038ff8424800 \ No newline at end of file +e54dded2012f0ab486ee138e9bd57c528af33980 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 26c2922ebd..4886ec1836 100644 --- a/src/build.c +++ b/src/build.c @@ -3018,7 +3018,7 @@ Index *sqlite3CreateIndex( pParse->checkSchema = 1; goto exit_create_index; } - assert( pTab->nCol<=0x7fff && j<=0x7fff ); + assert( j<=0x7fff ); pIndex->aiColumn[i] = (i16)j; if( pListItem->pExpr ){ int nColl; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b802d7aab7..092a14345f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3652,8 +3652,10 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); /* ** Threading interface */ +#if SQLITE_MAX_WORKER_THREADS>0 int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); int sqlite3ThreadJoin(SQLiteThread*, void**); +#endif /* ** Win32 interface From d30ab3d9dd17d37f152dfad1d7b9ffd7fd96bddc Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 9 Apr 2014 20:04:17 +0000 Subject: [PATCH 036/710] Experimental multi-threaded sorting changes to allow the sorter to begin returning items to the VDBE before all data is sorted. FossilOrigin-Name: f9d5e09afaf64d68a0e461c1c2f38179bcea4b1f --- manifest | 19 ++- manifest.uuid | 2 +- src/vdbesort.c | 443 ++++++++++++++++++++++++++++++++++++------------ test/sort2.test | 6 + 4 files changed, 348 insertions(+), 122 deletions(-) diff --git a/manifest b/manifest index 2eb2b1da22..8831053880 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2014-04-04T22:44:59.018 +C Experimental\smulti-threaded\ssorting\schanges\sto\sallow\sthe\ssorter\sto\sbegin\sreturning\sitems\sto\sthe\sVDBE\sbefore\sall\sdata\sis\ssorted. +D 2014-04-09T20:04:17.324 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 8da916fc74e78edd5bc95653206942e01710ac09 +F src/vdbesort.c 26823b626c3231a52e45f5e78a18cb8681bb1b88 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -818,7 +818,7 @@ F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a -F test/sort2.test 21cd865e31adecdc8fc81c8d95431e629676a8d8 +F test/sort2.test bbc2eb244fb862141a900a851056d48705b5997b F test/sort3.test c3f88d233452a129de519de311d109a0ad0da0af F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1163,7 +1163,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5e3dfa27c71a666e122e3cf64897038ff8424800 -R bbcde0d30a2bb025a3c15f4a10b2b404 -U drh -Z 9376e61a8443d421f4f6f69d3d5500ac +P e54dded2012f0ab486ee138e9bd57c528af33980 +R 803b4ddf4cddf7e21aeddc04109caaf0 +T *branch * threads-experimental +T *sym-threads-experimental * +T -sym-threads * +U dan +Z 3b5c615396ccbaaa23add5a8103bd906 diff --git a/manifest.uuid b/manifest.uuid index 44109c7358..6cf45357c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e54dded2012f0ab486ee138e9bd57c528af33980 \ No newline at end of file +f9d5e09afaf64d68a0e461c1c2f38179bcea4b1f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index f59e8f51f5..e558c42f11 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -96,6 +96,8 @@ typedef struct PmaReader PmaReader; /* Incrementally read one PMA */ typedef struct PmaWriter PmaWriter; /* Incrementally write on PMA */ typedef struct SorterRecord SorterRecord; /* A record being sorted */ typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ +typedef struct SorterFile SorterFile; +typedef struct IncrMerger IncrMerger; /* @@ -105,6 +107,11 @@ typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ #define SORT_SUBTASK_TO_PMA 2 /* Xfer pList to Packed-Memory-Array pTemp1 */ #define SORT_SUBTASK_CONS 3 /* Consolidate multiple PMAs */ +struct SorterFile { + sqlite3_file *pFd; + i64 iEof; +}; + /* ** Sorting is divided up into smaller subtasks. Each subtask is controlled ** by an instance of this object. A Subtask might run in either the main thread @@ -145,6 +152,7 @@ struct SortSubtask { int bDone; /* Set to true by pTask when finished */ sqlite3 *db; /* Database connection */ + VdbeSorter *pSorter; /* Sorter */ KeyInfo *pKeyInfo; /* How to compare records */ UnpackedRecord *pUnpacked; /* Space to unpack a record */ int pgsz; /* Main database page size */ @@ -155,9 +163,8 @@ struct SortSubtask { int nInMemory; /* Expected size of PMA based on pList */ u8 *aListMemory; /* Records memory (or NULL) */ - int nPMA; /* Number of PMAs currently in pTemp1 */ - i64 iTemp1Off; /* Offset to write to in pTemp1 */ - sqlite3_file *pTemp1; /* File to write PMAs to, or NULL */ + int nPMA; /* Number of PMAs currently in file */ + SorterFile file; }; @@ -239,8 +246,10 @@ struct VdbeSorter { int mnPmaSize; /* Minimum PMA size, in bytes */ int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ int bUsePMA; /* True if one or more PMAs created */ + int bUseThreads; /* True if one or more PMAs created */ SorterRecord *pRecord; /* Head of in-memory record list */ - MergeEngine *pMerger; /* For final merge of PMAs (by caller) */ + PmaReader *pReader; /* Read data from here after Rewind() */ + UnpackedRecord *pUnpacked; /* Used by VdbeSorterCompare() */ u8 *aMemory; /* Block of memory to alloc records from */ int iMemory; /* Offset of first free byte in aMemory */ int nMemory; /* Size of aMemory allocation in bytes */ @@ -265,6 +274,16 @@ struct PmaReader { u8 *aBuffer; /* Current read buffer */ int nBuffer; /* Size of read buffer in bytes */ u8 *aMap; /* Pointer to mapping of entire file */ + IncrMerger *pIncr; /* Incremental merger */ +}; + +struct IncrMerger { + int mxSz; /* Maximum size of files */ + SortSubtask *pTask; /* Task that owns this merger */ + int bEof; /* Set to true when merge is finished */ + SorterFile aFile[2]; /* aFile[0] for reading, [1] for writing */ + MergeEngine *pMerger; /* Merge engine thread reads data from */ + SQLiteThread *pThread; /* Thread currently populating aFile[1] */ }; /* @@ -326,6 +345,9 @@ struct SorterRecord { /* Maximum number of PMAs that a single MergeEngine can merge */ #define SORTER_MAX_MERGE_COUNT 16 +static int vdbeIncrSwap(IncrMerger*); +static void vdbeIncrFree(IncrMerger*); + /* ** Free all memory belonging to the PmaReader object passed as the second ** argument. All structure fields are set to zero before returning. @@ -334,6 +356,7 @@ static void vdbePmaReaderClear(PmaReader *pIter){ sqlite3_free(pIter->aAlloc); sqlite3_free(pIter->aBuffer); if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); + if( pIter->pIncr ) vdbeIncrFree(pIter->pIncr); memset(pIter, 0, sizeof(PmaReader)); } @@ -400,7 +423,7 @@ static int vdbePmaReadBlob( /* Extend the p->aAlloc[] allocation if required. */ if( p->nAllocnAlloc*2; + int nNew = MAX(128, p->nAlloc*2); while( nByte>nNew ) nNew = nNew*2; aNew = sqlite3Realloc(p->aAlloc, nNew); if( !aNew ) return SQLITE_NOMEM; @@ -464,22 +487,70 @@ static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ return SQLITE_OK; } +static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ + int rc = SQLITE_OK; + if( pFile->iEof<=(i64)(pTask->db->nMaxSorterMmap) ){ + rc = sqlite3OsFetch(pFile->pFd, 0, pFile->iEof, (void**)pp); + } + return rc; +} + +static int vdbePmaReaderReinit(PmaReader *pIter){ + IncrMerger *pIncr = pIter->pIncr; + SortSubtask *pTask = pIncr->pTask; + int rc = SQLITE_OK; + + assert( pIncr->bEof==0 ); + + if( pIter->aMap ){ + sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); + pIter->aMap = 0; + } + pIter->iReadOff = 0; + pIter->iEof = pIncr->aFile[0].iEof; + pIter->pFile = pIncr->aFile[0].pFd; + + rc = vdbeSorterMapFile(pTask, &pIncr->aFile[0], &pIter->aMap); + if( rc==SQLITE_OK ){ + if( pIter->aMap==0 && pIter->aBuffer==0 ){ + pIter->aBuffer = (u8*)sqlite3Malloc(pTask->pgsz); + if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; + pIter->nBuffer = pTask->pgsz; + } + } + + return rc; +} + /* ** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if ** no error occurs, or an SQLite error code if one does. */ static int vdbePmaReaderNext(PmaReader *pIter){ - int rc; /* Return Code */ + int rc = SQLITE_OK; /* Return Code */ u64 nRec = 0; /* Size of record in bytes */ if( pIter->iReadOff>=pIter->iEof ){ - /* This is an EOF condition */ - vdbePmaReaderClear(pIter); - return SQLITE_OK; + int bEof = 1; + if( pIter->pIncr ){ + rc = vdbeIncrSwap(pIter->pIncr); + if( rc==SQLITE_OK && pIter->pIncr->bEof==0 ){ + rc = vdbePmaReaderReinit(pIter); + bEof = 0; + } + } + + if( bEof ){ + /* This is an EOF condition */ + vdbePmaReaderClear(pIter); + return rc; + } } - rc = vdbePmaReadVarint(pIter, &nRec); + if( rc==SQLITE_OK ){ + rc = vdbePmaReadVarint(pIter, &nRec); + } if( rc==SQLITE_OK ){ pIter->nKey = (int)nRec; rc = vdbePmaReadBlob(pIter, (int)nRec, &pIter->aKey); @@ -493,10 +564,14 @@ static int vdbePmaReaderNext(PmaReader *pIter){ ** starting at offset iStart and ending at offset iEof-1. This function ** leaves the iterator pointing to the first key in the PMA (or EOF if the ** PMA is empty). +** +** If the pnByte parameter is NULL, then it is assumed that the file +** contains a single PMA, and that that PMA omits the initial length varint. */ static int vdbePmaReaderInit( - SortSubtask *pTask, /* Thread context */ - i64 iStart, /* Start offset in pTask->pTemp1 */ + SortSubtask *pTask, /* Task context */ + SorterFile *pFile, /* Sorter file to read from */ + i64 iStart, /* Start offset in pFile */ PmaReader *pIter, /* Iterator to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ ){ @@ -504,18 +579,18 @@ static int vdbePmaReaderInit( int nBuf = pTask->pgsz; void *pMap = 0; /* Mapping of temp file */ - assert( pTask->iTemp1Off>iStart ); + assert( pFile->iEof>iStart ); assert( pIter->aAlloc==0 ); assert( pIter->aBuffer==0 ); - pIter->pFile = pTask->pTemp1; + pIter->pFile = pFile->pFd; pIter->iReadOff = iStart; pIter->nAlloc = 128; pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc); if( pIter->aAlloc ){ /* Try to xFetch() a mapping of the entire temp file. If this is possible, ** the PMA will be read via the mapping. Otherwise, use xRead(). */ - if( pTask->iTemp1Off<=(i64)(pTask->db->nMaxSorterMmap) ){ - rc = sqlite3OsFetch(pIter->pFile, 0, pTask->iTemp1Off, &pMap); + if( pFile->iEof<=(i64)(pTask->db->nMaxSorterMmap) ){ + rc = sqlite3OsFetch(pIter->pFile, 0, pFile->iEof, &pMap); } }else{ rc = SQLITE_NOMEM; @@ -533,12 +608,12 @@ static int vdbePmaReaderInit( int iBuf = iStart % nBuf; if( iBuf ){ int nRead = nBuf - iBuf; - if( (iStart + nRead) > pTask->iTemp1Off ){ - nRead = (int)(pTask->iTemp1Off - iStart); + if( (iStart + nRead) > pFile->iEof ){ + nRead = (int)(pFile->iEof - iStart); } rc = sqlite3OsRead( - pTask->pTemp1, &pIter->aBuffer[iBuf], nRead, iStart - ); + pIter->pFile, &pIter->aBuffer[iBuf], nRead, iStart + ); assert( rc!=SQLITE_IOERR_SHORT_READ ); } } @@ -547,7 +622,7 @@ static int vdbePmaReaderInit( if( rc==SQLITE_OK ){ u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pTask->iTemp1Off; + pIter->iEof = pFile->iEof; rc = vdbePmaReadVarint(pIter, &nByte); pIter->iEof = pIter->iReadOff + nByte; *pnByte += nByte; @@ -669,11 +744,13 @@ int sqlite3VdbeSorterInit( pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nTask = nWorker + 1; + pSorter->bUseThreads = (pSorter->nTask>1); for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; pTask->pKeyInfo = pKeyInfo; pTask->pgsz = pgsz; pTask->db = db; + pTask->pSorter = pSorter; } if( !sqlite3TempInMemory(db) ){ @@ -723,9 +800,10 @@ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ pTask->aListMemory = 0; } pTask->pList = 0; - if( pTask->pTemp1 ){ - sqlite3OsCloseFree(pTask->pTemp1); - pTask->pTemp1 = 0; + if( pTask->file.pFd ){ + sqlite3OsCloseFree(pTask->file.pFd); + pTask->file.pFd = 0; + pTask->file.iEof = 0; } } @@ -761,7 +839,8 @@ static MergeEngine *vdbeMergeEngineNew(int nIter){ int nByte; /* Total bytes of space to allocate */ MergeEngine *pNew; /* Pointer to allocated object to return */ - assert( nIter<=SORTER_MAX_MERGE_COUNT ); + /* assert( nIter<=SORTER_MAX_MERGE_COUNT ); */ + while( NpMerger); - pSorter->pMerger = 0; + if( pSorter->pReader ){ + vdbePmaReaderClear(pSorter->pReader); + sqlite3DbFree(db, pSorter->pReader); + pSorter->pReader = 0; + } for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; vdbeSortSubtaskCleanup(db, pTask); @@ -806,6 +888,8 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ pSorter->nInMemory = 0; pSorter->bUsePMA = 0; pSorter->iMemory = 0; + sqlite3DbFree(db, pSorter->pUnpacked); + pSorter->pUnpacked = 0; } /* @@ -815,7 +899,6 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); - vdbeMergeEngineFree(pSorter->pMerger); sqlite3_free(pSorter->aMemory); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; @@ -1053,17 +1136,17 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ assert( pTask->nInMemory>0 ); /* If the first temporary PMA file has not been opened, open it now. */ - if( pTask->pTemp1==0 ){ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->pTemp1); - assert( rc!=SQLITE_OK || pTask->pTemp1 ); - assert( pTask->iTemp1Off==0 ); + if( pTask->file.pFd==0 ){ + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->file.pFd); + assert( rc!=SQLITE_OK || pTask->file.pFd ); + assert( pTask->file.iEof==0 ); assert( pTask->nPMA==0 ); } /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ vdbeSorterExtendFile(pTask->db, - pTask->pTemp1, pTask->iTemp1Off + pTask->nInMemory + 9 + pTask->file.pFd, pTask->file.iEof + pTask->nInMemory + 9 ); } @@ -1071,8 +1154,8 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ SorterRecord *p; SorterRecord *pNext = 0; - vdbePmaWriterInit(pTask->pTemp1, &writer, pTask->pgsz, - pTask->iTemp1Off); + vdbePmaWriterInit(pTask->file.pFd, &writer, pTask->pgsz, + pTask->file.iEof); pTask->nPMA++; vdbePmaWriteVarint(&writer, pTask->nInMemory); for(p=pTask->pList; p; p=pNext){ @@ -1082,7 +1165,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ if( pTask->aListMemory==0 ) sqlite3_free(p); } pTask->pList = p; - rc = vdbePmaWriterFinish(&writer, &pTask->iTemp1Off); + rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof); } assert( pTask->pList==0 || rc!=SQLITE_OK ); @@ -1164,6 +1247,23 @@ static int vdbeSorterNext( return rc; } +#if 0 +static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); +} +static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ + i64 t; + sqlite3OsCurrentTimeInt64(db->pVfs, &t); + fprintf(stderr, "%lld:X %s\n", t, zEvent); +} +#else +# define vdbeSorterWorkDebug(x,y) +# define vdbeSorterRewindDebug(x,y) +#endif + /* ** The main routine for sorter-thread operations. */ @@ -1177,6 +1277,8 @@ static void *vdbeSortSubtaskMain(void *pCtx){ ); assert( pTask->bDone==0 ); + vdbeSorterWorkDebug(pTask, "enter"); + if( pTask->pUnpacked==0 ){ char *pFree; pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( @@ -1211,7 +1313,7 @@ static void *vdbeSortSubtaskMain(void *pCtx){ /* Open a second temp file to write merged data to */ rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTemp2); if( rc==SQLITE_OK ){ - vdbeSorterExtendFile(pTask->db, pTemp2, pTask->iTemp1Off); + vdbeSorterExtendFile(pTask->db, pTemp2, pTask->file.iEof); }else{ vdbeMergeEngineFree(pMerger); break; @@ -1231,9 +1333,9 @@ static void *vdbeSortSubtaskMain(void *pCtx){ int iIter; for(iIter=0; iIteraIter[iIter]; - rc = vdbePmaReaderInit(pTask, iReadOff, pIter, &nOut); + rc = vdbePmaReaderInit(pTask, &pTask->file, iReadOff, pIter, &nOut); iReadOff = pIter->iEof; - if( iReadOff>=pTask->iTemp1Off || rc!=SQLITE_OK ) break; + if( iReadOff>=pTask->file.iEof || rc!=SQLITE_OK ) break; } for(iIter=pMerger->nTree-1; rc==SQLITE_OK && iIter>0; iIter--){ rc = vdbeSorterDoCompare(pTask, pMerger, iIter); @@ -1253,10 +1355,10 @@ static void *vdbeSortSubtaskMain(void *pCtx){ } vdbeMergeEngineFree(pMerger); - sqlite3OsCloseFree(pTask->pTemp1); - pTask->pTemp1 = pTemp2; + sqlite3OsCloseFree(pTask->file.pFd); + pTask->file.pFd = pTemp2; pTask->nPMA = (i / SORTER_MAX_MERGE_COUNT); - pTask->iTemp1Off = iWriteOff; + pTask->file.iEof = iWriteOff; } }else{ /* Sort the pTask->pList list */ @@ -1267,10 +1369,10 @@ static void *vdbeSortSubtaskMain(void *pCtx){ #ifdef SQLITE_DEBUG i64 nExpect = pTask->nInMemory + sqlite3VarintLen(pTask->nInMemory) - + pTask->iTemp1Off; + + pTask->file.iEof; #endif rc = vdbeSorterListToPMA(pTask); - assert( rc!=SQLITE_OK || (nExpect==pTask->iTemp1Off) ); + assert( rc!=SQLITE_OK || (nExpect==pTask->file.iEof) ); } } @@ -1280,6 +1382,7 @@ static void *vdbeSortSubtaskMain(void *pCtx){ assert( pTask->pUnpacked->errCode==SQLITE_NOMEM ); rc = SQLITE_NOMEM; } + vdbeSorterWorkDebug(pTask, "exit"); return SQLITE_INT_TO_PTR(rc); } @@ -1480,6 +1583,164 @@ static int vdbeSorterCountPMA(VdbeSorter *pSorter){ return nPMA; } +/* +** Read keys from pIncr->pMerger and populate pIncr->aFile[1]. The format +** of the data stored in aFile[1] is the same as that used by regular PMAs, +** except that the number-of-bytes varint is omitted from the start. +*/ +static int vdbeIncrPopulate(IncrMerger *pIncr){ + int rc = SQLITE_OK; + int rc2; + SorterFile *pOut = &pIncr->aFile[1]; + MergeEngine *pMerger = pIncr->pMerger; + PmaWriter writer; + assert( pIncr->bEof==0 ); + + vdbePmaWriterInit(pIncr->aFile[1].pFd, &writer, pIncr->pTask->pgsz, 0); + while( rc==SQLITE_OK ){ + int dummy; + PmaReader *pReader = &pMerger->aIter[ pMerger->aTree[1] ]; + int nKey = pReader->nKey; + i64 iEof = writer.iWriteOff + writer.iBufEnd; + + /* Check if the output file is full or if the input has been exhausted. + ** In either case exit the loop. */ + if( pReader->pFile==0 ) break; + if( iEof && (iEof + nKey)>pIncr->mxSz ) break; + + /* Write the next key to the output. */ + vdbePmaWriteVarint(&writer, nKey); + vdbePmaWriteBlob(&writer, pReader->aKey, nKey); + rc = vdbeSorterNext(pIncr->pTask, pIncr->pMerger, &dummy); + } + + rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); + if( rc==SQLITE_OK ) rc = rc2; + return rc; +} + +static void *vdbeIncrPopulateThreadMain(void *pCtx){ + IncrMerger *pIncr = (IncrMerger*)pCtx; + return SQLITE_INT_TO_PTR( vdbeIncrPopulate(pIncr) ); +} + +static int vdbeIncrBgPopulate(IncrMerger *pIncr){ + int rc; + assert( pIncr->pThread==0 ); + if( pIncr->pTask->pSorter->bUseThreads==0 ){ + rc = vdbeIncrPopulate(pIncr); + }else{ + void *pCtx = (void*)pIncr; + rc = sqlite3ThreadCreate(&pIncr->pThread, vdbeIncrPopulateThreadMain, pCtx); + } + return rc; +} + +static int vdbeIncrSwap(IncrMerger *pIncr){ + int rc = SQLITE_OK; + + if( pIncr->pThread ){ + void *pRet; + rc = sqlite3ThreadJoin(pIncr->pThread, &pRet); + if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); + pIncr->pThread = 0; + } + + if( rc==SQLITE_OK ){ + SorterFile f0 = pIncr->aFile[0]; + pIncr->aFile[0] = pIncr->aFile[1]; + pIncr->aFile[1] = f0; + + if( pIncr->aFile[0].iEof==0 ){ + pIncr->bEof = 1; + }else{ + rc = vdbeIncrBgPopulate(pIncr); + } + } + + return rc; +} + +static void vdbeIncrFree(IncrMerger *pIncr){ + if( pIncr->pThread ){ + void *pRet; + sqlite3ThreadJoin(pIncr->pThread, &pRet); + } + if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); + if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); + vdbeMergeEngineFree(pIncr->pMerger); + sqlite3_free(pIncr); +} + +/* +** Populate iterator *pIter so that it may be used to iterate through all +** keys stored in subtask pTask using the incremental merge method. +*/ +static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ + SortSubtask *pTask0 = &pSorter->aTask[0]; + int rc = SQLITE_OK; + MergeEngine *pMerger = 0; + IncrMerger *pIncr = 0; + int i; + int nPMA = 0; + + for(i=0; inTask; i++){ + nPMA += pSorter->aTask[i].nPMA; + } + pMerger = vdbeMergeEngineNew(nPMA); + if( pMerger==0 ){ + rc = SQLITE_NOMEM; + }else{ + int iIter = 0; + int iPMA; + for(i=0; inTask; i++){ + i64 iReadOff = 0; + SortSubtask *pTask = &pSorter->aTask[i]; + for(iPMA=0; iPMAnPMA; iPMA++){ + i64 nDummy = 0; + PmaReader *pIter = &pMerger->aIter[iIter++]; + rc = vdbePmaReaderInit(pTask, &pTask->file, iReadOff, pIter, &nDummy); + iReadOff = pIter->iEof; + } + } + for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ + rc = vdbeSorterDoCompare(pTask0, pMerger, i); + } + } + + if( rc==SQLITE_OK ){ + pIncr = (IncrMerger*)sqlite3_malloc(sizeof(IncrMerger)); + if( pIncr==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pIncr, 0, sizeof(IncrMerger)); + pIncr->mxSz = (pSorter->mxPmaSize / 2); + pIncr->pMerger = pMerger; + pIncr->pTask = pTask0; + } + } + + /* Open the two temp files. */ + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(pTask0->db->pVfs, &pIncr->aFile[0].pFd); + } + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(pTask0->db->pVfs, &pIncr->aFile[1].pFd); + } + + /* Launch a background thread to populate aFile[1]. */ + if( rc==SQLITE_OK ){ + rc = vdbeIncrBgPopulate(pIncr); + } + + pIter->pIncr = pIncr; + if( rc==SQLITE_OK ){ + rc = vdbePmaReaderNext(pIter); + } + return rc; +} + + /* ** Once the sorter has been populated by calls to sqlite3VdbeSorterWrite, ** this function is called to prepare for iterating through the records @@ -1520,70 +1781,21 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ /* Join all threads */ rc = vdbeSorterJoinAll(pSorter, rc); - /* If there are more than SORTER_MAX_MERGE_COUNT PMAs on disk, merge - ** some of them together so that this is no longer the case. */ - if( vdbeSorterCountPMA(pSorter)>SORTER_MAX_MERGE_COUNT ){ - int i; - for(i=0; rc==SQLITE_OK && inTask; i++){ - SortSubtask *pTask = &pSorter->aTask[i]; - if( pTask->pTemp1 ){ - pTask->nConsolidate = SORTER_MAX_MERGE_COUNT / pSorter->nTask; - pTask->eWork = SORT_SUBTASK_CONS; + vdbeSorterRewindDebug(db, "rewind"); -#if SQLITE_MAX_WORKER_THREADS>0 - if( i<(pSorter->nTask-1) ){ - void *pCtx = (void*)pTask; - rc = sqlite3ThreadCreate(&pTask->pThread, vdbeSortSubtaskMain, pCtx); - }else -#endif - { - rc = vdbeSorterRunTask(pTask); - } - } - } - } - - /* Join all threads */ - rc = vdbeSorterJoinAll(pSorter, rc); - - /* Assuming no errors have occurred, set up a merger structure to read - ** and merge all remaining PMAs. */ - assert( pSorter->pMerger==0 ); + /* Assuming no errors have occurred, set up a merger structure to + ** incrementally read and merge all remaining PMAs. */ + assert( pSorter->pReader==0 ); if( rc==SQLITE_OK ){ - int nIter = 0; /* Number of iterators used */ - int i; - MergeEngine *pMerger; - for(i=0; inTask; i++){ - nIter += pSorter->aTask[i].nPMA; - } - - pSorter->pMerger = pMerger = vdbeMergeEngineNew(nIter); - if( pMerger==0 ){ - rc = SQLITE_NOMEM; - }else{ - int iIter = 0; - int iThread = 0; - for(iThread=0; iThreadnTask; iThread++){ - int iPMA; - i64 iReadOff = 0; - SortSubtask *pTask = &pSorter->aTask[iThread]; - for(iPMA=0; iPMAnPMA && rc==SQLITE_OK; iPMA++){ - i64 nDummy = 0; - PmaReader *pIter = &pMerger->aIter[iIter++]; - rc = vdbePmaReaderInit(pTask, iReadOff, pIter, &nDummy); - iReadOff = pIter->iEof; - } - } - - for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(&pSorter->aTask[0], pMerger, i); - } - } + PmaReader *pReader; + pReader = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); + pSorter->pReader = pReader; + rc = vdbePmaReaderIncrInit(pSorter, pReader); + assert( rc!=SQLITE_OK || pReader->pFile ); + *pbEof = 0; } - if( rc==SQLITE_OK ){ - *pbEof = (pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]].pFile==0); - } + vdbeSorterRewindDebug(db, "rewinddone"); return rc; } @@ -1594,8 +1806,9 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; int rc; /* Return code */ - if( pSorter->pMerger ){ - rc = vdbeSorterNext(&pSorter->aTask[0], pSorter->pMerger, pbEof); + if( pSorter->pReader ){ + rc = vdbePmaReaderNext(pSorter->pReader); + *pbEof = (pSorter->pReader->pFile==0); }else{ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->u.pNext; @@ -1616,11 +1829,9 @@ static void *vdbeSorterRowkey( int *pnKey /* OUT: Size of current key in bytes */ ){ void *pKey; - if( pSorter->pMerger ){ - PmaReader *pIter; - pIter = &pSorter->pMerger->aIter[ pSorter->pMerger->aTree[1] ]; - *pnKey = pIter->nKey; - pKey = pIter->aKey; + if( pSorter->pReader ){ + *pnKey = pSorter->pReader->nKey; + pKey = pSorter->pReader->aKey; }else{ *pnKey = pSorter->pRecord->nVal; pKey = SRVAL(pSorter->pRecord); @@ -1669,13 +1880,19 @@ int sqlite3VdbeSorterCompare( int *pRes /* OUT: Result of comparison */ ){ VdbeSorter *pSorter = pCsr->pSorter; - UnpackedRecord *r2 = pSorter->aTask[0].pUnpacked; + UnpackedRecord *r2 = pSorter->pUnpacked; KeyInfo *pKeyInfo = pCsr->pKeyInfo; int i; void *pKey; int nKey; /* Sorter key to compare pVal with */ + if( r2==0 ){ + char *p; + r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo,0,0,&p); + assert( pSorter->pUnpacked==(UnpackedRecord*)p ); + if( r2==0 ) return SQLITE_NOMEM; + r2->nField = pKeyInfo->nField-nIgnore; + } assert( r2->nField>=pKeyInfo->nField-nIgnore ); - r2->nField = pKeyInfo->nField-nIgnore; pKey = vdbeSorterRowkey(pSorter, &nKey); sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, r2); diff --git a/test/sort2.test b/test/sort2.test index f8bfb0fe51..626630050c 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -47,6 +47,12 @@ do_execsql_test 2.2 { CREATE UNIQUE INDEX i1 ON t1(b, a); } +do_execsql_test 2.3 { + CREATE UNIQUE INDEX i2 ON t1(a); +} + +do_execsql_test 2.4 { PRAGMA integrity_check } {ok} + db close sqlite3_shutdown sqlite3_config_worker_threads 0 From 4be4c406faf605d78a9c737795c3abc6bb62621e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 11 Apr 2014 19:43:07 +0000 Subject: [PATCH 037/710] Avoid having the sorter merge too many PMAs at a time when incrementally merging data following a SorterRewind(). FossilOrigin-Name: 98bf0307b121b0776a7170108cc8d3f948a7ebfe --- manifest | 19 +- manifest.uuid | 2 +- src/shell.c | 2 +- src/vdbesort.c | 498 +++++++++++++++++++++++++++++++++++++----------- test/sort2.test | 102 ++++++---- 5 files changed, 455 insertions(+), 168 deletions(-) diff --git a/manifest b/manifest index 8831053880..f48df9c72d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\smulti-threaded\ssorting\schanges\sto\sallow\sthe\ssorter\sto\sbegin\sreturning\sitems\sto\sthe\sVDBE\sbefore\sall\sdata\sis\ssorted. -D 2014-04-09T20:04:17.324 +C Avoid\shaving\sthe\ssorter\smerge\stoo\smany\sPMAs\sat\sa\stime\swhen\sincrementally\smerging\sdata\sfollowing\sa\sSorterRewind(). +D 2014-04-11T19:43:07.755 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -218,7 +218,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 20055cf917222e660c4222fea306bd13a0623caa -F src/shell.c afc0b1a5a646d287142ef0c9a2a6e3139d57cba2 +F src/shell.c b44c3f17f0bf41b3431e9cc171706251156ae85f F src/sqlite.h.in 81221c50addbf698c3247154d92efd1095bfd885 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 26823b626c3231a52e45f5e78a18cb8681bb1b88 +F src/vdbesort.c 2984e3624383adf9c762558b8f85a17a626c11a7 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -818,7 +818,7 @@ F test/skipscan2.test 5a4db0799c338ddbacb154aaa5589c0254b36a8d F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a -F test/sort2.test bbc2eb244fb862141a900a851056d48705b5997b +F test/sort2.test 04e99d0d028b469c6cfab2c647c6c28755504063 F test/sort3.test c3f88d233452a129de519de311d109a0ad0da0af F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1163,10 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e54dded2012f0ab486ee138e9bd57c528af33980 -R 803b4ddf4cddf7e21aeddc04109caaf0 -T *branch * threads-experimental -T *sym-threads-experimental * -T -sym-threads * +P f9d5e09afaf64d68a0e461c1c2f38179bcea4b1f +R f6c598c1c558c5930404cda096730209 U dan -Z 3b5c615396ccbaaa23add5a8103bd906 +Z 3e9d4ee1a6e7b343cf831d1b18651067 diff --git a/manifest.uuid b/manifest.uuid index 6cf45357c4..b1943056c1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f9d5e09afaf64d68a0e461c1c2f38179bcea4b1f \ No newline at end of file +98bf0307b121b0776a7170108cc8d3f948a7ebfe \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index e032bd36d2..40ac24093a 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3535,7 +3535,7 @@ static void main_init(struct callback_data *data) { sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); sqlite3_config(SQLITE_CONFIG_MULTITHREAD); - sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 3); + sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 4); } /* diff --git a/src/vdbesort.c b/src/vdbesort.c index e558c42f11..16f6c618c6 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -164,7 +164,8 @@ struct SortSubtask { u8 *aListMemory; /* Records memory (or NULL) */ int nPMA; /* Number of PMAs currently in file */ - SorterFile file; + SorterFile file; /* Temp file for level-0 PMAs */ + SorterFile file2; /* Space for other PMAs */ }; @@ -240,6 +241,11 @@ struct MergeEngine { /* ** Main sorter structure. A single instance of this is allocated for each ** sorter cursor created by the VDBE. +** +** mxKeysize: +** As records are added to the sorter by calls to sqlite3VdbeSorterWrite(), +** this variable is updated so as to be set to the size on disk of the +** largest record in the sorter. */ struct VdbeSorter { int nInMemory; /* Current size of pRecord list as PMA */ @@ -249,6 +255,7 @@ struct VdbeSorter { int bUseThreads; /* True if one or more PMAs created */ SorterRecord *pRecord; /* Head of in-memory record list */ PmaReader *pReader; /* Read data from here after Rewind() */ + int mxKeysize; /* Largest serialized key seen so far */ UnpackedRecord *pUnpacked; /* Used by VdbeSorterCompare() */ u8 *aMemory; /* Block of memory to alloc records from */ int iMemory; /* Offset of first free byte in aMemory */ @@ -277,13 +284,21 @@ struct PmaReader { IncrMerger *pIncr; /* Incremental merger */ }; +/* +** Normally, a PmaReader object iterates through an existing PMA stored +** within a temp file. However, if the PmaReader.pIncr variable points to +** an object of the following type, it may be used to iterate/merge through +** multiple PMAs simultaneously. +*/ struct IncrMerger { - int mxSz; /* Maximum size of files */ SortSubtask *pTask; /* Task that owns this merger */ - int bEof; /* Set to true when merge is finished */ - SorterFile aFile[2]; /* aFile[0] for reading, [1] for writing */ - MergeEngine *pMerger; /* Merge engine thread reads data from */ SQLiteThread *pThread; /* Thread currently populating aFile[1] */ + MergeEngine *pMerger; /* Merge engine thread reads data from */ + i64 iStartOff; /* Offset to start writing file at */ + int mxSz; /* Maximum bytes of data to store */ + int bEof; /* Set to true when merge is finished */ + int bUseThread; /* True to use a bg thread for this object */ + SorterFile aFile[2]; /* aFile[0] for reading, [1] for writing */ }; /* @@ -506,16 +521,30 @@ static int vdbePmaReaderReinit(PmaReader *pIter){ sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); pIter->aMap = 0; } - pIter->iReadOff = 0; + pIter->iReadOff = pIncr->iStartOff; pIter->iEof = pIncr->aFile[0].iEof; pIter->pFile = pIncr->aFile[0].pFd; rc = vdbeSorterMapFile(pTask, &pIncr->aFile[0], &pIter->aMap); if( rc==SQLITE_OK ){ - if( pIter->aMap==0 && pIter->aBuffer==0 ){ - pIter->aBuffer = (u8*)sqlite3Malloc(pTask->pgsz); - if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; - pIter->nBuffer = pTask->pgsz; + if( pIter->aMap==0 ){ + /* TODO: Combine this code with similar code in vdbePmaReaderInit() */ + int iBuf = pIter->iReadOff % pTask->pgsz; + if( pIter->aBuffer==0 ){ + pIter->aBuffer = (u8*)sqlite3Malloc(pTask->pgsz); + if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; + pIter->nBuffer = pTask->pgsz; + } + if( iBuf ){ + int nRead = pTask->pgsz - iBuf; + if( (pIter->iReadOff + nRead) > pIter->iEof ){ + nRead = (int)(pIter->iEof - pIter->iReadOff); + } + rc = sqlite3OsRead( + pIter->pFile, &pIter->aBuffer[iBuf], nRead, pIter->iReadOff + ); + assert( rc!=SQLITE_IOERR_SHORT_READ ); + } } } @@ -577,7 +606,6 @@ static int vdbePmaReaderInit( ){ int rc = SQLITE_OK; int nBuf = pTask->pgsz; - void *pMap = 0; /* Mapping of temp file */ assert( pFile->iEof>iStart ); assert( pIter->aAlloc==0 ); @@ -589,33 +617,27 @@ static int vdbePmaReaderInit( if( pIter->aAlloc ){ /* Try to xFetch() a mapping of the entire temp file. If this is possible, ** the PMA will be read via the mapping. Otherwise, use xRead(). */ - if( pFile->iEof<=(i64)(pTask->db->nMaxSorterMmap) ){ - rc = sqlite3OsFetch(pIter->pFile, 0, pFile->iEof, &pMap); - } + rc = vdbeSorterMapFile(pTask, pFile, &pIter->aMap); }else{ rc = SQLITE_NOMEM; } - if( rc==SQLITE_OK ){ - if( pMap ){ - pIter->aMap = (u8*)pMap; + if( rc==SQLITE_OK && pIter->aMap==0 ){ + pIter->nBuffer = nBuf; + pIter->aBuffer = (u8*)sqlite3Malloc(nBuf); + if( !pIter->aBuffer ){ + rc = SQLITE_NOMEM; }else{ - pIter->nBuffer = nBuf; - pIter->aBuffer = (u8*)sqlite3Malloc(nBuf); - if( !pIter->aBuffer ){ - rc = SQLITE_NOMEM; - }else{ - int iBuf = iStart % nBuf; - if( iBuf ){ - int nRead = nBuf - iBuf; - if( (iStart + nRead) > pFile->iEof ){ - nRead = (int)(pFile->iEof - iStart); - } - rc = sqlite3OsRead( - pIter->pFile, &pIter->aBuffer[iBuf], nRead, iStart - ); - assert( rc!=SQLITE_IOERR_SHORT_READ ); + int iBuf = iStart % nBuf; + if( iBuf ){ + int nRead = nBuf - iBuf; + if( (iStart + nRead) > pFile->iEof ){ + nRead = (int)(pFile->iEof - iStart); } + rc = sqlite3OsRead( + pIter->pFile, &pIter->aBuffer[iBuf], nRead, iStart + ); + assert( rc!=SQLITE_IOERR_SHORT_READ ); } } } @@ -805,6 +827,11 @@ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ pTask->file.pFd = 0; pTask->file.iEof = 0; } + if( pTask->file2.pFd ){ + sqlite3OsCloseFree(pTask->file2.pFd); + pTask->file2.pFd = 0; + pTask->file2.iEof = 0; + } } /* @@ -839,7 +866,7 @@ static MergeEngine *vdbeMergeEngineNew(int nIter){ int nByte; /* Total bytes of space to allocate */ MergeEngine *pNew; /* Pointer to allocated object to return */ - /* assert( nIter<=SORTER_MAX_MERGE_COUNT ); */ + assert( nIter<=SORTER_MAX_MERGE_COUNT ); while( NnInMemory = 0; pSorter->bUsePMA = 0; pSorter->iMemory = 0; + pSorter->mxKeysize = 0; sqlite3DbFree(db, pSorter->pUnpacked); pSorter->pUnpacked = 0; } @@ -1259,11 +1287,35 @@ static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ sqlite3OsCurrentTimeInt64(db->pVfs, &t); fprintf(stderr, "%lld:X %s\n", t, zEvent); } +static void vdbeSorterPopulateDebug( + SortSubtask *pTask, + const char *zEvent +){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + fprintf(stderr, "%lld:bg%d %s\n", t, iTask, zEvent); +} #else # define vdbeSorterWorkDebug(x,y) # define vdbeSorterRewindDebug(x,y) +# define vdbeSorterPopulateDebug(x,y) #endif +static int vdbeSortAllocUnpacked(SortSubtask *pTask){ + if( pTask->pUnpacked==0 ){ + char *pFree; + pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( + pTask->pKeyInfo, 0, 0, &pFree + ); + assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); + if( pFree==0 ) return SQLITE_NOMEM; + pTask->pUnpacked->nField = pTask->pKeyInfo->nField; + pTask->pUnpacked->errCode = 0; + } + return SQLITE_OK; +} + /* ** The main routine for sorter-thread operations. */ @@ -1279,19 +1331,8 @@ static void *vdbeSortSubtaskMain(void *pCtx){ vdbeSorterWorkDebug(pTask, "enter"); - if( pTask->pUnpacked==0 ){ - char *pFree; - pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( - pTask->pKeyInfo, 0, 0, &pFree - ); - assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); - if( pFree==0 ){ - rc = SQLITE_NOMEM; - goto thread_out; - } - pTask->pUnpacked->nField = pTask->pKeyInfo->nField; - pTask->pUnpacked->errCode = 0; - } + rc = vdbeSortAllocUnpacked(pTask); + if( rc!=SQLITE_OK ) goto thread_out; if( pTask->eWork==SORT_SUBTASK_CONS ){ assert( pTask->pList==0 ); @@ -1533,6 +1574,9 @@ int sqlite3VdbeSorterWrite( } pSorter->nInMemory += nPMA; + if( nPMA>pSorter->mxKeysize ){ + pSorter->mxKeysize = nPMA; + } if( pSorter->aMemory ){ int nMin = pSorter->iMemory + nReq; @@ -1591,12 +1635,15 @@ static int vdbeSorterCountPMA(VdbeSorter *pSorter){ static int vdbeIncrPopulate(IncrMerger *pIncr){ int rc = SQLITE_OK; int rc2; + i64 iStart = pIncr->iStartOff; SorterFile *pOut = &pIncr->aFile[1]; MergeEngine *pMerger = pIncr->pMerger; PmaWriter writer; assert( pIncr->bEof==0 ); - vdbePmaWriterInit(pIncr->aFile[1].pFd, &writer, pIncr->pTask->pgsz, 0); + vdbeSorterPopulateDebug(pIncr->pTask, "enter"); + + vdbePmaWriterInit(pOut->pFd, &writer, pIncr->pTask->pgsz, iStart); while( rc==SQLITE_OK ){ int dummy; PmaReader *pReader = &pMerger->aIter[ pMerger->aTree[1] ]; @@ -1606,7 +1653,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ /* Check if the output file is full or if the input has been exhausted. ** In either case exit the loop. */ if( pReader->pFile==0 ) break; - if( iEof && (iEof + nKey)>pIncr->mxSz ) break; + if( (iEof + nKey + sqlite3VarintLen(nKey))>(iStart + pIncr->mxSz) ) break; /* Write the next key to the output. */ vdbePmaWriteVarint(&writer, nKey); @@ -1616,6 +1663,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); if( rc==SQLITE_OK ) rc = rc2; + vdbeSorterPopulateDebug(pIncr->pTask, "exit"); return rc; } @@ -1627,34 +1675,50 @@ static void *vdbeIncrPopulateThreadMain(void *pCtx){ static int vdbeIncrBgPopulate(IncrMerger *pIncr){ int rc; assert( pIncr->pThread==0 ); - if( pIncr->pTask->pSorter->bUseThreads==0 ){ + if( pIncr->bUseThread==0 ){ rc = vdbeIncrPopulate(pIncr); - }else{ + } +#if SQLITE_MAX_WORKER_THREADS>0 + else{ void *pCtx = (void*)pIncr; rc = sqlite3ThreadCreate(&pIncr->pThread, vdbeIncrPopulateThreadMain, pCtx); } +#endif return rc; } static int vdbeIncrSwap(IncrMerger *pIncr){ int rc = SQLITE_OK; - - if( pIncr->pThread ){ - void *pRet; - rc = sqlite3ThreadJoin(pIncr->pThread, &pRet); - if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); - pIncr->pThread = 0; - } - if( rc==SQLITE_OK ){ - SorterFile f0 = pIncr->aFile[0]; + if( pIncr->bUseThread ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->pThread ){ + void *pRet; + assert( pIncr->bUseThread ); + rc = sqlite3ThreadJoin(pIncr->pThread, &pRet); + if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); + pIncr->pThread = 0; + } +#endif + + if( rc==SQLITE_OK ){ + SorterFile f0 = pIncr->aFile[0]; + pIncr->aFile[0] = pIncr->aFile[1]; + pIncr->aFile[1] = f0; + } + + if( rc==SQLITE_OK ){ + if( pIncr->aFile[0].iEof==pIncr->iStartOff ){ + pIncr->bEof = 1; + }else{ + rc = vdbeIncrBgPopulate(pIncr); + } + } + }else{ + rc = vdbeIncrPopulate(pIncr); pIncr->aFile[0] = pIncr->aFile[1]; - pIncr->aFile[1] = f0; - - if( pIncr->aFile[0].iEof==0 ){ + if( pIncr->aFile[0].iEof==pIncr->iStartOff ){ pIncr->bEof = 1; - }else{ - rc = vdbeIncrBgPopulate(pIncr); } } @@ -1662,81 +1726,283 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ } static void vdbeIncrFree(IncrMerger *pIncr){ - if( pIncr->pThread ){ - void *pRet; - sqlite3ThreadJoin(pIncr->pThread, &pRet); + if( pIncr ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->pThread ){ + void *pRet; + sqlite3ThreadJoin(pIncr->pThread, &pRet); + } + if( pIncr->bUseThread ){ + if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); + if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); + } +#endif + vdbeMergeEngineFree(pIncr->pMerger); + sqlite3_free(pIncr); } - if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); - if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); - vdbeMergeEngineFree(pIncr->pMerger); - sqlite3_free(pIncr); +} + +static IncrMerger *vdbeIncrNew(SortSubtask *pTask, MergeEngine *pMerger){ + IncrMerger *pIncr = sqlite3_malloc(sizeof(IncrMerger)); + if( pIncr ){ + memset(pIncr, 0, sizeof(IncrMerger)); + pIncr->pMerger = pMerger; + pIncr->pTask = pTask; + pIncr->mxSz = MAX(pTask->pSorter->mxKeysize+9,pTask->pSorter->mxPmaSize/2); + pTask->file2.iEof += pIncr->mxSz; + +#if 0 + /* Open the two temp files. */ + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[0].pFd); + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[1].pFd); + } + if( rc!=SQLITE_OK ){ + vdbeIncrFree(pIncr); + pIncr = 0; + } +#endif + } + return pIncr; +} + +static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ + if( bUseThread ){ + pIncr->bUseThread = 1; + pIncr->pTask->file2.iEof -= pIncr->mxSz; + } +} + +static int vdbeIncrInit2(PmaReader *pIter){ + int rc = SQLITE_OK; + IncrMerger *pIncr = pIter->pIncr; + if( pIncr ){ + SortSubtask *pTask = pIncr->pTask; + int i; + MergeEngine *pMerger = pIncr->pMerger; + + for(i=0; rc==SQLITE_OK && inTree; i++){ + rc = vdbeIncrInit2(&pMerger->aIter[i]); + } + for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ + rc = vdbeSorterDoCompare(pIncr->pTask, pMerger, i); + } + + /* Set up the required files for pIncr */ + if( rc==SQLITE_OK ){ + if( pIncr->bUseThread==0 ){ + if( pTask->file2.pFd==0 ){ + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->file2.pFd); + assert( pTask->file2.iEof>0 ); + if( rc==SQLITE_OK ){ + vdbeSorterExtendFile(pTask->db,pTask->file2.pFd,pTask->file2.iEof); + pTask->file2.iEof = 0; + } + } + if( rc==SQLITE_OK ){ + pIncr->aFile[1].pFd = pTask->file2.pFd; + pIncr->iStartOff = pTask->file2.iEof; + pTask->file2.iEof += pIncr->mxSz; + } + }else{ + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[0].pFd); + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[1].pFd); + } + } + } + + if( rc==SQLITE_OK && pIncr->bUseThread ){ + rc = vdbeIncrBgPopulate(pIncr); + } + + if( rc==SQLITE_OK ){ + rc = vdbePmaReaderNext(pIter); + } + } + return rc; +} + +/* +** Allocate a new MergeEngine object to merge the contents of nPMA level-0 +** PMAs from pTask->file. If no error occurs, set *ppOut to point to +** the new object and return SQLITE_OK. Or, if an error does occur, set *ppOut +** to NULL and return an SQLite error code. +** +** When this function is called, *piOffset is set to the offset of the +** first PMA to read from pTask->file. Assuming no error occurs, it is +** set to the offset immediately following the last byte of the last +** PMA before returning. If an error does occur, then the final value of +** *piOffset is undefined. +*/ +static int vdbeMergeEngineLevel0( + SortSubtask *pTask, /* Sorter task to read from */ + int nPMA, /* Number of PMAs to read */ + i64 *piOffset, /* IN/OUT: Read offset in pTask->file */ + MergeEngine **ppOut /* OUT: New merge-engine */ +){ + MergeEngine *pNew; /* Merge engine to return */ + i64 iOff = *piOffset; + int i; + int rc = SQLITE_OK; + + *ppOut = pNew = vdbeMergeEngineNew(nPMA); + if( pNew==0 ) rc = SQLITE_NOMEM; + + for(i=0; iaIter[i]; + rc = vdbePmaReaderInit(pTask, &pTask->file, iOff, pIter, &nDummy); + iOff = pIter->iEof; + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pNew); + *ppOut = 0; + } + *piOffset = iOff; + return rc; +} + +typedef struct IncrBuilder IncrBuilder; +struct IncrBuilder { + int nPMA; /* Number of iterators used so far */ + MergeEngine *pMerger; /* Merge engine to populate. */ +}; + +static int vdbeAddToBuilder( + SortSubtask *pTask, + IncrBuilder *pBuilder, + MergeEngine *pMerger +){ + int rc = SQLITE_OK; + IncrMerger *pIncr; + + assert( pMerger ); + if( pBuilder->nPMA==SORTER_MAX_MERGE_COUNT ){ + rc = vdbeAddToBuilder(pTask, &pBuilder[1], pBuilder->pMerger); + pBuilder->pMerger = 0; + pBuilder->nPMA = 0; + } + + if( rc==SQLITE_OK && pBuilder->pMerger==0 ){ + pBuilder->pMerger = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); + if( pBuilder->pMerger==0 ) rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + pIncr = vdbeIncrNew(pTask, pMerger); + if( pIncr==0 ) rc = SQLITE_NOMEM; + pBuilder->pMerger->aIter[pBuilder->nPMA++].pIncr = pIncr; + } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pMerger); + } + + return rc; } /* ** Populate iterator *pIter so that it may be used to iterate through all -** keys stored in subtask pTask using the incremental merge method. +** keys stored in all PMAs created by this sorter. */ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ SortSubtask *pTask0 = &pSorter->aTask[0]; + MergeEngine *pMain = 0; + sqlite3 *db = pTask0->db; int rc = SQLITE_OK; - MergeEngine *pMerger = 0; - IncrMerger *pIncr = 0; - int i; - int nPMA = 0; + int iTask; - for(i=0; inTask; i++){ - nPMA += pSorter->aTask[i].nPMA; + IncrBuilder *aMerge; + const int nMerge = 32; + aMerge = sqlite3DbMallocZero(db, sizeof(aMerge[0])*nMerge); + if( aMerge==0 ) return SQLITE_NOMEM; + + if( pSorter->nTask>1 ){ + pMain = vdbeMergeEngineNew(pSorter->nTask); + if( pMain==0 ) rc = SQLITE_NOMEM; } - pMerger = vdbeMergeEngineNew(nPMA); - if( pMerger==0 ){ - rc = SQLITE_NOMEM; - }else{ - int iIter = 0; + + for(iTask=0; iTasknTask && rc==SQLITE_OK; iTask++){ + MergeEngine *pRoot = 0; int iPMA; - for(i=0; inTask; i++){ - i64 iReadOff = 0; - SortSubtask *pTask = &pSorter->aTask[i]; - for(iPMA=0; iPMAnPMA; iPMA++){ - i64 nDummy = 0; - PmaReader *pIter = &pMerger->aIter[iIter++]; - rc = vdbePmaReaderInit(pTask, &pTask->file, iReadOff, pIter, &nDummy); - iReadOff = pIter->iEof; + i64 iReadOff = 0; + SortSubtask *pTask = &pSorter->aTask[iTask]; + if( pTask->nPMA==0 ) continue; + for(iPMA=0; iPMAnPMA; iPMA += SORTER_MAX_MERGE_COUNT){ + MergeEngine *pMerger = 0; + int nReader = MIN(pTask->nPMA - iPMA, SORTER_MAX_MERGE_COUNT); + + rc = vdbeMergeEngineLevel0(pTask, nReader, &iReadOff, &pMerger); + if( rc!=SQLITE_OK ) break; + + if( iPMA==0 ){ + pRoot = pMerger; + }else{ + if( pRoot ){ + rc = vdbeAddToBuilder(pTask, &aMerge[0], pRoot); + pRoot = 0; + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pMerger); + break; + } + } + rc = vdbeAddToBuilder(pTask, &aMerge[0], pMerger); } } - for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pTask0, pMerger, i); + + if( pRoot==0 ){ + int i; + for(i=0; rc==SQLITE_OK && iaIter[iTask].pIncr = pNew; + if( pNew==0 ) rc = SQLITE_NOMEM; + } + memset(aMerge, 0, nMerge*sizeof(aMerge[0])); } } if( rc==SQLITE_OK ){ - pIncr = (IncrMerger*)sqlite3_malloc(sizeof(IncrMerger)); - if( pIncr==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(pIncr, 0, sizeof(IncrMerger)); - pIncr->mxSz = (pSorter->mxPmaSize / 2); - pIncr->pMerger = pMerger; - pIncr->pTask = pTask0; + SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; + + rc = vdbeSortAllocUnpacked(pLast); + if( rc==SQLITE_OK ){ + pIter->pIncr = vdbeIncrNew(pLast, pMain); + if( pIter->pIncr==0 ){ + rc = SQLITE_NOMEM; + }else{ + vdbeIncrSetThreads(pIter->pIncr, pSorter->bUseThreads); + for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ + IncrMerger *pIncr; + if( (pIncr = pMain->aIter[iTask].pIncr) ){ + vdbeIncrSetThreads(pIncr, pSorter->bUseThreads); + assert( pIncr->pTask!=pLast ); + } + } + } } } - - /* Open the two temp files. */ if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(pTask0->db->pVfs, &pIncr->aFile[0].pFd); - } - if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(pTask0->db->pVfs, &pIncr->aFile[1].pFd); + rc = vdbeIncrInit2(pIter); } - /* Launch a background thread to populate aFile[1]. */ - if( rc==SQLITE_OK ){ - rc = vdbeIncrBgPopulate(pIncr); - } - - pIter->pIncr = pIncr; - if( rc==SQLITE_OK ){ - rc = vdbePmaReaderNext(pIter); - } + sqlite3_free(aMerge); return rc; } diff --git a/test/sort2.test b/test/sort2.test index 626630050c..4fb6a9462b 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -15,47 +15,71 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix sort2 -db close -sqlite3_shutdown -sqlite3_config_worker_threads 7 -reset_db - -do_execsql_test 1 { - PRAGMA cache_size = 5; - WITH r(x,y) AS ( - SELECT 1, randomblob(100) - UNION ALL - SELECT x+1, randomblob(100) FROM r - LIMIT 100000 - ) - SELECT count(x), length(y) FROM r GROUP BY (x%5) +foreach {tn script} { + 1 { } + 2 { + catch { db close } + sqlite3_shutdown + sqlite3_config_worker_threads 7 + reset_db + } } { - 20000 100 20000 100 20000 100 20000 100 20000 100 + + eval $script + + do_execsql_test $tn.1 { + PRAGMA cache_size = 5; + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 100000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } { + 20000 100 20000 100 20000 100 20000 100 20000 100 + } + + do_execsql_test $tn.2.1 { + CREATE TABLE t1(a, b); + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 10000 + ) INSERT INTO t1 SELECT * FROM r; + } + + do_execsql_test $tn.2.2 { + CREATE UNIQUE INDEX i1 ON t1(b, a); + } + + do_execsql_test $tn.2.3 { + CREATE UNIQUE INDEX i2 ON t1(a); + } + + do_execsql_test $tn.2.4 { PRAGMA integrity_check } {ok} + + breakpoint + do_execsql_test $tn.3 { + PRAGMA cache_size = 5; + WITH r(x,y) AS ( + SELECT 1, randomblob(100) + UNION ALL + SELECT x+1, randomblob(100) FROM r + LIMIT 1000000 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } { + 200000 100 200000 100 200000 100 200000 100 200000 100 + } + + db close + sqlite3_shutdown + sqlite3_config_worker_threads 0 + sqlite3_initialize + } -do_execsql_test 2.1 { - CREATE TABLE t1(a, b); - WITH r(x,y) AS ( - SELECT 1, randomblob(100) - UNION ALL - SELECT x+1, randomblob(100) FROM r - LIMIT 10000 - ) INSERT INTO t1 SELECT * FROM r; -} - -do_execsql_test 2.2 { - CREATE UNIQUE INDEX i1 ON t1(b, a); -} - -do_execsql_test 2.3 { - CREATE UNIQUE INDEX i2 ON t1(a); -} - -do_execsql_test 2.4 { PRAGMA integrity_check } {ok} - -db close -sqlite3_shutdown -sqlite3_config_worker_threads 0 -sqlite3_initialize finish_test From 82a8a9f1204ac4fe3ba6b658199884f1852a2b4a Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 12 Apr 2014 19:34:44 +0000 Subject: [PATCH 038/710] Fix many issues with new code. FossilOrigin-Name: 62c406a042d7246f6df6b943421182a88483b2e3 --- manifest | 12 +- manifest.uuid | 2 +- src/vdbesort.c | 660 ++++++++++++++++++++++--------------------------- 3 files changed, 301 insertions(+), 373 deletions(-) diff --git a/manifest b/manifest index f48df9c72d..bca9ab362e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\shaving\sthe\ssorter\smerge\stoo\smany\sPMAs\sat\sa\stime\swhen\sincrementally\smerging\sdata\sfollowing\sa\sSorterRewind(). -D 2014-04-11T19:43:07.755 +C Fix\smany\sissues\swith\snew\scode. +D 2014-04-12T19:34:44.467 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 2984e3624383adf9c762558b8f85a17a626c11a7 +F src/vdbesort.c bc0d90e00abcc88997f463d4d41b7ba4a10cfd88 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P f9d5e09afaf64d68a0e461c1c2f38179bcea4b1f -R f6c598c1c558c5930404cda096730209 +P 98bf0307b121b0776a7170108cc8d3f948a7ebfe +R f3107fdb117ba86f9bee4609a5b08bfd U dan -Z 3e9d4ee1a6e7b343cf831d1b18651067 +Z 5a1b16a83fca264558f06f5fb6536949 diff --git a/manifest.uuid b/manifest.uuid index b1943056c1..e7ab0df5c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98bf0307b121b0776a7170108cc8d3f948a7ebfe \ No newline at end of file +62c406a042d7246f6df6b943421182a88483b2e3 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 16f6c618c6..1889c8fde1 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -88,6 +88,15 @@ #include "sqliteInt.h" #include "vdbeInt.h" +/* +** If SQLITE_DEBUG_SORTER_THREADS is defined, this module outputs various +** messages to stderr that may be helpful in understanding the performance +** characteristics of the sorter in multi-threaded mode. +*/ +#if 0 +# define SQLITE_DEBUG_SORTER_THREADS 1 +#endif + /* ** Private objects used by the sorter */ @@ -97,19 +106,48 @@ typedef struct PmaWriter PmaWriter; /* Incrementally write on PMA */ typedef struct SorterRecord SorterRecord; /* A record being sorted */ typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ typedef struct SorterFile SorterFile; +typedef struct SorterThread SorterThread; +typedef struct SorterList SorterList; typedef struct IncrMerger IncrMerger; +/* +** A container for a temp file handle and the current amount of data +** stored in the file. +*/ +struct SorterFile { + sqlite3_file *pFd; /* File handle */ + i64 iEof; /* Bytes of data stored in pFd */ +}; /* -** Candidate values for SortSubtask.eWork +** An object of this type is used to store the thread handle for each +** background thread launched by the sorter. Before the thread is launched, +** variable bDone is set to 0. Then, right before it exits, the thread +** itself sets bDone to 1. +** +** This is then used for two purposes: +** +** 1. When flushing the contents of memory to a level-0 PMA on disk, to +** attempt to select a SortSubtask for which there is not already an +** active background thread (since doing so causes the main thread +** to block until it finishes). +** +** 2. If SQLITE_DEBUG_SORTER_THREADS is defined, to determine if a call +** to sqlite3ThreadJoin() is likely to block. +** +** In both cases, the effects of the main thread seeing (bDone==0) even +** after the thread has finished are not dire. So we don't worry about +** memory barriers and such here. */ -#define SORT_SUBTASK_SORT 1 /* Sort records on pList */ -#define SORT_SUBTASK_TO_PMA 2 /* Xfer pList to Packed-Memory-Array pTemp1 */ -#define SORT_SUBTASK_CONS 3 /* Consolidate multiple PMAs */ +struct SorterThread { + SQLiteThread *pThread; + int bDone; +}; -struct SorterFile { - sqlite3_file *pFd; - i64 iEof; +struct SorterList { + SorterRecord *pList; /* Linked list of records */ + u8 *aMemory; /* If non-NULL, blob of memory for pList */ + int szPMA; /* Size of pList as PMA in bytes */ }; /* @@ -148,21 +186,13 @@ struct SorterFile { ** remain in temp file SortSubtask.pTemp1. */ struct SortSubtask { - SQLiteThread *pThread; /* Thread handle, or NULL */ - int bDone; /* Set to true by pTask when finished */ - + SorterThread thread; sqlite3 *db; /* Database connection */ VdbeSorter *pSorter; /* Sorter */ KeyInfo *pKeyInfo; /* How to compare records */ UnpackedRecord *pUnpacked; /* Space to unpack a record */ int pgsz; /* Main database page size */ - - u8 eWork; /* One of the SORT_SUBTASK_* constants */ - int nConsolidate; /* For SORT_SUBTASK_CONS, max final PMAs */ - SorterRecord *pList; /* List of records for pTask to sort */ - int nInMemory; /* Expected size of PMA based on pList */ - u8 *aListMemory; /* Records memory (or NULL) */ - + SorterList list; /* List for thread to write to a PMA */ int nPMA; /* Number of PMAs currently in file */ SorterFile file; /* Temp file for level-0 PMAs */ SorterFile file2; /* Space for other PMAs */ @@ -248,16 +278,19 @@ struct MergeEngine { ** largest record in the sorter. */ struct VdbeSorter { - int nInMemory; /* Current size of pRecord list as PMA */ int mnPmaSize; /* Minimum PMA size, in bytes */ int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ int bUsePMA; /* True if one or more PMAs created */ int bUseThreads; /* True if one or more PMAs created */ - SorterRecord *pRecord; /* Head of in-memory record list */ PmaReader *pReader; /* Read data from here after Rewind() */ int mxKeysize; /* Largest serialized key seen so far */ UnpackedRecord *pUnpacked; /* Used by VdbeSorterCompare() */ +#if 0 + int nInMemory; /* Current size of pRecord list as PMA */ + SorterRecord *pRecord; /* Head of in-memory record list */ u8 *aMemory; /* Block of memory to alloc records from */ +#endif + SorterList list; int iMemory; /* Offset of first free byte in aMemory */ int nMemory; /* Size of aMemory allocation in bytes */ int iPrev; /* Previous thread used to flush PMA */ @@ -292,7 +325,7 @@ struct PmaReader { */ struct IncrMerger { SortSubtask *pTask; /* Task that owns this merger */ - SQLiteThread *pThread; /* Thread currently populating aFile[1] */ + SorterThread thread; /* Thread for populating aFile[1] */ MergeEngine *pMerger; /* Merge engine thread reads data from */ i64 iStartOff; /* Offset to start writing file at */ int mxSz; /* Maximum bytes of data to store */ @@ -787,8 +820,8 @@ int sqlite3VdbeSorterInit( if( sqlite3GlobalConfig.pHeap==0 ){ assert( pSorter->iMemory==0 ); pSorter->nMemory = pgsz; - pSorter->aMemory = (u8*)sqlite3Malloc(pgsz); - if( !pSorter->aMemory ) rc = SQLITE_NOMEM; + pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz); + if( !pSorter->list.aMemory ) rc = SQLITE_NOMEM; } } } @@ -815,13 +848,13 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ sqlite3DbFree(db, pTask->pUnpacked); pTask->pUnpacked = 0; - if( pTask->aListMemory==0 ){ - vdbeSorterRecordFree(0, pTask->pList); + if( pTask->list.aMemory==0 ){ + vdbeSorterRecordFree(0, pTask->list.pList); }else{ - sqlite3_free(pTask->aListMemory); - pTask->aListMemory = 0; + sqlite3_free(pTask->list.aMemory); + pTask->list.aMemory = 0; } - pTask->pList = 0; + pTask->list.pList = 0; if( pTask->file.pFd ){ sqlite3OsCloseFree(pTask->file.pFd); pTask->file.pFd = 0; @@ -834,28 +867,96 @@ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ } } -/* -** Join all threads. -*/ +#ifdef SQLITE_DEBUG_SORTER_THREADS +static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); +} +static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ + i64 t; + sqlite3OsCurrentTimeInt64(db->pVfs, &t); + fprintf(stderr, "%lld:X %s\n", t, zEvent); +} +static void vdbeSorterPopulateDebug( + SortSubtask *pTask, + const char *zEvent +){ + i64 t; + int iTask = (pTask - pTask->pSorter->aTask); + sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + fprintf(stderr, "%lld:bg%d %s\n", t, iTask, zEvent); +} +static void vdbeSorterBlockDebug( + SortSubtask *pTask, + int bBlocked, + const char *zEvent +){ + if( bBlocked ){ + i64 t; + sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + fprintf(stderr, "%lld:main %s\n", t, zEvent); + } +} +#else +# define vdbeSorterWorkDebug(x,y) +# define vdbeSorterRewindDebug(x,y) +# define vdbeSorterPopulateDebug(x,y) +# define vdbeSorterBlockDebug(x,y,z) +#endif + #if SQLITE_MAX_WORKER_THREADS>0 +/* +** Join thread p. +*/ +static int vdbeSorterJoinThread(SortSubtask *pTask, SorterThread *p){ + int rc = SQLITE_OK; + if( p->pThread ){ +#ifdef SQLITE_DEBUG_SORTER_THREADS + int bDone = p->bDone; +#endif + void *pRet; + vdbeSorterBlockDebug(pTask, !bDone, "enter"); + rc = sqlite3ThreadJoin(p->pThread, &pRet); + vdbeSorterBlockDebug(pTask, !bDone, "exit"); + if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); + assert( p->bDone==1 ); + p->bDone = 0; + p->pThread = 0; + } + return rc; +} + +/* +** Launch a background thread to run xTask(pIn). +*/ +static int vdbeSorterCreateThread( + SorterThread *p, /* Thread object to populate */ + void *(*xTask)(void*), /* Routine to run in a separate thread */ + void *pIn /* Argument passed into xTask() */ +){ + assert( p->pThread==0 && p->bDone==0 ); + return sqlite3ThreadCreate(&p->pThread, xTask, pIn); +} + +/* +** Join all outstanding threads launched by SorterWrite() to create +** level-0 PMAs. +*/ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int rc = rcin; int i; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; - if( pTask->pThread ){ - void *pRet; - int rc2 = sqlite3ThreadJoin(pTask->pThread, &pRet); - pTask->pThread = 0; - pTask->bDone = 0; - if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); - } + int rc2 = vdbeSorterJoinThread(pTask, &pTask->thread); + if( rc==SQLITE_OK ) rc = rc2; } return rc; } #else # define vdbeSorterJoinAll(x,rcin) (rcin) +# define vdbeSorterJoinThread(pTask,p) SQLITE_OK #endif /* @@ -908,11 +1009,11 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ SortSubtask *pTask = &pSorter->aTask[i]; vdbeSortSubtaskCleanup(db, pTask); } - if( pSorter->aMemory==0 ){ - vdbeSorterRecordFree(0, pSorter->pRecord); + if( pSorter->list.aMemory==0 ){ + vdbeSorterRecordFree(0, pSorter->list.pList); } - pSorter->pRecord = 0; - pSorter->nInMemory = 0; + pSorter->list.pList = 0; + pSorter->list.szPMA = 0; pSorter->bUsePMA = 0; pSorter->iMemory = 0; pSorter->mxKeysize = 0; @@ -927,7 +1028,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ sqlite3VdbeSorterReset(db, pSorter); - sqlite3_free(pSorter->aMemory); + sqlite3_free(pSorter->list.aMemory); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; } @@ -952,6 +1053,21 @@ static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){ return rc; } +static int vdbeSortAllocUnpacked(SortSubtask *pTask){ + if( pTask->pUnpacked==0 ){ + char *pFree; + pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( + pTask->pKeyInfo, 0, 0, &pFree + ); + assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); + if( pFree==0 ) return SQLITE_NOMEM; + pTask->pUnpacked->nField = pTask->pKeyInfo->nField; + pTask->pUnpacked->errCode = 0; + } + return SQLITE_OK; +} + + /* ** Merge the two sorted lists p1 and p2 into a single list. ** Set *ppOut to the head of the new list. @@ -991,25 +1107,29 @@ static void vdbeSorterMerge( ** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if ** an error occurs. */ -static int vdbeSorterSort(SortSubtask *pTask){ +static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ int i; SorterRecord **aSlot; SorterRecord *p; + int rc; + + rc = vdbeSortAllocUnpacked(pTask); + if( rc!=SQLITE_OK ) return rc; aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); if( !aSlot ){ return SQLITE_NOMEM; } - p = pTask->pList; + p = pList->pList; while( p ){ SorterRecord *pNext; - if( pTask->aListMemory ){ - if( (u8*)p==pTask->aListMemory ){ + if( pList->aMemory ){ + if( (u8*)p==pList->aMemory ){ pNext = 0; }else{ - assert( p->u.iNextaListMemory) ); - pNext = (SorterRecord*)&pTask->aListMemory[p->u.iNext]; + assert( p->u.iNextaMemory) ); + pNext = (SorterRecord*)&pList->aMemory[p->u.iNext]; } }else{ pNext = p->u.pNext; @@ -1028,9 +1148,13 @@ static int vdbeSorterSort(SortSubtask *pTask){ for(i=0; i<64; i++){ vdbeSorterMerge(pTask, p, aSlot[i], &p); } - pTask->pList = p; + pList->pList = p; sqlite3_free(aSlot); + if( pTask->pUnpacked->errCode ){ + assert( pTask->pUnpacked->errCode==SQLITE_NOMEM ); + return SQLITE_NOMEM; + } return SQLITE_OK; } @@ -1144,8 +1268,9 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ /* -** Write the current contents of the in-memory linked-list to a PMA. Return -** SQLITE_OK if successful, or an SQLite error code otherwise. +** Write the current contents of in-memory linked-list pList to a level-0 +** PMA in the temp file belonging to sub-task pTask. Return SQLITE_OK if +** successful, or an SQLite error code otherwise. ** ** The format of a PMA is: ** @@ -1156,12 +1281,19 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ ** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ -static int vdbeSorterListToPMA(SortSubtask *pTask){ +static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ int rc = SQLITE_OK; /* Return code */ PmaWriter writer; /* Object used to write to the file */ +#ifdef SQLITE_DEBUG + /* Set iSz to the expected size of file pTask->file after writing the PMA. + ** This is used by an assert() statement at the end of this function. */ + i64 iSz = pList->szPMA + sqlite3VarintLen(pList->szPMA) + pTask->file.iEof; +#endif + + vdbeSorterWorkDebug(pTask, "enter"); memset(&writer, 0, sizeof(PmaWriter)); - assert( pTask->nInMemory>0 ); + assert( pList->szPMA>0 ); /* If the first temporary PMA file has not been opened, open it now. */ if( pTask->file.pFd==0 ){ @@ -1174,10 +1306,15 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ vdbeSorterExtendFile(pTask->db, - pTask->file.pFd, pTask->file.iEof + pTask->nInMemory + 9 + pTask->file.pFd, pTask->file.iEof + pList->szPMA + 9 ); } + /* Sort the list */ + if( rc==SQLITE_OK ){ + rc = vdbeSorterSort(pTask, pList); + } + if( rc==SQLITE_OK ){ SorterRecord *p; SorterRecord *pNext = 0; @@ -1185,18 +1322,20 @@ static int vdbeSorterListToPMA(SortSubtask *pTask){ vdbePmaWriterInit(pTask->file.pFd, &writer, pTask->pgsz, pTask->file.iEof); pTask->nPMA++; - vdbePmaWriteVarint(&writer, pTask->nInMemory); - for(p=pTask->pList; p; p=pNext){ + vdbePmaWriteVarint(&writer, pList->szPMA); + for(p=pList->pList; p; p=pNext){ pNext = p->u.pNext; vdbePmaWriteVarint(&writer, p->nVal); vdbePmaWriteBlob(&writer, SRVAL(p), p->nVal); - if( pTask->aListMemory==0 ) sqlite3_free(p); + if( pList->aMemory==0 ) sqlite3_free(p); } - pTask->pList = p; + pList->pList = p; rc = vdbePmaWriterFinish(&writer, &pTask->file.iEof); } - assert( pTask->pList==0 || rc!=SQLITE_OK ); + vdbeSorterWorkDebug(pTask, "exit"); + assert( rc!=SQLITE_OK || pList->pList==0 ); + assert( rc!=SQLITE_OK || pTask->file.iEof==iSz ); return rc; } @@ -1275,249 +1414,84 @@ static int vdbeSorterNext( return rc; } -#if 0 -static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ - i64 t; - int iTask = (pTask - pTask->pSorter->aTask); - sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); - fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); -} -static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ - i64 t; - sqlite3OsCurrentTimeInt64(db->pVfs, &t); - fprintf(stderr, "%lld:X %s\n", t, zEvent); -} -static void vdbeSorterPopulateDebug( - SortSubtask *pTask, - const char *zEvent -){ - i64 t; - int iTask = (pTask - pTask->pSorter->aTask); - sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); - fprintf(stderr, "%lld:bg%d %s\n", t, iTask, zEvent); -} -#else -# define vdbeSorterWorkDebug(x,y) -# define vdbeSorterRewindDebug(x,y) -# define vdbeSorterPopulateDebug(x,y) -#endif - -static int vdbeSortAllocUnpacked(SortSubtask *pTask){ - if( pTask->pUnpacked==0 ){ - char *pFree; - pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( - pTask->pKeyInfo, 0, 0, &pFree - ); - assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); - if( pFree==0 ) return SQLITE_NOMEM; - pTask->pUnpacked->nField = pTask->pKeyInfo->nField; - pTask->pUnpacked->errCode = 0; - } - return SQLITE_OK; -} - /* ** The main routine for sorter-thread operations. */ -static void *vdbeSortSubtaskMain(void *pCtx){ - int rc = SQLITE_OK; +static void *vdbeSorterFlushThread(void *pCtx){ SortSubtask *pTask = (SortSubtask*)pCtx; - - assert( pTask->eWork==SORT_SUBTASK_SORT - || pTask->eWork==SORT_SUBTASK_TO_PMA - || pTask->eWork==SORT_SUBTASK_CONS - ); - assert( pTask->bDone==0 ); - - vdbeSorterWorkDebug(pTask, "enter"); - - rc = vdbeSortAllocUnpacked(pTask); - if( rc!=SQLITE_OK ) goto thread_out; - - if( pTask->eWork==SORT_SUBTASK_CONS ){ - assert( pTask->pList==0 ); - while( pTask->nPMA>pTask->nConsolidate && rc==SQLITE_OK ){ - int nIter = MIN(pTask->nPMA, SORTER_MAX_MERGE_COUNT); - sqlite3_file *pTemp2 = 0; /* Second temp file to use */ - MergeEngine *pMerger; /* Object for reading/merging PMA data */ - i64 iReadOff = 0; /* Offset in pTemp1 to read from */ - i64 iWriteOff = 0; /* Offset in pTemp2 to write to */ - int i; - - /* Allocate a merger object to merge PMAs together. */ - pMerger = vdbeMergeEngineNew(nIter); - if( pMerger==0 ){ - rc = SQLITE_NOMEM; - break; - } - - /* Open a second temp file to write merged data to */ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTemp2); - if( rc==SQLITE_OK ){ - vdbeSorterExtendFile(pTask->db, pTemp2, pTask->file.iEof); - }else{ - vdbeMergeEngineFree(pMerger); - break; - } - - /* This loop runs once for each output PMA. Each output PMA is made - ** of data merged from up to SORTER_MAX_MERGE_COUNT input PMAs. */ - for(i=0; rc==SQLITE_OK && inPMA; i+=SORTER_MAX_MERGE_COUNT){ - PmaWriter writer; /* Object for writing data to pTemp2 */ - i64 nOut = 0; /* Bytes of data in output PMA */ - int bEof = 0; - int rc2; - - /* Configure the merger object to read and merge data from the next - ** SORTER_MAX_MERGE_COUNT PMAs in pTemp1 (or from all remaining PMAs, - ** if that is fewer). */ - int iIter; - for(iIter=0; iIteraIter[iIter]; - rc = vdbePmaReaderInit(pTask, &pTask->file, iReadOff, pIter, &nOut); - iReadOff = pIter->iEof; - if( iReadOff>=pTask->file.iEof || rc!=SQLITE_OK ) break; - } - for(iIter=pMerger->nTree-1; rc==SQLITE_OK && iIter>0; iIter--){ - rc = vdbeSorterDoCompare(pTask, pMerger, iIter); - } - - vdbePmaWriterInit(pTemp2, &writer, pTask->pgsz, iWriteOff); - vdbePmaWriteVarint(&writer, nOut); - while( rc==SQLITE_OK && bEof==0 ){ - PmaReader *pIter = &pMerger->aIter[ pMerger->aTree[1] ]; - assert( pIter->pFile!=0 ); /* pIter is not at EOF */ - vdbePmaWriteVarint(&writer, pIter->nKey); - vdbePmaWriteBlob(&writer, pIter->aKey, pIter->nKey); - rc = vdbeSorterNext(pTask, pMerger, &bEof); - } - rc2 = vdbePmaWriterFinish(&writer, &iWriteOff); - if( rc==SQLITE_OK ) rc = rc2; - } - - vdbeMergeEngineFree(pMerger); - sqlite3OsCloseFree(pTask->file.pFd); - pTask->file.pFd = pTemp2; - pTask->nPMA = (i / SORTER_MAX_MERGE_COUNT); - pTask->file.iEof = iWriteOff; - } - }else{ - /* Sort the pTask->pList list */ - rc = vdbeSorterSort(pTask); - - /* If required, write the list out to a PMA. */ - if( rc==SQLITE_OK && pTask->eWork==SORT_SUBTASK_TO_PMA ){ -#ifdef SQLITE_DEBUG - i64 nExpect = pTask->nInMemory - + sqlite3VarintLen(pTask->nInMemory) - + pTask->file.iEof; -#endif - rc = vdbeSorterListToPMA(pTask); - assert( rc!=SQLITE_OK || (nExpect==pTask->file.iEof) ); - } - } - - thread_out: - pTask->bDone = 1; - if( rc==SQLITE_OK && pTask->pUnpacked->errCode ){ - assert( pTask->pUnpacked->errCode==SQLITE_NOMEM ); - rc = SQLITE_NOMEM; - } - vdbeSorterWorkDebug(pTask, "exit"); + int rc; /* Return code */ + assert( pTask->thread.bDone==0 ); + rc = vdbeSorterListToPMA(pTask, &pTask->list); + pTask->thread.bDone = 1; return SQLITE_INT_TO_PTR(rc); } /* -** Run the activity scheduled by the object passed as the only argument -** in the current thread. -*/ -static int vdbeSorterRunTask(SortSubtask *pTask){ - int rc = SQLITE_PTR_TO_INT( vdbeSortSubtaskMain((void*)pTask) ); - assert( pTask->bDone ); - pTask->bDone = 0; - return rc; -} - -/* -** Flush the current contents of VdbeSorter.pRecord to a new PMA, possibly +** Flush the current contents of VdbeSorter.list to a new PMA, possibly ** using a background thread. -** -** If argument bFg is non-zero, the operation always uses the calling thread. */ -static int vdbeSorterFlushPMA(sqlite3 *db, const VdbeCursor *pCsr, int bFg){ - VdbeSorter *pSorter = pCsr->pSorter; +static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ +#if SQLITE_MAX_WORKER_THREADS==0 + pSorter->bUsePMA = 1; + return vdbeSorterListToPMA(&pSorter->aTask[0], &pSorter->list); +#else int rc = SQLITE_OK; int i; SortSubtask *pTask = 0; /* Thread context used to create new PMA */ int nWorker = (pSorter->nTask-1); + /* Set the flag to indicate that at least one PMA has been written. + ** Or will be, anyhow. */ pSorter->bUsePMA = 1; + + /* Select a sub-task to sort and flush the current list of in-memory + ** records to disk. If the sorter is running in multi-threaded mode, + ** round-robin between the first (pSorter->nTask-1) tasks. Except, if + ** the background thread from a sub-tasks previous turn is still running, + ** skip it. If the first (pSorter->nTask-1) sub-tasks are all still busy, + ** fall back to using the final sub-task. The first (pSorter->nTask-1) + ** sub-tasks are prefered as they use background threads - the final + ** sub-task uses the main thread. */ for(i=0; iiPrev + i + 1) % nWorker; pTask = &pSorter->aTask[iTest]; -#if SQLITE_MAX_WORKER_THREADS>0 - if( pTask->bDone ){ - void *pRet; - assert( pTask->pThread ); - rc = sqlite3ThreadJoin(pTask->pThread, &pRet); - pTask->pThread = 0; - pTask->bDone = 0; - if( rc==SQLITE_OK ){ - rc = SQLITE_PTR_TO_INT(pRet); - } + if( pTask->thread.bDone ){ + rc = vdbeSorterJoinThread(pTask, &pTask->thread); } -#endif - if( pTask->pThread==0 ) break; - pTask = 0; + if( pTask->thread.pThread==0 || rc!=SQLITE_OK ) break; } - if( pTask==0 ){ - pTask = &pSorter->aTask[nWorker]; - } - pSorter->iPrev = (pTask - pSorter->aTask); if( rc==SQLITE_OK ){ - assert( pTask->pThread==0 && pTask->bDone==0 ); - pTask->eWork = SORT_SUBTASK_TO_PMA; - pTask->pList = pSorter->pRecord; - pTask->nInMemory = pSorter->nInMemory; - pSorter->nInMemory = 0; - pSorter->pRecord = 0; - - if( pSorter->aMemory ){ - u8 *aMem = pTask->aListMemory; - pTask->aListMemory = pSorter->aMemory; - pSorter->aMemory = aMem; - } - -#if SQLITE_MAX_WORKER_THREADS>0 - if( !bFg && pTask!=&pSorter->aTask[nWorker] ){ - /* Launch a background thread for this operation */ - void *pCtx = (void*)pTask; - assert( pSorter->aMemory==0 || pTask->aListMemory!=0 ); - if( pTask->aListMemory ){ - if( pSorter->aMemory==0 ){ - pSorter->aMemory = sqlite3Malloc(pSorter->nMemory); - if( pSorter->aMemory==0 ) return SQLITE_NOMEM; - }else{ - pSorter->nMemory = sqlite3MallocSize(pSorter->aMemory); - } - } - rc = sqlite3ThreadCreate(&pTask->pThread, vdbeSortSubtaskMain, pCtx); - }else -#endif - { + if( i==nWorker ){ /* Use the foreground thread for this operation */ - rc = vdbeSorterRunTask(pTask); - if( rc==SQLITE_OK ){ - u8 *aMem = pTask->aListMemory; - pTask->aListMemory = pSorter->aMemory; - pSorter->aMemory = aMem; - assert( pTask->pList==0 ); + rc = vdbeSorterListToPMA(&pSorter->aTask[nWorker], &pSorter->list); + }else{ + /* Launch a background thread for this operation */ + u8 *aMem = pTask->list.aMemory; + void *pCtx = (void*)pTask; + + assert( pTask->thread.pThread==0 && pTask->thread.bDone==0 ); + assert( pTask->list.pList==0 ); + assert( pTask->list.aMemory==0 || pSorter->list.aMemory!=0 ); + + pSorter->iPrev = (pTask - pSorter->aTask); + pTask->list = pSorter->list; + pSorter->list.pList = 0; + pSorter->list.szPMA = 0; + if( aMem ){ + pSorter->list.aMemory = aMem; + pSorter->nMemory = sqlite3MallocSize(aMem); + }else{ + pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory); + if( !pSorter->list.aMemory ) return SQLITE_NOMEM; } + + rc = vdbeSorterCreateThread(&pTask->thread, vdbeSorterFlushThread, pCtx); } } return rc; +#endif } /* @@ -1557,28 +1531,28 @@ int sqlite3VdbeSorterWrite( nReq = pVal->n + sizeof(SorterRecord); nPMA = pVal->n + sqlite3VarintLen(pVal->n); if( pSorter->mxPmaSize ){ - if( pSorter->aMemory ){ + if( pSorter->list.aMemory ){ bFlush = pSorter->iMemory && (pSorter->iMemory+nReq) > pSorter->mxPmaSize; }else{ bFlush = ( - (pSorter->nInMemory > pSorter->mxPmaSize) - || (pSorter->nInMemory > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) + (pSorter->list.szPMA > pSorter->mxPmaSize) + || (pSorter->list.szPMA > pSorter->mnPmaSize && sqlite3HeapNearlyFull()) ); } if( bFlush ){ - rc = vdbeSorterFlushPMA(db, pCsr, 0); - pSorter->nInMemory = 0; + rc = vdbeSorterFlushPMA(pSorter); + pSorter->list.szPMA = 0; pSorter->iMemory = 0; - assert( rc!=SQLITE_OK || pSorter->pRecord==0 ); + assert( rc!=SQLITE_OK || pSorter->list.pList==0 ); } } - pSorter->nInMemory += nPMA; + pSorter->list.szPMA += nPMA; if( nPMA>pSorter->mxKeysize ){ pSorter->mxKeysize = nPMA; } - if( pSorter->aMemory ){ + if( pSorter->list.aMemory ){ int nMin = pSorter->iMemory + nReq; if( nMin>pSorter->nMemory ){ @@ -1588,45 +1562,33 @@ int sqlite3VdbeSorterWrite( if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; if( nNew < nMin ) nNew = nMin; - aNew = sqlite3Realloc(pSorter->aMemory, nNew); + aNew = sqlite3Realloc(pSorter->list.aMemory, nNew); if( !aNew ) return SQLITE_NOMEM; - pSorter->pRecord = (SorterRecord*)( - aNew + ((u8*)pSorter->pRecord - pSorter->aMemory) + pSorter->list.pList = (SorterRecord*)( + aNew + ((u8*)pSorter->list.pList - pSorter->list.aMemory) ); - pSorter->aMemory = aNew; + pSorter->list.aMemory = aNew; pSorter->nMemory = nNew; } - pNew = (SorterRecord*)&pSorter->aMemory[pSorter->iMemory]; + pNew = (SorterRecord*)&pSorter->list.aMemory[pSorter->iMemory]; pSorter->iMemory += ROUND8(nReq); - pNew->u.iNext = (u8*)(pSorter->pRecord) - pSorter->aMemory; + pNew->u.iNext = (u8*)(pSorter->list.pList) - pSorter->list.aMemory; }else{ pNew = (SorterRecord *)sqlite3Malloc(nReq); if( pNew==0 ){ return SQLITE_NOMEM; } - pNew->u.pNext = pSorter->pRecord; + pNew->u.pNext = pSorter->list.pList; } memcpy(SRVAL(pNew), pVal->z, pVal->n); pNew->nVal = pVal->n; - pSorter->pRecord = pNew; + pSorter->list.pList = pNew; return rc; } -/* -** Return the total number of PMAs in all temporary files. -*/ -static int vdbeSorterCountPMA(VdbeSorter *pSorter){ - int nPMA = 0; - int i; - for(i=0; inTask; i++){ - nPMA += pSorter->aTask[i].nPMA; - } - return nPMA; -} - /* ** Read keys from pIncr->pMerger and populate pIncr->aFile[1]. The format ** of the data stored in aFile[1] is the same as that used by regular PMAs, @@ -1667,39 +1629,27 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ return rc; } -static void *vdbeIncrPopulateThreadMain(void *pCtx){ +static void *vdbeIncrPopulateThread(void *pCtx){ IncrMerger *pIncr = (IncrMerger*)pCtx; - return SQLITE_INT_TO_PTR( vdbeIncrPopulate(pIncr) ); + void *pRet = SQLITE_INT_TO_PTR( vdbeIncrPopulate(pIncr) ); + pIncr->thread.bDone = 1; + return pRet; } -static int vdbeIncrBgPopulate(IncrMerger *pIncr){ - int rc; - assert( pIncr->pThread==0 ); - if( pIncr->bUseThread==0 ){ - rc = vdbeIncrPopulate(pIncr); - } #if SQLITE_MAX_WORKER_THREADS>0 - else{ - void *pCtx = (void*)pIncr; - rc = sqlite3ThreadCreate(&pIncr->pThread, vdbeIncrPopulateThreadMain, pCtx); - } -#endif - return rc; +static int vdbeIncrBgPopulate(IncrMerger *pIncr){ + void *pCtx = (void*)pIncr; + assert( pIncr->bUseThread ); + return vdbeSorterCreateThread(&pIncr->thread, vdbeIncrPopulateThread, pCtx); } +#endif static int vdbeIncrSwap(IncrMerger *pIncr){ int rc = SQLITE_OK; - if( pIncr->bUseThread ){ #if SQLITE_MAX_WORKER_THREADS>0 - if( pIncr->pThread ){ - void *pRet; - assert( pIncr->bUseThread ); - rc = sqlite3ThreadJoin(pIncr->pThread, &pRet); - if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); - pIncr->pThread = 0; - } -#endif + if( pIncr->bUseThread ){ + rc = vdbeSorterJoinThread(pIncr->pTask, &pIncr->thread); if( rc==SQLITE_OK ){ SorterFile f0 = pIncr->aFile[0]; @@ -1714,7 +1664,9 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ rc = vdbeIncrBgPopulate(pIncr); } } - }else{ + }else +#endif + { rc = vdbeIncrPopulate(pIncr); pIncr->aFile[0] = pIncr->aFile[1]; if( pIncr->aFile[0].iEof==pIncr->iStartOff ){ @@ -1728,10 +1680,7 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ static void vdbeIncrFree(IncrMerger *pIncr){ if( pIncr ){ #if SQLITE_MAX_WORKER_THREADS>0 - if( pIncr->pThread ){ - void *pRet; - sqlite3ThreadJoin(pIncr->pThread, &pRet); - } + vdbeSorterJoinThread(pIncr->pTask, &pIncr->thread); if( pIncr->bUseThread ){ if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); @@ -1750,18 +1699,6 @@ static IncrMerger *vdbeIncrNew(SortSubtask *pTask, MergeEngine *pMerger){ pIncr->pTask = pTask; pIncr->mxSz = MAX(pTask->pSorter->mxKeysize+9,pTask->pSorter->mxPmaSize/2); pTask->file2.iEof += pIncr->mxSz; - -#if 0 - /* Open the two temp files. */ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[0].pFd); - if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[1].pFd); - } - if( rc!=SQLITE_OK ){ - vdbeIncrFree(pIncr); - pIncr = 0; - } -#endif } return pIncr; } @@ -1784,9 +1721,6 @@ static int vdbeIncrInit2(PmaReader *pIter){ for(i=0; rc==SQLITE_OK && inTree; i++){ rc = vdbeIncrInit2(&pMerger->aIter[i]); } - for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pIncr->pTask, pMerger, i); - } /* Set up the required files for pIncr */ if( rc==SQLITE_OK ){ @@ -1812,6 +1746,10 @@ static int vdbeIncrInit2(PmaReader *pIter){ } } + for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ + rc = vdbeSorterDoCompare(pIncr->pTask, pMerger, i); + } + if( rc==SQLITE_OK && pIncr->bUseThread ){ rc = vdbeIncrBgPopulate(pIncr); } @@ -1998,9 +1936,7 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ } } } - if( rc==SQLITE_OK ){ - rc = vdbeIncrInit2(pIter); - } + if( rc==SQLITE_OK ) rc = vdbeIncrInit2(pIter); sqlite3_free(aMerge); return rc; @@ -2022,17 +1958,9 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** sort the VdbeSorter.pRecord list. The vdbe layer will read data directly ** from the in-memory list. */ if( pSorter->bUsePMA==0 ){ - if( pSorter->pRecord ){ - SortSubtask *pTask = &pSorter->aTask[0]; + if( pSorter->list.pList ){ *pbEof = 0; - pTask->pList = pSorter->pRecord; - pTask->eWork = SORT_SUBTASK_SORT; - assert( pTask->aListMemory==0 ); - pTask->aListMemory = pSorter->aMemory; - rc = vdbeSorterRunTask(pTask); - pTask->aListMemory = 0; - pSorter->pRecord = pTask->pList; - pTask->pList = 0; + rc = vdbeSorterSort(&pSorter->aTask[0], &pSorter->list); }else{ *pbEof = 1; } @@ -2040,8 +1968,8 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ } /* Write the current in-memory list to a PMA. */ - if( pSorter->pRecord ){ - rc = vdbeSorterFlushPMA(db, pCsr, 1); + if( pSorter->list.pList ){ + rc = vdbeSorterFlushPMA(pSorter); } /* Join all threads */ @@ -2076,11 +2004,11 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ rc = vdbePmaReaderNext(pSorter->pReader); *pbEof = (pSorter->pReader->pFile==0); }else{ - SorterRecord *pFree = pSorter->pRecord; - pSorter->pRecord = pFree->u.pNext; + SorterRecord *pFree = pSorter->list.pList; + pSorter->list.pList = pFree->u.pNext; pFree->u.pNext = 0; - if( pSorter->aMemory==0 ) vdbeSorterRecordFree(db, pFree); - *pbEof = !pSorter->pRecord; + if( pSorter->list.aMemory==0 ) vdbeSorterRecordFree(db, pFree); + *pbEof = !pSorter->list.pList; rc = SQLITE_OK; } return rc; @@ -2099,8 +2027,8 @@ static void *vdbeSorterRowkey( *pnKey = pSorter->pReader->nKey; pKey = pSorter->pReader->aKey; }else{ - *pnKey = pSorter->pRecord->nVal; - pKey = SRVAL(pSorter->pRecord); + *pnKey = pSorter->list.pList->nVal; + pKey = SRVAL(pSorter->list.pList); } return pKey; } From be3018c18f69c5cfbd689e3772a502fcc2edd0b4 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Apr 2014 07:30:39 +0000 Subject: [PATCH 039/710] Improve use of multiple threads in sqlite3VdbeSorterRewind(). FossilOrigin-Name: e1bdc4b810907cc0e55e0c923c8ebc777068cfe0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 44 +++++++++++++++++++++++++++++++++++++++----- 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index bca9ab362e..bb8ef3522e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\smany\sissues\swith\snew\scode. -D 2014-04-12T19:34:44.467 +C Improve\suse\sof\smultiple\sthreads\sin\ssqlite3VdbeSorterRewind(). +D 2014-04-14T07:30:39.899 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c bc0d90e00abcc88997f463d4d41b7ba4a10cfd88 +F src/vdbesort.c b047de6a9c89b122ad8649b083c848b6336b91cb F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 98bf0307b121b0776a7170108cc8d3f948a7ebfe -R f3107fdb117ba86f9bee4609a5b08bfd +P 62c406a042d7246f6df6b943421182a88483b2e3 +R c3e4d1b3a81e551661cfec7b71fa4391 U dan -Z 5a1b16a83fca264558f06f5fb6536949 +Z 2f53ddd699d6387555ef072104853e10 diff --git a/manifest.uuid b/manifest.uuid index e7ab0df5c2..1fc2ca4f8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62c406a042d7246f6df6b943421182a88483b2e3 \ No newline at end of file +e1bdc4b810907cc0e55e0c923c8ebc777068cfe0 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 1889c8fde1..32305ed792 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1710,7 +1710,10 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ } } -static int vdbeIncrInit2(PmaReader *pIter){ +#define INCRINIT2_NORMAL 0 +#define INCRINIT2_TASK 1 +#define INCRINIT2_ROOT 2 +static int vdbeIncrInit2(PmaReader *pIter, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pIter->pIncr; if( pIncr ){ @@ -1719,7 +1722,12 @@ static int vdbeIncrInit2(PmaReader *pIter){ MergeEngine *pMerger = pIncr->pMerger; for(i=0; rc==SQLITE_OK && inTree; i++){ - rc = vdbeIncrInit2(&pMerger->aIter[i]); + IncrMerger *p; + if( eMode==INCRINIT2_ROOT ){ + rc = vdbePmaReaderNext(&pMerger->aIter[i]); + }else{ + rc = vdbeIncrInit2(&pMerger->aIter[i], INCRINIT2_NORMAL); + } } /* Set up the required files for pIncr */ @@ -1751,16 +1759,32 @@ static int vdbeIncrInit2(PmaReader *pIter){ } if( rc==SQLITE_OK && pIncr->bUseThread ){ - rc = vdbeIncrBgPopulate(pIncr); + /* Use the current thread */ + assert( eMode==INCRINIT2_ROOT || eMode==INCRINIT2_TASK ); + rc = vdbeIncrPopulate(pIncr); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && eMode!=INCRINIT2_TASK ){ rc = vdbePmaReaderNext(pIter); } } return rc; } +static void *vdbeIncrInit2Thread(void *pCtx){ + PmaReader *pReader = (PmaReader*)pCtx; + void *pRet = SQLITE_INT_TO_PTR( vdbeIncrInit2(pReader, INCRINIT2_TASK) ); + pReader->pIncr->thread.bDone = 1; + return pRet; +} + +static int vdbeIncrBgInit2(PmaReader *pIter){ + void *pCtx = (void*)pIter; + return vdbeSorterCreateThread( + &pIter->pIncr->thread, vdbeIncrInit2Thread, pCtx + ); +} + /* ** Allocate a new MergeEngine object to merge the contents of nPMA level-0 ** PMAs from pTask->file. If no error occurs, set *ppOut to point to @@ -1933,10 +1957,20 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ assert( pIncr->pTask!=pLast ); } } + if( pSorter->nTask>1 ){ + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + PmaReader *p = &pMain->aIter[iTask]; + assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); + if( p->pIncr ){ rc = vdbeIncrBgInit2(p); } + } + } } } } - if( rc==SQLITE_OK ) rc = vdbeIncrInit2(pIter); + if( rc==SQLITE_OK ){ + int eMode = (pSorter->nTask>1 ? INCRINIT2_ROOT : INCRINIT2_NORMAL); + rc = vdbeIncrInit2(pIter, eMode); + } sqlite3_free(aMerge); return rc; From 92a20dde1f91f5485216dd10b51cc6de99fedd44 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Apr 2014 08:45:32 +0000 Subject: [PATCH 040/710] Minor fixes so that builds with SQLITE_MAX_WORKER_THREADS=0 work. FossilOrigin-Name: e400bbbf26cdfe88f6cb231e96cdcddb9a6bcc0f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 7 ++++++- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index bb8ef3522e..eb0882e7ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\suse\sof\smultiple\sthreads\sin\ssqlite3VdbeSorterRewind(). -D 2014-04-14T07:30:39.899 +C Minor\sfixes\sso\sthat\sbuilds\swith\sSQLITE_MAX_WORKER_THREADS=0\swork. +D 2014-04-14T08:45:32.394 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c b047de6a9c89b122ad8649b083c848b6336b91cb +F src/vdbesort.c 364cb94cf1eaefa1f79d86b7b1e4778c58192e1c F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 62c406a042d7246f6df6b943421182a88483b2e3 -R c3e4d1b3a81e551661cfec7b71fa4391 +P e1bdc4b810907cc0e55e0c923c8ebc777068cfe0 +R 15b762d2cb0315f25703d6fa43b514ac U dan -Z 2f53ddd699d6387555ef072104853e10 +Z 8e0f4aca92a8e556e8d49ca399252be9 diff --git a/manifest.uuid b/manifest.uuid index 1fc2ca4f8c..bd230a5f99 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1bdc4b810907cc0e55e0c923c8ebc777068cfe0 \ No newline at end of file +e400bbbf26cdfe88f6cb231e96cdcddb9a6bcc0f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 32305ed792..870e4e2e1c 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1771,6 +1771,7 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ return rc; } +#if SQLITE_MAX_WORKER_THREADS>0 static void *vdbeIncrInit2Thread(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; void *pRet = SQLITE_INT_TO_PTR( vdbeIncrInit2(pReader, INCRINIT2_TASK) ); @@ -1784,6 +1785,7 @@ static int vdbeIncrBgInit2(PmaReader *pIter){ &pIter->pIncr->thread, vdbeIncrInit2Thread, pCtx ); } +#endif /* ** Allocate a new MergeEngine object to merge the contents of nPMA level-0 @@ -1948,7 +1950,9 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ pIter->pIncr = vdbeIncrNew(pLast, pMain); if( pIter->pIncr==0 ){ rc = SQLITE_NOMEM; - }else{ + } +#if SQLITE_MAX_WORKER_THREADS>0 + else{ vdbeIncrSetThreads(pIter->pIncr, pSorter->bUseThreads); for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ IncrMerger *pIncr; @@ -1965,6 +1969,7 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ } } } +#endif } } if( rc==SQLITE_OK ){ From f77ceba59467de3149dab507ff58e356a6d26024 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 14 Apr 2014 18:41:21 +0000 Subject: [PATCH 041/710] Improve performance in single-threaded mode by having the final merge pass keys directly to the VDBE, instead of going via a final PMA. FossilOrigin-Name: 02610cd9b77caa2c181210056088beb3ad6ce30f --- manifest | 12 ++-- manifest.uuid | 2 +- src/vdbesort.c | 154 +++++++++++++++++++++++++++++-------------------- 3 files changed, 98 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index eb0882e7ed..08cf12e621 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sfixes\sso\sthat\sbuilds\swith\sSQLITE_MAX_WORKER_THREADS=0\swork. -D 2014-04-14T08:45:32.394 +C Improve\sperformance\sin\ssingle-threaded\smode\sby\shaving\sthe\sfinal\smerge\spass\skeys\sdirectly\sto\sthe\sVDBE,\sinstead\sof\sgoing\svia\sa\sfinal\sPMA. +D 2014-04-14T18:41:21.894 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 364cb94cf1eaefa1f79d86b7b1e4778c58192e1c +F src/vdbesort.c 15f59dc56c7c4509b4783ad20fb25479ac63d267 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e1bdc4b810907cc0e55e0c923c8ebc777068cfe0 -R 15b762d2cb0315f25703d6fa43b514ac +P e400bbbf26cdfe88f6cb231e96cdcddb9a6bcc0f +R b74f7a073d8c77e32a2dce332520aa10 U dan -Z 8e0f4aca92a8e556e8d49ca399252be9 +Z cd43207b847b4088208c6c3dd9c99bf8 diff --git a/manifest.uuid b/manifest.uuid index bd230a5f99..0cd782fad3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e400bbbf26cdfe88f6cb231e96cdcddb9a6bcc0f \ No newline at end of file +02610cd9b77caa2c181210056088beb3ad6ce30f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 870e4e2e1c..857b7ba690 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -280,21 +280,17 @@ struct MergeEngine { struct VdbeSorter { int mnPmaSize; /* Minimum PMA size, in bytes */ int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ - int bUsePMA; /* True if one or more PMAs created */ - int bUseThreads; /* True if one or more PMAs created */ PmaReader *pReader; /* Read data from here after Rewind() */ + MergeEngine *pMerger; /* Or here, if bUseThreads==0 */ int mxKeysize; /* Largest serialized key seen so far */ UnpackedRecord *pUnpacked; /* Used by VdbeSorterCompare() */ -#if 0 - int nInMemory; /* Current size of pRecord list as PMA */ - SorterRecord *pRecord; /* Head of in-memory record list */ - u8 *aMemory; /* Block of memory to alloc records from */ -#endif - SorterList list; - int iMemory; /* Offset of first free byte in aMemory */ - int nMemory; /* Size of aMemory allocation in bytes */ - int iPrev; /* Previous thread used to flush PMA */ - int nTask; /* Size of aTask[] array */ + SorterList list; /* List of in-memory records */ + int iMemory; /* Offset of free space in list.aMemory */ + int nMemory; /* Size of list.aMemory allocation in bytes */ + u8 bUsePMA; /* True if one or more PMAs created */ + u8 bUseThreads; /* True to use background threads */ + u8 iPrev; /* Previous thread used to flush PMA */ + u8 nTask; /* Size of aTask[] array */ SortSubtask aTask[1]; /* One or more subtasks */ }; @@ -1005,6 +1001,8 @@ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ sqlite3DbFree(db, pSorter->pReader); pSorter->pReader = 0; } + vdbeMergeEngineFree(pSorter->pMerger); + pSorter->pMerger = 0; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; vdbeSortSubtaskCleanup(db, pTask); @@ -1713,22 +1711,39 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ #define INCRINIT2_NORMAL 0 #define INCRINIT2_TASK 1 #define INCRINIT2_ROOT 2 + +static int vdbeIncrInit2(PmaReader *pIter, int eMode); + +static int vdbeIncrInitMerger( + SortSubtask *pTask, + MergeEngine *pMerger, + int eMode +){ + int i; + int rc = SQLITE_OK; + + for(i=0; rc==SQLITE_OK && inTree; i++){ + if( eMode==INCRINIT2_ROOT ){ + rc = vdbePmaReaderNext(&pMerger->aIter[i]); + }else{ + rc = vdbeIncrInit2(&pMerger->aIter[i], INCRINIT2_NORMAL); + } + } + + for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ + rc = vdbeSorterDoCompare(pTask, pMerger, i); + } + + return rc; +} + static int vdbeIncrInit2(PmaReader *pIter, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pIter->pIncr; if( pIncr ){ SortSubtask *pTask = pIncr->pTask; - int i; - MergeEngine *pMerger = pIncr->pMerger; - for(i=0; rc==SQLITE_OK && inTree; i++){ - IncrMerger *p; - if( eMode==INCRINIT2_ROOT ){ - rc = vdbePmaReaderNext(&pMerger->aIter[i]); - }else{ - rc = vdbeIncrInit2(&pMerger->aIter[i], INCRINIT2_NORMAL); - } - } + rc = vdbeIncrInitMerger(pTask, pIncr->pMerger, eMode); /* Set up the required files for pIncr */ if( rc==SQLITE_OK ){ @@ -1754,10 +1769,6 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ } } - for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pIncr->pTask, pMerger, i); - } - if( rc==SQLITE_OK && pIncr->bUseThread ){ /* Use the current thread */ assert( eMode==INCRINIT2_ROOT || eMode==INCRINIT2_TASK ); @@ -1871,7 +1882,7 @@ static int vdbeAddToBuilder( ** Populate iterator *pIter so that it may be used to iterate through all ** keys stored in all PMAs created by this sorter. */ -static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ +static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ SortSubtask *pTask0 = &pSorter->aTask[0]; MergeEngine *pMain = 0; sqlite3 *db = pTask0->db; @@ -1943,39 +1954,49 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter, PmaReader *pIter){ } if( rc==SQLITE_OK ){ - SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; - - rc = vdbeSortAllocUnpacked(pLast); - if( rc==SQLITE_OK ){ - pIter->pIncr = vdbeIncrNew(pLast, pMain); - if( pIter->pIncr==0 ){ - rc = SQLITE_NOMEM; +#if SQLITE_MAX_WORKER_THREADS + if( pSorter->bUseThreads ){ + PmaReader *pIter; + SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; + rc = vdbeSortAllocUnpacked(pLast); + if( rc==SQLITE_OK ){ + pIter = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); + pSorter->pReader = pIter; } -#if SQLITE_MAX_WORKER_THREADS>0 - else{ - vdbeIncrSetThreads(pIter->pIncr, pSorter->bUseThreads); - for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ - IncrMerger *pIncr; - if( (pIncr = pMain->aIter[iTask].pIncr) ){ - vdbeIncrSetThreads(pIncr, pSorter->bUseThreads); - assert( pIncr->pTask!=pLast ); - } + if( rc==SQLITE_OK ){ + pIter->pIncr = vdbeIncrNew(pLast, pMain); + if( pIter->pIncr==0 ){ + rc = SQLITE_NOMEM; } - if( pSorter->nTask>1 ){ - for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ - PmaReader *p = &pMain->aIter[iTask]; - assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); - if( p->pIncr ){ rc = vdbeIncrBgInit2(p); } + else{ + vdbeIncrSetThreads(pIter->pIncr, pSorter->bUseThreads); + for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ + IncrMerger *pIncr; + if( (pIncr = pMain->aIter[iTask].pIncr) ){ + vdbeIncrSetThreads(pIncr, pSorter->bUseThreads); + assert( pIncr->pTask!=pLast ); + } + } + if( pSorter->nTask>1 ){ + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + PmaReader *p = &pMain->aIter[iTask]; + assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); + if( p->pIncr ){ rc = vdbeIncrBgInit2(p); } + } } } } + if( rc==SQLITE_OK ){ + int eMode = (pSorter->nTask>1 ? INCRINIT2_ROOT : INCRINIT2_NORMAL); + rc = vdbeIncrInit2(pIter, eMode); + } + }else #endif + { + pSorter->pMerger = pMain; + rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT2_NORMAL); } } - if( rc==SQLITE_OK ){ - int eMode = (pSorter->nTask>1 ? INCRINIT2_ROOT : INCRINIT2_NORMAL); - rc = vdbeIncrInit2(pIter, eMode); - } sqlite3_free(aMerge); return rc; @@ -2020,11 +2041,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** incrementally read and merge all remaining PMAs. */ assert( pSorter->pReader==0 ); if( rc==SQLITE_OK ){ - PmaReader *pReader; - pReader = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); - pSorter->pReader = pReader; - rc = vdbePmaReaderIncrInit(pSorter, pReader); - assert( rc!=SQLITE_OK || pReader->pFile ); + rc = vdbePmaReaderIncrInit(pSorter); *pbEof = 0; } @@ -2039,9 +2056,17 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; int rc; /* Return code */ - if( pSorter->pReader ){ - rc = vdbePmaReaderNext(pSorter->pReader); - *pbEof = (pSorter->pReader->pFile==0); + assert( pSorter->bUsePMA || (pSorter->pReader==0 && pSorter->pMerger==0) ); + if( pSorter->bUsePMA ){ + assert( pSorter->pReader==0 || pSorter->pMerger==0 ); + assert( pSorter->bUseThreads==0 || pSorter->pReader ); + assert( pSorter->bUseThreads==1 || pSorter->pMerger ); + if( pSorter->bUseThreads ){ + rc = vdbePmaReaderNext(pSorter->pReader); + *pbEof = (pSorter->pReader->pFile==0); + }else{ + rc = vdbeSorterNext(&pSorter->aTask[0], pSorter->pMerger, pbEof); + } }else{ SorterRecord *pFree = pSorter->list.pList; pSorter->list.pList = pFree->u.pNext; @@ -2062,9 +2087,12 @@ static void *vdbeSorterRowkey( int *pnKey /* OUT: Size of current key in bytes */ ){ void *pKey; - if( pSorter->pReader ){ - *pnKey = pSorter->pReader->nKey; - pKey = pSorter->pReader->aKey; + if( pSorter->bUsePMA ){ + PmaReader *pReader = (pSorter->bUseThreads ? + pSorter->pReader : &pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]] + ); + *pnKey = pReader->nKey; + pKey = pReader->aKey; }else{ *pnKey = pSorter->list.pList->nVal; pKey = SRVAL(pSorter->list.pList); From 1a088a8ef58bb012e5ecb2e2e807530797db18c3 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 15 Apr 2014 19:52:34 +0000 Subject: [PATCH 042/710] Fix further code and documentation issues in vdbesort.c. FossilOrigin-Name: d03f5b8622d304f029f73c7cd0bee3182a81d081 --- manifest | 12 +- manifest.uuid | 2 +- src/vdbesort.c | 540 +++++++++++++++++++++++++++---------------------- 3 files changed, 310 insertions(+), 244 deletions(-) diff --git a/manifest b/manifest index f04882a0e8..0ca4cbc345 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\ssorter\sto\sbegin\sreturning\sdata\sto\sthe\sVDBE\sas\ssoon\sas\sit\sis\savailable,\sinstead\sof\swaiting\suntil\sall\skeys\shave\sbeen\ssorted. -D 2014-04-14T19:23:18.139 +C Fix\sfurther\scode\sand\sdocumentation\sissues\sin\svdbesort.c. +D 2014-04-15T19:52:34.797 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 15f59dc56c7c4509b4783ad20fb25479ac63d267 +F src/vdbesort.c ceb8e16055327d0c52bdd2087fcdd7d132fe314f F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1163,7 +1163,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e54dded2012f0ab486ee138e9bd57c528af33980 02610cd9b77caa2c181210056088beb3ad6ce30f -R b74f7a073d8c77e32a2dce332520aa10 +P cb0ab20c48962cdee03115efa93d7d501780ac73 +R 44c59b06ba4f012b63c2d0e26d30a7e6 U dan -Z ebd968da8bc859eb7bfdd863767ac91c +Z 2cc8bdff8b0228a6efe530348d9564c8 diff --git a/manifest.uuid b/manifest.uuid index 1714edb86b..b7e5cbc645 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb0ab20c48962cdee03115efa93d7d501780ac73 \ No newline at end of file +d03f5b8622d304f029f73c7cd0bee3182a81d081 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 857b7ba690..8b6cce6259 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -10,7 +10,7 @@ ** ************************************************************************* ** This file contains code for the VdbeSorter object, used in concert with -** a VdbeCursor to sort large numbers of keys for CREATE TABLE statements +** a VdbeCursor to sort large numbers of keys for CREATE INDEX statements ** or by SELECT statements with ORDER BY clauses that cannot be satisfied ** using indexes and without LIMIT clauses. ** @@ -57,33 +57,84 @@ ** ** The interfaces above must be called in a particular order. Write() can ** only occur in between Init()/Reset() and Rewind(). Next(), Rowkey(), and -** Compare() can only occur in between Rewind() and Close()/Reset(). +** Compare() can only occur in between Rewind() and Close()/Reset(). i.e. +** +** Init() +** for each record: Write() +** Rewind() +** Rowkey()/Compare() +** Next() +** Close() ** ** Algorithm: ** -** Records to be sorted are initially held in memory, in the order in -** which they arrive from Write(). When the amount of memory needed exceeds -** a threshold, all in-memory records are sorted and then appended to -** a temporary file as a "Packed-Memory-Array" or "PMA" and the memory is -** reset. There is a single temporary file used for all PMAs. The PMAs -** are packed one after another in the file. The VdbeSorter object keeps -** track of the number of PMAs written. +** Records passed to the sorter via calls to Write() are initially held +** unsorted in main memory. Assuming the amount of memory used never exceeds +** a threshold, when Rewind() is called the set of records is sorted using +** an in-memory merge sort. In this case, no temporary files are required +** and subsequent calls to Rowkey(), Next() and Compare() read records +** directly from main memory. ** -** When the Rewind() is seen, any records still held in memory are sorted. -** If no PMAs have been written (if all records are still held in memory) -** then subsequent Rowkey(), Next(), and Compare() operations work directly -** from memory. But if PMAs have been written things get a little more -** complicated. +** If the amount of space used to store records in main memory exceeds the +** threshold, then the set of records currently in memory are sorted and +** written to a temporary file in "Packed Memory Array" (PMA) format. +** A PMA created at this point is known as a "level-0 PMA". Higher levels +** of PMAs may be created by merging existing PMAs together - for example +** merging two or more level-0 PMAs together creates a level-1 PMA. ** -** When Rewind() is seen after PMAs have been written, any records still -** in memory are sorted and written as a final PMA. Then all the PMAs -** are merged together into a single massive PMA that Next(), Rowkey(), -** and Compare() walk to extract the records in sorted order. +** The threshold for the amount of main memory to use before flushing +** records to a PMA is roughly the same as the limit configured for the +** page-cache of the main database. Specifically, the threshold is set to +** the value returned multiplied by "PRAGMA main.page_size" multipled by +** that returned by "PRAGMA main.cache_size", in bytes. ** -** If SQLITE_MAX_WORKER_THREADS is non-zero, various steps of the above -** algorithm might be performed in parallel by separate threads. Threads -** are only used when one or more PMA spill to disk. If the sort is small -** enough to fit entirely in memory, everything happens on the main thread. +** If the sorter is running in single-threaded mode, then all PMAs generated +** are appended to a single temporary file. Or, if the sorter is running in +** multi-threaded mode then up to (N+1) temporary files may be opened, where +** N is the configured number of worker threads. In this case, instead of +** sorting the records and writing the PMA to a temporary file itself, the +** calling thread usually launches a worker thread to do so. Except, if +** there are already N worker threads running, the main thread does the work +** itself. +** +** The sorter is running in multi-threaded mode if (a) the library was built +** with pre-processor symbol SQLITE_MAX_WORKER_THREADS set to a value greater +** than zero, and (b) worker threads have been enabled at runtime by calling +** sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, ...). +** +** When Rewind() is called, any data remaining in memory is flushed to a +** final PMA. So at this point the data is stored in some number of sorted +** PMAs within temporary files on disk. Within a single file sorter is +** running in single threaded mode, or distributed between one or more files +** for multi-threaded sorters. +** +** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the +** sorter is running in single-threaded mode, then these PMAs are merged +** incrementally as keys are retreived from the sorter by the VDBE. See +** comments above object MergeEngine below for details. +** +** Or, if running in multi-threaded mode, then a background thread is +** launched to merge the existing PMAs. Once the background thread has +** merged T bytes of data into a single sorted PMA, the main thread +** begins reading keys from that PMA while the background thread proceeds +** with merging the next T bytes of data. And so on. +** +** Parameter T is set to half the value of the memory threshold used +** by Write() above to determine when to create a new PMA. +** +** If there are more than SORTER_MAX_MERGE_COUNT PMAs in total when +** Rewind() is called, then a hierarchy of incremental-merges is used. +** First, T bytes of data from the first SORTER_MAX_MERGE_COUNT PMAs on +** disk are merged together. Then T bytes of data from the second set, and +** so on, such that no operation ever merges more than SORTER_MAX_MERGE_COUNT +** PMAs at a time. This done is to improve locality. +** +** If running in multi-threaded mode and there are more than +** SORTER_MAX_MERGE_COUNT PMAs on disk when Rewind() is called, then more +** than one background thread may be created. Specifically, there may be +** one background thread for each temporary file on disk, and one background +** thread to merge the output of each of the others to a single PMA for +** the main thread to read from. */ #include "sqliteInt.h" #include "vdbeInt.h" @@ -102,12 +153,11 @@ */ typedef struct MergeEngine MergeEngine; /* Merge PMAs together */ typedef struct PmaReader PmaReader; /* Incrementally read one PMA */ -typedef struct PmaWriter PmaWriter; /* Incrementally write on PMA */ +typedef struct PmaWriter PmaWriter; /* Incrementally write one PMA */ typedef struct SorterRecord SorterRecord; /* A record being sorted */ typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ -typedef struct SorterFile SorterFile; -typedef struct SorterThread SorterThread; -typedef struct SorterList SorterList; +typedef struct SorterFile SorterFile; /* Temporary file object wrapper */ +typedef struct SorterList SorterList; /* In-memory list of records */ typedef struct IncrMerger IncrMerger; /* @@ -120,85 +170,14 @@ struct SorterFile { }; /* -** An object of this type is used to store the thread handle for each -** background thread launched by the sorter. Before the thread is launched, -** variable bDone is set to 0. Then, right before it exits, the thread -** itself sets bDone to 1. -** -** This is then used for two purposes: -** -** 1. When flushing the contents of memory to a level-0 PMA on disk, to -** attempt to select a SortSubtask for which there is not already an -** active background thread (since doing so causes the main thread -** to block until it finishes). -** -** 2. If SQLITE_DEBUG_SORTER_THREADS is defined, to determine if a call -** to sqlite3ThreadJoin() is likely to block. -** -** In both cases, the effects of the main thread seeing (bDone==0) even -** after the thread has finished are not dire. So we don't worry about -** memory barriers and such here. +** In memory linked list of records. */ -struct SorterThread { - SQLiteThread *pThread; - int bDone; -}; - struct SorterList { SorterRecord *pList; /* Linked list of records */ u8 *aMemory; /* If non-NULL, blob of memory for pList */ int szPMA; /* Size of pList as PMA in bytes */ }; -/* -** Sorting is divided up into smaller subtasks. Each subtask is controlled -** by an instance of this object. A Subtask might run in either the main thread -** or in a background thread. -** -** Exactly VdbeSorter.nTask instances of this object are allocated -** as part of each VdbeSorter object. Instances are never allocated any other -** way. VdbeSorter.nTask is set to the number of worker threads allowed -** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). -** -** When a background thread is launched to perform work, SortSubtask.bDone -** is set to 0 and the SortSubtask.pTask variable set to point to the -** thread handle. SortSubtask.bDone is set to 1 (to indicate to the main -** thread that joining SortSubtask.pTask will not block) before the thread -** exits. SortSubtask.pTask and bDone are always cleared after the -** background thread has been joined. -** -** One object (specifically, VdbeSorter.aTask[VdbeSorter.nTask-1]) -** is reserved for the foreground thread. -** -** The nature of the work performed is determined by SortSubtask.eWork, -** as follows: -** -** SORT_SUBTASK_SORT: -** Sort the linked list of records at SortSubtask.pList. -** -** SORT_SUBTASK_TO_PMA: -** Sort the linked list of records at SortSubtask.pList, and write -** the results to a new PMA in temp file SortSubtask.pTemp1. Open -** the temp file if it is not already open. -** -** SORT_SUBTASK_CONS: -** Merge existing PMAs until SortSubtask.nConsolidate or fewer -** remain in temp file SortSubtask.pTemp1. -*/ -struct SortSubtask { - SorterThread thread; - sqlite3 *db; /* Database connection */ - VdbeSorter *pSorter; /* Sorter */ - KeyInfo *pKeyInfo; /* How to compare records */ - UnpackedRecord *pUnpacked; /* Space to unpack a record */ - int pgsz; /* Main database page size */ - SorterList list; /* List for thread to write to a PMA */ - int nPMA; /* Number of PMAs currently in file */ - SorterFile file; /* Temp file for level-0 PMAs */ - SorterFile file2; /* Space for other PMAs */ -}; - - /* ** The MergeEngine object is used to combine two or more smaller PMAs into ** one big PMA using a merge operation. Separate PMAs all need to be @@ -268,6 +247,45 @@ struct MergeEngine { PmaReader *aIter; /* Array of iterators to merge data from */ }; +/* +** Exactly VdbeSorter.nTask instances of this object are allocated +** as part of each VdbeSorter object. Instances are never allocated any +** other way. VdbeSorter.nTask is set to the number of worker threads allowed +** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). +** +** Essentially, this structure contains all those fields of the VdbeSorter +** structure for which each thread requires a separate instance. For example, +** each thread requries its own UnpackedRecord object to unpack records in +** as part of comparison operations. +** +** Before a background thread is launched, variable bDone is set to 0. Then, +** right before it exits, the thread itself sets bDone to 1. This is used for +** two purposes: +** +** 1. When flushing the contents of memory to a level-0 PMA on disk, to +** attempt to select a SortSubtask for which there is not already an +** active background thread (since doing so causes the main thread +** to block until it finishes). +** +** 2. If SQLITE_DEBUG_SORTER_THREADS is defined, to determine if a call +** to sqlite3ThreadJoin() is likely to block. Cases that are likely to +** block provoke debugging output. +** +** In both cases, the effects of the main thread seeing (bDone==0) even +** after the thread has finished are not dire. So we don't worry about +** memory barriers and such here. +*/ +struct SortSubtask { + SQLiteThread *pThread; /* Background thread, if any */ + int bDone; /* Set if thread is finished but not joined */ + VdbeSorter *pSorter; /* Sorter that owns this sub-task */ + UnpackedRecord *pUnpacked; /* Space to unpack a record */ + SorterList list; /* List for thread to write to a PMA */ + int nPMA; /* Number of PMAs currently in file */ + SorterFile file; /* Temp file for level-0 PMAs */ + SorterFile file2; /* Space for other PMAs */ +}; + /* ** Main sorter structure. A single instance of this is allocated for each ** sorter cursor created by the VDBE. @@ -280,9 +298,12 @@ struct MergeEngine { struct VdbeSorter { int mnPmaSize; /* Minimum PMA size, in bytes */ int mxPmaSize; /* Maximum PMA size, in bytes. 0==no limit */ + int mxKeysize; /* Largest serialized key seen so far */ + int pgsz; /* Main database page size */ PmaReader *pReader; /* Read data from here after Rewind() */ MergeEngine *pMerger; /* Or here, if bUseThreads==0 */ - int mxKeysize; /* Largest serialized key seen so far */ + sqlite3 *db; /* Database connection */ + KeyInfo *pKeyInfo; /* How to compare records */ UnpackedRecord *pUnpacked; /* Used by VdbeSorterCompare() */ SorterList list; /* List of in-memory records */ int iMemory; /* Offset of free space in list.aMemory */ @@ -318,10 +339,35 @@ struct PmaReader { ** within a temp file. However, if the PmaReader.pIncr variable points to ** an object of the following type, it may be used to iterate/merge through ** multiple PMAs simultaneously. +** +** There are two types of IncrMerger object - single (bUseThread==0) and +** multi-threaded (bUseThread==1). +** +** A multi-threaded IncrMerger object uses two temporary files - aFile[0] +** and aFile[1]. Neither file is allowed to grow to more than mxSz bytes in +** size. When the IncrMerger is initialized, it reads enough data from +** pMerger to populate aFile[0]. It then sets variables within the +** corresponding PmaReader object to read from that file and kicks off +** a background thread to populate aFile[1] with the next mxSz bytes of +** sorted record data from pMerger. +** +** When the PmaReader reaches the end of aFile[0], it blocks until the +** background thread has finished populating aFile[1]. It then exchanges +** the contents of the aFile[0] and aFile[1] variables within this structure, +** sets the PmaReader fields to read from the new aFile[0] and kicks off +** another background thread to populate the new aFile[1]. And so on, until +** the contents of pMerger are exhausted. +** +** A single-threaded IncrMerger does not open any temporary files of its +** own. Instead, it has exclusive access to mxSz bytes of space beginning +** at offset iStartOff of file pTask->file2. And instead of using a +** background thread to prepare data for the PmaReader, with a single +** threaded IncrMerger the allocate part of pTask->file2 is "refilled" with +** keys from pMerger by the calling thread whenever the PmaReader runs out +** of data. */ struct IncrMerger { SortSubtask *pTask; /* Task that owns this merger */ - SorterThread thread; /* Thread for populating aFile[1] */ MergeEngine *pMerger; /* Merge engine thread reads data from */ i64 iStartOff; /* Offset to start writing file at */ int mxSz; /* Maximum bytes of data to store */ @@ -354,10 +400,10 @@ struct PmaWriter { ** ** How the linked list is connected depends on how memory is being managed ** by this module. If using a separate allocation for each in-memory record -** (VdbeSorter.aMemory==0), then the list is always connected using the +** (VdbeSorter.list.aMemory==0), then the list is always connected using the ** SorterRecord.u.pNext pointers. ** -** Or, if using the single large allocation method (VdbeSorter.aMemory!=0), +** Or, if using the single large allocation method (VdbeSorter.list.aMemory!=0), ** then while records are being accumulated the list is linked using the ** SorterRecord.u.iNext offset. This is because the aMemory[] array may ** be sqlite3Realloc()ed while records are being accumulated. Once the VM @@ -390,7 +436,7 @@ struct SorterRecord { #define SORTER_MAX_MERGE_COUNT 16 static int vdbeIncrSwap(IncrMerger*); -static void vdbeIncrFree(IncrMerger*); +static void vdbeIncrFree(IncrMerger *); /* ** Free all memory belonging to the PmaReader object passed as the second @@ -531,56 +577,69 @@ static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ return SQLITE_OK; } +/* +** Attempt to memory map file pFile. If successful, set *pp to point to the +** new mapping and return SQLITE_OK. If the mapping is not attempted +** (because the file is too large or the VFS layer is configured not to use +** mmap), return SQLITE_OK and set *pp to NULL. +** +** Or, if an error occurs, return an SQLite error code. The final value of +** *pp is undefined in this case. +*/ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ int rc = SQLITE_OK; - if( pFile->iEof<=(i64)(pTask->db->nMaxSorterMmap) ){ + if( pFile->iEof<=(i64)(pTask->pSorter->db->nMaxSorterMmap) ){ rc = sqlite3OsFetch(pFile->pFd, 0, pFile->iEof, (void**)pp); } return rc; } -static int vdbePmaReaderReinit(PmaReader *pIter){ - IncrMerger *pIncr = pIter->pIncr; - SortSubtask *pTask = pIncr->pTask; +/* +** Seek iterator pIter to offset iOff within file pFile. Return SQLITE_OK +** if successful, or an SQLite error code if an error occurs. +*/ +static int vdbePmaReaderSeek( + SortSubtask *pTask, /* Task context */ + PmaReader *pIter, /* Iterate to populate */ + SorterFile *pFile, /* Sorter file to read from */ + i64 iOff /* Offset in pFile */ +){ int rc = SQLITE_OK; - assert( pIncr->bEof==0 ); + assert( pIter->pIncr==0 || pIter->pIncr->bEof==0 ); if( pIter->aMap ){ sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); pIter->aMap = 0; } - pIter->iReadOff = pIncr->iStartOff; - pIter->iEof = pIncr->aFile[0].iEof; - pIter->pFile = pIncr->aFile[0].pFd; + pIter->iReadOff = iOff; + pIter->iEof = pFile->iEof; + pIter->pFile = pFile->pFd; - rc = vdbeSorterMapFile(pTask, &pIncr->aFile[0], &pIter->aMap); - if( rc==SQLITE_OK ){ - if( pIter->aMap==0 ){ - /* TODO: Combine this code with similar code in vdbePmaReaderInit() */ - int iBuf = pIter->iReadOff % pTask->pgsz; - if( pIter->aBuffer==0 ){ - pIter->aBuffer = (u8*)sqlite3Malloc(pTask->pgsz); - if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; - pIter->nBuffer = pTask->pgsz; - } - if( iBuf ){ - int nRead = pTask->pgsz - iBuf; - if( (pIter->iReadOff + nRead) > pIter->iEof ){ - nRead = (int)(pIter->iEof - pIter->iReadOff); - } - rc = sqlite3OsRead( - pIter->pFile, &pIter->aBuffer[iBuf], nRead, pIter->iReadOff - ); - assert( rc!=SQLITE_IOERR_SHORT_READ ); + rc = vdbeSorterMapFile(pTask, pFile, &pIter->aMap); + if( rc==SQLITE_OK && pIter->aMap==0 ){ + int pgsz = pTask->pSorter->pgsz; + int iBuf = pIter->iReadOff % pgsz; + if( pIter->aBuffer==0 ){ + pIter->aBuffer = (u8*)sqlite3Malloc(pgsz); + if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; + pIter->nBuffer = pgsz; + } + if( iBuf ){ + int nRead = pgsz - iBuf; + if( (pIter->iReadOff + nRead) > pIter->iEof ){ + nRead = (int)(pIter->iEof - pIter->iReadOff); } + rc = sqlite3OsRead( + pIter->pFile, &pIter->aBuffer[iBuf], nRead, pIter->iReadOff + ); + assert( rc!=SQLITE_IOERR_SHORT_READ ); } } return rc; } - /* ** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if ** no error occurs, or an SQLite error code if one does. @@ -590,11 +649,14 @@ static int vdbePmaReaderNext(PmaReader *pIter){ u64 nRec = 0; /* Size of record in bytes */ if( pIter->iReadOff>=pIter->iEof ){ + IncrMerger *pIncr = pIter->pIncr; int bEof = 1; - if( pIter->pIncr ){ - rc = vdbeIncrSwap(pIter->pIncr); - if( rc==SQLITE_OK && pIter->pIncr->bEof==0 ){ - rc = vdbePmaReaderReinit(pIter); + if( pIncr ){ + rc = vdbeIncrSwap(pIncr); + if( rc==SQLITE_OK && pIncr->bEof==0 ){ + rc = vdbePmaReaderSeek( + pIncr->pTask, pIter, &pIncr->aFile[0], pIncr->iStartOff + ); bEof = 0; } } @@ -633,47 +695,16 @@ static int vdbePmaReaderInit( PmaReader *pIter, /* Iterator to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ ){ - int rc = SQLITE_OK; - int nBuf = pTask->pgsz; + int rc; assert( pFile->iEof>iStart ); - assert( pIter->aAlloc==0 ); + assert( pIter->aAlloc==0 && pIter->nAlloc==0 ); assert( pIter->aBuffer==0 ); - pIter->pFile = pFile->pFd; - pIter->iReadOff = iStart; - pIter->nAlloc = 128; - pIter->aAlloc = (u8*)sqlite3Malloc(pIter->nAlloc); - if( pIter->aAlloc ){ - /* Try to xFetch() a mapping of the entire temp file. If this is possible, - ** the PMA will be read via the mapping. Otherwise, use xRead(). */ - rc = vdbeSorterMapFile(pTask, pFile, &pIter->aMap); - }else{ - rc = SQLITE_NOMEM; - } - - if( rc==SQLITE_OK && pIter->aMap==0 ){ - pIter->nBuffer = nBuf; - pIter->aBuffer = (u8*)sqlite3Malloc(nBuf); - if( !pIter->aBuffer ){ - rc = SQLITE_NOMEM; - }else{ - int iBuf = iStart % nBuf; - if( iBuf ){ - int nRead = nBuf - iBuf; - if( (iStart + nRead) > pFile->iEof ){ - nRead = (int)(pFile->iEof - iStart); - } - rc = sqlite3OsRead( - pIter->pFile, &pIter->aBuffer[iBuf], nRead, iStart - ); - assert( rc!=SQLITE_IOERR_SHORT_READ ); - } - } - } + assert( pIter->aMap==0 ); + rc = vdbePmaReaderSeek(pTask, pIter, pFile, iStart); if( rc==SQLITE_OK ){ u64 nByte; /* Size of PMA in bytes */ - pIter->iEof = pFile->iEof; rc = vdbePmaReadVarint(pIter, &nByte); pIter->iEof = pIter->iReadOff + nByte; *pnByte += nByte; @@ -706,7 +737,7 @@ static int vdbeSorterCompare( ){ UnpackedRecord *r2 = pTask->pUnpacked; if( pKey2 ){ - sqlite3VdbeRecordUnpack(pTask->pKeyInfo, nKey2, pKey2, r2); + sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); } return sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); } @@ -788,19 +819,16 @@ int sqlite3VdbeSorterInit( if( pSorter==0 ){ rc = SQLITE_NOMEM; }else{ - pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); + pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; if( nField && nWorker==0 ) pKeyInfo->nField = nField; - pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); - + pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); pSorter->nTask = nWorker + 1; pSorter->bUseThreads = (pSorter->nTask>1); + pSorter->db = db; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; - pTask->pKeyInfo = pKeyInfo; - pTask->pgsz = pgsz; - pTask->db = db; pTask->pSorter = pSorter; } @@ -904,22 +932,22 @@ static void vdbeSorterBlockDebug( #if SQLITE_MAX_WORKER_THREADS>0 /* -** Join thread p. +** Join thread pTask->thread. */ -static int vdbeSorterJoinThread(SortSubtask *pTask, SorterThread *p){ +static int vdbeSorterJoinThread(SortSubtask *pTask){ int rc = SQLITE_OK; - if( p->pThread ){ + if( pTask->pThread ){ #ifdef SQLITE_DEBUG_SORTER_THREADS - int bDone = p->bDone; + int bDone = pTask->bDone; #endif void *pRet; vdbeSorterBlockDebug(pTask, !bDone, "enter"); - rc = sqlite3ThreadJoin(p->pThread, &pRet); + rc = sqlite3ThreadJoin(pTask->pThread, &pRet); vdbeSorterBlockDebug(pTask, !bDone, "exit"); if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); - assert( p->bDone==1 ); - p->bDone = 0; - p->pThread = 0; + assert( pTask->bDone==1 ); + pTask->bDone = 0; + pTask->pThread = 0; } return rc; } @@ -928,12 +956,12 @@ static int vdbeSorterJoinThread(SortSubtask *pTask, SorterThread *p){ ** Launch a background thread to run xTask(pIn). */ static int vdbeSorterCreateThread( - SorterThread *p, /* Thread object to populate */ + SortSubtask *pTask, /* Thread will use this task object */ void *(*xTask)(void*), /* Routine to run in a separate thread */ void *pIn /* Argument passed into xTask() */ ){ - assert( p->pThread==0 && p->bDone==0 ); - return sqlite3ThreadCreate(&p->pThread, xTask, pIn); + assert( pTask->pThread==0 && pTask->bDone==0 ); + return sqlite3ThreadCreate(&pTask->pThread, xTask, pIn); } /* @@ -945,14 +973,14 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int i; for(i=0; inTask; i++){ SortSubtask *pTask = &pSorter->aTask[i]; - int rc2 = vdbeSorterJoinThread(pTask, &pTask->thread); + int rc2 = vdbeSorterJoinThread(pTask); if( rc==SQLITE_OK ) rc = rc2; } return rc; } #else # define vdbeSorterJoinAll(x,rcin) (rcin) -# define vdbeSorterJoinThread(pTask,p) SQLITE_OK +# define vdbeSorterJoinThread(pTask) SQLITE_OK #endif /* @@ -990,6 +1018,24 @@ static void vdbeMergeEngineFree(MergeEngine *pMerger){ sqlite3_free(pMerger); } +/* +** Free all resources associated with the IncrMerger object indicated by +** the first argument. +*/ +static void vdbeIncrFree(IncrMerger *pIncr){ + if( pIncr ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + vdbeSorterJoinThread(pIncr->pTask); + if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); + if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); + } +#endif + vdbeMergeEngineFree(pIncr->pMerger); + sqlite3_free(pIncr); + } +} + /* ** Reset a sorting cursor back to its original empty state. */ @@ -1051,15 +1097,20 @@ static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){ return rc; } +/* +** If it has not already been allocated, allocate the UnpackedRecord +** structure at pTask->pUnpacked. Return SQLITE_OK if successful (or +** if no allocation was required), or SQLITE_NOMEM otherwise. +*/ static int vdbeSortAllocUnpacked(SortSubtask *pTask){ if( pTask->pUnpacked==0 ){ char *pFree; pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( - pTask->pKeyInfo, 0, 0, &pFree + pTask->pSorter->pKeyInfo, 0, 0, &pFree ); assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); if( pFree==0 ) return SQLITE_NOMEM; - pTask->pUnpacked->nField = pTask->pKeyInfo->nField; + pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField; pTask->pUnpacked->errCode = 0; } return SQLITE_OK; @@ -1280,6 +1331,7 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ ** key). The varint is the number of bytes in the blob of data. */ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ + sqlite3 *db = pTask->pSorter->db; int rc = SQLITE_OK; /* Return code */ PmaWriter writer; /* Object used to write to the file */ @@ -1295,7 +1347,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ /* If the first temporary PMA file has not been opened, open it now. */ if( pTask->file.pFd==0 ){ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->file.pFd); + rc = vdbeSorterOpenTempFile(db->pVfs, &pTask->file.pFd); assert( rc!=SQLITE_OK || pTask->file.pFd ); assert( pTask->file.iEof==0 ); assert( pTask->nPMA==0 ); @@ -1303,9 +1355,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ - vdbeSorterExtendFile(pTask->db, - pTask->file.pFd, pTask->file.iEof + pList->szPMA + 9 - ); + vdbeSorterExtendFile(db, pTask->file.pFd, pTask->file.iEof+pList->szPMA+9); } /* Sort the list */ @@ -1317,7 +1367,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ SorterRecord *p; SorterRecord *pNext = 0; - vdbePmaWriterInit(pTask->file.pFd, &writer, pTask->pgsz, + vdbePmaWriterInit(pTask->file.pFd, &writer, pTask->pSorter->pgsz, pTask->file.iEof); pTask->nPMA++; vdbePmaWriteVarint(&writer, pList->szPMA); @@ -1413,14 +1463,14 @@ static int vdbeSorterNext( } /* -** The main routine for sorter-thread operations. +** The main routine for background threads that write level-0 PMAs. */ static void *vdbeSorterFlushThread(void *pCtx){ SortSubtask *pTask = (SortSubtask*)pCtx; int rc; /* Return code */ - assert( pTask->thread.bDone==0 ); + assert( pTask->bDone==0 ); rc = vdbeSorterListToPMA(pTask, &pTask->list); - pTask->thread.bDone = 1; + pTask->bDone = 1; return SQLITE_INT_TO_PTR(rc); } @@ -1453,10 +1503,10 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ for(i=0; iiPrev + i + 1) % nWorker; pTask = &pSorter->aTask[iTest]; - if( pTask->thread.bDone ){ - rc = vdbeSorterJoinThread(pTask, &pTask->thread); + if( pTask->bDone ){ + rc = vdbeSorterJoinThread(pTask); } - if( pTask->thread.pThread==0 || rc!=SQLITE_OK ) break; + if( pTask->pThread==0 || rc!=SQLITE_OK ) break; } if( rc==SQLITE_OK ){ @@ -1468,7 +1518,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ u8 *aMem = pTask->list.aMemory; void *pCtx = (void*)pTask; - assert( pTask->thread.pThread==0 && pTask->thread.bDone==0 ); + assert( pTask->pThread==0 && pTask->bDone==0 ); assert( pTask->list.pList==0 ); assert( pTask->list.aMemory==0 || pSorter->list.aMemory!=0 ); @@ -1484,7 +1534,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ if( !pSorter->list.aMemory ) return SQLITE_NOMEM; } - rc = vdbeSorterCreateThread(&pTask->thread, vdbeSorterFlushThread, pCtx); + rc = vdbeSorterCreateThread(pTask, vdbeSorterFlushThread, pCtx); } } @@ -1597,13 +1647,14 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ int rc2; i64 iStart = pIncr->iStartOff; SorterFile *pOut = &pIncr->aFile[1]; + SortSubtask *pTask = pIncr->pTask; MergeEngine *pMerger = pIncr->pMerger; PmaWriter writer; assert( pIncr->bEof==0 ); - vdbeSorterPopulateDebug(pIncr->pTask, "enter"); + vdbeSorterPopulateDebug(pTask, "enter"); - vdbePmaWriterInit(pOut->pFd, &writer, pIncr->pTask->pgsz, iStart); + vdbePmaWriterInit(pOut->pFd, &writer, pTask->pSorter->pgsz, iStart); while( rc==SQLITE_OK ){ int dummy; PmaReader *pReader = &pMerger->aIter[ pMerger->aTree[1] ]; @@ -1618,36 +1669,60 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ /* Write the next key to the output. */ vdbePmaWriteVarint(&writer, nKey); vdbePmaWriteBlob(&writer, pReader->aKey, nKey); - rc = vdbeSorterNext(pIncr->pTask, pIncr->pMerger, &dummy); + rc = vdbeSorterNext(pTask, pIncr->pMerger, &dummy); } rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); if( rc==SQLITE_OK ) rc = rc2; - vdbeSorterPopulateDebug(pIncr->pTask, "exit"); + vdbeSorterPopulateDebug(pTask, "exit"); return rc; } +#if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for background threads that populate aFile[1] of +** multi-threaded IncrMerger objects. +*/ static void *vdbeIncrPopulateThread(void *pCtx){ IncrMerger *pIncr = (IncrMerger*)pCtx; void *pRet = SQLITE_INT_TO_PTR( vdbeIncrPopulate(pIncr) ); - pIncr->thread.bDone = 1; + pIncr->pTask->bDone = 1; return pRet; } -#if SQLITE_MAX_WORKER_THREADS>0 +/* +** Launch a background thread to populate aFile[1] of pIncr. +*/ static int vdbeIncrBgPopulate(IncrMerger *pIncr){ - void *pCtx = (void*)pIncr; + void *p = (void*)pIncr; assert( pIncr->bUseThread ); - return vdbeSorterCreateThread(&pIncr->thread, vdbeIncrPopulateThread, pCtx); + return vdbeSorterCreateThread(pIncr->pTask, vdbeIncrPopulateThread, p); } #endif +/* +** This function is called when the PmaReader corresponding to pIncr has +** finished reading the contents of aFile[0]. Its purpose is to "refill" +** aFile[0] such that the iterator should start rereading it from the +** beginning. +** +** For single-threaded objects, this is accomplished by literally reading +** keys from pIncr->pMerger and repopulating aFile[0]. +** +** For multi-threaded objects, all that is required is to wait until the +** background thread is finished (if it is not already) and then swap +** aFile[0] and aFile[1] in place. If the contents of pMerger have not +** been exhausted, this function also launches a new background thread +** to populate the new aFile[1]. +** +** SQLITE_OK is returned on success, or an SQLite error code otherwise. +*/ static int vdbeIncrSwap(IncrMerger *pIncr){ int rc = SQLITE_OK; #if SQLITE_MAX_WORKER_THREADS>0 if( pIncr->bUseThread ){ - rc = vdbeSorterJoinThread(pIncr->pTask, &pIncr->thread); + rc = vdbeSorterJoinThread(pIncr->pTask); if( rc==SQLITE_OK ){ SorterFile f0 = pIncr->aFile[0]; @@ -1675,20 +1750,9 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ return rc; } -static void vdbeIncrFree(IncrMerger *pIncr){ - if( pIncr ){ -#if SQLITE_MAX_WORKER_THREADS>0 - vdbeSorterJoinThread(pIncr->pTask, &pIncr->thread); - if( pIncr->bUseThread ){ - if( pIncr->aFile[0].pFd ) sqlite3OsCloseFree(pIncr->aFile[0].pFd); - if( pIncr->aFile[1].pFd ) sqlite3OsCloseFree(pIncr->aFile[1].pFd); - } -#endif - vdbeMergeEngineFree(pIncr->pMerger); - sqlite3_free(pIncr); - } -} - +/* +** Allocate and return a new IncrMerger object to read data from pMerger. +*/ static IncrMerger *vdbeIncrNew(SortSubtask *pTask, MergeEngine *pMerger){ IncrMerger *pIncr = sqlite3_malloc(sizeof(IncrMerger)); if( pIncr ){ @@ -1701,6 +1765,9 @@ static IncrMerger *vdbeIncrNew(SortSubtask *pTask, MergeEngine *pMerger){ return pIncr; } +/* +** Set the "use-threads" flag on object pIncr. +*/ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ if( bUseThread ){ pIncr->bUseThread = 1; @@ -1742,6 +1809,7 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ IncrMerger *pIncr = pIter->pIncr; if( pIncr ){ SortSubtask *pTask = pIncr->pTask; + sqlite3 *db = pTask->pSorter->db; rc = vdbeIncrInitMerger(pTask, pIncr->pMerger, eMode); @@ -1749,10 +1817,10 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ if( rc==SQLITE_OK ){ if( pIncr->bUseThread==0 ){ if( pTask->file2.pFd==0 ){ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pTask->file2.pFd); + rc = vdbeSorterOpenTempFile(db->pVfs, &pTask->file2.pFd); assert( pTask->file2.iEof>0 ); if( rc==SQLITE_OK ){ - vdbeSorterExtendFile(pTask->db,pTask->file2.pFd,pTask->file2.iEof); + vdbeSorterExtendFile(db, pTask->file2.pFd, pTask->file2.iEof); pTask->file2.iEof = 0; } } @@ -1762,9 +1830,9 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ pTask->file2.iEof += pIncr->mxSz; } }else{ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[0].pFd); + rc = vdbeSorterOpenTempFile(db->pVfs, &pIncr->aFile[0].pFd); if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(pTask->db->pVfs, &pIncr->aFile[1].pFd); + rc = vdbeSorterOpenTempFile(db->pVfs, &pIncr->aFile[1].pFd); } } } @@ -1786,15 +1854,13 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ static void *vdbeIncrInit2Thread(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; void *pRet = SQLITE_INT_TO_PTR( vdbeIncrInit2(pReader, INCRINIT2_TASK) ); - pReader->pIncr->thread.bDone = 1; + pReader->pIncr->pTask->bDone = 1; return pRet; } static int vdbeIncrBgInit2(PmaReader *pIter){ void *pCtx = (void*)pIter; - return vdbeSorterCreateThread( - &pIter->pIncr->thread, vdbeIncrInit2Thread, pCtx - ); + return vdbeSorterCreateThread(pIter->pIncr->pTask, vdbeIncrInit2Thread, pCtx); } #endif @@ -1885,7 +1951,7 @@ static int vdbeAddToBuilder( static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ SortSubtask *pTask0 = &pSorter->aTask[0]; MergeEngine *pMain = 0; - sqlite3 *db = pTask0->db; + sqlite3 *db = pTask0->pSorter->db; int rc = SQLITE_OK; int iTask; From 22ace891799e7c7ff4159acfc7c3c4481cc9a2a3 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 15 Apr 2014 20:52:27 +0000 Subject: [PATCH 043/710] Fix some problems to do with OOM conditions in vdbesort.c. Some problems remain. FossilOrigin-Name: 2f94f9ce9bf11f1599bbc640b3fc8c15da588416 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/vdbesort.c | 16 +++++++++++++++- test/sortfault.test | 40 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+), 8 deletions(-) create mode 100644 test/sortfault.test diff --git a/manifest b/manifest index 0ca4cbc345..234641dfb4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sfurther\scode\sand\sdocumentation\sissues\sin\svdbesort.c. -D 2014-04-15T19:52:34.797 +C Fix\ssome\sproblems\sto\sdo\swith\sOOM\sconditions\sin\svdbesort.c.\sSome\sproblems\sremain. +D 2014-04-15T20:52:27.270 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c ceb8e16055327d0c52bdd2087fcdd7d132fe314f +F src/vdbesort.c 56c663772aaa7cd790ba2eb8f3bd3b4a4ac1fa4d F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -820,6 +820,7 @@ F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a F test/sort2.test 04e99d0d028b469c6cfab2c647c6c28755504063 F test/sort3.test c3f88d233452a129de519de311d109a0ad0da0af +F test/sortfault.test 2e2337aa5db6ab5cd546368cf2410676c11cb577 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1163,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P cb0ab20c48962cdee03115efa93d7d501780ac73 -R 44c59b06ba4f012b63c2d0e26d30a7e6 +P d03f5b8622d304f029f73c7cd0bee3182a81d081 +R a54791a581239e379f5a047014cbfa51 U dan -Z 2cc8bdff8b0228a6efe530348d9564c8 +Z b9d6664d305a1dc63db907b778150c9a diff --git a/manifest.uuid b/manifest.uuid index b7e5cbc645..aa04dbee2d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d03f5b8622d304f029f73c7cd0bee3182a81d081 \ No newline at end of file +2f94f9ce9bf11f1599bbc640b3fc8c15da588416 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 8b6cce6259..0922ea88b0 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -625,7 +625,7 @@ static int vdbePmaReaderSeek( if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; pIter->nBuffer = pgsz; } - if( iBuf ){ + if( rc==SQLITE_OK && iBuf ){ int nRead = pgsz - iBuf; if( (pIter->iReadOff + nRead) > pIter->iEof ){ nRead = (int)(pIter->iEof - pIter->iReadOff); @@ -1990,6 +1990,7 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ } } rc = vdbeAddToBuilder(pTask, &aMerge[0], pMerger); + if( rc!=SQLITE_OK ) break; } } @@ -2017,6 +2018,10 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ } memset(aMerge, 0, nMerge*sizeof(aMerge[0])); } + + if( rc!=SQLITE_OK ){ + vdbeMergeEngineFree(pRoot); + } } if( rc==SQLITE_OK ){ @@ -2050,6 +2055,7 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ if( p->pIncr ){ rc = vdbeIncrBgInit2(p); } } } + pMain = 0; } } if( rc==SQLITE_OK ){ @@ -2061,9 +2067,17 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ { pSorter->pMerger = pMain; rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT2_NORMAL); + pMain = 0; } } + if( rc!=SQLITE_OK ){ + int i; + for(i=0; rc==SQLITE_OK && i Date: Wed, 16 Apr 2014 16:43:05 +0000 Subject: [PATCH 044/710] Rework the way trees of MergeEngine objects are built in vdbesort.c to make it easier to follow. Fix memory leaks that could follow an OOM or IO error. Add various comments to explain functions in vdbesort.c. FossilOrigin-Name: 69026ec7dc3bd3e33bbe17c221a53cf1dd0f8945 --- manifest | 12 +-- manifest.uuid | 2 +- src/vdbesort.c | 288 ++++++++++++++++++++++++++++++------------------- 3 files changed, 184 insertions(+), 118 deletions(-) diff --git a/manifest b/manifest index 234641dfb4..0981a1907c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sproblems\sto\sdo\swith\sOOM\sconditions\sin\svdbesort.c.\sSome\sproblems\sremain. -D 2014-04-15T20:52:27.270 +C Rework\sthe\sway\strees\sof\sMergeEngine\sobjects\sare\sbuilt\sin\svdbesort.c\sto\smake\sit\seasier\sto\sfollow.\sFix\smemory\sleaks\sthat\scould\sfollow\san\sOOM\sor\sIO\serror.\sAdd\svarious\scomments\sto\sexplain\sfunctions\sin\svdbesort.c. +D 2014-04-16T16:43:05.014 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 56c663772aaa7cd790ba2eb8f3bd3b4a4ac1fa4d +F src/vdbesort.c 30505a846dd55e7ff62bccad455d465342487887 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1164,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d03f5b8622d304f029f73c7cd0bee3182a81d081 -R a54791a581239e379f5a047014cbfa51 +P 2f94f9ce9bf11f1599bbc640b3fc8c15da588416 +R 826dd10e19d54d131d0af65a48be33dc U dan -Z b9d6664d305a1dc63db907b778150c9a +Z b3fd9ee2f5beb1e7affcfabe59de18f9 diff --git a/manifest.uuid b/manifest.uuid index aa04dbee2d..ef89cf487e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f94f9ce9bf11f1599bbc640b3fc8c15da588416 \ No newline at end of file +69026ec7dc3bd3e33bbe17c221a53cf1dd0f8945 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 0922ea88b0..4f03643007 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1752,17 +1752,28 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ /* ** Allocate and return a new IncrMerger object to read data from pMerger. +** +** If an OOM condition is encountered, return NULL. In this case free the +** pMerger argument before returning. */ -static IncrMerger *vdbeIncrNew(SortSubtask *pTask, MergeEngine *pMerger){ - IncrMerger *pIncr = sqlite3_malloc(sizeof(IncrMerger)); +static int vdbeIncrNew( + SortSubtask *pTask, + MergeEngine *pMerger, + IncrMerger **ppOut +){ + int rc = SQLITE_OK; + IncrMerger *pIncr = *ppOut = (IncrMerger*)sqlite3_malloc(sizeof(IncrMerger)); if( pIncr ){ memset(pIncr, 0, sizeof(IncrMerger)); pIncr->pMerger = pMerger; pIncr->pTask = pTask; pIncr->mxSz = MAX(pTask->pSorter->mxKeysize+9,pTask->pSorter->mxPmaSize/2); pTask->file2.iEof += pIncr->mxSz; + }else{ + vdbeMergeEngineFree(pMerger); + rc = SQLITE_NOMEM; } - return pIncr; + return rc; } /* @@ -1778,16 +1789,32 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ #define INCRINIT2_NORMAL 0 #define INCRINIT2_TASK 1 #define INCRINIT2_ROOT 2 - static int vdbeIncrInit2(PmaReader *pIter, int eMode); +/* +** Initialize the merger argument passed as the second argument. Once this +** function returns, the first key of merged data may be read from the merger +** object in the usual fashion. +** +** If argument eMode is INCRINIT2_ROOT, then it is assumed that any IncrMerge +** objects attached to the PmaReader objects that the merger reads from have +** already been populated, but that they have not yet populated aFile[0] and +** set the PmaReader objects up to read from it. In this case all that is +** required is to call vdbePmaReaderNext() on each iterator to point it at +** its first key. +** +** Otherwise, if eMode is any value other than INCRINIT2_ROOT, then use +** vdbeIncrInit2() to initialize each PmaReader that feeds data to pMerger. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ static int vdbeIncrInitMerger( SortSubtask *pTask, MergeEngine *pMerger, - int eMode + int eMode /* One of the INCRINIT2_XXX constants */ ){ - int i; - int rc = SQLITE_OK; + int rc = SQLITE_OK; /* Return code */ + int i; /* For iterating through PmaReader objects */ for(i=0; rc==SQLITE_OK && inTree; i++){ if( eMode==INCRINIT2_ROOT ){ @@ -1851,6 +1878,9 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ } #if SQLITE_MAX_WORKER_THREADS>0 +/* +** The main routine for vdbeIncrInit2() operations run in background threads. +*/ static void *vdbeIncrInit2Thread(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; void *pRet = SQLITE_INT_TO_PTR( vdbeIncrInit2(pReader, INCRINIT2_TASK) ); @@ -1858,6 +1888,14 @@ static void *vdbeIncrInit2Thread(void *pCtx){ return pRet; } +/* +** Use a background thread to invoke vdbeIncrInit2(INCRINIT2_TASK) on the +** the PmaReader object passed as the first argument. +** +** This call will initialize the various fields of the pIter->pIncr +** structure and, if it is a multi-threaded IncrMerger, launch a +** background thread to populate aFile[1]. +*/ static int vdbeIncrBgInit2(PmaReader *pIter){ void *pCtx = (void*)pIter; return vdbeSorterCreateThread(pIter->pIncr->pTask, vdbeIncrInit2Thread, pCtx); @@ -1905,42 +1943,146 @@ static int vdbeMergeEngineLevel0( return rc; } -typedef struct IncrBuilder IncrBuilder; -struct IncrBuilder { - int nPMA; /* Number of iterators used so far */ - MergeEngine *pMerger; /* Merge engine to populate. */ -}; +/* +** Return the depth of a tree comprising nPMA PMAs, assuming a fanout of +** SORTER_MAX_MERGE_COUNT. The returned value does not include leaf nodes. +** +** i.e. +** +** nPMA<=16 -> TreeDepth() == 0 +** nPMA<=256 -> TreeDepth() == 1 +** nPMA<=65536 -> TreeDepth() == 2 +*/ +static int vdbeSorterTreeDepth(int nPMA){ + int nDepth = 0; + i64 nDiv = SORTER_MAX_MERGE_COUNT; + while( nDiv < (i64)nPMA ){ + nDiv = nDiv * SORTER_MAX_MERGE_COUNT; + nDepth++; + } + return nDepth; +} -static int vdbeAddToBuilder( - SortSubtask *pTask, - IncrBuilder *pBuilder, - MergeEngine *pMerger +/* +** pRoot is the root of an incremental merge-tree with depth nDepth (according +** to vdbeSorterTreeDepth()). pLeaf is the iSeq'th leaf to be added to the +** tree, counting from zero. This function adds pLeaf to the tree. +** +** If successful, SQLITE_OK is returned. If an error occurs, an SQLite error +** code is returned and pLeaf is freed. +*/ +static int vdbeSorterAddToTree( + SortSubtask *pTask, /* Task context */ + int nDepth, /* Depth of tree according to TreeDepth() */ + int iSeq, /* Sequence number of leaf within tree */ + MergeEngine *pRoot, /* Root of tree */ + MergeEngine *pLeaf /* Leaf to add to tree */ ){ int rc = SQLITE_OK; + int nDiv = 1; + int i; + MergeEngine *p = pRoot; IncrMerger *pIncr; - assert( pMerger ); - if( pBuilder->nPMA==SORTER_MAX_MERGE_COUNT ){ - rc = vdbeAddToBuilder(pTask, &pBuilder[1], pBuilder->pMerger); - pBuilder->pMerger = 0; - pBuilder->nPMA = 0; + rc = vdbeIncrNew(pTask, pLeaf, &pIncr); + + for(i=1; ipMerger==0 ){ - pBuilder->pMerger = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); - if( pBuilder->pMerger==0 ) rc = SQLITE_NOMEM; + for(i=1; iaIter[iIter]; + + if( pIter->pIncr==0 ){ + MergeEngine *pNew = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = vdbeIncrNew(pTask, pNew, &pIter->pIncr); + } + } + + p = pIter->pIncr->pMerger; + nDiv = nDiv / SORTER_MAX_MERGE_COUNT; } if( rc==SQLITE_OK ){ - pIncr = vdbeIncrNew(pTask, pMerger); - if( pIncr==0 ) rc = SQLITE_NOMEM; - pBuilder->pMerger->aIter[pBuilder->nPMA++].pIncr = pIncr; + p->aIter[iSeq % SORTER_MAX_MERGE_COUNT].pIncr = pIncr; + }else{ + vdbeIncrFree(pIncr); + } + return rc; +} + +/* +** This function is called as part of a SorterRewind() operation on a sorter +** that has already written two or more level-0 PMAs to one or more temp +** files. It builds a tree of MergeEngine/IncrMerger/PmaReader objects that +** can be used to incrementally merge all PMAs on disk. +** +** If successful, SQLITE_OK is returned and *ppOut set to point to the +** MergeEngine object at the root of the tree before returning. Or, if an +** error occurs, an SQLite error code is returned and the final value +** of *ppOut is undefined. +*/ +static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ + MergeEngine *pMain = 0; + int rc = SQLITE_OK; + int iTask; + + /* If the sorter uses more than one task, then create the top-level + ** MergeEngine here. This MergeEngine will read data from exactly + ** one PmaReader per sub-task. */ + assert( pSorter->bUseThreads || pSorter->nTask==1 ); + if( pSorter->nTask>1 ){ + pMain = vdbeMergeEngineNew(pSorter->nTask); + if( pMain==0 ) rc = SQLITE_NOMEM; + } + + for(iTask=0; iTasknTask && rc==SQLITE_OK; iTask++){ + SortSubtask *pTask = &pSorter->aTask[iTask]; + if( pTask->nPMA ){ + MergeEngine *pRoot = 0; /* Root node of tree for this task */ + int nDepth = vdbeSorterTreeDepth(pTask->nPMA); + i64 iReadOff = 0; + + if( pTask->nPMA<=SORTER_MAX_MERGE_COUNT ){ + rc = vdbeMergeEngineLevel0(pTask, pTask->nPMA, &iReadOff, &pRoot); + }else{ + int i; + int iSeq = 0; + pRoot = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); + if( pRoot==0 ) rc = SQLITE_NOMEM; + for(i=0; inPMA && rc==SQLITE_OK; i += SORTER_MAX_MERGE_COUNT){ + MergeEngine *pMerger = 0; /* New level-0 PMA merger */ + int nReader; /* Number of level-0 PMAs to merge */ + + nReader = MIN(pTask->nPMA - i, SORTER_MAX_MERGE_COUNT); + rc = vdbeMergeEngineLevel0(pTask, nReader, &iReadOff, &pMerger); + if( rc==SQLITE_OK ){ + rc = vdbeSorterAddToTree(pTask, nDepth, iSeq++, pRoot, pMerger); + } + } + } + + if( rc==SQLITE_OK ){ + if( pMain==0 ){ + pMain = pRoot; + }else{ + rc = vdbeIncrNew(pTask, pRoot, &pMain->aIter[iTask].pIncr); + } + }else{ + vdbeMergeEngineFree(pRoot); + } + } } if( rc!=SQLITE_OK ){ - vdbeMergeEngineFree(pMerger); + vdbeMergeEngineFree(pMain); + pMain = 0; } - + *ppOut = pMain; return rc; } @@ -1949,81 +2091,13 @@ static int vdbeAddToBuilder( ** keys stored in all PMAs created by this sorter. */ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ + int rc; /* Return code */ SortSubtask *pTask0 = &pSorter->aTask[0]; MergeEngine *pMain = 0; sqlite3 *db = pTask0->pSorter->db; - int rc = SQLITE_OK; int iTask; - IncrBuilder *aMerge; - const int nMerge = 32; - aMerge = sqlite3DbMallocZero(db, sizeof(aMerge[0])*nMerge); - if( aMerge==0 ) return SQLITE_NOMEM; - - if( pSorter->nTask>1 ){ - pMain = vdbeMergeEngineNew(pSorter->nTask); - if( pMain==0 ) rc = SQLITE_NOMEM; - } - - for(iTask=0; iTasknTask && rc==SQLITE_OK; iTask++){ - MergeEngine *pRoot = 0; - int iPMA; - i64 iReadOff = 0; - SortSubtask *pTask = &pSorter->aTask[iTask]; - if( pTask->nPMA==0 ) continue; - for(iPMA=0; iPMAnPMA; iPMA += SORTER_MAX_MERGE_COUNT){ - MergeEngine *pMerger = 0; - int nReader = MIN(pTask->nPMA - iPMA, SORTER_MAX_MERGE_COUNT); - - rc = vdbeMergeEngineLevel0(pTask, nReader, &iReadOff, &pMerger); - if( rc!=SQLITE_OK ) break; - - if( iPMA==0 ){ - pRoot = pMerger; - }else{ - if( pRoot ){ - rc = vdbeAddToBuilder(pTask, &aMerge[0], pRoot); - pRoot = 0; - if( rc!=SQLITE_OK ){ - vdbeMergeEngineFree(pMerger); - break; - } - } - rc = vdbeAddToBuilder(pTask, &aMerge[0], pMerger); - if( rc!=SQLITE_OK ) break; - } - } - - if( pRoot==0 ){ - int i; - for(i=0; rc==SQLITE_OK && iaIter[iTask].pIncr = pNew; - if( pNew==0 ) rc = SQLITE_NOMEM; - } - memset(aMerge, 0, nMerge*sizeof(aMerge[0])); - } - - if( rc!=SQLITE_OK ){ - vdbeMergeEngineFree(pRoot); - } - } - + rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS if( pSorter->bUseThreads ){ @@ -2035,11 +2109,8 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ pSorter->pReader = pIter; } if( rc==SQLITE_OK ){ - pIter->pIncr = vdbeIncrNew(pLast, pMain); - if( pIter->pIncr==0 ){ - rc = SQLITE_NOMEM; - } - else{ + rc = vdbeIncrNew(pLast, pMain, &pIter->pIncr); + if( rc==SQLITE_OK ){ vdbeIncrSetThreads(pIter->pIncr, pSorter->bUseThreads); for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ IncrMerger *pIncr; @@ -2055,8 +2126,8 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ if( p->pIncr ){ rc = vdbeIncrBgInit2(p); } } } - pMain = 0; } + pMain = 0; } if( rc==SQLITE_OK ){ int eMode = (pSorter->nTask>1 ? INCRINIT2_ROOT : INCRINIT2_NORMAL); @@ -2065,20 +2136,15 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ }else #endif { - pSorter->pMerger = pMain; rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT2_NORMAL); + pSorter->pMerger = pMain; pMain = 0; } } if( rc!=SQLITE_OK ){ - int i; - for(i=0; rc==SQLITE_OK && i Date: Wed, 16 Apr 2014 17:41:22 +0000 Subject: [PATCH 045/710] Change the name of vdbeIncrInit2 to vdbePmaReaderIncrInit. Add a header comment to the same function. FossilOrigin-Name: 6622d87675c1d7992b1e6d52ae7da1007a1568a4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 41 ++++++++++++++++++++++++----------------- 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 0981a1907c..b476a4c7f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rework\sthe\sway\strees\sof\sMergeEngine\sobjects\sare\sbuilt\sin\svdbesort.c\sto\smake\sit\seasier\sto\sfollow.\sFix\smemory\sleaks\sthat\scould\sfollow\san\sOOM\sor\sIO\serror.\sAdd\svarious\scomments\sto\sexplain\sfunctions\sin\svdbesort.c. -D 2014-04-16T16:43:05.014 +C Change\sthe\sname\sof\svdbeIncrInit2\sto\svdbePmaReaderIncrInit.\sAdd\sa\sheader\scomment\sto\sthe\ssame\sfunction. +D 2014-04-16T17:41:22.128 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 30505a846dd55e7ff62bccad455d465342487887 +F src/vdbesort.c 70e42f93ebc6c4b08fff55f3f3bc6b25916cfeda F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1164,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2f94f9ce9bf11f1599bbc640b3fc8c15da588416 -R 826dd10e19d54d131d0af65a48be33dc +P 69026ec7dc3bd3e33bbe17c221a53cf1dd0f8945 +R ed75157562bc112af42ea8e3070ab11f U dan -Z b3fd9ee2f5beb1e7affcfabe59de18f9 +Z ff87db58800fcd075476469fa2fb9f2f diff --git a/manifest.uuid b/manifest.uuid index ef89cf487e..52fd8bbfbc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69026ec7dc3bd3e33bbe17c221a53cf1dd0f8945 \ No newline at end of file +6622d87675c1d7992b1e6d52ae7da1007a1568a4 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 4f03643007..348eb447fd 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1789,7 +1789,7 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ #define INCRINIT2_NORMAL 0 #define INCRINIT2_TASK 1 #define INCRINIT2_ROOT 2 -static int vdbeIncrInit2(PmaReader *pIter, int eMode); +static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); /* ** Initialize the merger argument passed as the second argument. Once this @@ -1804,7 +1804,8 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode); ** its first key. ** ** Otherwise, if eMode is any value other than INCRINIT2_ROOT, then use -** vdbeIncrInit2() to initialize each PmaReader that feeds data to pMerger. +** vdbePmaReaderIncrInit() to initialize each PmaReader that feeds data +** to pMerger. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ @@ -1820,7 +1821,7 @@ static int vdbeIncrInitMerger( if( eMode==INCRINIT2_ROOT ){ rc = vdbePmaReaderNext(&pMerger->aIter[i]); }else{ - rc = vdbeIncrInit2(&pMerger->aIter[i], INCRINIT2_NORMAL); + rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT2_NORMAL); } } @@ -1831,7 +1832,7 @@ static int vdbeIncrInitMerger( return rc; } -static int vdbeIncrInit2(PmaReader *pIter, int eMode){ +static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pIter->pIncr; if( pIncr ){ @@ -1879,26 +1880,27 @@ static int vdbeIncrInit2(PmaReader *pIter, int eMode){ #if SQLITE_MAX_WORKER_THREADS>0 /* -** The main routine for vdbeIncrInit2() operations run in background threads. +** The main routine for vdbePmaReaderIncrInit() operations run in +** background threads. */ -static void *vdbeIncrInit2Thread(void *pCtx){ +static void *vdbePmaReaderBgInit(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; - void *pRet = SQLITE_INT_TO_PTR( vdbeIncrInit2(pReader, INCRINIT2_TASK) ); + void *pRet = SQLITE_INT_TO_PTR(vdbePmaReaderIncrInit(pReader,INCRINIT2_TASK)); pReader->pIncr->pTask->bDone = 1; return pRet; } /* -** Use a background thread to invoke vdbeIncrInit2(INCRINIT2_TASK) on the -** the PmaReader object passed as the first argument. +** Use a background thread to invoke vdbePmaReaderIncrInit(INCRINIT2_TASK) +** on the the PmaReader object passed as the first argument. ** ** This call will initialize the various fields of the pIter->pIncr ** structure and, if it is a multi-threaded IncrMerger, launch a ** background thread to populate aFile[1]. */ -static int vdbeIncrBgInit2(PmaReader *pIter){ +static int vdbePmaReaderBgIncrInit(PmaReader *pIter){ void *pCtx = (void*)pIter; - return vdbeSorterCreateThread(pIter->pIncr->pTask, vdbeIncrInit2Thread, pCtx); + return vdbeSorterCreateThread(pIter->pIncr->pTask, vdbePmaReaderBgInit, pCtx); } #endif @@ -2087,10 +2089,15 @@ static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ } /* -** Populate iterator *pIter so that it may be used to iterate through all -** keys stored in all PMAs created by this sorter. +** This function is called as part of an sqlite3VdbeSorterRewind() operation +** on a sorter that has written two or more PMAs to temporary files. It sets +** up either VdbeSorter.pMerger (for single threaded sorters) or pReader +** (for multi-threaded sorters) so that it can be used to iterate through +** all records stored in the sorter. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ +static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ int rc; /* Return code */ SortSubtask *pTask0 = &pSorter->aTask[0]; MergeEngine *pMain = 0; @@ -2123,7 +2130,7 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ PmaReader *p = &pMain->aIter[iTask]; assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); - if( p->pIncr ){ rc = vdbeIncrBgInit2(p); } + if( p->pIncr ){ rc = vdbePmaReaderBgIncrInit(p); } } } } @@ -2131,7 +2138,7 @@ static int vdbePmaReaderIncrInit(VdbeSorter *pSorter){ } if( rc==SQLITE_OK ){ int eMode = (pSorter->nTask>1 ? INCRINIT2_ROOT : INCRINIT2_NORMAL); - rc = vdbeIncrInit2(pIter, eMode); + rc = vdbePmaReaderIncrInit(pIter, eMode); } }else #endif @@ -2187,7 +2194,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** incrementally read and merge all remaining PMAs. */ assert( pSorter->pReader==0 ); if( rc==SQLITE_OK ){ - rc = vdbePmaReaderIncrInit(pSorter); + rc = vdbeSorterSetupMerge(pSorter); *pbEof = 0; } From 31a0bfde9b6994c3040a7f87b29487532d094319 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 16 Apr 2014 19:04:23 +0000 Subject: [PATCH 046/710] Clarify the purpose of the nField argument passed to sqlite3VdbeSorterInit(). FossilOrigin-Name: c0c8cff17b7311bbc4fd081313a5552f927c3833 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 16 ++++++++++++++++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index b476a4c7f2..df788673d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\svdbeIncrInit2\sto\svdbePmaReaderIncrInit.\sAdd\sa\sheader\scomment\sto\sthe\ssame\sfunction. -D 2014-04-16T17:41:22.128 +C Clarify\sthe\spurpose\sof\sthe\snField\sargument\spassed\sto\ssqlite3VdbeSorterInit(). +D 2014-04-16T19:04:23.830 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 70e42f93ebc6c4b08fff55f3f3bc6b25916cfeda +F src/vdbesort.c fcd15eb438a3462f4dbeebf506ee9050947236bf F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1164,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 69026ec7dc3bd3e33bbe17c221a53cf1dd0f8945 -R ed75157562bc112af42ea8e3070ab11f +P 6622d87675c1d7992b1e6d52ae7da1007a1568a4 +R dd88eb57ef2cb3488d23de10e0a80225 U dan -Z ff87db58800fcd075476469fa2fb9f2f +Z 28b7dc83a03f9f7e8d724df0f79c29e1 diff --git a/manifest.uuid b/manifest.uuid index 52fd8bbfbc..8d63094336 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6622d87675c1d7992b1e6d52ae7da1007a1568a4 \ No newline at end of file +c0c8cff17b7311bbc4fd081313a5552f927c3833 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 348eb447fd..63f17e4e77 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -794,6 +794,22 @@ static int vdbeSorterDoCompare( /* ** Initialize the temporary index cursor just opened as a sorter cursor. +** +** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nField) +** to determine the number of fields that should be compared from the +** records being sorted. However, if the value passed as argument nField +** is non-zero and the sorter is able to guarantee a stable sort, nField +** is used instead. This is used when sorting records for a CREATE INDEX +** statement. In this case, keys are always delivered to the sorter in +** order of the primary key, which happens to be make up the final part +** of the records being sorted. So if the sort is stable, there is never +** any reason to compare PK fields and they can be ignored for a small +** performance boost. +** +** The sorter can guarantee a stable sort when running in single-threaded +** mode, but not in multi-threaded mode. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ int sqlite3VdbeSorterInit( sqlite3 *db, /* Database connection (for malloc()) */ From a9f43d735b52276133151180a292891827158bda Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 17 Apr 2014 08:57:17 +0000 Subject: [PATCH 047/710] Fix build problems in vdbesort.c. Add further comments and changes to make things easier to understand. FossilOrigin-Name: 12b190db1d20c34390c768614b40ff58a9d3b74c --- manifest | 12 ++-- manifest.uuid | 2 +- src/vdbesort.c | 154 +++++++++++++++++++++++++++++++------------------ 3 files changed, 105 insertions(+), 63 deletions(-) diff --git a/manifest b/manifest index df788673d3..5fdab138c8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clarify\sthe\spurpose\sof\sthe\snField\sargument\spassed\sto\ssqlite3VdbeSorterInit(). -D 2014-04-16T19:04:23.830 +C Fix\sbuild\sproblems\sin\svdbesort.c.\sAdd\sfurther\scomments\sand\schanges\sto\smake\sthings\seasier\sto\sunderstand. +D 2014-04-17T08:57:17.925 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c d8dc38965507a34b0e150c0d7fc82b02f8cf25ea F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c fcd15eb438a3462f4dbeebf506ee9050947236bf +F src/vdbesort.c a41721a8e97735597afb80c930f04f195d86a817 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1164,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 6622d87675c1d7992b1e6d52ae7da1007a1568a4 -R dd88eb57ef2cb3488d23de10e0a80225 +P c0c8cff17b7311bbc4fd081313a5552f927c3833 +R 5c2193354b4ddf9cc33c16483e4c137f U dan -Z 28b7dc83a03f9f7e8d724df0f79c29e1 +Z c671751bbf58b5cb738173e8c3e25105 diff --git a/manifest.uuid b/manifest.uuid index 8d63094336..b5687b30fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c0c8cff17b7311bbc4fd081313a5552f927c3833 \ No newline at end of file +12b190db1d20c34390c768614b40ff58a9d3b74c \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 63f17e4e77..ec9849bf16 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -911,7 +911,7 @@ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ i64 t; int iTask = (pTask - pTask->pSorter->aTask); - sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); } static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ @@ -925,7 +925,7 @@ static void vdbeSorterPopulateDebug( ){ i64 t; int iTask = (pTask - pTask->pSorter->aTask); - sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); fprintf(stderr, "%lld:bg%d %s\n", t, iTask, zEvent); } static void vdbeSorterBlockDebug( @@ -935,7 +935,7 @@ static void vdbeSorterBlockDebug( ){ if( bBlocked ){ i64 t; - sqlite3OsCurrentTimeInt64(pTask->db->pVfs, &t); + sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); fprintf(stderr, "%lld:main %s\n", t, zEvent); } } @@ -1094,21 +1094,52 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ } } +#if SQLITE_MAX_MMAP_SIZE>0 +/* +** The first argument is a file-handle open on a temporary file. The file +** is guaranteed to be nByte bytes or smaller in size. This function +** attempts to extend the file to nByte bytes in size and to ensure that +** the VFS has memory mapped it. +** +** Whether or not the file does end up memory mapped of course depends on +** the specific VFS implementation. +*/ +static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ + if( nByte<=(i64)(db->nMaxSorterMmap) ){ + int rc = sqlite3OsTruncate(pFile, nByte); + if( rc==SQLITE_OK ){ + void *p = 0; + sqlite3OsFetch(pFile, 0, nByte, &p); + sqlite3OsUnfetch(pFile, 0, p); + } + } +} +#else +# define vdbeSorterExtendFile(x,y,z) SQLITE_OK +#endif + /* ** Allocate space for a file-handle and open a temporary file. If successful, ** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK. ** Otherwise, set *ppFile to 0 and return an SQLite error code. */ -static int vdbeSorterOpenTempFile(sqlite3_vfs *pVfs, sqlite3_file **ppFile){ +static int vdbeSorterOpenTempFile( + sqlite3 *db, /* Database handle doing sort */ + i64 nExtend, /* Attempt to extend file to this size */ + sqlite3_file **ppFile +){ int rc; - rc = sqlite3OsOpenMalloc(pVfs, 0, ppFile, + rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &rc ); if( rc==SQLITE_OK ){ i64 max = SQLITE_MAX_MMAP_SIZE; - sqlite3OsFileControlHint( *ppFile, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); + sqlite3OsFileControlHint(*ppFile, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); + if( nExtend>0 ){ + vdbeSorterExtendFile(db, *ppFile, nExtend); + } } return rc; } @@ -1307,31 +1338,6 @@ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ vdbePmaWriteBlob(p, aByte, nByte); } -#if SQLITE_MAX_MMAP_SIZE>0 -/* -** The first argument is a file-handle open on a temporary file. The file -** is guaranteed to be nByte bytes or smaller in size. This function -** attempts to extend the file to nByte bytes in size and to ensure that -** the VFS has memory mapped it. -** -** Whether or not the file does end up memory mapped of course depends on -** the specific VFS implementation. -*/ -static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ - if( nByte<=(i64)(db->nMaxSorterMmap) ){ - int rc = sqlite3OsTruncate(pFile, nByte); - if( rc==SQLITE_OK ){ - void *p = 0; - sqlite3OsFetch(pFile, 0, nByte, &p); - sqlite3OsUnfetch(pFile, 0, p); - } - } -} -#else -# define vdbeSorterExtendFile(x,y,z) SQLITE_OK -#endif - - /* ** Write the current contents of in-memory linked-list pList to a level-0 ** PMA in the temp file belonging to sub-task pTask. Return SQLITE_OK if @@ -1363,7 +1369,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ /* If the first temporary PMA file has not been opened, open it now. */ if( pTask->file.pFd==0 ){ - rc = vdbeSorterOpenTempFile(db->pVfs, &pTask->file.pFd); + rc = vdbeSorterOpenTempFile(db, 0, &pTask->file.pFd); assert( rc!=SQLITE_OK || pTask->file.pFd ); assert( pTask->file.iEof==0 ); assert( pTask->nPMA==0 ); @@ -1802,9 +1808,9 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ } } -#define INCRINIT2_NORMAL 0 -#define INCRINIT2_TASK 1 -#define INCRINIT2_ROOT 2 +#define INCRINIT_NORMAL 0 +#define INCRINIT_TASK 1 +#define INCRINIT_ROOT 2 static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); /* @@ -1812,14 +1818,14 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); ** function returns, the first key of merged data may be read from the merger ** object in the usual fashion. ** -** If argument eMode is INCRINIT2_ROOT, then it is assumed that any IncrMerge +** If argument eMode is INCRINIT_ROOT, then it is assumed that any IncrMerge ** objects attached to the PmaReader objects that the merger reads from have ** already been populated, but that they have not yet populated aFile[0] and ** set the PmaReader objects up to read from it. In this case all that is ** required is to call vdbePmaReaderNext() on each iterator to point it at ** its first key. ** -** Otherwise, if eMode is any value other than INCRINIT2_ROOT, then use +** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use ** vdbePmaReaderIncrInit() to initialize each PmaReader that feeds data ** to pMerger. ** @@ -1828,16 +1834,16 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); static int vdbeIncrInitMerger( SortSubtask *pTask, MergeEngine *pMerger, - int eMode /* One of the INCRINIT2_XXX constants */ + int eMode /* One of the INCRINIT_XXX constants */ ){ int rc = SQLITE_OK; /* Return code */ int i; /* For iterating through PmaReader objects */ for(i=0; rc==SQLITE_OK && inTree; i++){ - if( eMode==INCRINIT2_ROOT ){ + if( eMode==INCRINIT_ROOT ){ rc = vdbePmaReaderNext(&pMerger->aIter[i]); }else{ - rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT2_NORMAL); + rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT_NORMAL); } } @@ -1848,6 +1854,39 @@ static int vdbeIncrInitMerger( return rc; } +/* +** If the PmaReader passed as the first argument is not an incremental-reader +** (if pIter->pIncr==0), then this function is a no-op. Otherwise, it serves +** to open and/or initialize the temp file related fields of the IncrMerge +** object at (pIter->pIncr). +** +** If argument eMode is set to INCRINIT_NORMAL, then PmaReader iterators +** in the sub-tree headed by pIter are also initialized. Data is then loaded +** into the buffers belonging to this iterator, pIter, and it is set to +** point to the first key in its range. +** +** If argument eMode is set to INCRINIT_TASK, then PmaReader is guaranteed +** to be a multi-threaded iterator and this function is being called in a +** background thread. In this case all iterators in the sub-tree are +** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to +** pIter is populated. However, the iterator itself is not set up to point +** to its first key. A call to vdbePmaReaderNext() is still required to do +** that. +** +** The reason this function does not call vdbePmaReaderNext() immediately +** in the INCRINIT_TASK case is that vdbePmaReaderNext() assumes that has +** to block on thread (pTask->thread) before accessing aFile[1]. But, since +** this entire function is being run by thread (pTask->thread), that will +** lead to the current background thread attempting to join itself. +** +** Finally, if argument eMode is set to INCRINIT_ROOT, it may be assumed +** that pIter->pIncr is a multi-threaded IncrMerge objects, and that all +** child-trees have already been initialized using IncrInit(INCRINIT_TASK). +** In this case vdbePmaReaderNext() is called on all child iterators and +** the current iterator set to point to the first key in its range. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pIter->pIncr; @@ -1855,39 +1894,42 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; + assert( eMode==INCRINIT_NORMAL || pIncr->bUseThread==1 ); rc = vdbeIncrInitMerger(pTask, pIncr->pMerger, eMode); - /* Set up the required files for pIncr */ + /* Set up the required files for pIncr. A multi-theaded IncrMerge object + ** requires two temp files to itself, whereas a single-threaded object + ** only requires a region of pTask->file2. */ if( rc==SQLITE_OK ){ + int mxSz = pIncr->mxSz; if( pIncr->bUseThread==0 ){ if( pTask->file2.pFd==0 ){ - rc = vdbeSorterOpenTempFile(db->pVfs, &pTask->file2.pFd); assert( pTask->file2.iEof>0 ); - if( rc==SQLITE_OK ){ - vdbeSorterExtendFile(db, pTask->file2.pFd, pTask->file2.iEof); - pTask->file2.iEof = 0; - } + rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd); + pTask->file2.iEof = 0; } if( rc==SQLITE_OK ){ pIncr->aFile[1].pFd = pTask->file2.pFd; pIncr->iStartOff = pTask->file2.iEof; - pTask->file2.iEof += pIncr->mxSz; + pTask->file2.iEof += mxSz; } }else{ - rc = vdbeSorterOpenTempFile(db->pVfs, &pIncr->aFile[0].pFd); + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(db->pVfs, &pIncr->aFile[1].pFd); + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); } } } if( rc==SQLITE_OK && pIncr->bUseThread ){ - /* Use the current thread */ - assert( eMode==INCRINIT2_ROOT || eMode==INCRINIT2_TASK ); + /* Use the current thread to populate aFile[1], even though this + ** iterator is multi-threaded. The reason being that this function + ** is already running in background thread pIncr->pTask->thread. */ + assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); rc = vdbeIncrPopulate(pIncr); } - if( rc==SQLITE_OK && eMode!=INCRINIT2_TASK ){ + if( rc==SQLITE_OK && eMode!=INCRINIT_TASK ){ rc = vdbePmaReaderNext(pIter); } } @@ -1901,13 +1943,13 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ */ static void *vdbePmaReaderBgInit(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; - void *pRet = SQLITE_INT_TO_PTR(vdbePmaReaderIncrInit(pReader,INCRINIT2_TASK)); + void *pRet = SQLITE_INT_TO_PTR(vdbePmaReaderIncrInit(pReader,INCRINIT_TASK)); pReader->pIncr->pTask->bDone = 1; return pRet; } /* -** Use a background thread to invoke vdbePmaReaderIncrInit(INCRINIT2_TASK) +** Use a background thread to invoke vdbePmaReaderIncrInit(INCRINIT_TASK) ** on the the PmaReader object passed as the first argument. ** ** This call will initialize the various fields of the pIter->pIncr @@ -2153,13 +2195,13 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ pMain = 0; } if( rc==SQLITE_OK ){ - int eMode = (pSorter->nTask>1 ? INCRINIT2_ROOT : INCRINIT2_NORMAL); + int eMode = (pSorter->nTask>1 ? INCRINIT_ROOT : INCRINIT_NORMAL); rc = vdbePmaReaderIncrInit(pIter, eMode); } }else #endif { - rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT2_NORMAL); + rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT_NORMAL); pSorter->pMerger = pMain; pMain = 0; } From 958d261bb8fa8b559c76e15f20c90c3623c1c55a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 18 Apr 2014 13:40:07 +0000 Subject: [PATCH 048/710] Fix harmless compiler warnings. FossilOrigin-Name: f8f72ecb9052a4cace1db75879fb8b5131ea4f50 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 5 +++-- src/vdbeInt.h | 4 ++-- src/vdbesort.c | 29 +++++++++++++++++------------ 5 files changed, 31 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index ae525eafb2..58e22e9235 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\strunk\schanges\sinto\sthe\sthreads\sbranch. -D 2014-04-18T12:38:54.966 +C Fix\sharmless\scompiler\swarnings. +D 2014-04-18T13:40:07.511 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -279,14 +279,14 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c cba6c791a3621de633cc018f09c81aa472072d42 +F src/vdbe.c b50cd3009a2e3067746c73dce36153f19df2e42e F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 -F src/vdbeInt.h ba1069627d0ab75e9ddb8f9c10958b86cdbd333d +F src/vdbeInt.h c78ace64dc37495806dd50596eded1f6cd2b5a64 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 8289ed68e2262844334461ccb1b91c4d55b29b0b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c a41721a8e97735597afb80c930f04f195d86a817 +F src/vdbesort.c f93c8aaff5398a702a7b49aae128031e050300b9 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1164,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 12b190db1d20c34390c768614b40ff58a9d3b74c 2c5363873a6f990a0abaacac6303acd46b48befc -R bb69be65b78f81eaebedcfbb221ccbdc +P 8729aa3e3ed1da2e15408ef8705cbe185cd2a5ac +R 1501533958e541baa710100e336c4bc8 U drh -Z b1d3cff21f914d2f704f3c429938823e +Z 328a0fa311f69db138a2df8298545ace diff --git a/manifest.uuid b/manifest.uuid index 4eee3d389b..3ae7a2ee4a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8729aa3e3ed1da2e15408ef8705cbe185cd2a5ac \ No newline at end of file +f8f72ecb9052a4cace1db75879fb8b5131ea4f50 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index c8b5d64477..fa34a0b7f3 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4234,6 +4234,7 @@ case OP_SorterCompare: { assert( pOp->p4type==P4_INT32 ); pIn3 = &aMem[pOp->p3]; nIgnore = pOp->p4.i; + res = 0; rc = sqlite3VdbeSorterCompare(pC, pIn3, nIgnore, &res); VdbeBranchTaken(res!=0,2); if( res ){ @@ -4483,7 +4484,7 @@ case OP_Rewind: { /* jump */ assert( isSorter(pC)==(pOp->opcode==OP_SorterSort) ); res = 1; if( isSorter(pC) ){ - rc = sqlite3VdbeSorterRewind(db, pC, &res); + rc = sqlite3VdbeSorterRewind(pC, &res); }else{ pCrsr = pC->pCursor; assert( pCrsr ); @@ -4642,7 +4643,7 @@ case OP_IdxInsert: { /* in2 */ rc = ExpandBlob(pIn2); if( rc==SQLITE_OK ){ if( isSorter(pC) ){ - rc = sqlite3VdbeSorterWrite(db, pC, pIn2); + rc = sqlite3VdbeSorterWrite(pC, pIn2); }else{ nKey = pIn2->n; zKey = pIn2->z; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 03a303ca1b..97c49446b2 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -442,8 +442,8 @@ void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); -int sqlite3VdbeSorterRewind(sqlite3 *, const VdbeCursor *, int *); -int sqlite3VdbeSorterWrite(sqlite3 *, const VdbeCursor *, Mem *); +int sqlite3VdbeSorterRewind(const VdbeCursor *, int *); +int sqlite3VdbeSorterWrite(const VdbeCursor *, Mem *); int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int, int *); #if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE>0 diff --git a/src/vdbesort.c b/src/vdbesort.c index ec9849bf16..6fc4e4e109 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -589,7 +589,7 @@ static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ int rc = SQLITE_OK; if( pFile->iEof<=(i64)(pTask->pSorter->db->nMaxSorterMmap) ){ - rc = sqlite3OsFetch(pFile->pFd, 0, pFile->iEof, (void**)pp); + rc = sqlite3OsFetch(pFile->pFd, 0, (int)pFile->iEof, (void**)pp); } return rc; } @@ -914,9 +914,9 @@ static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ sqlite3OsCurrentTimeInt64(pTask->pSorter->db->pVfs, &t); fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); } -static void vdbeSorterRewindDebug(sqlite3 *db, const char *zEvent){ +static void vdbeSorterRewindDebug(const char *zEvent){ i64 t; - sqlite3OsCurrentTimeInt64(db->pVfs, &t); + sqlite3OsCurrentTimeInt64(sqlite3_vfs_find(0), &t); fprintf(stderr, "%lld:X %s\n", t, zEvent); } static void vdbeSorterPopulateDebug( @@ -941,7 +941,7 @@ static void vdbeSorterBlockDebug( } #else # define vdbeSorterWorkDebug(x,y) -# define vdbeSorterRewindDebug(x,y) +# define vdbeSorterRewindDebug(y) # define vdbeSorterPopulateDebug(x,y) # define vdbeSorterBlockDebug(x,y,z) #endif @@ -1109,7 +1109,7 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ int rc = sqlite3OsTruncate(pFile, nByte); if( rc==SQLITE_OK ){ void *p = 0; - sqlite3OsFetch(pFile, 0, nByte, &p); + sqlite3OsFetch(pFile, 0, (int)nByte, &p); sqlite3OsUnfetch(pFile, 0, p); } } @@ -1484,6 +1484,7 @@ static int vdbeSorterNext( return rc; } +#if SQLITE_MAX_WORKER_THREADS>0 /* ** The main routine for background threads that write level-0 PMAs. */ @@ -1495,6 +1496,7 @@ static void *vdbeSorterFlushThread(void *pCtx){ pTask->bDone = 1; return SQLITE_INT_TO_PTR(rc); } +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ /* ** Flush the current contents of VdbeSorter.list to a new PMA, possibly @@ -1561,14 +1563,13 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ } return rc; -#endif +#endif /* SQLITE_MAX_WORKER_THREADS!=0 */ } /* ** Add a record to the sorter. */ int sqlite3VdbeSorterWrite( - sqlite3 *db, /* Database handle */ const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ @@ -1643,7 +1644,7 @@ int sqlite3VdbeSorterWrite( pNew = (SorterRecord*)&pSorter->list.aMemory[pSorter->iMemory]; pSorter->iMemory += ROUND8(nReq); - pNew->u.iNext = (u8*)(pSorter->list.pList) - pSorter->list.aMemory; + pNew->u.iNext = (int)((u8*)(pSorter->list.pList) - pSorter->list.aMemory); }else{ pNew = (SorterRecord *)sqlite3Malloc(nReq); if( pNew==0 ){ @@ -1798,6 +1799,7 @@ static int vdbeIncrNew( return rc; } +#if SQLITE_MAX_WORKER_THREADS>0 /* ** Set the "use-threads" flag on object pIncr. */ @@ -1807,6 +1809,7 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ pIncr->pTask->file2.iEof -= pIncr->mxSz; } } +#endif /* SQLITE_MAX_WORKER_THREADS>0 */ #define INCRINIT_NORMAL 0 #define INCRINIT_TASK 1 @@ -2159,13 +2162,15 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ int rc; /* Return code */ SortSubtask *pTask0 = &pSorter->aTask[0]; MergeEngine *pMain = 0; +#if SQLITE_MAX_WORKER_THREADS sqlite3 *db = pTask0->pSorter->db; - int iTask; +#endif rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS if( pSorter->bUseThreads ){ + int iTask; PmaReader *pIter; SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; rc = vdbeSortAllocUnpacked(pLast); @@ -2219,7 +2224,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ ** this function is called to prepare for iterating through the records ** in sorted order. */ -int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ +int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ VdbeSorter *pSorter = pCsr->pSorter; int rc = SQLITE_OK; /* Return code */ @@ -2246,7 +2251,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ /* Join all threads */ rc = vdbeSorterJoinAll(pSorter, rc); - vdbeSorterRewindDebug(db, "rewind"); + vdbeSorterRewindDebug("rewind"); /* Assuming no errors have occurred, set up a merger structure to ** incrementally read and merge all remaining PMAs. */ @@ -2256,7 +2261,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ *pbEof = 0; } - vdbeSorterRewindDebug(db, "rewinddone"); + vdbeSorterRewindDebug("rewinddone"); return rc; } From 46a06bbe47c043cbc034b4e94f7b08eefb6172be Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 18 Apr 2014 13:57:39 +0000 Subject: [PATCH 049/710] Add to speedtest1.c the --threads option for setting the SQLITE_CONFIG_WORKER_THREADS configuration. FossilOrigin-Name: 5fce40c44aacf883df2e8e9472c399a6e92197b3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/speedtest1.c | 10 ++++++++++ 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 58e22e9235..644d369a1a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2014-04-18T13:40:07.511 +C Add\sto\sspeedtest1.c\sthe\s--threads\soption\sfor\ssetting\sthe\nSQLITE_CONFIG_WORKER_THREADS\sconfiguration. +D 2014-04-18T13:57:39.195 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -829,7 +829,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 90446861e566a9965a8d005381a3c964ff333646 +F test/speedtest1.c bd150a4c29e26ff4ddbdd505e0a0d96347ffca51 F test/spellfix.test 61309f5efbec53603b3f86457d34a504f80abafe F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1164,7 +1164,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8729aa3e3ed1da2e15408ef8705cbe185cd2a5ac -R 1501533958e541baa710100e336c4bc8 +P f8f72ecb9052a4cace1db75879fb8b5131ea4f50 +R ec8b910c95550d011d611a610b4db207 U drh -Z 328a0fa311f69db138a2df8298545ace +Z 39c3a2898ecaace6ee1641d24df0dd67 diff --git a/manifest.uuid b/manifest.uuid index 3ae7a2ee4a..f698cc6543 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8f72ecb9052a4cace1db75879fb8b5131ea4f50 \ No newline at end of file +5fce40c44aacf883df2e8e9472c399a6e92197b3 \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 28c24e0885..7d38b37b0d 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -27,6 +27,7 @@ static const char zHelp[] = " --stats Show statistics at the end\n" " --testset T Run test-set T\n" " --trace Turn on SQL tracing\n" + " --threads N Use up to N threads for sorting\n" " --utf16be Set text encoding to UTF-16BE\n" " --utf16le Set text encoding to UTF-16LE\n" " --without-rowid Use WITHOUT ROWID where appropriate\n" @@ -962,6 +963,7 @@ int main(int argc, char **argv){ int nPCache = 0, szPCache = 0;/* --pcache configuration */ int nScratch = 0, szScratch=0;/* --scratch configuration */ int showStats = 0; /* True for --stats */ + int nThread = 0; /* --threads value */ const char *zTSet = "main"; /* Which --testset torun */ int doTrace = 0; /* True for --trace */ const char *zEncoding = 0; /* --utf16be or --utf16le */ @@ -1046,6 +1048,9 @@ int main(int argc, char **argv){ zTSet = argv[++i]; }else if( strcmp(z,"trace")==0 ){ doTrace = 1; + }else if( strcmp(z,"threads")==0 ){ + if( i>=argc-1 ) fatal_error("missing argument on %s\n", argv[i]); + nThread = integerValue(argv[++i]); }else if( strcmp(z,"utf16le")==0 ){ zEncoding = "utf16le"; }else if( strcmp(z,"utf16be")==0 ){ @@ -1092,6 +1097,11 @@ int main(int argc, char **argv){ rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, pScratch, szScratch, nScratch); if( rc ) fatal_error("scratch configuration failed: %d\n", rc); } +#ifdef SQLITE_CONFIG_WORKER_THREADS + if( nThread>0 ){ + sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, nThread); + } +#endif if( nLook>0 ){ sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0); } From 3de4df27dda707357d3fc62bf1c318a7a2e9dfeb Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 24 Apr 2014 12:28:28 +0000 Subject: [PATCH 050/710] Improvements to comments. Store some extra information in SqliteThread that is useful for debugging. FossilOrigin-Name: 9fb5e212089d85cdd3b4787dd69c72e6d84560b6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/threads.c | 10 +++++++--- src/vdbesort.c | 17 ++++++++++------- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 2caa7ed483..b71dec8c60 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\strunk\schanges\sinto\sthe\sthreads\sbranch. -D 2014-04-23T12:57:55.344 +C Improvements\sto\scomments.\s\sStore\ssome\sextra\sinformation\sin\sSqliteThread\sthat\nis\suseful\sfor\sdebugging. +D 2014-04-24T12:28:28.628 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -272,7 +272,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c b8e7232f2b9c9d148d6886117160de394d172f85 +F src/threads.c e35de159f9ced746266ff4b2129b7a828e59119c F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 8289ed68e2262844334461ccb1b91c4d55b29b0b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c f93c8aaff5398a702a7b49aae128031e050300b9 +F src/vdbesort.c 2b13026e5d4afa4737a312715aa1cd7e2f4d07c2 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1165,7 +1165,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5fce40c44aacf883df2e8e9472c399a6e92197b3 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb -R 07151f40a802d55a98a20acba34885fa +P e2c9f71a451e44040624b9f255b4510743513019 +R 85ab93c26e2f3dfc5f24d719f3aa0846 U drh -Z 85be1e117efe898356ded29b053b1ff8 +Z 26e90d38e785197a28b1e4097f1c9b04 diff --git a/manifest.uuid b/manifest.uuid index 3cda1ddaf8..6ee6407693 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2c9f71a451e44040624b9f255b4510743513019 \ No newline at end of file +9fb5e212089d85cdd3b4787dd69c72e6d84560b6 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index eeab5379ce..64975801be 100644 --- a/src/threads.c +++ b/src/threads.c @@ -37,9 +37,11 @@ /* A running thread */ struct SQLiteThread { - pthread_t tid; - int done; - void *pOut; + pthread_t tid; /* Thread ID */ + int done; /* Set to true when thread finishes */ + void *pOut; /* Result returned by the thread */ + void *(*xTask)(void*); /* The thread routine */ + void *pIn; /* Argument to the thread */ }; /* Create a new thread */ @@ -56,6 +58,8 @@ int sqlite3ThreadCreate( p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); + p->xTask = xTask; + p->pIn = pIn; if( sqlite3GlobalConfig.bCoreMutex==0 || pthread_create(&p->tid, 0, xTask, pIn)!=0 ){ diff --git a/src/vdbesort.c b/src/vdbesort.c index 6fc4e4e109..46de735284 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -15,7 +15,7 @@ ** using indexes and without LIMIT clauses. ** ** The VdbeSorter object implements a multi-threaded external merge sort -** algorithm that is efficient even if the number of element being sorted +** algorithm that is efficient even if the number of elements being sorted ** exceeds the available memory. ** ** Here is the (internal, non-API) interface between this module and the @@ -104,9 +104,7 @@ ** ** When Rewind() is called, any data remaining in memory is flushed to a ** final PMA. So at this point the data is stored in some number of sorted -** PMAs within temporary files on disk. Within a single file sorter is -** running in single threaded mode, or distributed between one or more files -** for multi-threaded sorters. +** PMAs within temporary files on disk. ** ** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the ** sorter is running in single-threaded mode, then these PMAs are merged @@ -158,7 +156,7 @@ typedef struct SorterRecord SorterRecord; /* A record being sorted */ typedef struct SortSubtask SortSubtask; /* A sub-task in the sort process */ typedef struct SorterFile SorterFile; /* Temporary file object wrapper */ typedef struct SorterList SorterList; /* In-memory list of records */ -typedef struct IncrMerger IncrMerger; +typedef struct IncrMerger IncrMerger; /* Read & merge multiple PMAs */ /* ** A container for a temp file handle and the current amount of data @@ -170,11 +168,16 @@ struct SorterFile { }; /* -** In memory linked list of records. +** An in-memory list of objects to be sorted. +** +** If aMemory==0 then each object is allocated separately and the objects +** are connected using SorterRecord.u.pNext. If aMemory!=0 then all objects +** are stored in the aMemory[] bulk memory, one right after the other, and +** are connected using SorterRecord.u.iNext. */ struct SorterList { SorterRecord *pList; /* Linked list of records */ - u8 *aMemory; /* If non-NULL, blob of memory for pList */ + u8 *aMemory; /* If non-NULL, bulk memory to hold pList */ int szPMA; /* Size of pList as PMA in bytes */ }; From f690c142806a545dbea5c3ed8cc3680ef2cbd78b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 24 Apr 2014 16:25:25 +0000 Subject: [PATCH 051/710] Improved header comment on the vdbesort.c module. No changes to code. FossilOrigin-Name: bf09ce24d054bc68c226064f5f28d97e0e648a13 --- manifest | 12 +++++------ manifest.uuid | 2 +- src/vdbesort.c | 56 +++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 58 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0187e05510..b7837cbb94 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reopen\sthe\sorderby-planning\sbranch\sand\smerge\sin\sthe\slatest\strunk\senhancements\nand\sfixes. -D 2014-04-24T15:06:25.003 +C Improved\sheader\scomment\son\sthe\svdbesort.c\smodule.\s\sNo\schanges\sto\scode. +D 2014-04-24T16:25:25.812 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c ad0f9f717a73b870c12c0a0f47781b8b042a5348 +F src/vdbesort.c 469ae9af4115779b527b47edd53bd9a0943f7906 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1161,7 +1161,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 58f7ca29303c280229f86d01f418e1de5f5aebba 65d2544af9adc1e2f1d193e57f8be0422fb0d5eb -R b46cdf624d79e68eb1974021b4d3f992 +P 6077ddcd93318e24b9756adaaf293ba9fb3cedf7 +R f7111ea680230454a4775d5c337b77f9 U drh -Z d3c484ede684435a1175d26a414ba92a +Z b4087c0ba17f1174753bea8bd453086a diff --git a/manifest.uuid b/manifest.uuid index 4866c44be0..a672982fcb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6077ddcd93318e24b9756adaaf293ba9fb3cedf7 \ No newline at end of file +bf09ce24d054bc68c226064f5f28d97e0e648a13 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 791a2465ca..6a34cefa61 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1,5 +1,5 @@ /* -** 2011 July 9 +** 2011-07-09 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -10,9 +10,55 @@ ** ************************************************************************* ** This file contains code for the VdbeSorter object, used in concert with -** a VdbeCursor to sort large numbers of keys (as may be required, for -** example, by CREATE INDEX statements on tables too large to fit in main -** memory). +** a VdbeCursor to sort large numbers of keys for CREATE TABLE statements +** or by SELECT statements with ORDER BY clauses that cannot be satisfied +** using indexes and without LIMIT clauses. +** +** The VdbeSorter object implements a external merge sort +** algorithm that is efficient even if the aggregate size of +** the elements being sorted exceeds the available memory. +** +** Here is the (internal, non-API) interface between this module and the +** rest of the SQLite system: +** +** sqlite3VdbeSorterInit() Create a new VdbeSorter object. +** +** sqlite3VdbeSorterWrite() Add a single new row to the VdbeSorter +** object. The row is a binary blob in the +** OP_MakeRecord format that contains both +** the ORDER BY key columns and result columns +** in the case of a SELECT w/ ORDER BY, or +** the complete record for an index entry +** in the case of a CREATE INDEX. +** +** sqlite3VdbeSorterRewind() Sort all content previously added. +** Position the read cursor on the +** first sorted element. +** +** sqlite3VdbeSorterNext() Advance the read cursor to the next sorted +** element. +** +** sqlite3VdbeSorterRowkey() Return the complete binary blob for the +** row currently under the read cursor. +** +** sqlite3VdbeSorterCompare() Compare the binary blob for the row +** currently under the read cursor against +** another binary blob X and report if +** X is strictly less than the read cursor. +** Used to enforce uniqueness in a +** CREATE UNIQUE INDEX statement. +** +** sqlite3VdbeSorterClose() Close the VdbeSorter object and reclaim +** all resources. +** +** sqlite3VdbeSorterReset() Refurbish the VdbeSorter for reuse. This +** is like Close() followed by Init() only +** much faster. +** +** The interfaces above must be called in a particular order. Write() can +** only occur in between Init()/Reset() and Rewind(). Next(), Rowkey(), and +** Compare() can only occur in between Rewind() and Close()/Reset(). +** */ #include "sqliteInt.h" @@ -893,7 +939,7 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ */ int sqlite3VdbeSorterWrite( sqlite3 *db, /* Database handle */ - const VdbeCursor *pCsr, /* Sorter cursor */ + const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal /* Memory cell containing record */ ){ VdbeSorter *pSorter = pCsr->pSorter; From 7a0fd192ccf4efba952b38a0d5a7b38d18ba19fe Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 15:25:24 +0000 Subject: [PATCH 052/710] Fix a faulty assert() statement. FossilOrigin-Name: 9196ce407379ca3b151b601b98848771e5cb4e8f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6166275bb9..789773a941 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\senhancements\sand\sfixes\sinto\sthe\sorderby-planning\sbranch. -D 2014-05-02T13:09:06.754 +C Fix\sa\sfaulty\sassert()\sstatement. +D 2014-05-02T15:25:24.157 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 469ae9af4115779b527b47edd53bd9a0943f7906 +F src/vdbesort.c d0fc5ecd19650b335e58632e60c9b8585b839c65 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P bf09ce24d054bc68c226064f5f28d97e0e648a13 3bc43594aaeee9225c0590677fcce480bedcb37b -R 54b60e57b7c4903b7427b2471b549ff9 +P 84862d3a095629d20c8e7b8a16f4dc26cd41ab6d +R f6e2655ab9521eb2aa3af53a52824666 U drh -Z 3f220c7986d95ea2653ad2dab9b5d42e +Z 7c7e4b02db28ce1795a4d7aa65a0e17d diff --git a/manifest.uuid b/manifest.uuid index 2809ea21e5..a5c49533a7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84862d3a095629d20c8e7b8a16f4dc26cd41ab6d \ No newline at end of file +9196ce407379ca3b151b601b98848771e5cb4e8f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 6a34cefa61..70fbbcf425 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -733,10 +733,10 @@ static int vdbeSorterSort(const VdbeCursor *pCsr){ while( p ){ SorterRecord *pNext; if( pSorter->aMemory ){ - assert( p->u.iNextnMemory ); if( (u8*)p==pSorter->aMemory ){ pNext = 0; }else{ + assert( p->u.iNextnMemory ); pNext = (SorterRecord*)&pSorter->aMemory[p->u.iNext]; } }else{ From 012e133772cec1e77fb132eab406d0af0ad636c9 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 May 2014 16:03:57 +0000 Subject: [PATCH 053/710] Remove a faulty assert() from vdbesort.c. FossilOrigin-Name: d95d68aa1d14b750888d50068380cc107f9070df --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b71dec8c60..07d5ceadf1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\scomments.\s\sStore\ssome\sextra\sinformation\sin\sSqliteThread\sthat\nis\suseful\sfor\sdebugging. -D 2014-04-24T12:28:28.628 +C Remove\sa\sfaulty\sassert()\sfrom\svdbesort.c. +D 2014-05-02T16:03:57.598 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 8289ed68e2262844334461ccb1b91c4d55b29b0b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 2b13026e5d4afa4737a312715aa1cd7e2f4d07c2 +F src/vdbesort.c 0095545ae3786d00c9104d036f5d092953a1e2d3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1165,7 +1165,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e2c9f71a451e44040624b9f255b4510743513019 -R 85ab93c26e2f3dfc5f24d719f3aa0846 -U drh -Z 26e90d38e785197a28b1e4097f1c9b04 +P 9fb5e212089d85cdd3b4787dd69c72e6d84560b6 +R ee91b52dc43b32be4c405d028722e931 +U dan +Z 8b01b62c4d596e9ee4b370ddc9560981 diff --git a/manifest.uuid b/manifest.uuid index 6ee6407693..89f9d98d45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9fb5e212089d85cdd3b4787dd69c72e6d84560b6 \ No newline at end of file +d95d68aa1d14b750888d50068380cc107f9070df \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 46de735284..daf2477532 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1900,7 +1900,6 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; - assert( eMode==INCRINIT_NORMAL || pIncr->bUseThread==1 ); rc = vdbeIncrInitMerger(pTask, pIncr->pMerger, eMode); /* Set up the required files for pIncr. A multi-theaded IncrMerge object From c7f6c148d9b4dcf7c19b3edd19c82d8738d4048e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 16:22:55 +0000 Subject: [PATCH 054/710] Failure to extend a temp file for use with mmap() in vdbesort.c is benign. FossilOrigin-Name: d4d396387d373bd1e82eda2c7c2e7ca35ec099c4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 11 +++++------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 789773a941..d6c9c71387 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sstatement. -D 2014-05-02T15:25:24.157 +C Failure\sto\sextend\sa\stemp\sfile\sfor\suse\swith\smmap()\sin\svdbesort.c\sis\sbenign. +D 2014-05-02T16:22:55.791 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c d0fc5ecd19650b335e58632e60c9b8585b839c65 +F src/vdbesort.c 6bcf73fb160ee5bb8ce8a8ec61fda268b081dbb7 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 84862d3a095629d20c8e7b8a16f4dc26cd41ab6d -R f6e2655ab9521eb2aa3af53a52824666 +P 9196ce407379ca3b151b601b98848771e5cb4e8f +R 51e21c516ddbd5830d43ea54f991a56e U drh -Z 7c7e4b02db28ce1795a4d7aa65a0e17d +Z 19bdd7c6b9f54ab4b2fbcff19d13a623 diff --git a/manifest.uuid b/manifest.uuid index a5c49533a7..e8d80914a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9196ce407379ca3b151b601b98848771e5cb4e8f \ No newline at end of file +d4d396387d373bd1e82eda2c7c2e7ca35ec099c4 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 70fbbcf425..86fef69256 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -857,17 +857,16 @@ static void fileWriterWriteVarint(FileWriter *p, u64 iVal){ ** Whether or not the file does end up memory mapped of course depends on ** the specific VFS implementation. */ -static int vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ +static void vdbeSorterExtendFile(sqlite3_file *pFile, i64 nByte){ int rc = sqlite3OsTruncate(pFile, nByte); if( rc==SQLITE_OK ){ void *p = 0; sqlite3OsFetch(pFile, 0, nByte, &p); sqlite3OsUnfetch(pFile, 0, p); } - return rc; } #else -# define vdbeSorterExtendFile(x,y) SQLITE_OK +# define vdbeSorterExtendFile(x,y) #endif /* @@ -907,8 +906,8 @@ static int vdbeSorterListToPMA(sqlite3 *db, const VdbeCursor *pCsr){ /* Try to get the file to memory map */ if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile( - pSorter->pTemp1, pSorter->iWriteOff + pSorter->nInMemory + 9 + vdbeSorterExtendFile( + pSorter->pTemp1, pSorter->iWriteOff + pSorter->nInMemory + 9 ); } @@ -1132,7 +1131,7 @@ int sqlite3VdbeSorterRewind(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ assert( iWrite2==0 ); rc = vdbeSorterOpenTempFile(db, &pTemp2); if( rc==SQLITE_OK ){ - rc = vdbeSorterExtendFile(pTemp2, pSorter->iWriteOff); + vdbeSorterExtendFile(pTemp2, pSorter->iWriteOff); } } From a7bf23c6b562711263b559c76a70f78036304726 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 2 May 2014 17:12:41 +0000 Subject: [PATCH 055/710] Fix some broken asserts in btree.c and vdbeaux.c that may fail following an OOM error. FossilOrigin-Name: e15f47064bef431c0afd8bf93eb4e160c23ad562 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 2 +- src/vdbeaux.c | 13 ++++++++----- test/malloc.test | 20 ++++++++++++++++++++ 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 07d5ceadf1..1b2b4c23fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sfaulty\sassert()\sfrom\svdbesort.c. -D 2014-05-02T16:03:57.598 +C Fix\ssome\sbroken\sasserts\sin\sbtree.c\sand\svdbeaux.c\sthat\smay\sfail\sfollowing\san\sOOM\serror. +D 2014-05-02T17:12:41.912 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 1e8fa0e597f73cc1b144285065f175f5a661ee75 +F src/btree.c e14daeed62b5b1499835a6523f9a47d660c297c8 F src/btree.h d79306df4ed9181b48916737fe8871a4392c4594 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 9ea11e29230d8f454580870465354c707bd42851 @@ -283,7 +283,7 @@ F src/vdbe.c b50cd3009a2e3067746c73dce36153f19df2e42e F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h c78ace64dc37495806dd50596eded1f6cd2b5a64 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 8289ed68e2262844334461ccb1b91c4d55b29b0b +F src/vdbeaux.c 6e6993fa4be39ab96bdd565994bee8ec3bef06ab F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 0095545ae3786d00c9104d036f5d092953a1e2d3 @@ -663,7 +663,7 @@ F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 F test/lookaside.test 93f07bac140c5bb1d49f3892d2684decafdc7af2 F test/main.test 39c4bb8a157f57298ed1659d6df89d9f35aaf2c8 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 -F test/malloc.test 26ae08a09cc15a98d147ee63925e3a66048e71c9 +F test/malloc.test ada8ae15193ffc6415bf431a6db31d4ec507dbfe F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 F test/malloc5.test fafce0aa9157060445cd1a56ad50fc79d82f28c3 @@ -1165,7 +1165,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9fb5e212089d85cdd3b4787dd69c72e6d84560b6 -R ee91b52dc43b32be4c405d028722e931 +P d95d68aa1d14b750888d50068380cc107f9070df +R 2642a9b7016b11d8b125fe44a5d1acd4 U dan -Z 8b01b62c4d596e9ee4b370ddc9560981 +Z 60595d854f9f1dca901aef6c64df6a77 diff --git a/manifest.uuid b/manifest.uuid index 89f9d98d45..7ac8504d07 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d95d68aa1d14b750888d50068380cc107f9070df \ No newline at end of file +e15f47064bef431c0afd8bf93eb4e160c23ad562 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index a1d888a137..f8f0373b65 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4735,7 +4735,7 @@ int sqlite3BtreeMovetoUnpacked( } assert( (pIdxKey->errCode!=SQLITE_CORRUPT || c==0) - && (pIdxKey->errCode!=SQLITE_NOMEM || !pCur->pBtree->db->mallocFailed) + && (pIdxKey->errCode!=SQLITE_NOMEM || pCur->pBtree->db->mallocFailed) ); if( c<0 ){ lwr = idx+1; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index e7f4b12798..360525a5da 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3593,8 +3593,9 @@ int sqlite3VdbeRecordCompare( /* rc==0 here means that one or both of the keys ran out of fields and ** all the fields up to that point were equal. Return the the default_rc ** value. */ - assert( CORRUPT_DB + assert( CORRUPT_DB || pKeyInfo->db==0 || pPKey2->default_rc==vdbeRecordCompareDebug(nKey1, pKey1, pPKey2) + || pKeyInfo->db->mallocFailed ); return pPKey2->default_rc; } @@ -3692,10 +3693,11 @@ static int vdbeRecordCompareInt( res = pPKey2->default_rc; } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) + assert( pPKey2->pKeyInfo->db==0 + || (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || CORRUPT_DB + || CORRUPT_DB || pPKey2->pKeyInfo->db->mallocFailed ); return res; } @@ -3756,10 +3758,11 @@ static int vdbeRecordCompareString( } } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) + assert( pPKey2->pKeyInfo->db==0 + || (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || CORRUPT_DB + || CORRUPT_DB || pPKey2->pKeyInfo->db->mallocFailed ); return res; } diff --git a/test/malloc.test b/test/malloc.test index 10d2a18c96..a2a7682777 100644 --- a/test/malloc.test +++ b/test/malloc.test @@ -902,6 +902,26 @@ do_faultsim_test 40.3 -faults oom-trans* -body { faultsim_integrity_check } +reset_db +add_test_utf16bin_collate db +set big [string repeat x 200] +do_execsql_test 41.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a COLLATE utf16bin); + INSERT INTO t1 VALUES('fghij' || $::big); + INSERT INTO t1 VALUES('pqrst' || $::big); + INSERT INTO t1 VALUES('abcde' || $::big); + INSERT INTO t1 VALUES('uvwxy' || $::big); + INSERT INTO t1 VALUES('klmno' || $::big); + CREATE INDEX i1 ON t1(a); +} +do_faultsim_test 41.2 -faults oom* -body { + execsql { SELECT * FROM t1 WHERE a = ('abcde' || $::big)} +} -test { + faultsim_test_result [list 0 "abcde$::big"] + faultsim_integrity_check +} + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} From 79211e194d465fba319faae7fbf70774a12415ae Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 17:33:16 +0000 Subject: [PATCH 056/710] Simplify assert() statements used to verify correct operation of record comparison routines. FossilOrigin-Name: 3300d62dcbe74842cf86ca436959fe4e77a89f84 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 40 +++++++++++++++++++--------------------- 3 files changed, 26 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index d6c9c71387..8fd72b4b86 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Failure\sto\sextend\sa\stemp\sfile\sfor\suse\swith\smmap()\sin\svdbesort.c\sis\sbenign. -D 2014-05-02T16:22:55.791 +C Simplify\sassert()\sstatements\sused\sto\sverify\scorrect\soperation\sof\s\nrecord\scomparison\sroutines. +D 2014-05-02T17:33:16.279 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/vdbe.c b3510cc71f706beffc66e2aa4bbda54bcd5e9668 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 +F src/vdbeaux.c c9a8c917776c941af99285594d5c30d99e21c99a F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 6bcf73fb160ee5bb8ce8a8ec61fda268b081dbb7 @@ -1166,7 +1166,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9196ce407379ca3b151b601b98848771e5cb4e8f -R 51e21c516ddbd5830d43ea54f991a56e +P d4d396387d373bd1e82eda2c7c2e7ca35ec099c4 +R defe6572bf21263fd4b2b33706bcd8e7 U drh -Z 19bdd7c6b9f54ab4b2fbcff19d13a623 +Z 26e6dec065eafceab8263c10d377378f diff --git a/manifest.uuid b/manifest.uuid index e8d80914a0..ea44721a8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4d396387d373bd1e82eda2c7c2e7ca35ec099c4 \ No newline at end of file +3300d62dcbe74842cf86ca436959fe4e77a89f84 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0a6b536720..172d12042b 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3137,10 +3137,14 @@ void sqlite3VdbeRecordUnpack( ** sqlite3VdbeSerialGet() and sqlite3MemCompare() functions. It is used ** in assert() statements to ensure that the optimized code in ** sqlite3VdbeRecordCompare() returns results with these two primitives. +** +** Return true if the result of comparison is equivalent to desiredResult. +** Return false if there is a disagreement. */ static int vdbeRecordCompareDebug( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2 /* Right key */ + const UnpackedRecord *pPKey2, /* Right key */ + int desiredResult /* Correct answer */ ){ u32 d1; /* Offset into aKey[] of next data element */ u32 idx1; /* Offset into aKey[] of next header element */ @@ -3202,7 +3206,7 @@ static int vdbeRecordCompareDebug( if( pKeyInfo->aSortOrder[i] ){ rc = -rc; /* Invert the result for DESC sort order. */ } - return rc; + goto debugCompareEnd; } i++; }while( idx1nField ); @@ -3216,7 +3220,15 @@ static int vdbeRecordCompareDebug( /* rc==0 here means that one of the keys ran out of fields and ** all the fields up to that point were equal. Return the the default_rc ** value. */ - return pPKey2->default_rc; + rc = pPKey2->default_rc; + +debugCompareEnd: + if( desiredResult==0 && rc==0 ) return 1; + if( desiredResult<0 && rc<0 ) return 1; + if( desiredResult>0 && rc>0 ) return 1; + if( CORRUPT_DB ) return 1; + if( pKeyInfo->db->mallocFailed ) return 1; + return 0; } #endif @@ -3564,11 +3576,7 @@ int sqlite3VdbeRecordCompare( if( pKeyInfo->aSortOrder[i] ){ rc = -rc; } - assert( CORRUPT_DB - || (rc<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (rc>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || pKeyInfo->db->mallocFailed - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, rc) ); assert( mem1.zMalloc==0 ); /* See comment below */ return rc; } @@ -3587,9 +3595,7 @@ int sqlite3VdbeRecordCompare( /* rc==0 here means that one or both of the keys ran out of fields and ** all the fields up to that point were equal. Return the the default_rc ** value. */ - assert( CORRUPT_DB - || pPKey2->default_rc==vdbeRecordCompareDebug(nKey1, pKey1, pPKey2) - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) ); return pPKey2->default_rc; } @@ -3686,11 +3692,7 @@ static int vdbeRecordCompareInt( res = pPKey2->default_rc; } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) - || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || CORRUPT_DB - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) ); return res; } @@ -3750,11 +3752,7 @@ static int vdbeRecordCompareString( } } - assert( (res==0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)==0) - || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) - || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) - || CORRUPT_DB - ); + assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, res) ); return res; } From 84de690b4f7c46b9dd70bdf6b24b4089f19475ef Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 2 May 2014 18:46:52 +0000 Subject: [PATCH 057/710] Do not run the vdbeRecordCompareDebug() assert if pKeyInfo->db is NULL since in that case there would be no way to check for a memory allocation failure. FossilOrigin-Name: 63ed2d6acb82be8a74dbf6a61388be6da6113985 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ac820ac5a9..d82c20faf7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sorderby-planning\swith\sthis\sbranch. -D 2014-05-02T18:05:38.568 +C Do\snot\srun\sthe\svdbeRecordCompareDebug()\sassert\sif\spKeyInfo->db\sis\sNULL\ssince\nin\sthat\scase\sthere\swould\sbe\sno\sway\sto\scheck\sfor\sa\smemory\sallocation\sfailure. +D 2014-05-02T18:46:52.058 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,7 +284,7 @@ F src/vdbe.c 89ab2ded5123e823b47293aedd7931a4742fb6bd F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h c78ace64dc37495806dd50596eded1f6cd2b5a64 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c f651c4de1193326597b3da2b8f91d993e8a6e97b +F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 0095545ae3786d00c9104d036f5d092953a1e2d3 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e15f47064bef431c0afd8bf93eb4e160c23ad562 3300d62dcbe74842cf86ca436959fe4e77a89f84 -R 6fa5ff1df255fa8dfad56d587950c625 -U dan -Z 63362d13bfcb7640057f1ca0f3721e45 +P d9549de31741239ece060e448b592ce8fc5b8042 +R bd79c5eed170ef831b62fddde9ae2198 +U drh +Z beeab334f565837c4ac52f73c3e92e7b diff --git a/manifest.uuid b/manifest.uuid index 55e5e7ae81..fc320531a2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9549de31741239ece060e448b592ce8fc5b8042 \ No newline at end of file +63ed2d6acb82be8a74dbf6a61388be6da6113985 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 882bb6908e..3f52dd6017 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3156,6 +3156,7 @@ static int vdbeRecordCompareDebug( Mem mem1; pKeyInfo = pPKey2->pKeyInfo; + if( pKeyInfo->db==0 ) return 1; mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */ From e500747cfee58695a2f02df0b9cf095746f0a574 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 2 May 2014 19:12:37 +0000 Subject: [PATCH 058/710] Fix typo in Windows makefile. Make sure the WaitForSingleObjectEx system call is always available. FossilOrigin-Name: d7ed529fa2aa5cb13edaabca2acaad06dffef569 --- Makefile.msc | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_win.c | 4 ---- 4 files changed, 10 insertions(+), 14 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 226d464785..db47f02755 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -564,7 +564,7 @@ LIBOBJS0 = vdbe.lo parse.lo alter.lo analyze.lo attach.lo auth.lo \ notify.lo opcodes.lo os.lo os_unix.lo os_win.lo \ pager.lo pcache.lo pcache1.lo pragma.lo prepare.lo printf.lo \ random.lo resolve.lo rowset.lo rtree.lo select.lo status.lo \ - table.lo threads.o tokenize.lo trigger.lo \ + table.lo threads.lo tokenize.lo trigger.lo \ update.lo util.lo vacuum.lo \ vdbeapi.lo vdbeaux.lo vdbeblob.lo vdbemem.lo vdbesort.lo \ vdbetrace.lo wal.lo walker.lo where.lo utf.lo vtab.lo diff --git a/manifest b/manifest index d82c20faf7..68b54aaa89 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Do\snot\srun\sthe\svdbeRecordCompareDebug()\sassert\sif\spKeyInfo->db\sis\sNULL\ssince\nin\sthat\scase\sthere\swould\sbe\sno\sway\sto\scheck\sfor\sa\smemory\sallocation\sfailure. -D 2014-05-02T18:46:52.058 +C Fix\stypo\sin\sWindows\smakefile.\s\sMake\ssure\sthe\sWaitForSingleObjectEx\ssystem\scall\sis\salways\savailable. +D 2014-05-02T19:12:37.015 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 153eb9b9725bc7b8e4dbe963219298e0c4a644b0 +F Makefile.msc 7d6981e8e14d8189a2a2f9b4a485f40b2032cb2e F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c @@ -205,7 +205,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f -F src/os_win.c 8245fe9184300e641d02e29a8ca95cefe0cb0fd0 +F src/os_win.c 187fad4d385b3b26ec6fd4b703b1b087ad6a5c4d F src/pager.c ab62a24218d87dda1be641f6c5ad291bff78fd94 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d9549de31741239ece060e448b592ce8fc5b8042 -R bd79c5eed170ef831b62fddde9ae2198 -U drh -Z beeab334f565837c4ac52f73c3e92e7b +P 63ed2d6acb82be8a74dbf6a61388be6da6113985 +R 820b790c56c4122e03b250f07fbb06ef +U mistachkin +Z 87a0e192077836669877b548f0fe55d8 diff --git a/manifest.uuid b/manifest.uuid index fc320531a2..30c06534cc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -63ed2d6acb82be8a74dbf6a61388be6da6113985 \ No newline at end of file +d7ed529fa2aa5cb13edaabca2acaad06dffef569 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 44eb7e686f..10fea2638c 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -948,11 +948,7 @@ static struct win_syscall { #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ DWORD))aSyscall[63].pCurrent) -#if SQLITE_OS_WINRT { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, -#else - { "WaitForSingleObjectEx", (SYSCALL)0, 0 }, -#endif #define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ BOOL))aSyscall[64].pCurrent) From a09c8855b7d41088d40df18b4b7b41a0c0c7ad28 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 3 May 2014 11:22:09 +0000 Subject: [PATCH 059/710] Add the SQLITE_DEFAULT_WORKER_THREADS compile-time option. Fix a NULL-pointer dereference that can occur following OOM. FossilOrigin-Name: e0dea89b3e9f295f80210fcca007681bf1b08692 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/global.c | 2 +- src/sqliteInt.h | 7 +++++++ src/vdbesort.c | 1 + 5 files changed, 19 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 68b54aaa89..a8edf3f089 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\sWindows\smakefile.\s\sMake\ssure\sthe\sWaitForSingleObjectEx\ssystem\scall\sis\salways\savailable. -D 2014-05-02T19:12:37.015 +C Add\sthe\sSQLITE_DEFAULT_WORKER_THREADS\scompile-time\soption.\s\nFix\sa\sNULL-pointer\sdereference\sthat\scan\soccur\sfollowing\sOOM. +D 2014-05-03T11:22:09.616 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -178,7 +178,7 @@ F src/expr.c 4f9e497c66e2f25a4d139357a778c84d5713207c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 -F src/global.c deadd872189b92aca4ee2566332a86315839f811 +F src/global.c b7943ff485c31660ec0b17c68467034804df01b1 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -223,7 +223,7 @@ F src/shell.c 6946aea9f21af551fa84bc6b2a8de55d93bf0004 F src/sqlite.h.in 579aebacdea59386d9cdf01fd4a16f4cafbb248f F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 76bd7ba54008909d9e437807565a54b1075983dc +F src/sqliteInt.h 851003126071d4a3bac86a0db75c48197fbd0ff0 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 0095545ae3786d00c9104d036f5d092953a1e2d3 +F src/vdbesort.c d205b56d0a1c2cbd8f6c8c4f513337ab0096d0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 63ed2d6acb82be8a74dbf6a61388be6da6113985 -R 820b790c56c4122e03b250f07fbb06ef -U mistachkin -Z 87a0e192077836669877b548f0fe55d8 +P d7ed529fa2aa5cb13edaabca2acaad06dffef569 +R bf6d771799174fcada0f5c36289734b7 +U drh +Z 521faa6705b79213c3ba83b826f10b41 diff --git a/manifest.uuid b/manifest.uuid index 30c06534cc..03cf71aa4d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d7ed529fa2aa5cb13edaabca2acaad06dffef569 \ No newline at end of file +e0dea89b3e9f295f80210fcca007681bf1b08692 \ No newline at end of file diff --git a/src/global.c b/src/global.c index 091b750803..4cd637f83b 100644 --- a/src/global.c +++ b/src/global.c @@ -167,7 +167,7 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ - 0, /* nWorker */ + SQLITE_DEFAULT_WORKER_THREADS, /* nWorker */ /* All the rest should always be initialized to zero */ 0, /* isInit */ 0, /* inProgress */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0c8fe65003..43f5d34d66 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -433,6 +433,13 @@ #ifndef SQLITE_MAX_WORKER_THREADS # define SQLITE_MAX_WORKER_THREADS 0 #endif +#ifndef SQLITE_DEFAULT_WORKER_THREADS +# define SQLITE_DEFAULT_WORKER_THREADS 0 +#endif +#if SQLITE_DEFAULT_WORKER_THREADS>SQLITE_MAX_WORKER_THREADS +# undef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS SQLITE_DEFAULT_WORKER_THREADS +#endif /* diff --git a/src/vdbesort.c b/src/vdbesort.c index daf2477532..413e85c3b5 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -2179,6 +2179,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ if( rc==SQLITE_OK ){ pIter = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); pSorter->pReader = pIter; + if( pIter==0 ) rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ rc = vdbeIncrNew(pLast, pMain, &pIter->pIncr); From 7bd3c89114aa93b791a232fa39c6ec7f25cd54b4 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 3 May 2014 12:00:01 +0000 Subject: [PATCH 060/710] Add two new static mutexes, SQLITE_MUTEX_STATIC_APP1 and _APP2, for use by the application program. First intended use is in test programs for the memory allocation logic where one does not want to allocating a _FAST or _RECURSIVE mutex since that would involve using the memory allocation system under test. FossilOrigin-Name: 13686035dd1cf67ad9c6d282ab13c3259e7273d1 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/mutex_unix.c | 6 +++++- src/mutex_w32.c | 6 +++++- src/sqlite.h.in | 8 ++++++-- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index a8edf3f089..ed70315ed0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_DEFAULT_WORKER_THREADS\scompile-time\soption.\s\nFix\sa\sNULL-pointer\sdereference\sthat\scan\soccur\sfollowing\sOOM. -D 2014-05-03T11:22:09.616 +C Add\stwo\snew\sstatic\smutexes,\sSQLITE_MUTEX_STATIC_APP1\sand\s_APP2,\sfor\suse\sby\nthe\sapplication\sprogram.\s\sFirst\sintended\suse\sis\sin\stest\sprograms\sfor\sthe\nmemory\sallocation\slogic\swhere\sone\sdoes\snot\swant\sto\sallocating\sa\s_FAST\nor\s_RECURSIVE\smutex\ssince\sthat\swould\sinvolve\susing\sthe\smemory\sallocation\nsystem\sunder\stest. +D 2014-05-03T12:00:01.738 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -198,8 +198,8 @@ F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 -F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc -F src/mutex_w32.c 6108c88e1cb38d8fbb3534b170793815cbedbf97 +F src/mutex_unix.c 56e22c1bc6aabfa2f9736317a8f56acd5d0c5f7c +F src/mutex_w32.c f648cebb542b7a7ab98cecaa79259e8519e8f492 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f @@ -220,7 +220,7 @@ F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c a5ed3fdc82ebab5b9b095ea1971515a7f8a303d2 F src/shell.c 6946aea9f21af551fa84bc6b2a8de55d93bf0004 -F src/sqlite.h.in 579aebacdea59386d9cdf01fd4a16f4cafbb248f +F src/sqlite.h.in ed6d0cc90da850340c3863c84351e6e164c0ef00 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 851003126071d4a3bac86a0db75c48197fbd0ff0 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d7ed529fa2aa5cb13edaabca2acaad06dffef569 -R bf6d771799174fcada0f5c36289734b7 +P e0dea89b3e9f295f80210fcca007681bf1b08692 +R 29c0adc487699967d8fac6f96f2dadac U drh -Z 521faa6705b79213c3ba83b826f10b41 +Z ccb706fc99bd01bffced28b368a2dc97 diff --git a/manifest.uuid b/manifest.uuid index 03cf71aa4d..62ecc8633b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e0dea89b3e9f295f80210fcca007681bf1b08692 \ No newline at end of file +13686035dd1cf67ad9c6d282ab13c3259e7273d1 \ No newline at end of file diff --git a/src/mutex_unix.c b/src/mutex_unix.c index eca7295831..ea1203c047 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -96,10 +96,12 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU **
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -133,6 +135,8 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; sqlite3_mutex *p; diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 4b88c17452..7c17cc501e 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -99,6 +99,8 @@ static sqlite3_mutex winMutex_staticMutexes[6] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; static int winMutex_isInit = 0; @@ -159,10 +161,12 @@ static int winMutexEnd(void){ **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU **
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f09c374a74..2f39101a31 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5828,10 +5828,12 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU -**
  • SQLITE_MUTEX_STATIC_LRU2 +**
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 ** )^ ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) @@ -6035,6 +6037,8 @@ int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ #define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ #define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ +#define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ /* ** CAPI3REF: Retrieve the mutex for a database connection From 2fc7bc08bdde4b3da490c3990e1f06168eded03d Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 3 May 2014 13:53:37 +0000 Subject: [PATCH 061/710] Get SQLITE_MUTEX_STATIC_APP1 and _APP2 working for the debugMutex implementation. FossilOrigin-Name: f49ba1c926c63ee1c4609930138389fca182c845 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex.c | 2 +- src/mutex_noop.c | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ed70315ed0..a2cac08c7a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stwo\snew\sstatic\smutexes,\sSQLITE_MUTEX_STATIC_APP1\sand\s_APP2,\sfor\suse\sby\nthe\sapplication\sprogram.\s\sFirst\sintended\suse\sis\sin\stest\sprograms\sfor\sthe\nmemory\sallocation\slogic\swhere\sone\sdoes\snot\swant\sto\sallocating\sa\s_FAST\nor\s_RECURSIVE\smutex\ssince\sthat\swould\sinvolve\susing\sthe\smemory\sallocation\nsystem\sunder\stest. -D 2014-05-03T12:00:01.738 +C Get\sSQLITE_MUTEX_STATIC_APP1\sand\s_APP2\sworking\sfor\sthe\sdebugMutex\nimplementation. +D 2014-05-03T13:53:37.085 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,9 +195,9 @@ F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c 74670012946c4adc8a6ad84d03acc80959c3e529 F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 -F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc +F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea -F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 +F src/mutex_noop.c 4222773e5f61e506f232aedc9ad9e16ca00c1399 F src/mutex_unix.c 56e22c1bc6aabfa2f9736317a8f56acd5d0c5f7c F src/mutex_w32.c f648cebb542b7a7ab98cecaa79259e8519e8f492 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e0dea89b3e9f295f80210fcca007681bf1b08692 -R 29c0adc487699967d8fac6f96f2dadac +P 13686035dd1cf67ad9c6d282ab13c3259e7273d1 +R 7c657c53542f045bb02fc7656a8fcff4 U drh -Z ccb706fc99bd01bffced28b368a2dc97 +Z af16da93d438f6495f6a372d4f94c5d4 diff --git a/manifest.uuid b/manifest.uuid index 62ecc8633b..1fae5ede0f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13686035dd1cf67ad9c6d282ab13c3259e7273d1 \ No newline at end of file +f49ba1c926c63ee1c4609930138389fca182c845 \ No newline at end of file diff --git a/src/mutex.c b/src/mutex.c index b567e7c27e..bad5a7c113 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -81,7 +81,7 @@ int sqlite3MutexEnd(void){ */ sqlite3_mutex *sqlite3_mutex_alloc(int id){ #ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return 0; + if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; #endif return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } diff --git a/src/mutex_noop.c b/src/mutex_noop.c index 456e82a25e..4cfc6f3a76 100644 --- a/src/mutex_noop.c +++ b/src/mutex_noop.c @@ -107,7 +107,7 @@ static int debugMutexEnd(void){ return SQLITE_OK; } ** that means that a mutex could not be allocated. */ static sqlite3_mutex *debugMutexAlloc(int id){ - static sqlite3_debug_mutex aStatic[6]; + static sqlite3_debug_mutex aStatic[8]; sqlite3_debug_mutex *pNew = 0; switch( id ){ case SQLITE_MUTEX_FAST: From 0d51def29e2734627d8ea4009dcfbbf837de30b7 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 May 2014 14:28:14 +0000 Subject: [PATCH 062/710] Fix a problem in the sorter causing it to return spurious SQLITE_NOMEM errors when configured to use memsys3 or memsys5. FossilOrigin-Name: 3a66c4e1bf311d38668dfcdcd77867feff6db7bd --- manifest | 28 ++++----- manifest.uuid | 2 +- src/test1.c | 35 ++++++++++++ src/test_config.c | 8 +-- src/vdbesort.c | 2 +- test/permutations.test | 6 ++ test/sort.test | 126 ++++++++++++++++++++++++++++++++++++++++- test/sort2.test | 3 + test/sort3.test | 4 ++ test/sortfault.test | 37 +++++++----- 10 files changed, 213 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index a2cac08c7a..53d89eafaf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sSQLITE_MUTEX_STATIC_APP1\sand\s_APP2\sworking\sfor\sthe\sdebugMutex\nimplementation. -D 2014-05-03T13:53:37.085 +C Fix\sa\sproblem\sin\sthe\ssorter\scausing\sit\sto\sreturn\sspurious\sSQLITE_NOMEM\serrors\swhen\sconfigured\sto\suse\smemsys3\sor\smemsys5. +D 2014-05-03T14:28:14.676 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd -F src/test1.c 0cd73ae82fdf7add76ca603e3575380ae7539ae2 +F src/test1.c bd88cc00bff2f15279d808e84501f06148c144f9 F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -241,7 +241,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c ebd0a42983b696f2b121515d577753cf2afdc9b0 +F src/test_config.c bf2e0bf49ebd8fe3dccabb4542157f9571fe48fa F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c d205b56d0a1c2cbd8f6c8c4f513337ab0096d0b3 +F src/vdbesort.c 3e8827bb9d12465556357c24641f8805a7e2bba0 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -741,7 +741,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test a214a42b4767bbbc7cd0fd965ea6198044ab414d +F test/permutations.test 46a18489379943fcbd2ef07faa3858a88adb30cb F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -821,10 +821,10 @@ F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 79dc647c4e9b123a64e57b7080b7f9a2df43f87a -F test/sort2.test 04e99d0d028b469c6cfab2c647c6c28755504063 -F test/sort3.test c3f88d233452a129de519de311d109a0ad0da0af -F test/sortfault.test 2e2337aa5db6ab5cd546368cf2410676c11cb577 +F test/sort.test 8330b31b160483b52bb502a3ac4013f3f9028d73 +F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 +F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 +F test/sortfault.test 7fdc4a9bd76280a659c5782cdc6d95806d62d512 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 13686035dd1cf67ad9c6d282ab13c3259e7273d1 -R 7c657c53542f045bb02fc7656a8fcff4 -U drh -Z af16da93d438f6495f6a372d4f94c5d4 +P f49ba1c926c63ee1c4609930138389fca182c845 +R ecf8f8bb6befc88e6cfdbe14a65100aa +U dan +Z 4d5ad6cf671998327bccef4dcd098843 diff --git a/manifest.uuid b/manifest.uuid index 1fae5ede0f..500fa40c2b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f49ba1c926c63ee1c4609930138389fca182c845 \ No newline at end of file +3a66c4e1bf311d38668dfcdcd77867feff6db7bd \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 44e96c2c1f..403a7fafa7 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6336,6 +6336,40 @@ static int tclLoadStaticExtensionCmd( return TCL_OK; } +/* +** sorter_test_fakeheap BOOL +** +*/ +static int sorter_test_fakeheap( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + int bArg; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "BOOL"); + return TCL_ERROR; + } + + if( Tcl_GetBooleanFromObj(interp, objv[1], &bArg) ){ + return TCL_ERROR; + } + + if( bArg ){ + if( sqlite3GlobalConfig.pHeap==0 ){ + sqlite3GlobalConfig.pHeap = SQLITE_INT_TO_PTR(-1); + } + }else{ + if( sqlite3GlobalConfig.pHeap==SQLITE_INT_TO_PTR(-1) ){ + sqlite3GlobalConfig.pHeap = 0; + } + } + + Tcl_ResetResult(interp); + return TCL_OK; +} + /* ** Register commands with the TCL interpreter. @@ -6569,6 +6603,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "getrusage", test_getrusage }, #endif { "load_static_extension", tclLoadStaticExtensionCmd }, + { "sorter_test_fakeheap", sorter_test_fakeheap }, }; static int bitmask_size = sizeof(Bitmask)*8; int i; diff --git a/src/test_config.c b/src/test_config.c index 2201fba007..33898a759c 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -99,11 +99,9 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "mmap", "0", TCL_GLOBAL_ONLY); #endif -#if SQLITE_MAX_WORKER_THREADS>0 - Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "1", TCL_GLOBAL_ONLY); -#else - Tcl_SetVar2(interp, "sqlite_options", "worker_threads", "0", TCL_GLOBAL_ONLY); -#endif + Tcl_SetVar2(interp, "sqlite_options", "worker_threads", + STRINGVALUE(SQLITE_MAX_WORKER_THREADS), TCL_GLOBAL_ONLY + ); #if 1 /* def SQLITE_MEMDEBUG */ Tcl_SetVar2(interp, "sqlite_options", "memdebug", "1", TCL_GLOBAL_ONLY); diff --git a/src/vdbesort.c b/src/vdbesort.c index 413e85c3b5..5efd6f112c 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1556,7 +1556,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ if( aMem ){ pSorter->list.aMemory = aMem; pSorter->nMemory = sqlite3MallocSize(aMem); - }else{ + }else if( pSorter->list.aMemory ){ pSorter->list.aMemory = sqlite3Malloc(pSorter->nMemory); if( !pSorter->list.aMemory ) return SQLITE_NOMEM; } diff --git a/test/permutations.test b/test/permutations.test index 4487af055b..b547e7b1bc 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -332,6 +332,12 @@ test_suite "coverage-analyze" -description { analyze.test analyzeB.test mallocA.test } +test_suite "coverage-sorter" -description { + Coverage tests for file vdbesort.c. +} -files { + sort.test sortfault.test +} + lappend ::testsuitelist xxx #------------------------------------------------------------------------- diff --git a/test/sort.test b/test/sort.test index ccbfdda2b3..c6c7fc6e41 100644 --- a/test/sort.test +++ b/test/sort.test @@ -8,10 +8,10 @@ # May you share freely, never taking more than you give. # #*********************************************************************** -# This file implements regression tests for SQLite library. The -# focus of this file is testing the CREATE TABLE statement. # -# $Id: sort.test,v 1.25 2005/11/14 22:29:06 drh Exp $ +# This file implements regression tests for SQLite library. The +# focus of this file is testing the sorter (code in vdbesort.c). +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -485,5 +485,125 @@ do_execsql_test sort-13.3 { SELECT a, b FROM t10 ORDER BY a; } [db eval {SELECT a, b FROM t10 ORDER BY a, b}] +#------------------------------------------------------------------------- +# Sort some large ( > 4KiB) records. +# +proc cksum {x} { + set i1 1 + set i2 2 + binary scan $x c* L + foreach {a b} $L { + set i1 [expr (($i2<<3) + $a) & 0x7FFFFFFF] + set i2 [expr (($i1<<3) + $b) & 0x7FFFFFFF] + } + list $i1 $i2 +} +db func cksum cksum + +do_execsql_test sort-14.0 { + PRAGMA cache_size = 5; + CREATE TABLE t11(a, b); + INSERT INTO t11 VALUES(randomblob(5000), NULL); + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --2 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --3 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --4 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --5 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --6 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --7 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --8 + INSERT INTO t11 SELECT randomblob(5000), NULL FROM t11; --9 + UPDATE t11 SET b = cksum(a); +} + +foreach {tn mmap_limit} { + 1 0 + 2 1000000 +} { + do_test sort-14.$tn { + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit + set prev "" + db eval { SELECT * FROM t11 ORDER BY b } { + if {$b != [cksum $a]} {error "checksum failed"} + if {[string compare $b $prev] < 0} {error "sort failed"} + set prev $b + } + set {} {} + } {} +} + +#------------------------------------------------------------------------- +# +foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { + 1 0 3 file true false + 2 0 3 file true true + 3 0 0 file true false + 4 1000000 3 file true false + 5 0 0 memory false true +} { + db close + + sqlite3_shutdown + sqlite3_config_worker_threads $nWorker + if {$coremutex} { + sqlite3_config multithread + } else { + sqlite3_config singlethread + } + sqlite3_initialize + + sorter_test_fakeheap $fakeheap + + reset_db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit + execsql "PRAGMA temp_store = $tmpstore" + + set ten [string repeat X 10300] + set one [string repeat y 200] + + do_execsql_test 15.$tn.1 { + PRAGMA cache_size = 5; + WITH rr AS ( + SELECT 4, $ten UNION ALL + SELECT 2, $one UNION ALL + SELECT 1, $ten UNION ALL + SELECT 3, $one + ) + SELECT * FROM rr ORDER BY 1; + } [list 1 $ten 2 $one 3 $one 4 $ten] + + do_execsql_test 15.$tn.2 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(4); + INSERT INTO t1 VALUES(5); + INSERT INTO t1 VALUES(3); + INSERT INTO t1 VALUES(2); + INSERT INTO t1 VALUES(6); + INSERT INTO t1 VALUES(1); + CREATE INDEX i1 ON t1(a); + SELECT * FROM t1 ORDER BY a; + } {1 2 3 4 5 6} + + do_execsql_test 15.$tn.3 { + PRAGMA cache_size = 5; + WITH rr AS ( + SELECT 4, $ten UNION ALL + SELECT 2, $one + ) + SELECT * FROM rr ORDER BY 1; + } [list 2 $one 4 $ten] + + sorter_test_fakeheap 0 +} + +db close +sqlite3_shutdown +#sqlite3_config_worker_threads $sqlite_options(worker_threads) +sqlite3_config_worker_threads 0 +set t(0) singlethread +set t(1) multithread +set t(2) serialized +sqlite3_config $t($sqlite_options(threadsafe)) +sqlite3_initialize finish_test + diff --git a/test/sort2.test b/test/sort2.test index 4fb6a9462b..e4e40dab74 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -10,6 +10,9 @@ #*********************************************************************** # This file implements regression tests for SQLite library. # +# Specifically, the tests in this file attempt to verify that +# multi-threaded sorting works. +# set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/sort3.test b/test/sort3.test index 9963dcccd3..80d8bbca3f 100644 --- a/test/sort3.test +++ b/test/sort3.test @@ -10,6 +10,10 @@ #*********************************************************************** # This file implements regression tests for SQLite library. # +# The tests in this file verify that sorting works when the library is +# configured to use mmap(), but the temporary files generated by the +# sorter are too large to be completely mapped. +# set testdir [file dirname $argv0] source $testdir/tester.tcl diff --git a/test/sortfault.test b/test/sortfault.test index bdf57b45ba..abe9af6854 100644 --- a/test/sortfault.test +++ b/test/sortfault.test @@ -10,6 +10,9 @@ #*********************************************************************** # This file implements regression tests for SQLite library. # +# Specifically, it tests the effects of fault injection on the sorter +# module (code in vdbesort.c). +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -20,20 +23,26 @@ do_execsql_test 1.0 { PRAGMA cache_size = 5; } -do_faultsim_test 1 -prep { - sqlite3 db test.db -} -body { - execsql { - WITH r(x,y) AS ( - SELECT 1, randomblob(1000) - UNION ALL - SELECT x+1, randomblob(1000) FROM r - LIMIT 500 - ) - SELECT count(x), length(y) FROM r GROUP BY (x%5) - } -} -test { - faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}} +foreach {tn mmap_limit} { + 1 0 + 2 100000 +} { + do_faultsim_test 1.$tn -prep { + sqlite3 db test.db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit + } -body { + execsql { + WITH r(x,y) AS ( + SELECT 1, randomblob(1000) + UNION ALL + SELECT x+1, randomblob(1000) FROM r + LIMIT 500 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } + } -test { + faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}} + } } finish_test From e18e90ebafe89d444c0524f91088b0c9316cbbc0 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 May 2014 19:33:00 +0000 Subject: [PATCH 063/710] Fix a race condition in the sorter. FossilOrigin-Name: 32ccf3ae18531682dfd039fa8df6ad9a907ac455 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbesort.c | 21 ++++++++++++++++++--- test/sort.test | 3 --- test/sortfault.test | 38 ++++++++++++++++++++++++++++---------- 5 files changed, 55 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 53d89eafaf..917d67a6b3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\ssorter\scausing\sit\sto\sreturn\sspurious\sSQLITE_NOMEM\serrors\swhen\sconfigured\sto\suse\smemsys3\sor\smemsys5. -D 2014-05-03T14:28:14.676 +C Fix\sa\srace\scondition\sin\sthe\ssorter. +D 2014-05-03T19:33:00.713 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 3e8827bb9d12465556357c24641f8805a7e2bba0 +F src/vdbesort.c c443cdf00fc8b90b17fbeaa1ad833d3091b0bf5c F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -821,10 +821,10 @@ F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 8330b31b160483b52bb502a3ac4013f3f9028d73 +F test/sort.test 2af626b7963eddb8c93b6c87babf81396c379ef5 F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 -F test/sortfault.test 7fdc4a9bd76280a659c5782cdc6d95806d62d512 +F test/sortfault.test f875d29c58b2eafdca1b51c0810075570d5a3cbc F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P f49ba1c926c63ee1c4609930138389fca182c845 -R ecf8f8bb6befc88e6cfdbe14a65100aa +P 3a66c4e1bf311d38668dfcdcd77867feff6db7bd +R 07ad433b1f531db0a3f09b334b377488 U dan -Z 4d5ad6cf671998327bccef4dcd098843 +Z 8b4c7c15ef224489143b17033ef5ced9 diff --git a/manifest.uuid b/manifest.uuid index 500fa40c2b..6bfb75656c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a66c4e1bf311d38668dfcdcd77867feff6db7bd \ No newline at end of file +32ccf3ae18531682dfd039fa8df6ad9a907ac455 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 5efd6f112c..5676d96e37 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -651,6 +651,7 @@ static int vdbePmaReaderNext(PmaReader *pIter){ int rc = SQLITE_OK; /* Return Code */ u64 nRec = 0; /* Size of record in bytes */ + if( pIter->iReadOff>=pIter->iEof ){ IncrMerger *pIncr = pIter->pIncr; int bEof = 1; @@ -1844,10 +1845,18 @@ static int vdbeIncrInitMerger( ){ int rc = SQLITE_OK; /* Return code */ int i; /* For iterating through PmaReader objects */ + int nTree = pMerger->nTree; - for(i=0; rc==SQLITE_OK && inTree; i++){ + for(i=0; rc==SQLITE_OK && iaIter[i]); + /* Iterators should be normally initialized in order, as if they are + ** reading from the same temp file this makes for more linear file IO. + ** However, in the INCRINIT_ROOT case, if iterator aIter[nTask-1] is + ** in use it will block the vdbePmaReaderNext() call while it uses + ** the main thread to fill its buffer. So calling PmaReaderNext() + ** on this iterator before any of the multi-threaded iterators takes + ** better advantage of multi-processor hardware. */ + rc = vdbePmaReaderNext(&pMerger->aIter[nTree-i-1]); }else{ rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT_NORMAL); } @@ -2196,7 +2205,13 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ PmaReader *p = &pMain->aIter[iTask]; assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); - if( p->pIncr ){ rc = vdbePmaReaderBgIncrInit(p); } + if( p->pIncr ){ + if( iTask==pSorter->nTask-1 ){ + rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); + }else{ + rc = vdbePmaReaderBgIncrInit(p); + } + } } } } diff --git a/test/sort.test b/test/sort.test index c6c7fc6e41..0c87dd6441 100644 --- a/test/sort.test +++ b/test/sort.test @@ -541,7 +541,6 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { 5 0 0 memory false true } { db close - sqlite3_shutdown sqlite3_config_worker_threads $nWorker if {$coremutex} { @@ -550,7 +549,6 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { sqlite3_config singlethread } sqlite3_initialize - sorter_test_fakeheap $fakeheap reset_db @@ -597,7 +595,6 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { db close sqlite3_shutdown -#sqlite3_config_worker_threads $sqlite_options(worker_threads) sqlite3_config_worker_threads 0 set t(0) singlethread set t(1) multithread diff --git a/test/sortfault.test b/test/sortfault.test index abe9af6854..7c58bb6833 100644 --- a/test/sortfault.test +++ b/test/sortfault.test @@ -23,27 +23,45 @@ do_execsql_test 1.0 { PRAGMA cache_size = 5; } -foreach {tn mmap_limit} { - 1 0 - 2 100000 +foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} { + 1 0 0 file multithread false + 2 100000 0 file multithread false } { + catch { db close } + sqlite3_shutdown + sqlite3_config_worker_threads $nWorker + sqlite3_config $threadsmode + sqlite3_initialize + sorter_test_fakeheap $fakeheap + + set str [string repeat a 1000] + do_faultsim_test 1.$tn -prep { sqlite3 db test.db sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit + execsql { PRAGMA cache_size = 5 } } -body { execsql { WITH r(x,y) AS ( - SELECT 1, randomblob(1000) + SELECT 1, $::str UNION ALL - SELECT x+1, randomblob(1000) FROM r - LIMIT 500 - ) - SELECT count(x), length(y) FROM r GROUP BY (x%5) - } + SELECT x+1, $::str FROM r + LIMIT 200 + ) + SELECT count(x), length(y) FROM r GROUP BY (x%5) + } } -test { - faultsim_test_result {0 {100 1000 100 1000 100 1000 100 1000 100 1000}} + faultsim_test_result {0 {40 1000 40 1000 40 1000 40 1000 40 1000}} } } +catch { db close } +sqlite3_shutdown +sqlite3_config_worker_threads 0 +set t(0) singlethread +set t(1) multithread +set t(2) serialized +sqlite3_config $t($sqlite_options(threadsafe)) +sqlite3_initialize finish_test From f7f425d6006c5181cf5e3771d35766c5de456271 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 May 2014 20:43:13 +0000 Subject: [PATCH 064/710] Add an extra fault-injection test to sortfault.test. Remove an unreachable branch from vdbesort.c. FossilOrigin-Name: a33a366ba8a0da81ddd895d552a348441ef8529a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 12 +++++------- test/sortfault.test | 22 ++++++++++++++++++++++ 4 files changed, 35 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 917d67a6b3..903e7795ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\srace\scondition\sin\sthe\ssorter. -D 2014-05-03T19:33:00.713 +C Add\san\sextra\sfault-injection\stest\sto\ssortfault.test.\sRemove\san\sunreachable\sbranch\sfrom\svdbesort.c. +D 2014-05-03T20:43:13.986 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c c443cdf00fc8b90b17fbeaa1ad833d3091b0bf5c +F src/vdbesort.c db67b5b54c476163f3068a194a734c394ecb675e F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -824,7 +824,7 @@ F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 2af626b7963eddb8c93b6c87babf81396c379ef5 F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 -F test/sortfault.test f875d29c58b2eafdca1b51c0810075570d5a3cbc +F test/sortfault.test f2f94227d97f7efb6351aff29e77498e877d6216 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 3a66c4e1bf311d38668dfcdcd77867feff6db7bd -R 07ad433b1f531db0a3f09b334b377488 +P 32ccf3ae18531682dfd039fa8df6ad9a907ac455 +R 5ede29f57c1094d89ff6cfb82c23c871 U dan -Z 8b4c7c15ef224489143b17033ef5ced9 +Z 83a8a7f60269415a80d26ca6a02bda49 diff --git a/manifest.uuid b/manifest.uuid index 6bfb75656c..4a6d01e399 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -32ccf3ae18531682dfd039fa8df6ad9a907ac455 \ No newline at end of file +a33a366ba8a0da81ddd895d552a348441ef8529a \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 5676d96e37..d0a6be84c9 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1807,11 +1807,9 @@ static int vdbeIncrNew( /* ** Set the "use-threads" flag on object pIncr. */ -static void vdbeIncrSetThreads(IncrMerger *pIncr, int bUseThread){ - if( bUseThread ){ - pIncr->bUseThread = 1; - pIncr->pTask->file2.iEof -= pIncr->mxSz; - } +static void vdbeIncrSetThreads(IncrMerger *pIncr){ + pIncr->bUseThread = 1; + pIncr->pTask->file2.iEof -= pIncr->mxSz; } #endif /* SQLITE_MAX_WORKER_THREADS>0 */ @@ -2193,11 +2191,11 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ if( rc==SQLITE_OK ){ rc = vdbeIncrNew(pLast, pMain, &pIter->pIncr); if( rc==SQLITE_OK ){ - vdbeIncrSetThreads(pIter->pIncr, pSorter->bUseThreads); + vdbeIncrSetThreads(pIter->pIncr); for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ IncrMerger *pIncr; if( (pIncr = pMain->aIter[iTask].pIncr) ){ - vdbeIncrSetThreads(pIncr, pSorter->bUseThreads); + vdbeIncrSetThreads(pIncr); assert( pIncr->pTask!=pLast ); } } diff --git a/test/sortfault.test b/test/sortfault.test index 7c58bb6833..2f746fcc24 100644 --- a/test/sortfault.test +++ b/test/sortfault.test @@ -26,6 +26,7 @@ do_execsql_test 1.0 { foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} { 1 0 0 file multithread false 2 100000 0 file multithread false + 3 100000 1 file multithread false } { catch { db close } sqlite3_shutdown @@ -53,6 +54,27 @@ foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} { } -test { faultsim_test_result {0 {40 1000 40 1000 40 1000 40 1000 40 1000}} } + + + do_faultsim_test 2.$tn -prep { + sqlite3 db test.db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit + add_test_utf16bin_collate db + execsql { PRAGMA cache_size = 5 } + } -body { + execsql { + WITH r(x,y) AS ( + SELECT 100, $::str + UNION ALL + SELECT x-1, $::str FROM r + LIMIT 100 + ) + SELECT count(x), length(y) FROM r GROUP BY y COLLATE utf16bin, (x%5) + } + } -test { + faultsim_test_result {0 {20 1000 20 1000 20 1000 20 1000 20 1000}} + } + } catch { db close } From d94d4ee7bd462e9a00e47207cac8a0119e7bffa8 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 May 2014 09:08:54 +0000 Subject: [PATCH 065/710] Add tests so that the "coverage-sorter" test permutation covers all branches in vdbesort.c. Fix a few minor problems in the same file. FossilOrigin-Name: bde28b702dabd02269e333535cc41481351c5efc --- main.mk | 1 + manifest | 20 +++++----- manifest.uuid | 2 +- src/threads.c | 10 +++++ src/vdbesort.c | 54 +++++++++++++------------- test/sort.test | 50 ++++++++++++++++++++---- test/sortfault.test | 94 +++++++++++++++++++++++++++++++++++++++++---- 7 files changed, 178 insertions(+), 53 deletions(-) diff --git a/main.mk b/main.mk index 76285b0d76..d61f715e80 100644 --- a/main.mk +++ b/main.mk @@ -312,6 +312,7 @@ TESTSRC2 = \ $(TOP)/src/pcache.c \ $(TOP)/src/pcache1.c \ $(TOP)/src/select.c \ + $(TOP)/src/threads.c \ $(TOP)/src/tokenize.c \ $(TOP)/src/utf.c \ $(TOP)/src/util.c \ diff --git a/manifest b/manifest index 903e7795ef..a8debded75 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sextra\sfault-injection\stest\sto\ssortfault.test.\sRemove\san\sunreachable\sbranch\sfrom\svdbesort.c. -D 2014-05-03T20:43:13.986 +C Add\stests\sso\sthat\sthe\s"coverage-sorter"\stest\spermutation\scovers\sall\sbranches\sin\svdbesort.c.\sFix\sa\sfew\sminor\sproblems\sin\sthe\ssame\sfile. +D 2014-05-05T09:08:54.007 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -144,7 +144,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt f439556c5ce01ced70987e5ee86549a45165d9ff -F main.mk 97673939d94f210b7218a8236df6c8799d4a0da6 +F main.mk f51f401a87fd07be255bcecae8b03105ca2024e8 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -273,7 +273,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c e35de159f9ced746266ff4b2129b7a828e59119c +F src/threads.c b9daffcb6ae3a9c080da37b905e790d45a8f99db F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c db67b5b54c476163f3068a194a734c394ecb675e +F src/vdbesort.c af752fa4c125ab2bfeee1c4e0b50504e79aa7051 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -821,10 +821,10 @@ F test/skipscan1.test bed8cbe9d554c8c27afb6c88500f704c86a9196f F test/skipscan2.test d77f79cdbba25f0f6f35298136cff21a7d7a553a F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 2af626b7963eddb8c93b6c87babf81396c379ef5 +F test/sort.test 688468cef8c9a66fcc1d54235de8e4deac745690 F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 -F test/sortfault.test f2f94227d97f7efb6351aff29e77498e877d6216 +F test/sortfault.test 1a12b6e27d475f50658a8164aaa34f0080a86b36 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 32ccf3ae18531682dfd039fa8df6ad9a907ac455 -R 5ede29f57c1094d89ff6cfb82c23c871 +P a33a366ba8a0da81ddd895d552a348441ef8529a +R 48fa4af9d614a9372c1dc3ff956ba091 U dan -Z 83a8a7f60269415a80d26ca6a02bda49 +Z 1076615c7d8998a9f1b953b2c0f517bb diff --git a/manifest.uuid b/manifest.uuid index 4a6d01e399..8a5cb48075 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a33a366ba8a0da81ddd895d552a348441ef8529a \ No newline at end of file +bde28b702dabd02269e333535cc41481351c5efc \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 64975801be..6b59ce90d2 100644 --- a/src/threads.c +++ b/src/threads.c @@ -208,6 +208,7 @@ int sqlite3ThreadCreate( /* Get the results of the thread */ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ + assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; if( p->xTask ){ @@ -216,6 +217,15 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ *ppOut = p->pResult; } sqlite3_free(p); + +#if defined(SQLITE_TEST) + { + void *pTstAlloc = sqlite3Malloc(10); + if (!pTstAlloc) return SQLITE_NOMEM; + sqlite3_free(pTstAlloc); + } +#endif + return SQLITE_OK; } diff --git a/src/vdbesort.c b/src/vdbesort.c index d0a6be84c9..90a611914d 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1251,11 +1251,10 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ pList->pList = p; sqlite3_free(aSlot); - if( pTask->pUnpacked->errCode ){ - assert( pTask->pUnpacked->errCode==SQLITE_NOMEM ); - return SQLITE_NOMEM; - } - return SQLITE_OK; + assert( pTask->pUnpacked->errCode==SQLITE_OK + || pTask->pUnpacked->errCode==SQLITE_NOMEM + ); + return pTask->pUnpacked->errCode; } /* @@ -1485,7 +1484,7 @@ static int vdbeSorterNext( *pbEof = (pMerger->aIter[pMerger->aTree[1]].pFile==0); } - return rc; + return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); } #if SQLITE_MAX_WORKER_THREADS>0 @@ -1534,7 +1533,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ if( pTask->bDone ){ rc = vdbeSorterJoinThread(pTask); } - if( pTask->pThread==0 || rc!=SQLITE_OK ) break; + if( rc!=SQLITE_OK || pTask->pThread==0 ) break; } if( rc==SQLITE_OK ){ @@ -1864,7 +1863,7 @@ static int vdbeIncrInitMerger( rc = vdbeSorterDoCompare(pTask, pMerger, i); } - return rc; + return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); } /* @@ -2074,9 +2073,10 @@ static int vdbeSorterAddToTree( rc = vdbeIncrNew(pTask, pNew, &pIter->pIncr); } } - - p = pIter->pIncr->pMerger; - nDiv = nDiv / SORTER_MAX_MERGE_COUNT; + if( rc==SQLITE_OK ){ + p = pIter->pIncr->pMerger; + nDiv = nDiv / SORTER_MAX_MERGE_COUNT; + } } if( rc==SQLITE_OK ){ @@ -2178,6 +2178,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ rc = vdbeSorterMergeTreeBuild(pSorter, &pMain); if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS + assert( pSorter->bUseThreads==0 || pSorter->nTask>1 ); if( pSorter->bUseThreads ){ int iTask; PmaReader *pIter; @@ -2199,16 +2200,14 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ assert( pIncr->pTask!=pLast ); } } - if( pSorter->nTask>1 ){ - for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ - PmaReader *p = &pMain->aIter[iTask]; - assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); - if( p->pIncr ){ - if( iTask==pSorter->nTask-1 ){ - rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); - }else{ - rc = vdbePmaReaderBgIncrInit(p); - } + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ + PmaReader *p = &pMain->aIter[iTask]; + assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); + if( p->pIncr ){ + if( iTask==pSorter->nTask-1 ){ + rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); + }else{ + rc = vdbePmaReaderBgIncrInit(p); } } } @@ -2216,8 +2215,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ pMain = 0; } if( rc==SQLITE_OK ){ - int eMode = (pSorter->nTask>1 ? INCRINIT_ROOT : INCRINIT_NORMAL); - rc = vdbePmaReaderIncrInit(pIter, eMode); + rc = vdbePmaReaderIncrInit(pIter, INCRINIT_ROOT); } }else #endif @@ -2259,10 +2257,12 @@ int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ return rc; } - /* Write the current in-memory list to a PMA. */ - if( pSorter->list.pList ){ - rc = vdbeSorterFlushPMA(pSorter); - } + /* Write the current in-memory list to a PMA. When the VdbeSorterWrite() + ** function flushes the contents of memory to disk, it immediately always + ** creates a new list consisting of a single key immediately afterwards. + ** So the list is never empty at this point. */ + assert( pSorter->list.pList ); + rc = vdbeSorterFlushPMA(pSorter); /* Join all threads */ rc = vdbeSorterJoinAll(pSorter, rc); diff --git a/test/sort.test b/test/sort.test index 0c87dd6441..e75740e9c3 100644 --- a/test/sort.test +++ b/test/sort.test @@ -533,12 +533,14 @@ foreach {tn mmap_limit} { #------------------------------------------------------------------------- # -foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { - 1 0 3 file true false - 2 0 3 file true true - 3 0 0 file true false - 4 1000000 3 file true false - 5 0 0 memory false true +foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap softheaplimit} { + 1 0 3 file true false 0 + 2 0 3 file true true 0 + 3 0 0 file true false 0 + 4 1000000 3 file true false 0 + 5 0 0 memory false true 0 + 6 0 0 file false true 1000000 + 7 0 0 file false true 10000 } { db close sqlite3_shutdown @@ -550,6 +552,7 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { } sqlite3_initialize sorter_test_fakeheap $fakeheap + sqlite3_soft_heap_limit $softheaplimit reset_db sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit @@ -558,8 +561,13 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { set ten [string repeat X 10300] set one [string repeat y 200] + if {$softheaplimit} { + execsql { PRAGMA cache_size = 20 }; + } else { + execsql { PRAGMA cache_size = 5 }; + } + do_execsql_test 15.$tn.1 { - PRAGMA cache_size = 5; WITH rr AS ( SELECT 4, $ten UNION ALL SELECT 2, $one UNION ALL @@ -582,7 +590,6 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap} { } {1 2 3 4 5 6} do_execsql_test 15.$tn.3 { - PRAGMA cache_size = 5; WITH rr AS ( SELECT 4, $ten UNION ALL SELECT 2, $one @@ -601,6 +608,33 @@ set t(1) multithread set t(2) serialized sqlite3_config $t($sqlite_options(threadsafe)) sqlite3_initialize +sqlite3_soft_heap_limit 0 + +reset_db +do_catchsql_test 16.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(1, NULL, 3); + INSERT INTO t1 VALUES(NULL, 2, 3); + INSERT INTO t1 VALUES(1, 2, NULL); + INSERT INTO t1 VALUES(4, 5, 6); + CREATE UNIQUE INDEX i1 ON t1(b, a, c); +} {0 {}} +reset_db +do_catchsql_test 16.2 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(1, NULL, 3); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(1, 2, NULL); + INSERT INTO t1 VALUES(4, 5, 6); + CREATE UNIQUE INDEX i1 ON t1(b, a, c); +} {1 {UNIQUE constraint failed: t1.b, t1.a, t1.c}} + +reset_db +do_execsql_test 17.1 { + SELECT * FROM sqlite_master ORDER BY sql; +} {} finish_test diff --git a/test/sortfault.test b/test/sortfault.test index 2f746fcc24..4c199ab212 100644 --- a/test/sortfault.test +++ b/test/sortfault.test @@ -18,24 +18,32 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix sortfault - do_execsql_test 1.0 { PRAGMA cache_size = 5; } -foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} { - 1 0 0 file multithread false - 2 100000 0 file multithread false - 3 100000 1 file multithread false +foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap lookaside} { + 1 0 0 file multithread false false + 2 100000 0 file multithread false false + 3 100000 1 file multithread false false + 4 2000000 0 file singlethread false true } { + if {$sqlite_options(threadsafe)} { set threadsmode singlethread } + catch { db close } sqlite3_shutdown sqlite3_config_worker_threads $nWorker sqlite3_config $threadsmode + if { $lookaside } { + sqlite3_config_lookaside 100 500 + } else { + sqlite3_config_lookaside 0 0 + } sqlite3_initialize sorter_test_fakeheap $fakeheap set str [string repeat a 1000] + puts $threadsmode do_faultsim_test 1.$tn -prep { sqlite3 db test.db @@ -55,8 +63,7 @@ foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} { faultsim_test_result {0 {40 1000 40 1000 40 1000 40 1000 40 1000}} } - - do_faultsim_test 2.$tn -prep { + do_faultsim_test 2.$tn -faults oom* -prep { sqlite3 db test.db sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit add_test_utf16bin_collate db @@ -75,6 +82,30 @@ foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap} { faultsim_test_result {0 {20 1000 20 1000 20 1000 20 1000 20 1000}} } + if {$mmap_limit > 1000000} { + set str2 [string repeat $str 10] + + sqlite3_memdebug_vfs_oom_test 0 + sqlite3 db test.db + sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $::mmap_limit + execsql { PRAGMA cache_size = 5 } + + do_faultsim_test 3.$tn -faults oom-trans* -body { + execsql { + WITH r(x,y) AS ( + SELECT 300, $::str2 + UNION ALL + SELECT x-1, $::str2 FROM r + LIMIT 300 + ) + SELECT count(x), length(y) FROM r GROUP BY y, (x%5) + } + } -test { + faultsim_test_result {0 {60 10000 60 10000 60 10000 60 10000 60 10000}} + } + + sqlite3_memdebug_vfs_oom_test 1 + } } catch { db close } @@ -84,6 +115,55 @@ set t(0) singlethread set t(1) multithread set t(2) serialized sqlite3_config $t($sqlite_options(threadsafe)) +sqlite3_config_lookaside 100 500 sqlite3_initialize + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); +} +do_test 4.1 { + for {set i 0} {$i < 256} {incr i} { + execsql { + INSERT INTO t1 SELECT + ((a<<3) + b) & 2147483647, + ((b<<3) + c) & 2147483647, + ((c<<3) + a) & 2147483647 + FROM t1 ORDER BY rowid DESC LIMIT 1; + } + } +} {} + +faultsim_save_and_close + +do_faultsim_test 4.2 -faults oom* -prep { + faultsim_restore_and_reopen +} -body { + execsql { CREATE UNIQUE INDEX i1 ON t1(a,b,c) } +} -test { + faultsim_test_result {0 {}} +} + +#------------------------------------------------------------------------- +# +reset_db +set a [string repeat a 500] +set b [string repeat b 500] +set c [string repeat c 500] +do_execsql_test 5.0 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES($a, $b, $c); + INSERT INTO t1 VALUES($c, $b, $a); +} + +do_faultsim_test 5.1 -faults oom* -body { + execsql { SELECT * FROM t1 ORDER BY a } +} -test { + faultsim_test_result [list 0 [list $::a $::b $::c $::c $::b $::a]] +} + finish_test From 0d3a4085e5f33768726abe9450ca95af182ee082 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 May 2014 15:58:40 +0000 Subject: [PATCH 066/710] Fix a race condition in the sorter code. FossilOrigin-Name: 2d2edfe58db101d42a96772b856e6e55b401aab6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a8debded75..e411d8e3de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sso\sthat\sthe\s"coverage-sorter"\stest\spermutation\scovers\sall\sbranches\sin\svdbesort.c.\sFix\sa\sfew\sminor\sproblems\sin\sthe\ssame\sfile. -D 2014-05-05T09:08:54.007 +C Fix\sa\srace\scondition\sin\sthe\ssorter\scode. +D 2014-05-05T15:58:40.542 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -287,7 +287,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c af752fa4c125ab2bfeee1c4e0b50504e79aa7051 +F src/vdbesort.c b36070436d33372237541e9eaa6068e1a61bc402 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1170,7 +1170,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a33a366ba8a0da81ddd895d552a348441ef8529a -R 48fa4af9d614a9372c1dc3ff956ba091 +P bde28b702dabd02269e333535cc41481351c5efc +R e53afa0ba1bd427219aede77058d3d04 U dan -Z 1076615c7d8998a9f1b953b2c0f517bb +Z ba1cae0a2780ce3568cf6b58bd76a820 diff --git a/manifest.uuid b/manifest.uuid index 8a5cb48075..82ab91d82e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bde28b702dabd02269e333535cc41481351c5efc \ No newline at end of file +2d2edfe58db101d42a96772b856e6e55b401aab6 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 90a611914d..9c57135dd3 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -991,7 +991,15 @@ static int vdbeSorterCreateThread( static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ int rc = rcin; int i; - for(i=0; inTask; i++){ + + /* This function is always called by the main user thread. + ** + ** If this function is being called after SorterRewind() has been called, + ** it is possible that thread pSorter->aTask[pSorter->nTask-1].pThread + ** is currently attempt to join one of the other threads. To avoid a race + ** condition where this thread also attempts to join the same object, join + ** thread pSorter->aTask[pSorter->nTask-1].pThread first. */ + for(i=pSorter->nTask-1; i>=0; i--){ SortSubtask *pTask = &pSorter->aTask[i]; int rc2 = vdbeSorterJoinThread(pTask); if( rc==SQLITE_OK ) rc = rc2; From 449cb9a5af9762d6eba3343802c601723f19929f Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 May 2014 20:03:50 +0000 Subject: [PATCH 067/710] Add test file sort4.test, containing brute force tests for the multi-theaded sorter. FossilOrigin-Name: 9cc364c42cc64ab7b55b5c55e303fb63a456cf00 --- manifest | 13 ++-- manifest.uuid | 2 +- test/permutations.test | 2 +- test/sort4.test | 145 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 154 insertions(+), 8 deletions(-) create mode 100644 test/sort4.test diff --git a/manifest b/manifest index e411d8e3de..ac504dddc6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\srace\scondition\sin\sthe\ssorter\scode. -D 2014-05-05T15:58:40.542 +C Add\stest\sfile\ssort4.test,\scontaining\sbrute\sforce\stests\sfor\sthe\smulti-theaded\ssorter. +D 2014-05-05T20:03:50.967 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -741,7 +741,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 46a18489379943fcbd2ef07faa3858a88adb30cb +F test/permutations.test 465bc22a873ced74a6d1dedc0dde5da424be6e6a F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -824,6 +824,7 @@ F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 688468cef8c9a66fcc1d54235de8e4deac745690 F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 +F test/sort4.test a29761a7d05f194e516e91a03d38f8194ac9849d F test/sortfault.test 1a12b6e27d475f50658a8164aaa34f0080a86b36 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1170,7 +1171,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P bde28b702dabd02269e333535cc41481351c5efc -R e53afa0ba1bd427219aede77058d3d04 +P 2d2edfe58db101d42a96772b856e6e55b401aab6 +R a3f7784643670968322db5c49315411c U dan -Z ba1cae0a2780ce3568cf6b58bd76a820 +Z cca87b3c456f151b38b423a6310da351 diff --git a/manifest.uuid b/manifest.uuid index 82ab91d82e..f77d0e5e6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d2edfe58db101d42a96772b856e6e55b401aab6 \ No newline at end of file +9cc364c42cc64ab7b55b5c55e303fb63a456cf00 \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index b547e7b1bc..3cf4aa80b2 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -112,7 +112,7 @@ set allquicktests [test_set $alltests -exclude { incrvacuum_ioerr.test autovacuum_crash.test btree8.test shared_err.test vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test - sort3.test + sort3.test sort4.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] diff --git a/test/sort4.test b/test/sort4.test new file mode 100644 index 0000000000..3fb335cb2f --- /dev/null +++ b/test/sort4.test @@ -0,0 +1,145 @@ +# 2014 May 6. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# The tests in this file are brute force tests of the multi-threaded +# sorter. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sort4 + +catch { db close } +sqlite3_shutdown +sqlite3_config_worker_threads 3 +sqlite3_initialize +reset_db + +#-------------------------------------------------------------------- +# Set up a table "t1" containing $nRow rows. Each row contains also +# contains blob fields that total to at least $nPayload bytes of +# content. +# +proc populate_table {nRow nPayload} { + set nCol 0 + + set n 0 + for {set nCol 0} {$n < $nPayload} {incr nCol} { + incr n [expr (4 << $nCol)] + } + + set cols [lrange [list xxx c0 c1 c2 c3 c4 c5 c6 c7] 1 $nCol] + set data [lrange [list xxx \ + randomblob(4) randomblob(8) randomblob(16) randomblob(32) \ + randomblob(64) randomblob(128) randomblob(256) randomblob(512) \ + ] 1 $nCol] + + execsql { DROP TABLE IF EXISTS t1 } + + db transaction { + execsql "CREATE TABLE t1(a, [join $cols ,], b);" + set insert "INSERT INTO t1 VALUES(:k, [join $data ,], :k)" + for {set i 0} {$i < $nRow} {incr i} { + set k [expr int(rand()*1000000000)] + execsql $insert + } + } +} + +# Helper for [do_sorter_test] +# +proc sorter_test {nRow nRead {nPayload 100} {cache_size 10}} { + db eval "PRAGMA cache_size = $cache_size" + set res [list] + + set nLoad [expr ($nRow > $nRead) ? $nRead : $nRow] + + set nPayload [expr (($nPayload+3)/4) * 4] + set cols [list] + foreach {mask col} { + 0x04 c0 0x08 c1 0x10 c2 0x20 c3 + 0x40 c4 0x80 c5 0x100 c6 0x200 c7 + } { + if {$nPayload & $mask} { lappend cols $col } + } + + set n 0 + db eval "SELECT a, [join $cols ,], b FROM t1 WHERE rowid<=$nRow ORDER BY a" { + if {$a!=$b} { error "a!=b (a=$a b=$b)" } + lappend res $a + incr n + if {$n==$nLoad} break + } + + + set sql {SELECT a FROM t1 WHERE rowid<=$nRow ORDER BY a LIMIT $nRead} + if {$res != [db eval $sql]} { + puts $res + puts [db eval {SELECT a FROM t1 WHERE rowid<=$nLoad ORDER BY a}] + error "data no good" + } + + set {} {} +} + +# Usage: +# +# do_sorter_test ... +# +# where are any of the following switches: +# +# -rows N (number of rows to have sorter sort) +# -read N (number of rows to read out of sorter) +# -payload N (bytes of payload to read with each row) +# -cachesize N (Value for "PRAGMA cache_size = ?") +# -repeats N (number of times to repeat test) +# +proc do_sorter_test {tn args} { + set a(-rows) 1000 + set a(-repeats) 1 + set a(-read) 100 + set a(-payload) 100 + set a(-cachesize) 100 + + foreach {s val} $args { + if {[info exists a($s)]==0} { + unset a(-cachesize) + set optlist "[join [array names a] ,] or -cachesize" + error "Unknown option $s, expected $optlist" + } + set a($s) $val + } + + for {set i 0} {$i < $a(-repeats)} {incr i} { + set cmd [list sorter_test $a(-rows) $a(-read) $a(-payload) $a(-cachesize)] + do_test $tn.$i $cmd {} + } +} + +do_test 1 { + execsql "PRAGMA page_size = 4096" + populate_table 100000 500 +} {} + +do_sorter_test 2 -repeats 10 -rows 1000 -read 100 +do_sorter_test 3 -repeats 10 -rows 100000 -read 1000 +do_sorter_test 4 -repeats 10 -rows 100000 -read 1000 -payload 500 +do_sorter_test 5 -repeats 10 -rows 100000 -read 100000 -payload 8 +do_sorter_test 6 -repeats 10 -rows 100000 -read 10 -payload 8 + +catch { db close } +sqlite3_shutdown +sqlite3_config_worker_threads 0 +sqlite3_initialize +finish_test + + From dfea45330e5187e37a7de7750fcef6884630a1bf Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 6 May 2014 15:38:07 +0000 Subject: [PATCH 068/710] Re-implement the core of the multi-threaded sorter tests in sort4.test using C. Run each test in sort4.test ten times, or repeat all tests for 300 seconds as part of the "multithread" permutation test. FossilOrigin-Name: 208b2b04d4d282bec4424ea7160a123ba549d118 --- manifest | 16 +++---- manifest.uuid | 2 +- src/test1.c | 74 ++++++++++++++++++++++++++++++ test/permutations.test | 2 +- test/sort4.test | 101 +++++++++++++++++++++++++++++------------ 5 files changed, 156 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index ac504dddc6..e5ca2b242a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sfile\ssort4.test,\scontaining\sbrute\sforce\stests\sfor\sthe\smulti-theaded\ssorter. -D 2014-05-05T20:03:50.967 +C Re-implement\sthe\score\sof\sthe\smulti-threaded\ssorter\stests\sin\ssort4.test\susing\sC.\sRun\seach\stest\sin\ssort4.test\sten\stimes,\sor\srepeat\sall\stests\sfor\s300\sseconds\sas\spart\sof\sthe\s"multithread"\spermutation\stest. +D 2014-05-06T15:38:07.762 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd -F src/test1.c bd88cc00bff2f15279d808e84501f06148c144f9 +F src/test1.c a0e59104fec5626bb1d1bd746157f43e85245e4b F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -741,7 +741,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 465bc22a873ced74a6d1dedc0dde5da424be6e6a +F test/permutations.test 33e7e239ba494fdb30e2f4ffc64c508b145ff42f F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -824,7 +824,7 @@ F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 688468cef8c9a66fcc1d54235de8e4deac745690 F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 -F test/sort4.test a29761a7d05f194e516e91a03d38f8194ac9849d +F test/sort4.test 5cce4601abc9b1b63ac25d087723b257710577bb F test/sortfault.test 1a12b6e27d475f50658a8164aaa34f0080a86b36 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1171,7 +1171,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2d2edfe58db101d42a96772b856e6e55b401aab6 -R a3f7784643670968322db5c49315411c +P 9cc364c42cc64ab7b55b5c55e303fb63a456cf00 +R 3ed4f872bc36ee2cfbd000cc3f4dde43 U dan -Z cca87b3c456f151b38b423a6310da351 +Z 5684026d864e64d9d77cb71d6b4cd16b diff --git a/manifest.uuid b/manifest.uuid index f77d0e5e6f..4e0ab1c048 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9cc364c42cc64ab7b55b5c55e303fb63a456cf00 \ No newline at end of file +208b2b04d4d282bec4424ea7160a123ba549d118 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 403a7fafa7..2d0efb949e 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6370,6 +6370,79 @@ static int sorter_test_fakeheap( return TCL_OK; } +/* +** sorter_test_sort4_helper DB SQL1 NSTEP SQL2 +** +** Compile SQL statement $SQL1 and step it $NSTEP times. For each row, +** check that the leftmost and rightmost columns returned are both integers, +** and that both contain the same value. +** +** Then execute statement $SQL2. Check that the statement returns the same +** set of integers in the same order as in the previous step (using $SQL1). +*/ +static int sorter_test_sort4_helper( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + const char *zSql1; + const char *zSql2; + int nStep; + int iStep; + int iCksum1 = 0; + int iCksum2 = 0; + int rc; + int iB; + sqlite3 *db; + sqlite3_stmt *pStmt; + + if( objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB SQL1 NSTEP SQL2"); + return TCL_ERROR; + } + + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zSql1 = Tcl_GetString(objv[2]); + if( Tcl_GetIntFromObj(interp, objv[3], &nStep) ) return TCL_ERROR; + zSql2 = Tcl_GetString(objv[4]); + + rc = sqlite3_prepare_v2(db, zSql1, -1, &pStmt, 0); + if( rc!=SQLITE_OK ) goto sql_error; + + iB = sqlite3_column_count(pStmt)-1; + for(iStep=0; iStep, b INTEGER); +# +# For each row, the values of columns "a" and "b" are set to the same +# pseudo-randomly selected integer. The "extra-columns", of which there +# are at most eight, are named c0, c1, c2 etc. Column c0 contains a 4 +# byte string. Column c1 an 8 byte string. Field c2 16 bytes, and so on. +# +# This table is intended to be used for testing queries of the form: +# +# SELECT a, , b FROM t1 ORDER BY a; +# +# The test code checks that rows are returned in order, and that the +# values of "a" and "b" are the same for each row (the idea being that +# if field "b" at the end of the sorter record has not been corrupted, +# the rest of the record is probably Ok as well). # proc populate_table {nRow nPayload} { set nCol 0 @@ -57,8 +80,7 @@ proc populate_table {nRow nPayload} { # Helper for [do_sorter_test] # -proc sorter_test {nRow nRead {nPayload 100} {cache_size 10}} { - db eval "PRAGMA cache_size = $cache_size" +proc sorter_test {nRow nRead nPayload} { set res [list] set nLoad [expr ($nRow > $nRead) ? $nRead : $nRow] @@ -72,22 +94,25 @@ proc sorter_test {nRow nRead {nPayload 100} {cache_size 10}} { if {$nPayload & $mask} { lappend cols $col } } - set n 0 - db eval "SELECT a, [join $cols ,], b FROM t1 WHERE rowid<=$nRow ORDER BY a" { - if {$a!=$b} { error "a!=b (a=$a b=$b)" } - lappend res $a - incr n - if {$n==$nLoad} break - } - - - set sql {SELECT a FROM t1 WHERE rowid<=$nRow ORDER BY a LIMIT $nRead} - if {$res != [db eval $sql]} { - puts $res - puts [db eval {SELECT a FROM t1 WHERE rowid<=$nLoad ORDER BY a}] - error "data no good" - } + # Create two SELECT statements. Statement $sql1 uses the sorter to sort + # $nRow records of a bit over $nPayload bytes each read from the "t1" + # table created by [populate_table] proc above. Rows are sorted in order + # of the integer field in each "t1" record. + # + # The second SQL statement sorts the same set of rows as the first, but + # uses a LIMIT clause, causing SQLite to use a temp table instead of the + # sorter for sorting. + # + set sql1 "SELECT a, [join $cols ,], b FROM t1 WHERE rowid<=$nRow ORDER BY a" + set sql2 "SELECT a FROM t1 WHERE rowid<=$nRow ORDER BY a LIMIT $nRead" + # Pass the two SQL statements to a helper command written in C. This + # command steps statement $sql1 $nRead times and compares the integer + # values in the rows returned with the results of executing $sql2. If + # the comparison fails (indicating some bug in the sorter), a Tcl + # exception is thrown. + # + sorter_test_sort4_helper db $sql1 $nRead $sql2 set {} {} } @@ -119,22 +144,41 @@ proc do_sorter_test {tn args} { set a($s) $val } - for {set i 0} {$i < $a(-repeats)} {incr i} { - set cmd [list sorter_test $a(-rows) $a(-read) $a(-payload) $a(-cachesize)] - do_test $tn.$i $cmd {} - } + db eval "PRAGMA cache_size = $a(-cachesize)" + + do_test $tn [subst -nocommands { + for {set i 0} {[set i] < $a(-repeats)} {incr i} { + sorter_test $a(-rows) $a(-read) $a(-payload) + } + }] {} } +proc clock_seconds {} { + db one {SELECT strftime('%s')} +} + +#------------------------------------------------------------------------- +# Begin tests here. + +# Create a test database. do_test 1 { execsql "PRAGMA page_size = 4096" populate_table 100000 500 } {} -do_sorter_test 2 -repeats 10 -rows 1000 -read 100 -do_sorter_test 3 -repeats 10 -rows 100000 -read 1000 -do_sorter_test 4 -repeats 10 -rows 100000 -read 1000 -payload 500 -do_sorter_test 5 -repeats 10 -rows 100000 -read 100000 -payload 8 -do_sorter_test 6 -repeats 10 -rows 100000 -read 10 -payload 8 +set iTimeLimit [expr [clock_seconds] + $SORT4TIMEOUT] + +for {set tn 2} {1} {incr tn} { + do_sorter_test $tn.2 -repeats 10 -rows 1000 -read 100 + do_sorter_test $tn.3 -repeats 10 -rows 100000 -read 1000 + do_sorter_test $tn.4 -repeats 10 -rows 100000 -read 1000 -payload 500 + do_sorter_test $tn.5 -repeats 10 -rows 100000 -read 100000 -payload 8 + do_sorter_test $tn.6 -repeats 10 -rows 100000 -read 10 -payload 8 + + set iNow [clock_seconds] + if {$iNow>=$iTimeLimit} break + do_test "$testprefix-([expr $iTimeLimit-$iNow] seconds remain)" {} {} +} catch { db close } sqlite3_shutdown @@ -142,4 +186,3 @@ sqlite3_config_worker_threads 0 sqlite3_initialize finish_test - From 5a7dbc70eac74506513f2e69141039fb8ee0efd3 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 6 May 2014 16:21:30 +0000 Subject: [PATCH 069/710] Add a little extra variety to the tests in sort4.test. FossilOrigin-Name: 7de6aee6a5cb5c7f89dced89f2ebf38f8be7a4fa --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/sort4.test | 24 +++++++++++++++++------- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index e5ca2b242a..e4b717ce1a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Re-implement\sthe\score\sof\sthe\smulti-threaded\ssorter\stests\sin\ssort4.test\susing\sC.\sRun\seach\stest\sin\ssort4.test\sten\stimes,\sor\srepeat\sall\stests\sfor\s300\sseconds\sas\spart\sof\sthe\s"multithread"\spermutation\stest. -D 2014-05-06T15:38:07.762 +C Add\sa\slittle\sextra\svariety\sto\sthe\stests\sin\ssort4.test. +D 2014-05-06T16:21:30.441 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -824,7 +824,7 @@ F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 688468cef8c9a66fcc1d54235de8e4deac745690 F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 -F test/sort4.test 5cce4601abc9b1b63ac25d087723b257710577bb +F test/sort4.test 971452fd4e2928e6fc05c3868396ad7d5f9ce2ad F test/sortfault.test 1a12b6e27d475f50658a8164aaa34f0080a86b36 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1171,7 +1171,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9cc364c42cc64ab7b55b5c55e303fb63a456cf00 -R 3ed4f872bc36ee2cfbd000cc3f4dde43 +P 208b2b04d4d282bec4424ea7160a123ba549d118 +R b3799ca8720be78abdc748ed84ab7526 U dan -Z 5684026d864e64d9d77cb71d6b4cd16b +Z f689b9a36b0734ece63c5c5e094898f6 diff --git a/manifest.uuid b/manifest.uuid index 4e0ab1c048..0fc1ce1baf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -208b2b04d4d282bec4424ea7160a123ba549d118 \ No newline at end of file +7de6aee6a5cb5c7f89dced89f2ebf38f8be7a4fa \ No newline at end of file diff --git a/test/sort4.test b/test/sort4.test index 7bee0aad27..4e8336cd84 100644 --- a/test/sort4.test +++ b/test/sort4.test @@ -127,6 +127,7 @@ proc sorter_test {nRow nRead nPayload} { # -payload N (bytes of payload to read with each row) # -cachesize N (Value for "PRAGMA cache_size = ?") # -repeats N (number of times to repeat test) +# -fakeheap BOOL (true to use separate allocations for in-memory records) # proc do_sorter_test {tn args} { set a(-rows) 1000 @@ -134,6 +135,7 @@ proc do_sorter_test {tn args} { set a(-read) 100 set a(-payload) 100 set a(-cachesize) 100 + set a(-fakeheap) 0 foreach {s val} $args { if {[info exists a($s)]==0} { @@ -143,14 +145,20 @@ proc do_sorter_test {tn args} { } set a($s) $val } + if {[permutation] == "memsys3" || [permutation] == "memsys5"} { + set a(-fakeheap) 0 + } + if {$a(-fakeheap)} { sorter_test_fakeheap 1 } + db eval "PRAGMA cache_size = $a(-cachesize)" - do_test $tn [subst -nocommands { for {set i 0} {[set i] < $a(-repeats)} {incr i} { sorter_test $a(-rows) $a(-read) $a(-payload) } }] {} + + if {$a(-fakeheap)} { sorter_test_fakeheap 0 } } proc clock_seconds {} { @@ -168,12 +176,14 @@ do_test 1 { set iTimeLimit [expr [clock_seconds] + $SORT4TIMEOUT] -for {set tn 2} {1} {incr tn} { - do_sorter_test $tn.2 -repeats 10 -rows 1000 -read 100 - do_sorter_test $tn.3 -repeats 10 -rows 100000 -read 1000 - do_sorter_test $tn.4 -repeats 10 -rows 100000 -read 1000 -payload 500 - do_sorter_test $tn.5 -repeats 10 -rows 100000 -read 100000 -payload 8 - do_sorter_test $tn.6 -repeats 10 -rows 100000 -read 10 -payload 8 +for {set t 2} {1} {incr tn} { + do_sorter_test $t.2 -repeats 10 -rows 1000 -read 100 + do_sorter_test $t.3 -repeats 10 -rows 100000 -read 1000 + do_sorter_test $t.4 -repeats 10 -rows 100000 -read 1000 -payload 500 + do_sorter_test $t.5 -repeats 10 -rows 100000 -read 100000 -payload 8 + do_sorter_test $t.6 -repeats 10 -rows 100000 -read 10 -payload 8 + do_sorter_test $t.7 -repeats 10 -rows 10000 -read 10000 -payload 8 -fakeheap 1 + do_sorter_test $t.8 -repeats 10 -rows 100000 -read 10000 -cachesize 250 set iNow [clock_seconds] if {$iNow>=$iTimeLimit} break From df24cf81a6ccd41a75c348e8e9c342e502527c5c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 8 May 2014 22:01:08 +0000 Subject: [PATCH 070/710] Fix static variable declaration issue on Windows. FossilOrigin-Name: a41d29691307067523c8637b486941c5f7c33775 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex_w32.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e4b717ce1a..4aa2464bf2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\slittle\sextra\svariety\sto\sthe\stests\sin\ssort4.test. -D 2014-05-06T16:21:30.441 +C Fix\sstatic\svariable\sdeclaration\sissue\son\sWindows. +D 2014-05-08T22:01:08.441 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -199,7 +199,7 @@ F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 4222773e5f61e506f232aedc9ad9e16ca00c1399 F src/mutex_unix.c 56e22c1bc6aabfa2f9736317a8f56acd5d0c5f7c -F src/mutex_w32.c f648cebb542b7a7ab98cecaa79259e8519e8f492 +F src/mutex_w32.c c4726d3bfe80996665a39d27598ebd056e2d6b89 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f @@ -1171,7 +1171,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 208b2b04d4d282bec4424ea7160a123ba549d118 -R b3799ca8720be78abdc748ed84ab7526 -U dan -Z f689b9a36b0734ece63c5c5e094898f6 +P 7de6aee6a5cb5c7f89dced89f2ebf38f8be7a4fa +R 117bf7d69b405485b3cb8ed74140d14c +U mistachkin +Z 6be3c46bf656f034e9d870a5f4b03d3e diff --git a/manifest.uuid b/manifest.uuid index 0fc1ce1baf..80fca0f480 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7de6aee6a5cb5c7f89dced89f2ebf38f8be7a4fa \ No newline at end of file +a41d29691307067523c8637b486941c5f7c33775 \ No newline at end of file diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 7c17cc501e..05d2d5516b 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -93,7 +93,7 @@ static int winMutexNotheld(sqlite3_mutex *p){ /* ** Initialize and deinitialize the mutex subsystem. */ -static sqlite3_mutex winMutex_staticMutexes[6] = { +static sqlite3_mutex winMutex_staticMutexes[] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, From dcb1a840ff101edd3b3d7388aba431fcf733d68e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 9 May 2014 11:15:57 +0000 Subject: [PATCH 071/710] Add new static mutex SQLITE_MUTEX_STATIC_APP3. FossilOrigin-Name: ee0ab09c80a648e9202757fc04122952375e7c8c --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/mutex_noop.c | 2 +- src/mutex_unix.c | 2 ++ src/mutex_w32.c | 2 ++ src/sqlite.h.in | 1 + 6 files changed, 17 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 4aa2464bf2..60a99091c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sstatic\svariable\sdeclaration\sissue\son\sWindows. -D 2014-05-08T22:01:08.441 +C Add\snew\sstatic\smutex\sSQLITE_MUTEX_STATIC_APP3. +D 2014-05-09T11:15:57.314 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in ad0921c4b2780d01868cf69b419a4f102308d125 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -197,9 +197,9 @@ F src/mem5.c 74670012946c4adc8a6ad84d03acc80959c3e529 F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea -F src/mutex_noop.c 4222773e5f61e506f232aedc9ad9e16ca00c1399 -F src/mutex_unix.c 56e22c1bc6aabfa2f9736317a8f56acd5d0c5f7c -F src/mutex_w32.c c4726d3bfe80996665a39d27598ebd056e2d6b89 +F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 +F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 +F src/mutex_w32.c 6509b34042b0a8cdd8ea849f5987e187a969f225 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f @@ -220,7 +220,7 @@ F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c a5ed3fdc82ebab5b9b095ea1971515a7f8a303d2 F src/shell.c 6946aea9f21af551fa84bc6b2a8de55d93bf0004 -F src/sqlite.h.in ed6d0cc90da850340c3863c84351e6e164c0ef00 +F src/sqlite.h.in 3f3934dd2ff0adbd79d259fbbb2eee38a2c12367 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 851003126071d4a3bac86a0db75c48197fbd0ff0 @@ -1171,7 +1171,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7de6aee6a5cb5c7f89dced89f2ebf38f8be7a4fa -R 117bf7d69b405485b3cb8ed74140d14c -U mistachkin -Z 6be3c46bf656f034e9d870a5f4b03d3e +P a41d29691307067523c8637b486941c5f7c33775 +R dfeb990bf911fd0420714aaa7823f840 +U dan +Z 6074041396d198afcfcf58f3847bacd2 diff --git a/manifest.uuid b/manifest.uuid index 80fca0f480..a410acecfb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a41d29691307067523c8637b486941c5f7c33775 \ No newline at end of file +ee0ab09c80a648e9202757fc04122952375e7c8c \ No newline at end of file diff --git a/src/mutex_noop.c b/src/mutex_noop.c index 4cfc6f3a76..1a900c225a 100644 --- a/src/mutex_noop.c +++ b/src/mutex_noop.c @@ -107,7 +107,7 @@ static int debugMutexEnd(void){ return SQLITE_OK; } ** that means that a mutex could not be allocated. */ static sqlite3_mutex *debugMutexAlloc(int id){ - static sqlite3_debug_mutex aStatic[8]; + static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_APP3 - 1]; sqlite3_debug_mutex *pNew = 0; switch( id ){ case SQLITE_MUTEX_FAST: diff --git a/src/mutex_unix.c b/src/mutex_unix.c index ea1203c047..c8663144e8 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -102,6 +102,7 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } **
  • SQLITE_MUTEX_STATIC_PMEM **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -137,6 +138,7 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; sqlite3_mutex *p; diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 05d2d5516b..9a1663e596 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -101,6 +101,7 @@ static sqlite3_mutex winMutex_staticMutexes[] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; static int winMutex_isInit = 0; @@ -167,6 +168,7 @@ static int winMutexEnd(void){ **
  • SQLITE_MUTEX_STATIC_PMEM **
  • SQLITE_MUTEX_STATIC_APP1 **
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 2f39101a31..d3fe5694d8 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6039,6 +6039,7 @@ int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ #define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ #define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ /* ** CAPI3REF: Retrieve the mutex for a database connection From b0f935e4da79d87c5f2fb1c518bf48d24d7e865f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 12 May 2014 15:30:00 +0000 Subject: [PATCH 072/710] In the sorter, only use large memory allocations if scratch memory has not been configured. Add #ifdefs to disable unused code when SQLITE_MAX_WORKER_THREADS is zero. Other sorter changes in support of testability. FossilOrigin-Name: d7e2b0d9cb099eda3341bc934bedff9facfe88bd --- manifest | 16 +++++++-------- manifest.uuid | 2 +- src/main.c | 2 ++ src/vdbesort.c | 53 ++++++++++++++++++++++++++++++++++++-------------- 4 files changed, 49 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 7252ec3ac5..0a8a624dbd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\slatest\strunk\schanges\sinto\sthe\sthreads\sbranch. -D 2014-05-09T15:00:32.924 +C In\sthe\ssorter,\sonly\suse\slarge\smemory\sallocations\sif\sscratch\smemory\shas\snot\nbeen\sconfigured.\s\sAdd\s#ifdefs\sto\sdisable\sunused\scode\swhen\s\nSQLITE_MAX_WORKER_THREADS\sis\szero.\s\sOther\ssorter\schanges\sin\ssupport\nof\stestability. +D 2014-05-12T15:30:00.944 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -187,7 +187,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 2ba4f6eeb84cf8efd8bb632c1abe203dc86adcac +F src/main.c 274a72f5de174a8eb20d01bbd83393347c092592 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -289,7 +289,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c b36070436d33372237541e9eaa6068e1a61bc402 +F src/vdbesort.c 0daa029978b50d9c1d103d179ac1e2b38d9daa20 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1175,7 +1175,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P ee0ab09c80a648e9202757fc04122952375e7c8c 116bed5af664899a73b46dca528ac0c021fc50c3 -R e8e0d3ce5fd31e02042e8e0c64134243 -U dan -Z b9842070469fd517448a2bc084149835 +P 9ac8f1e7115bc50663235adedeb0d3e1234c5740 +R 6fb72f0b0bf6c011a8428254c9753d68 +U drh +Z 8d87c7cf2c9f964aeaecc8906e4a1927 diff --git a/manifest.uuid b/manifest.uuid index cc8556fc36..695d315026 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ac8f1e7115bc50663235adedeb0d3e1234c5740 \ No newline at end of file +d7e2b0d9cb099eda3341bc934bedff9facfe88bd \ No newline at end of file diff --git a/src/main.c b/src/main.c index d76bcb607e..4f8a128508 100644 --- a/src/main.c +++ b/src/main.c @@ -516,9 +516,11 @@ int sqlite3_config(int op, ...){ #endif case SQLITE_CONFIG_WORKER_THREADS: { +#if SQLITE_MAX_WORKER_THREADS>0 int n = va_arg(ap, int); if( n>SQLITE_MAX_WORKER_THREADS ) n = SQLITE_MAX_WORKER_THREADS; if( n>=0 ) sqlite3GlobalConfig.nWorker = n; +#endif break; } diff --git a/src/vdbesort.c b/src/vdbesort.c index 9c57135dd3..01404e4f12 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -449,7 +449,7 @@ static void vdbePmaReaderClear(PmaReader *pIter){ sqlite3_free(pIter->aAlloc); sqlite3_free(pIter->aBuffer); if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); - if( pIter->pIncr ) vdbeIncrFree(pIter->pIncr); + vdbeIncrFree(pIter->pIncr); memset(pIter, 0, sizeof(PmaReader)); } @@ -828,7 +828,11 @@ int sqlite3VdbeSorterInit( int szKeyInfo; /* Size of pCsr->pKeyInfo in bytes */ int sz; /* Size of pSorter in bytes */ int rc = SQLITE_OK; +#if SQLITE_MAX_WORKER_THREADS==0 + const int nWorker = 0; +#else int nWorker = (sqlite3GlobalConfig.bCoreMutex?sqlite3GlobalConfig.nWorker:0); +#endif assert( pCsr->pKeyInfo && pCsr->pBt==0 ); szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); @@ -858,10 +862,13 @@ int sqlite3VdbeSorterInit( if( mxCachemxPmaSize = mxCache * pgsz; - /* If the application is using memsys3 or memsys5, use a separate - ** allocation for each sort-key in memory. Otherwise, use a single big - ** allocation at pSorter->aMemory for all sort-keys. */ - if( sqlite3GlobalConfig.pHeap==0 ){ + /* If the application has not configure scratch memory using + ** SQLITE_CONFIG_SCRATCH then we assume it is OK to do large memory + ** allocations. If scratch memory has been configured, then assume + ** large memory allocations should be avoided to prevent heap + ** fragmentation. + */ + if( sqlite3GlobalConfig.pScratch==0 ){ assert( pSorter->iMemory==0 ); pSorter->nMemory = pgsz; pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz); @@ -1921,7 +1928,15 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ ** only requires a region of pTask->file2. */ if( rc==SQLITE_OK ){ int mxSz = pIncr->mxSz; - if( pIncr->bUseThread==0 ){ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pIncr->bUseThread ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); + if( rc==SQLITE_OK ){ + rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); + } + }else +#endif + /*if( !pIncr->bUseThread )*/{ if( pTask->file2.pFd==0 ){ assert( pTask->file2.iEof>0 ); rc = vdbeSorterOpenTempFile(db, pTask->file2.iEof, &pTask->file2.pFd); @@ -1932,14 +1947,10 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ pIncr->iStartOff = pTask->file2.iEof; pTask->file2.iEof += mxSz; } - }else{ - rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[0].pFd); - if( rc==SQLITE_OK ){ - rc = vdbeSorterOpenTempFile(db, mxSz, &pIncr->aFile[1].pFd); - } } } +#if SQLITE_MAX_WORKER_THREADS>0 if( rc==SQLITE_OK && pIncr->bUseThread ){ /* Use the current thread to populate aFile[1], even though this ** iterator is multi-threaded. The reason being that this function @@ -1947,6 +1958,7 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); rc = vdbeIncrPopulate(pIncr); } +#endif if( rc==SQLITE_OK && eMode!=INCRINIT_TASK ){ rc = vdbePmaReaderNext(pIter); @@ -2111,6 +2123,7 @@ static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ int rc = SQLITE_OK; int iTask; +#if SQLITE_MAX_WORKER_THREADS>0 /* If the sorter uses more than one task, then create the top-level ** MergeEngine here. This MergeEngine will read data from exactly ** one PmaReader per sub-task. */ @@ -2119,6 +2132,7 @@ static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ pMain = vdbeMergeEngineNew(pSorter->nTask); if( pMain==0 ) rc = SQLITE_NOMEM; } +#endif for(iTask=0; iTasknTask && rc==SQLITE_OK; iTask++){ SortSubtask *pTask = &pSorter->aTask[iTask]; @@ -2301,10 +2315,13 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ assert( pSorter->pReader==0 || pSorter->pMerger==0 ); assert( pSorter->bUseThreads==0 || pSorter->pReader ); assert( pSorter->bUseThreads==1 || pSorter->pMerger ); +#if SQLITE_MAX_WORKER_THREADS>0 if( pSorter->bUseThreads ){ rc = vdbePmaReaderNext(pSorter->pReader); *pbEof = (pSorter->pReader->pFile==0); - }else{ + }else +#endif + /*if( !pSorter->bUseThreads )*/ { rc = vdbeSorterNext(&pSorter->aTask[0], pSorter->pMerger, pbEof); } }else{ @@ -2328,9 +2345,15 @@ static void *vdbeSorterRowkey( ){ void *pKey; if( pSorter->bUsePMA ){ - PmaReader *pReader = (pSorter->bUseThreads ? - pSorter->pReader : &pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]] - ); + PmaReader *pReader; +#if SQLITE_MAX_WORKER_THREADS>0 + if( pSorter->bUseThreads ){ + pReader = pSorter->pReader; + }else +#endif + /*if( !pSorter->bUseThreads )*/{ + pReader = &pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]]; + } *pnKey = pReader->nKey; pKey = pReader->aKey; }else{ From 6cc375938318a5a68a2fe5e6a79bcc6beb208b38 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 15 May 2014 16:56:56 +0000 Subject: [PATCH 073/710] Use #ifdef to omit code that is not used when SQLITE_MAX_WORKER_THREADS is zero. FossilOrigin-Name: 2e8d287d4f41d395a488bf65b20e956b0e311177 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 0a8a624dbd..2bf8b35c71 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\ssorter,\sonly\suse\slarge\smemory\sallocations\sif\sscratch\smemory\shas\snot\nbeen\sconfigured.\s\sAdd\s#ifdefs\sto\sdisable\sunused\scode\swhen\s\nSQLITE_MAX_WORKER_THREADS\sis\szero.\s\sOther\ssorter\schanges\sin\ssupport\nof\stestability. -D 2014-05-12T15:30:00.944 +C Use\s#ifdef\sto\somit\scode\sthat\sis\snot\sused\swhen\sSQLITE_MAX_WORKER_THREADS\sis\szero. +D 2014-05-15T16:56:56.755 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 0daa029978b50d9c1d103d179ac1e2b38d9daa20 +F src/vdbesort.c 0cb40c336c04e2ada936fc4e17a6c6cb885af8c6 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1175,7 +1175,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P 9ac8f1e7115bc50663235adedeb0d3e1234c5740 -R 6fb72f0b0bf6c011a8428254c9753d68 +P d7e2b0d9cb099eda3341bc934bedff9facfe88bd +R e9a885bf89bf1a3baa55ce358814bd90 U drh -Z 8d87c7cf2c9f964aeaecc8906e4a1927 +Z 872834e0bfb672176645b5e47ef04abb diff --git a/manifest.uuid b/manifest.uuid index 695d315026..9ae9c294db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d7e2b0d9cb099eda3341bc934bedff9facfe88bd \ No newline at end of file +2e8d287d4f41d395a488bf65b20e956b0e311177 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 01404e4f12..ef8e7f6349 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1077,11 +1077,14 @@ static void vdbeIncrFree(IncrMerger *pIncr){ void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ int i; (void)vdbeSorterJoinAll(pSorter, SQLITE_OK); + assert( pSorter->bUseThreads || pSorter->pReader==0 ); +#if SQLITE_MAX_WORKER_THREADS>0 if( pSorter->pReader ){ vdbePmaReaderClear(pSorter->pReader); sqlite3DbFree(db, pSorter->pReader); pSorter->pReader = 0; } +#endif vdbeMergeEngineFree(pSorter->pMerger); pSorter->pMerger = 0; for(i=0; inTask; i++){ From 8f0dab37cb6800ade1d73386009cc8384240c6bb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 May 2014 12:18:08 +0000 Subject: [PATCH 074/710] Use #ifdef logic to avoid an always-true branch when SQLITE_MAX_WORKER_THREADS=0 FossilOrigin-Name: 88cfe6d7de5f19f484304d0db585eac5de6c00ae --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 2bf8b35c71..5971b7a8b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\s#ifdef\sto\somit\scode\sthat\sis\snot\sused\swhen\sSQLITE_MAX_WORKER_THREADS\sis\szero. -D 2014-05-15T16:56:56.755 +C Use\s#ifdef\slogic\sto\savoid\san\salways-true\sbranch\swhen\nSQLITE_MAX_WORKER_THREADS=0 +D 2014-05-16T12:18:08.658 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 0cb40c336c04e2ada936fc4e17a6c6cb885af8c6 +F src/vdbesort.c 7946115f7c3e0d6156121d5e39512e3e980da134 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1175,7 +1175,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix a94fb9b1b1ef06efc2898975cdfcfa9643731f5e -P d7e2b0d9cb099eda3341bc934bedff9facfe88bd -R e9a885bf89bf1a3baa55ce358814bd90 +P 2e8d287d4f41d395a488bf65b20e956b0e311177 +R e7717e2e9621a181baf0cb2712e277d2 U drh -Z 872834e0bfb672176645b5e47ef04abb +Z 12a20c396c3fdadc5d2762a7d632efff diff --git a/manifest.uuid b/manifest.uuid index 9ae9c294db..ec40eff80f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e8d287d4f41d395a488bf65b20e956b0e311177 \ No newline at end of file +88cfe6d7de5f19f484304d0db585eac5de6c00ae \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index ef8e7f6349..3bd19e1c36 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -829,7 +829,7 @@ int sqlite3VdbeSorterInit( int sz; /* Size of pSorter in bytes */ int rc = SQLITE_OK; #if SQLITE_MAX_WORKER_THREADS==0 - const int nWorker = 0; +# define nWorker 0 #else int nWorker = (sqlite3GlobalConfig.bCoreMutex?sqlite3GlobalConfig.nWorker:0); #endif @@ -879,6 +879,7 @@ int sqlite3VdbeSorterInit( return rc; } +#undef nWorker /* Defined at the top of this function */ /* ** Free the list of sorted records starting at pRecord. @@ -1806,9 +1807,8 @@ static int vdbeIncrNew( IncrMerger **ppOut ){ int rc = SQLITE_OK; - IncrMerger *pIncr = *ppOut = (IncrMerger*)sqlite3_malloc(sizeof(IncrMerger)); + IncrMerger *pIncr = *ppOut = (IncrMerger*)sqlite3MallocZero(sizeof(*pIncr)); if( pIncr ){ - memset(pIncr, 0, sizeof(IncrMerger)); pIncr->pMerger = pMerger; pIncr->pTask = pTask; pIncr->mxSz = MAX(pTask->pSorter->mxKeysize+9,pTask->pSorter->mxPmaSize/2); From 190d6959673c8855243911808b9c5efd98cf7764 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 May 2014 17:31:42 +0000 Subject: [PATCH 075/710] Add a pair of sqlite3FaultSim(100) calls to vdbesort.c to facilitate testing of obscure and hard to reach error conditions. FossilOrigin-Name: cceac14fd83ddd8f868c1767cdc66635607cb159 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 008a8c89ec..963e9310be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\schanges\sfrom\strunk,\sand\sespecially\sthe\snew\ssqlite3FaultSim()\ninterface. -D 2014-05-16T14:27:05.717 +C Add\sa\spair\sof\ssqlite3FaultSim(100)\scalls\sto\svdbesort.c\sto\sfacilitate\stesting\nof\sobscure\sand\shard\sto\sreach\serror\sconditions. +D 2014-05-16T17:31:42.435 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 7946115f7c3e0d6156121d5e39512e3e980da134 +F src/vdbesort.c f126a837c4bea832f3b91796081131b22d7b9c7e F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1175,7 +1175,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 88cfe6d7de5f19f484304d0db585eac5de6c00ae 0d43a7ad9abe821e33e0bf83a997aa4461b1e3f2 -R ab0eda2a23c6a8769439757acdbfc7f8 +P 43fcbd9116401f30781fdcbe55d1674d6b96311b +R 2be9e504fbe8e48aafd8e17341d9b966 U drh -Z b6ed5f464b035536d72301437f5c1385 +Z 9396ea859eaf958c2b1d37dca9017593 diff --git a/manifest.uuid b/manifest.uuid index 48bb400eb1..5102b5e8a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -43fcbd9116401f30781fdcbe55d1674d6b96311b \ No newline at end of file +cceac14fd83ddd8f868c1767cdc66635607cb159 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 3bd19e1c36..7c844065b4 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1032,7 +1032,7 @@ static MergeEngine *vdbeMergeEngineNew(int nIter){ while( NnTree = N; pNew->aIter = (PmaReader*)&pNew[1]; @@ -1807,7 +1807,8 @@ static int vdbeIncrNew( IncrMerger **ppOut ){ int rc = SQLITE_OK; - IncrMerger *pIncr = *ppOut = (IncrMerger*)sqlite3MallocZero(sizeof(*pIncr)); + IncrMerger *pIncr = *ppOut = (IncrMerger*) + (sqlite3FaultSim(100) ? 0 : sqlite3MallocZero(sizeof(*pIncr))); if( pIncr ){ pIncr->pMerger = pMerger; pIncr->pTask = pTask; From 5f4a4790485f14102c55ba4ea38413b554e5e275 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 May 2014 20:24:51 +0000 Subject: [PATCH 076/710] Rearrange some conditionals and add #if statements to make the code more testable. FossilOrigin-Name: 17afd77057f8695733a9a60225646c1d8813b1a0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 2 +- src/vdbesort.c | 29 ++++++++++++++++++++--------- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 963e9310be..01d72475f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\spair\sof\ssqlite3FaultSim(100)\scalls\sto\svdbesort.c\sto\sfacilitate\stesting\nof\sobscure\sand\shard\sto\sreach\serror\sconditions. -D 2014-05-16T17:31:42.435 +C Rearrange\ssome\sconditionals\sand\sadd\s#if\sstatements\sto\smake\sthe\scode\smore\ntestable. +D 2014-05-16T20:24:51.024 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,14 +282,14 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c 049fe1d3c0e2209c1bee107aec2fcff6285f909f F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 89ab2ded5123e823b47293aedd7931a4742fb6bd +F src/vdbe.c 4d11273a6a6644d948c534a17bdeaf42a2793d30 F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 F src/vdbeInt.h c78ace64dc37495806dd50596eded1f6cd2b5a64 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c f126a837c4bea832f3b91796081131b22d7b9c7e +F src/vdbesort.c 15d1405fcfbb2f4a2ad667872be211153d69fa36 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1175,7 +1175,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 43fcbd9116401f30781fdcbe55d1674d6b96311b -R 2be9e504fbe8e48aafd8e17341d9b966 +P cceac14fd83ddd8f868c1767cdc66635607cb159 +R 48ad2b172d9e67d362d3e27e91c64651 U drh -Z 9396ea859eaf958c2b1d37dca9017593 +Z 9751e875adf689a22af045a3a24a99e5 diff --git a/manifest.uuid b/manifest.uuid index 5102b5e8a8..d6d88fd0bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cceac14fd83ddd8f868c1767cdc66635607cb159 \ No newline at end of file +17afd77057f8695733a9a60225646c1d8813b1a0 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index b8bf727b53..a2a59406a4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1128,7 +1128,7 @@ case OP_Move: { assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); - VdbeMemRelease(pOut); + sqlite3VdbeMemRelease(pOut); zMalloc = pOut->zMalloc; memcpy(pOut, pIn1, sizeof(Mem)); #ifdef SQLITE_DEBUG diff --git a/src/vdbesort.c b/src/vdbesort.c index 7c844065b4..aa08879aef 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -108,8 +108,9 @@ ** ** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the ** sorter is running in single-threaded mode, then these PMAs are merged -** incrementally as keys are retreived from the sorter by the VDBE. See -** comments above object MergeEngine below for details. +** incrementally as keys are retreived from the sorter by the VDBE. The +** MergeEngine object, described in further detail below, performs this +** merge. ** ** Or, if running in multi-threaded mode, then a background thread is ** launched to merge the existing PMAs. Once the background thread has @@ -900,11 +901,17 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ sqlite3DbFree(db, pTask->pUnpacked); pTask->pUnpacked = 0; - if( pTask->list.aMemory==0 ){ - vdbeSorterRecordFree(0, pTask->list.pList); - }else{ +#if SQLITE_MAX_WORKER_THREADS>0 + /* pTask->list.aMemory can only be non-zero if it was handed memory + ** from the main thread. That only occurs SQLITE_MAX_WORKER_THREADS>0 */ + if( pTask->list.aMemory ){ sqlite3_free(pTask->list.aMemory); pTask->list.aMemory = 0; + }else +#endif + { + assert( pTask->list.aMemory==0 ); + vdbeSorterRecordFree(0, pTask->list.pList); } pTask->list.pList = 0; if( pTask->file.pFd ){ @@ -2138,7 +2145,7 @@ static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ } #endif - for(iTask=0; iTasknTask && rc==SQLITE_OK; iTask++){ + for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ SortSubtask *pTask = &pSorter->aTask[iTask]; if( pTask->nPMA ){ MergeEngine *pRoot = 0; /* Root node of tree for this task */ @@ -2165,10 +2172,14 @@ static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ } if( rc==SQLITE_OK ){ - if( pMain==0 ){ - pMain = pRoot; - }else{ +#if SQLITE_MAX_WORKER_THREADS>0 + if( pMain!=0 ){ rc = vdbeIncrNew(pTask, pRoot, &pMain->aIter[iTask].pIncr); + }else +#endif + { + assert( pMain==0 ); + pMain = pRoot; } }else{ vdbeMergeEngineFree(pRoot); From de823bedefdbadcffb818c5bafdd90f344fc94bb Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 May 2014 11:03:53 +0000 Subject: [PATCH 077/710] In vdbesort.c, change the names of PmaReader variables "pIter" to "pReadr". Other related comment changes. The generated object code should be the same. FossilOrigin-Name: 99efb235a08784020535a770d7e7d96d9ccee12d --- manifest | 12 +- manifest.uuid | 2 +- src/vdbesort.c | 330 ++++++++++++++++++++++++------------------------- 3 files changed, 172 insertions(+), 172 deletions(-) diff --git a/manifest b/manifest index 01dfb41694..48c7be4c03 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\schanges\sfrom\strunk. -D 2014-05-19T23:17:33.708 +C In\svdbesort.c,\schange\sthe\snames\sof\sPmaReader\svariables\s"pIter"\sto\s"pReadr".\nOther\srelated\scomment\schanges.\s\sThe\sgenerated\sobject\scode\sshould\sbe\sthe\ssame. +D 2014-05-20T11:03:53.818 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 44d4d1f5711f71eaf0d624de5c3e4976fe4e180b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 15d1405fcfbb2f4a2ad667872be211153d69fa36 +F src/vdbesort.c a8b8798f23baa4b293804597532a94470e44f248 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1177,7 +1177,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 17afd77057f8695733a9a60225646c1d8813b1a0 8180e320ee4090e41511836678e49a98c0b228e8 -R caf94a64ae09103b6f818ad8457d731e +P 6eefdad946da6a5f4052ac51d327777890fa3f18 +R 7346935112c4ac3d3d573eb5af19111a U drh -Z 654b19280087cc59a60d4e21f62a0c05 +Z 3f1d10c1aba3efc362d8ec22de1a631d diff --git a/manifest.uuid b/manifest.uuid index ea7a5b8d59..0e33e66f25 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6eefdad946da6a5f4052ac51d327777890fa3f18 \ No newline at end of file +99efb235a08784020535a770d7e7d96d9ccee12d \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index aa08879aef..e9776914c2 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -188,11 +188,11 @@ struct SorterList { ** combined into one big PMA in order to be able to step through the sorted ** records in order. ** -** The aIter[] array contains a PmaReader object for each of the PMAs being -** merged. An aIter[] object either points to a valid key or else is at EOF. +** The aReadr[] array contains a PmaReader object for each of the PMAs being +** merged. An aReadr[] object either points to a valid key or else is at EOF. ** For the purposes of the paragraphs below, we assume that the array is ** actually N elements in size, where N is the smallest power of 2 greater -** to or equal to the number of PMAs being merged. The extra aIter[] elements +** to or equal to the number of PMAs being merged. The extra aReadr[] elements ** are treated as if they are empty (always at EOF). ** ** The aTree[] array is also N elements in size. The value of N is stored in @@ -200,7 +200,7 @@ struct SorterList { ** ** The final (N/2) elements of aTree[] contain the results of comparing ** pairs of PMA keys together. Element i contains the result of -** comparing aIter[2*i-N] and aIter[2*i-N+1]. Whichever key is smaller, the +** comparing aReadr[2*i-N] and aReadr[2*i-N+1]. Whichever key is smaller, the ** aTree element is set to the index of it. ** ** For the purposes of this comparison, EOF is considered greater than any @@ -208,34 +208,34 @@ struct SorterList { ** values), it doesn't matter which index is stored. ** ** The (N/4) elements of aTree[] that precede the final (N/2) described -** above contains the index of the smallest of each block of 4 iterators. -** And so on. So that aTree[1] contains the index of the iterator that +** above contains the index of the smallest of each block of 4 PmaReaders +** And so on. So that aTree[1] contains the index of the PmaReader that ** currently points to the smallest key value. aTree[0] is unused. ** ** Example: ** -** aIter[0] -> Banana -** aIter[1] -> Feijoa -** aIter[2] -> Elderberry -** aIter[3] -> Currant -** aIter[4] -> Grapefruit -** aIter[5] -> Apple -** aIter[6] -> Durian -** aIter[7] -> EOF +** aReadr[0] -> Banana +** aReadr[1] -> Feijoa +** aReadr[2] -> Elderberry +** aReadr[3] -> Currant +** aReadr[4] -> Grapefruit +** aReadr[5] -> Apple +** aReadr[6] -> Durian +** aReadr[7] -> EOF ** ** aTree[] = { X, 5 0, 5 0, 3, 5, 6 } ** ** The current element is "Apple" (the value of the key indicated by -** iterator 5). When the Next() operation is invoked, iterator 5 will +** PmaReader 5). When the Next() operation is invoked, PmaReader 5 will ** be advanced to the next key in its segment. Say the next key is ** "Eggplant": ** -** aIter[5] -> Eggplant +** aReadr[5] -> Eggplant ** -** The contents of aTree[] are updated first by comparing the new iterator -** 5 key to the current key of iterator 4 (still "Grapefruit"). The iterator +** The contents of aTree[] are updated first by comparing the new PmaReader +** 5 key to the current key of PmaReader 4 (still "Grapefruit"). The PmaReader ** 5 value is still smaller, so aTree[6] is set to 5. And so on up the tree. -** The value of iterator 6 - "Durian" - is now smaller than that of iterator +** The value of PmaReader 6 - "Durian" - is now smaller than that of PmaReader ** 5, so aTree[3] is set to 6. Key 0 is smaller than key 6 (BananaaAlloc); - sqlite3_free(pIter->aBuffer); - if( pIter->aMap ) sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); - vdbeIncrFree(pIter->pIncr); - memset(pIter, 0, sizeof(PmaReader)); +static void vdbePmaReaderClear(PmaReader *pReadr){ + sqlite3_free(pReadr->aAlloc); + sqlite3_free(pReadr->aBuffer); + if( pReadr->aMap ) sqlite3OsUnfetch(pReadr->pFile, 0, pReadr->aMap); + vdbeIncrFree(pReadr->pIncr); + memset(pReadr, 0, sizeof(PmaReader)); } /* @@ -464,7 +464,7 @@ static void vdbePmaReaderClear(PmaReader *pIter){ ** next call to this function. */ static int vdbePmaReadBlob( - PmaReader *p, /* Iterator */ + PmaReader *p, /* PmaReader from which to take the blob */ int nByte, /* Bytes of data to read */ u8 **ppOut /* OUT: Pointer to buffer containing data */ ){ @@ -495,7 +495,7 @@ static int vdbePmaReadBlob( } assert( nRead>0 ); - /* Read data from the file. Return early if an error occurs. */ + /* Readr data from the file. Return early if an error occurs. */ rc = sqlite3OsRead(p->pFile, p->aBuffer, nRead, p->iReadOff); assert( rc!=SQLITE_IOERR_SHORT_READ ); if( rc!=SQLITE_OK ) return rc; @@ -599,43 +599,43 @@ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ } /* -** Seek iterator pIter to offset iOff within file pFile. Return SQLITE_OK +** Seek PmaReader pReadr to offset iOff within file pFile. Return SQLITE_OK ** if successful, or an SQLite error code if an error occurs. */ static int vdbePmaReaderSeek( SortSubtask *pTask, /* Task context */ - PmaReader *pIter, /* Iterate to populate */ + PmaReader *pReadr, /* Iterate to populate */ SorterFile *pFile, /* Sorter file to read from */ i64 iOff /* Offset in pFile */ ){ int rc = SQLITE_OK; - assert( pIter->pIncr==0 || pIter->pIncr->bEof==0 ); + assert( pReadr->pIncr==0 || pReadr->pIncr->bEof==0 ); - if( pIter->aMap ){ - sqlite3OsUnfetch(pIter->pFile, 0, pIter->aMap); - pIter->aMap = 0; + if( pReadr->aMap ){ + sqlite3OsUnfetch(pReadr->pFile, 0, pReadr->aMap); + pReadr->aMap = 0; } - pIter->iReadOff = iOff; - pIter->iEof = pFile->iEof; - pIter->pFile = pFile->pFd; + pReadr->iReadOff = iOff; + pReadr->iEof = pFile->iEof; + pReadr->pFile = pFile->pFd; - rc = vdbeSorterMapFile(pTask, pFile, &pIter->aMap); - if( rc==SQLITE_OK && pIter->aMap==0 ){ + rc = vdbeSorterMapFile(pTask, pFile, &pReadr->aMap); + if( rc==SQLITE_OK && pReadr->aMap==0 ){ int pgsz = pTask->pSorter->pgsz; - int iBuf = pIter->iReadOff % pgsz; - if( pIter->aBuffer==0 ){ - pIter->aBuffer = (u8*)sqlite3Malloc(pgsz); - if( pIter->aBuffer==0 ) rc = SQLITE_NOMEM; - pIter->nBuffer = pgsz; + int iBuf = pReadr->iReadOff % pgsz; + if( pReadr->aBuffer==0 ){ + pReadr->aBuffer = (u8*)sqlite3Malloc(pgsz); + if( pReadr->aBuffer==0 ) rc = SQLITE_NOMEM; + pReadr->nBuffer = pgsz; } if( rc==SQLITE_OK && iBuf ){ int nRead = pgsz - iBuf; - if( (pIter->iReadOff + nRead) > pIter->iEof ){ - nRead = (int)(pIter->iEof - pIter->iReadOff); + if( (pReadr->iReadOff + nRead) > pReadr->iEof ){ + nRead = (int)(pReadr->iEof - pReadr->iReadOff); } rc = sqlite3OsRead( - pIter->pFile, &pIter->aBuffer[iBuf], nRead, pIter->iReadOff + pReadr->pFile, &pReadr->aBuffer[iBuf], nRead, pReadr->iReadOff ); assert( rc!=SQLITE_IOERR_SHORT_READ ); } @@ -645,22 +645,22 @@ static int vdbePmaReaderSeek( } /* -** Advance iterator pIter to the next key in its PMA. Return SQLITE_OK if +** Advance PmaReader pReadr to the next key in its PMA. Return SQLITE_OK if ** no error occurs, or an SQLite error code if one does. */ -static int vdbePmaReaderNext(PmaReader *pIter){ +static int vdbePmaReaderNext(PmaReader *pReadr){ int rc = SQLITE_OK; /* Return Code */ u64 nRec = 0; /* Size of record in bytes */ - if( pIter->iReadOff>=pIter->iEof ){ - IncrMerger *pIncr = pIter->pIncr; + if( pReadr->iReadOff>=pReadr->iEof ){ + IncrMerger *pIncr = pReadr->pIncr; int bEof = 1; if( pIncr ){ rc = vdbeIncrSwap(pIncr); if( rc==SQLITE_OK && pIncr->bEof==0 ){ rc = vdbePmaReaderSeek( - pIncr->pTask, pIter, &pIncr->aFile[0], pIncr->iStartOff + pIncr->pTask, pReadr, &pIncr->aFile[0], pIncr->iStartOff ); bEof = 0; } @@ -668,26 +668,26 @@ static int vdbePmaReaderNext(PmaReader *pIter){ if( bEof ){ /* This is an EOF condition */ - vdbePmaReaderClear(pIter); + vdbePmaReaderClear(pReadr); return rc; } } if( rc==SQLITE_OK ){ - rc = vdbePmaReadVarint(pIter, &nRec); + rc = vdbePmaReadVarint(pReadr, &nRec); } if( rc==SQLITE_OK ){ - pIter->nKey = (int)nRec; - rc = vdbePmaReadBlob(pIter, (int)nRec, &pIter->aKey); + pReadr->nKey = (int)nRec; + rc = vdbePmaReadBlob(pReadr, (int)nRec, &pReadr->aKey); } return rc; } /* -** Initialize iterator pIter to scan through the PMA stored in file pFile +** Initialize PmaReader pReadr to scan through the PMA stored in file pFile ** starting at offset iStart and ending at offset iEof-1. This function -** leaves the iterator pointing to the first key in the PMA (or EOF if the +** leaves the PmaReader pointing to the first key in the PMA (or EOF if the ** PMA is empty). ** ** If the pnByte parameter is NULL, then it is assumed that the file @@ -697,26 +697,26 @@ static int vdbePmaReaderInit( SortSubtask *pTask, /* Task context */ SorterFile *pFile, /* Sorter file to read from */ i64 iStart, /* Start offset in pFile */ - PmaReader *pIter, /* Iterator to populate */ + PmaReader *pReadr, /* PmaReader to populate */ i64 *pnByte /* IN/OUT: Increment this value by PMA size */ ){ int rc; assert( pFile->iEof>iStart ); - assert( pIter->aAlloc==0 && pIter->nAlloc==0 ); - assert( pIter->aBuffer==0 ); - assert( pIter->aMap==0 ); + assert( pReadr->aAlloc==0 && pReadr->nAlloc==0 ); + assert( pReadr->aBuffer==0 ); + assert( pReadr->aMap==0 ); - rc = vdbePmaReaderSeek(pTask, pIter, pFile, iStart); + rc = vdbePmaReaderSeek(pTask, pReadr, pFile, iStart); if( rc==SQLITE_OK ){ u64 nByte; /* Size of PMA in bytes */ - rc = vdbePmaReadVarint(pIter, &nByte); - pIter->iEof = pIter->iReadOff + nByte; + rc = vdbePmaReadVarint(pReadr, &nByte); + pReadr->iEof = pReadr->iReadOff + nByte; *pnByte += nByte; } if( rc==SQLITE_OK ){ - rc = vdbePmaReaderNext(pIter); + rc = vdbePmaReaderNext(pReadr); } return rc; } @@ -748,7 +748,7 @@ static int vdbeSorterCompare( } /* -** This function is called to compare two iterator keys when merging +** This function is called to compare two PmaReader keys when merging ** multiple b-tree segments. Parameter iOut is the index of the aTree[] ** value to recalculate. */ @@ -773,8 +773,8 @@ static int vdbeSorterDoCompare( i2 = pMerger->aTree[iOut*2+1]; } - p1 = &pMerger->aIter[i1]; - p2 = &pMerger->aIter[i2]; + p1 = &pMerger->aReadr[i1]; + p2 = &pMerger->aReadr[i2]; if( p1->pFile==0 ){ iRes = i2; @@ -1027,23 +1027,23 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ #endif /* -** Allocate a new MergeEngine object with space for nIter iterators. +** Allocate a new MergeEngine object with space for nReader PmaReaders. */ -static MergeEngine *vdbeMergeEngineNew(int nIter){ - int N = 2; /* Smallest power of two >= nIter */ +static MergeEngine *vdbeMergeEngineNew(int nReader){ + int N = 2; /* Smallest power of two >= nReader */ int nByte; /* Total bytes of space to allocate */ MergeEngine *pNew; /* Pointer to allocated object to return */ - assert( nIter<=SORTER_MAX_MERGE_COUNT ); + assert( nReader<=SORTER_MAX_MERGE_COUNT ); - while( NnTree = N; - pNew->aIter = (PmaReader*)&pNew[1]; - pNew->aTree = (int*)&pNew->aIter[N]; + pNew->aReadr = (PmaReader*)&pNew[1]; + pNew->aTree = (int*)&pNew->aReadr[N]; } return pNew; } @@ -1055,7 +1055,7 @@ static void vdbeMergeEngineFree(MergeEngine *pMerger){ int i; if( pMerger ){ for(i=0; inTree; i++){ - vdbePmaReaderClear(&pMerger->aIter[i]); + vdbePmaReaderClear(&pMerger->aReadr[i]); } } sqlite3_free(pMerger); @@ -1439,8 +1439,8 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ } /* -** Advance the MergeEngine iterator passed as the second argument to -** the next entry. Set *pbEof to true if this means the iterator has +** Advance the MergeEngine PmaReader passed as the second argument to +** the next entry. Set *pbEof to true if this means the PmaReader has ** reached EOF. ** ** Return SQLITE_OK if successful or an error code if an error occurs. @@ -1451,63 +1451,63 @@ static int vdbeSorterNext( int *pbEof ){ int rc; - int iPrev = pMerger->aTree[1];/* Index of iterator to advance */ + int iPrev = pMerger->aTree[1];/* Index of PmaReader to advance */ - /* Advance the current iterator */ - rc = vdbePmaReaderNext(&pMerger->aIter[iPrev]); + /* Advance the current PmaReader */ + rc = vdbePmaReaderNext(&pMerger->aReadr[iPrev]); /* Update contents of aTree[] */ if( rc==SQLITE_OK ){ int i; /* Index of aTree[] to recalculate */ - PmaReader *pIter1; /* First iterator to compare */ - PmaReader *pIter2; /* Second iterator to compare */ - u8 *pKey2; /* To pIter2->aKey, or 0 if record cached */ + PmaReader *pReadr1; /* First PmaReader to compare */ + PmaReader *pReadr2; /* Second PmaReader to compare */ + u8 *pKey2; /* To pReadr2->aKey, or 0 if record cached */ - /* Find the first two iterators to compare. The one that was just + /* Find the first two PmaReaders to compare. The one that was just ** advanced (iPrev) and the one next to it in the array. */ - pIter1 = &pMerger->aIter[(iPrev & 0xFFFE)]; - pIter2 = &pMerger->aIter[(iPrev | 0x0001)]; - pKey2 = pIter2->aKey; + pReadr1 = &pMerger->aReadr[(iPrev & 0xFFFE)]; + pReadr2 = &pMerger->aReadr[(iPrev | 0x0001)]; + pKey2 = pReadr2->aKey; for(i=(pMerger->nTree+iPrev)/2; i>0; i=i/2){ - /* Compare pIter1 and pIter2. Store the result in variable iRes. */ + /* Compare pReadr1 and pReadr2. Store the result in variable iRes. */ int iRes; - if( pIter1->pFile==0 ){ + if( pReadr1->pFile==0 ){ iRes = +1; - }else if( pIter2->pFile==0 ){ + }else if( pReadr2->pFile==0 ){ iRes = -1; }else{ iRes = vdbeSorterCompare(pTask, - pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey + pReadr1->aKey, pReadr1->nKey, pKey2, pReadr2->nKey ); } - /* If pIter1 contained the smaller value, set aTree[i] to its index. - ** Then set pIter2 to the next iterator to compare to pIter1. In this - ** case there is no cache of pIter2 in pTask->pUnpacked, so set - ** pKey2 to point to the record belonging to pIter2. + /* If pReadr1 contained the smaller value, set aTree[i] to its index. + ** Then set pReadr2 to the next PmaReader to compare to pReadr1. In this + ** case there is no cache of pReadr2 in pTask->pUnpacked, so set + ** pKey2 to point to the record belonging to pReadr2. ** - ** Alternatively, if pIter2 contains the smaller of the two values, - ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() + ** Alternatively, if pReadr2 contains the smaller of the two values, + ** set aTree[i] to its index and update pReadr1. If vdbeSorterCompare() ** was actually called above, then pTask->pUnpacked now contains - ** a value equivalent to pIter2. So set pKey2 to NULL to prevent - ** vdbeSorterCompare() from decoding pIter2 again. + ** a value equivalent to pReadr2. So set pKey2 to NULL to prevent + ** vdbeSorterCompare() from decoding pReadr2 again. ** ** If the two values were equal, then the value from the oldest - ** PMA should be considered smaller. The VdbeSorter.aIter[] array - ** is sorted from oldest to newest, so pIter1 contains older values - ** than pIter2 iff (pIter1aTree[i] = (int)(pIter1 - pMerger->aIter); - pIter2 = &pMerger->aIter[ pMerger->aTree[i ^ 0x0001] ]; - pKey2 = pIter2->aKey; + ** PMA should be considered smaller. The VdbeSorter.aReadr[] array + ** is sorted from oldest to newest, so pReadr1 contains older values + ** than pReadr2 iff (pReadr1aTree[i] = (int)(pReadr1 - pMerger->aReadr); + pReadr2 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; + pKey2 = pReadr2->aKey; }else{ - if( pIter1->pFile ) pKey2 = 0; - pMerger->aTree[i] = (int)(pIter2 - pMerger->aIter); - pIter1 = &pMerger->aIter[ pMerger->aTree[i ^ 0x0001] ]; + if( pReadr1->pFile ) pKey2 = 0; + pMerger->aTree[i] = (int)(pReadr2 - pMerger->aReadr); + pReadr1 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; } } - *pbEof = (pMerger->aIter[pMerger->aTree[1]].pFile==0); + *pbEof = (pMerger->aReadr[pMerger->aTree[1]].pFile==0); } return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); @@ -1709,7 +1709,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ vdbePmaWriterInit(pOut->pFd, &writer, pTask->pSorter->pgsz, iStart); while( rc==SQLITE_OK ){ int dummy; - PmaReader *pReader = &pMerger->aIter[ pMerger->aTree[1] ]; + PmaReader *pReader = &pMerger->aReadr[ pMerger->aTree[1] ]; int nKey = pReader->nKey; i64 iEof = writer.iWriteOff + writer.iBufEnd; @@ -1755,7 +1755,7 @@ static int vdbeIncrBgPopulate(IncrMerger *pIncr){ /* ** This function is called when the PmaReader corresponding to pIncr has ** finished reading the contents of aFile[0]. Its purpose is to "refill" -** aFile[0] such that the iterator should start rereading it from the +** aFile[0] such that the PmaReader should start rereading it from the ** beginning. ** ** For single-threaded objects, this is accomplished by literally reading @@ -1841,7 +1841,7 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr){ #define INCRINIT_NORMAL 0 #define INCRINIT_TASK 1 #define INCRINIT_ROOT 2 -static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); +static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); /* ** Initialize the merger argument passed as the second argument. Once this @@ -1852,7 +1852,7 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode); ** objects attached to the PmaReader objects that the merger reads from have ** already been populated, but that they have not yet populated aFile[0] and ** set the PmaReader objects up to read from it. In this case all that is -** required is to call vdbePmaReaderNext() on each iterator to point it at +** required is to call vdbePmaReaderNext() on each PmaReader to point it at ** its first key. ** ** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use @@ -1867,21 +1867,21 @@ static int vdbeIncrInitMerger( int eMode /* One of the INCRINIT_XXX constants */ ){ int rc = SQLITE_OK; /* Return code */ - int i; /* For iterating through PmaReader objects */ + int i; /* For looping over PmaReader objects */ int nTree = pMerger->nTree; for(i=0; rc==SQLITE_OK && iaIter[nTree-i-1]); + rc = vdbePmaReaderNext(&pMerger->aReadr[nTree-i-1]); }else{ - rc = vdbePmaReaderIncrInit(&pMerger->aIter[i], INCRINIT_NORMAL); + rc = vdbePmaReaderIncrInit(&pMerger->aReadr[i], INCRINIT_NORMAL); } } @@ -1894,20 +1894,20 @@ static int vdbeIncrInitMerger( /* ** If the PmaReader passed as the first argument is not an incremental-reader -** (if pIter->pIncr==0), then this function is a no-op. Otherwise, it serves +** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it serves ** to open and/or initialize the temp file related fields of the IncrMerge -** object at (pIter->pIncr). +** object at (pReadr->pIncr). ** -** If argument eMode is set to INCRINIT_NORMAL, then PmaReader iterators -** in the sub-tree headed by pIter are also initialized. Data is then loaded -** into the buffers belonging to this iterator, pIter, and it is set to +** If argument eMode is set to INCRINIT_NORMAL, then PmaReader PmaReaders +** in the sub-tree headed by pReadr are also initialized. Data is then loaded +** into the buffers belonging to this PmaReader, pReadr, and it is set to ** point to the first key in its range. ** ** If argument eMode is set to INCRINIT_TASK, then PmaReader is guaranteed -** to be a multi-threaded iterator and this function is being called in a -** background thread. In this case all iterators in the sub-tree are +** to be a multi-threaded PmaReader and this function is being called in a +** background thread. In this case all PmaReaders in the sub-tree are ** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to -** pIter is populated. However, the iterator itself is not set up to point +** pReadr is populated. However, the PmaReader itself is not set up to point ** to its first key. A call to vdbePmaReaderNext() is still required to do ** that. ** @@ -1918,16 +1918,16 @@ static int vdbeIncrInitMerger( ** lead to the current background thread attempting to join itself. ** ** Finally, if argument eMode is set to INCRINIT_ROOT, it may be assumed -** that pIter->pIncr is a multi-threaded IncrMerge objects, and that all +** that pReadr->pIncr is a multi-threaded IncrMerge objects, and that all ** child-trees have already been initialized using IncrInit(INCRINIT_TASK). -** In this case vdbePmaReaderNext() is called on all child iterators and -** the current iterator set to point to the first key in its range. +** In this case vdbePmaReaderNext() is called on all child PmaReaders and +** the current PmaReader set to point to the first key in its range. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ +static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode){ int rc = SQLITE_OK; - IncrMerger *pIncr = pIter->pIncr; + IncrMerger *pIncr = pReadr->pIncr; if( pIncr ){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; @@ -1964,7 +1964,7 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ #if SQLITE_MAX_WORKER_THREADS>0 if( rc==SQLITE_OK && pIncr->bUseThread ){ /* Use the current thread to populate aFile[1], even though this - ** iterator is multi-threaded. The reason being that this function + ** PmaReader is multi-threaded. The reason being that this function ** is already running in background thread pIncr->pTask->thread. */ assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); rc = vdbeIncrPopulate(pIncr); @@ -1972,7 +1972,7 @@ static int vdbePmaReaderIncrInit(PmaReader *pIter, int eMode){ #endif if( rc==SQLITE_OK && eMode!=INCRINIT_TASK ){ - rc = vdbePmaReaderNext(pIter); + rc = vdbePmaReaderNext(pReadr); } } return rc; @@ -1994,13 +1994,13 @@ static void *vdbePmaReaderBgInit(void *pCtx){ ** Use a background thread to invoke vdbePmaReaderIncrInit(INCRINIT_TASK) ** on the the PmaReader object passed as the first argument. ** -** This call will initialize the various fields of the pIter->pIncr +** This call will initialize the various fields of the pReadr->pIncr ** structure and, if it is a multi-threaded IncrMerger, launch a ** background thread to populate aFile[1]. */ -static int vdbePmaReaderBgIncrInit(PmaReader *pIter){ - void *pCtx = (void*)pIter; - return vdbeSorterCreateThread(pIter->pIncr->pTask, vdbePmaReaderBgInit, pCtx); +static int vdbePmaReaderBgIncrInit(PmaReader *pReadr){ + void *pCtx = (void*)pReadr; + return vdbeSorterCreateThread(pReadr->pIncr->pTask, vdbePmaReaderBgInit, pCtx); } #endif @@ -2019,7 +2019,7 @@ static int vdbePmaReaderBgIncrInit(PmaReader *pIter){ static int vdbeMergeEngineLevel0( SortSubtask *pTask, /* Sorter task to read from */ int nPMA, /* Number of PMAs to read */ - i64 *piOffset, /* IN/OUT: Read offset in pTask->file */ + i64 *piOffset, /* IN/OUT: Readr offset in pTask->file */ MergeEngine **ppOut /* OUT: New merge-engine */ ){ MergeEngine *pNew; /* Merge engine to return */ @@ -2032,9 +2032,9 @@ static int vdbeMergeEngineLevel0( for(i=0; iaIter[i]; - rc = vdbePmaReaderInit(pTask, &pTask->file, iOff, pIter, &nDummy); - iOff = pIter->iEof; + PmaReader *pReadr = &pNew->aReadr[i]; + rc = vdbePmaReaderInit(pTask, &pTask->file, iOff, pReadr, &nDummy); + iOff = pReadr->iEof; } if( rc!=SQLITE_OK ){ @@ -2094,24 +2094,24 @@ static int vdbeSorterAddToTree( for(i=1; iaIter[iIter]; + PmaReader *pReadr = &p->aReadr[iIter]; - if( pIter->pIncr==0 ){ + if( pReadr->pIncr==0 ){ MergeEngine *pNew = vdbeMergeEngineNew(SORTER_MAX_MERGE_COUNT); if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ - rc = vdbeIncrNew(pTask, pNew, &pIter->pIncr); + rc = vdbeIncrNew(pTask, pNew, &pReadr->pIncr); } } if( rc==SQLITE_OK ){ - p = pIter->pIncr->pMerger; + p = pReadr->pIncr->pMerger; nDiv = nDiv / SORTER_MAX_MERGE_COUNT; } } if( rc==SQLITE_OK ){ - p->aIter[iSeq % SORTER_MAX_MERGE_COUNT].pIncr = pIncr; + p->aReadr[iSeq % SORTER_MAX_MERGE_COUNT].pIncr = pIncr; }else{ vdbeIncrFree(pIncr); } @@ -2174,7 +2174,7 @@ static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS>0 if( pMain!=0 ){ - rc = vdbeIncrNew(pTask, pRoot, &pMain->aIter[iTask].pIncr); + rc = vdbeIncrNew(pTask, pRoot, &pMain->aReadr[iTask].pIncr); }else #endif { @@ -2218,27 +2218,27 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ assert( pSorter->bUseThreads==0 || pSorter->nTask>1 ); if( pSorter->bUseThreads ){ int iTask; - PmaReader *pIter; + PmaReader *pReadr; SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; rc = vdbeSortAllocUnpacked(pLast); if( rc==SQLITE_OK ){ - pIter = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); - pSorter->pReader = pIter; - if( pIter==0 ) rc = SQLITE_NOMEM; + pReadr = (PmaReader*)sqlite3DbMallocZero(db, sizeof(PmaReader)); + pSorter->pReader = pReadr; + if( pReadr==0 ) rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ - rc = vdbeIncrNew(pLast, pMain, &pIter->pIncr); + rc = vdbeIncrNew(pLast, pMain, &pReadr->pIncr); if( rc==SQLITE_OK ){ - vdbeIncrSetThreads(pIter->pIncr); + vdbeIncrSetThreads(pReadr->pIncr); for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ IncrMerger *pIncr; - if( (pIncr = pMain->aIter[iTask].pIncr) ){ + if( (pIncr = pMain->aReadr[iTask].pIncr) ){ vdbeIncrSetThreads(pIncr); assert( pIncr->pTask!=pLast ); } } for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ - PmaReader *p = &pMain->aIter[iTask]; + PmaReader *p = &pMain->aReadr[iTask]; assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); if( p->pIncr ){ if( iTask==pSorter->nTask-1 ){ @@ -2252,7 +2252,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ pMain = 0; } if( rc==SQLITE_OK ){ - rc = vdbePmaReaderIncrInit(pIter, INCRINIT_ROOT); + rc = vdbePmaReaderIncrInit(pReadr, INCRINIT_ROOT); } }else #endif @@ -2367,7 +2367,7 @@ static void *vdbeSorterRowkey( }else #endif /*if( !pSorter->bUseThreads )*/{ - pReader = &pSorter->pMerger->aIter[pSorter->pMerger->aTree[1]]; + pReader = &pSorter->pMerger->aReadr[pSorter->pMerger->aTree[1]]; } *pnKey = pReader->nKey; pKey = pReader->aKey; From cb6effafecd4bbd73a55002e4df081c2e687451b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 20 May 2014 19:11:50 +0000 Subject: [PATCH 078/710] Improvements to the testability of the threads.c module. FossilOrigin-Name: 386e088868b44b02646e452147838d2e97b093ee --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/threads.c | 11 ++++++----- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 48c7be4c03..86d0d6e9cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\svdbesort.c,\schange\sthe\snames\sof\sPmaReader\svariables\s"pIter"\sto\s"pReadr".\nOther\srelated\scomment\schanges.\s\sThe\sgenerated\sobject\scode\sshould\sbe\sthe\ssame. -D 2014-05-20T11:03:53.818 +C Improvements\sto\sthe\stestability\sof\sthe\sthreads.c\smodule. +D 2014-05-20T19:11:50.626 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in de92112472618cb869d27249966bad1783e4a853 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -275,7 +275,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c b9daffcb6ae3a9c080da37b905e790d45a8f99db +F src/threads.c 3c63f60ce0aae4c40ed4b8dc745490ff42a05308 F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 @@ -1177,7 +1177,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6eefdad946da6a5f4052ac51d327777890fa3f18 -R 7346935112c4ac3d3d573eb5af19111a +P 99efb235a08784020535a770d7e7d96d9ccee12d +R 41b3a138387644de9249d3787f3c7d11 U drh -Z 3f1d10c1aba3efc362d8ec22de1a631d +Z 514d5c30144e64644ef12bde520c79ea diff --git a/manifest.uuid b/manifest.uuid index 0e33e66f25..bffaf5bd3b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -99efb235a08784020535a770d7e7d96d9ccee12d \ No newline at end of file +386e088868b44b02646e452147838d2e97b093ee \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 6b59ce90d2..1cfe2cc6ce 100644 --- a/src/threads.c +++ b/src/threads.c @@ -54,15 +54,16 @@ int sqlite3ThreadCreate( assert( ppThread!=0 ); assert( xTask!=0 ); + /* This routine is never used in single-threaded mode */ + assert( sqlite3GlobalConfig.bCoreMutex!=0 ); + *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM; memset(p, 0, sizeof(*p)); p->xTask = xTask; p->pIn = pIn; - if( sqlite3GlobalConfig.bCoreMutex==0 - || pthread_create(&p->tid, 0, xTask, pIn)!=0 - ){ + if( sqlite3FaultSim(200) ? 1 : pthread_create(&p->tid, 0, xTask, pIn) ){ p->done = 1; p->pOut = xTask(pIn); } @@ -80,10 +81,10 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ *ppOut = p->pOut; rc = SQLITE_OK; }else{ - rc = pthread_join(p->tid, ppOut); + rc = pthread_join(p->tid, ppOut) ? SQLITE_ERROR : SQLITE_OK; } sqlite3_free(p); - return rc ? SQLITE_ERROR : SQLITE_OK; + return rc; } #endif /* SQLITE_OS_UNIX && defined(SQLITE_MUTEX_PTHREADS) */ From c1971541e4e98434e36d82f4c2a7b18ea17078be Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 23 Jun 2014 23:28:13 +0000 Subject: [PATCH 079/710] Add the .fullschema command to the sqlite3.exe utility. This command shows the schema and the content of the sqlite_stat tables, all in one go. Useful when reporting problems with the query planner. FossilOrigin-Name: ebec48921c092e20c9d7608242b63db40b40be5e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 39 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fe8a9edf8e..9a02b2a068 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sSQLITE_OMIT_WSD\sbuilds. -D 2014-06-23T10:18:50.447 +C Add\sthe\s.fullschema\scommand\sto\sthe\ssqlite3.exe\sutility.\s\sThis\scommand\sshows\nthe\sschema\sand\sthe\scontent\sof\sthe\ssqlite_stat\stables,\sall\sin\sone\sgo.\s\sUseful\nwhen\sreporting\sproblems\swith\sthe\squery\splanner. +D 2014-06-23T23:28:13.644 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 6762c62e11b504aa014edceab8886495165e3a77 -F src/shell.c 058940e8a7198ea15ae90f60ffb0e2d67287b46c +F src/shell.c 56de2dfa3f25def4bf03098f7e2256fbb42f6e3c F src/sqlite.h.in a98eb3e8c86c934ea6f5bcfc6b69653dde2f4ed4 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1179,7 +1179,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 612b6d1b1f74eaf618520b90811eca10f978fc71 -R 283ec0802d51fd4e82222d529f6a8475 -U dan -Z de359222916ca6f6bd684ca986937509 +P 07dda49c1bf8997a18c3368acb81b6d863ea38d6 +R 4c1bf033e13f45ce881081fa1f198f33 +U drh +Z 8c91ced329194a39e8cdf52b6d2ec4dd diff --git a/manifest.uuid b/manifest.uuid index 1a8777f6b6..bc939b8328 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07dda49c1bf8997a18c3368acb81b6d863ea38d6 \ No newline at end of file +ebec48921c092e20c9d7608242b63db40b40be5e \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 00d4597545..ff87d817b9 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1583,6 +1583,7 @@ static char zHelp[] = ".exit Exit this program\n" ".explain ?on|off? Turn output mode suitable for EXPLAIN on or off.\n" " With no args, it turns EXPLAIN on.\n" + ".fullschema Show schema and the content of sqlite_stat tables\n" ".headers on|off Turn display of headers on or off\n" ".help Show this message\n" ".import FILE TABLE Import data from FILE into TABLE\n" @@ -2412,6 +2413,44 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } }else + if( c=='f' && strncmp(azArg[0], "fullschema", n)==0 ){ + struct callback_data data; + char *zErrMsg = 0; + if( nArg!=1 ){ + fprintf(stderr, "Usage: .fullschema\n"); + rc = 1; + goto meta_command_exit; + } + open_db(p, 0); + memcpy(&data, p, sizeof(data)); + data.showHeader = 0; + data.mode = MODE_Semi; + rc = sqlite3_exec(p->db, + "SELECT sql FROM" + " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" + " FROM sqlite_master UNION ALL" + " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " + "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%'" + "ORDER BY rowid", + callback, &data, &zErrMsg + ); + sqlite3_exec(p->db, "SELECT 'ANALYZE sqlite_master;'", + callback, &data, &zErrMsg); + data.mode = MODE_Insert; + data.zDestTable = "sqlite_stat1"; + shell_exec(p->db, "SELECT * FROM sqlite_stat1", + shell_callback, &data,&zErrMsg); + data.zDestTable = "sqlite_stat3"; + shell_exec(p->db, "SELECT * FROM sqlite_stat3", + shell_callback, &data,&zErrMsg); + data.zDestTable = "sqlite_stat4"; + shell_exec(p->db, "SELECT * FROM sqlite_stat4", + shell_callback, &data, &zErrMsg); + data.mode = MODE_Semi; + shell_exec(p->db, "SELECT 'ANALYZE sqlite_master;'", + shell_callback, &data, &zErrMsg); + }else + if( c=='h' && strncmp(azArg[0], "headers", n)==0 ){ if( nArg==2 ){ p->showHeader = booleanValue(azArg[1]); From f127814eaea2b71c63f620dab60be4ba7a37a72f Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 24 Jun 2014 00:59:15 +0000 Subject: [PATCH 080/710] Add the showstat4.exe utility program for decoding and displaying the content of the sqlite_stat4 table in a database. FossilOrigin-Name: b4d9f6053d1d95fdc1eab8ce610b51e7df8d896d --- manifest | 11 ++-- manifest.uuid | 2 +- tool/showstat4.c | 155 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 tool/showstat4.c diff --git a/manifest b/manifest index 9a02b2a068..14bc9eca79 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s.fullschema\scommand\sto\sthe\ssqlite3.exe\sutility.\s\sThis\scommand\sshows\nthe\sschema\sand\sthe\scontent\sof\sthe\ssqlite_stat\stables,\sall\sin\sone\sgo.\s\sUseful\nwhen\sreporting\sproblems\swith\sthe\squery\splanner. -D 2014-06-23T23:28:13.644 +C Add\sthe\sshowstat4.exe\sutility\sprogram\sfor\sdecoding\sand\sdisplaying\sthe\s\ncontent\sof\sthe\ssqlite_stat4\stable\sin\sa\sdatabase. +D 2014-06-24T00:59:15.223 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1160,6 +1160,7 @@ F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/showdb.c b018a8a69d07050fc0fe9afcf17313cdef0cc599 F tool/showjournal.c b62cecaab86a4053d944c276bb5232e4d17ece02 +F tool/showstat4.c 47662af1e2c9567fd821da910b3a247c5c138021 F tool/showwal.c 3f7f7da5ec0cba51b1449a75f700493377da57b5 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b @@ -1179,7 +1180,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 07dda49c1bf8997a18c3368acb81b6d863ea38d6 -R 4c1bf033e13f45ce881081fa1f198f33 +P ebec48921c092e20c9d7608242b63db40b40be5e +R 0ce5547c26474f508ca16bdace146cd3 U drh -Z 8c91ced329194a39e8cdf52b6d2ec4dd +Z 0e7559d34a6f25f8be2f321b29251255 diff --git a/manifest.uuid b/manifest.uuid index bc939b8328..142fcbe956 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ebec48921c092e20c9d7608242b63db40b40be5e \ No newline at end of file +b4d9f6053d1d95fdc1eab8ce610b51e7df8d896d \ No newline at end of file diff --git a/tool/showstat4.c b/tool/showstat4.c new file mode 100644 index 0000000000..c85ab1c770 --- /dev/null +++ b/tool/showstat4.c @@ -0,0 +1,155 @@ +/* +** This utility program decodes and displays the content of the +** sqlite_stat4 table in the database file named on the command +** line. +*/ +#include +#include +#include +#include +#include "sqlite3.h" + +typedef sqlite3_int64 i64; /* 64-bit signed integer type */ + + +/* +** Convert the var-int format into i64. Return the number of bytes +** in the var-int. Write the var-int value into *pVal. +*/ +static int decodeVarint(const unsigned char *z, i64 *pVal){ + i64 v = 0; + int i; + for(i=0; i<8; i++){ + v = (v<<7) + (z[i]&0x7f); + if( (z[i]&0x80)==0 ){ *pVal = v; return i+1; } + } + v = (v<<8) + (z[i]&0xff); + *pVal = v; + return 9; +} + + + +int main(int argc, char **argv){ + sqlite3 *db; + sqlite3_stmt *pStmt; + char *zIdx = 0; + int rc, j, x, y, mxHdr; + const unsigned char *aSample; + int nSample; + i64 iVal; + const char *zSep; + + if( argc!=2 ){ + fprintf(stderr, "Usage: %s DATABASE-FILE\n", argv[0]); + exit(1); + } + rc = sqlite3_open(argv[1], &db); + if( rc!=SQLITE_OK || db==0 ){ + fprintf(stderr, "Cannot open database file [%s]\n", argv[1]); + exit(1); + } + rc = sqlite3_prepare_v2(db, + "SELECT tbl||'.'||idx, nEq, nLT, nDLt, sample " + "FROM sqlite_stat4 ORDER BY 1", -1, + &pStmt, 0); + if( rc!=SQLITE_OK || pStmt==0 ){ + fprintf(stderr, "%s\n", sqlite3_errmsg(db)); + sqlite3_close(db); + exit(1); + } + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + if( zIdx==0 || strcmp(zIdx, (const char*)sqlite3_column_text(pStmt,0))!=0 ){ + if( zIdx ) printf("\n"); + sqlite3_free(zIdx); + zIdx = sqlite3_mprintf("%s", sqlite3_column_text(pStmt,0)); + printf("%s:\n", zIdx); + }else{ + printf(" -----------------------------------------------------------\n"); + } + printf(" nEq = %s\n", sqlite3_column_text(pStmt,1)); + printf(" nLt = %s\n", sqlite3_column_text(pStmt,2)); + printf(" nDLt = %s\n", sqlite3_column_text(pStmt,3)); + printf(" sample = x'"); + aSample = sqlite3_column_blob(pStmt,4); + nSample = sqlite3_column_bytes(pStmt,4); + for(j=0; jnSample ){ + printf(" \n"); + continue; + } + y = mxHdr = (int)iVal; + while( xmxHdr ) break; + if( iVal<0 ) break; + switch( iVal ){ + case 0: sz = 0; break; + case 1: sz = 1; break; + case 2: sz = 2; break; + case 3: sz = 3; break; + case 4: sz = 4; break; + case 5: sz = 6; break; + case 6: sz = 8; break; + case 7: sz = 8; break; + case 8: sz = 0; break; + case 9: sz = 0; break; + case 10: + case 11: sz = 0; break; + default: sz = (int)(iVal-12)/2; break; + } + if( y+sz>nSample ) break; + if( iVal==0 ){ + printf("%sNULL", zSep); + }else if( iVal<=7 ){ + v = (signed char)aSample[y]; + for(j=1; j Date: Tue, 24 Jun 2014 20:19:21 +0000 Subject: [PATCH 081/710] Fix showstat4.c so that it decodes typecodes 8 and 9 correctly. FossilOrigin-Name: 9ca737c0b41f87998d842e7772c3e483bb291c50 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/showstat4.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 14bc9eca79..fea0a6e2cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sshowstat4.exe\sutility\sprogram\sfor\sdecoding\sand\sdisplaying\sthe\s\ncontent\sof\sthe\ssqlite_stat4\stable\sin\sa\sdatabase. -D 2014-06-24T00:59:15.223 +C Fix\sshowstat4.c\sso\sthat\sit\sdecodes\stypecodes\s8\sand\s9\scorrectly. +D 2014-06-24T20:19:21.030 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1160,7 +1160,7 @@ F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/showdb.c b018a8a69d07050fc0fe9afcf17313cdef0cc599 F tool/showjournal.c b62cecaab86a4053d944c276bb5232e4d17ece02 -F tool/showstat4.c 47662af1e2c9567fd821da910b3a247c5c138021 +F tool/showstat4.c c39279d6bd37cb999b634f0064f6f86ad7af008f F tool/showwal.c 3f7f7da5ec0cba51b1449a75f700493377da57b5 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b @@ -1180,7 +1180,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ebec48921c092e20c9d7608242b63db40b40be5e -R 0ce5547c26474f508ca16bdace146cd3 +P b4d9f6053d1d95fdc1eab8ce610b51e7df8d896d +R 508dd98757238bd14305734cabeeae37 U drh -Z 0e7559d34a6f25f8be2f321b29251255 +Z 96b01d8567714808bc64d5917b4d4853 diff --git a/manifest.uuid b/manifest.uuid index 142fcbe956..5313f68aef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b4d9f6053d1d95fdc1eab8ce610b51e7df8d896d \ No newline at end of file +9ca737c0b41f87998d842e7772c3e483bb291c50 \ No newline at end of file diff --git a/tool/showstat4.c b/tool/showstat4.c index c85ab1c770..668d2106af 100644 --- a/tool/showstat4.c +++ b/tool/showstat4.c @@ -106,6 +106,8 @@ int main(int argc, char **argv){ if( y+sz>nSample ) break; if( iVal==0 ){ printf("%sNULL", zSep); + }else if( iVal==8 || iVal==9 ){ + printf("%s%d", zSep, ((int)iVal)-8); }else if( iVal<=7 ){ v = (signed char)aSample[y]; for(j=1; j Date: Thu, 26 Jun 2014 20:21:46 +0000 Subject: [PATCH 082/710] Attempt to use sqlite_stat4 data to estimate the number of rows visited by a range query that uses a skip-scan. This code is largely untested. FossilOrigin-Name: 01dc8102592427b71a18c2cb82301d2266dd59c2 --- manifest | 21 ++-- manifest.uuid | 2 +- src/sqliteInt.h | 2 + src/vdbemem.c | 137 +++++++++++++++++++------ src/where.c | 261 +++++++++++++++++++++++++++++++++--------------- 5 files changed, 300 insertions(+), 123 deletions(-) diff --git a/manifest b/manifest index fea0a6e2cc..5a780681cb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sshowstat4.c\sso\sthat\sit\sdecodes\stypecodes\s8\sand\s9\scorrectly. -D 2014-06-24T20:19:21.030 +C Attempt\sto\suse\ssqlite_stat4\sdata\sto\sestimate\sthe\snumber\sof\srows\svisited\sby\sa\srange\squery\sthat\suses\sa\sskip-scan.\sThis\scode\sis\slargely\suntested. +D 2014-06-26T20:21:46.005 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -227,7 +227,7 @@ F src/shell.c 56de2dfa3f25def4bf03098f7e2256fbb42f6e3c F src/sqlite.h.in a98eb3e8c86c934ea6f5bcfc6b69653dde2f4ed4 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h fccdc735c27b3dc12322fec7cdad8bc76be8d00b +F src/sqliteInt.h e88614d7371b80ff69dbbb5e4b9813ee93dfd890 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -289,14 +289,14 @@ F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac -F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 +F src/vdbemem.c 8f28cb5bdd5b8748dba67aab5a07a47386fe40dc F src/vdbesort.c 44441d73b08b3a638dcdb725afffb87c6574ad27 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 7b9e13cff91a2f14ac61e6a1bc3a83b5113e6298 +F src/where.c 643abd2dce6650d537c240c1ecdfc4090271091f F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1180,7 +1180,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b4d9f6053d1d95fdc1eab8ce610b51e7df8d896d -R 508dd98757238bd14305734cabeeae37 -U drh -Z 96b01d8567714808bc64d5917b4d4853 +P 9ca737c0b41f87998d842e7772c3e483bb291c50 +R 48ab9caf5628261ea5cdb35094645ef7 +T *branch * stat4-skipscan +T *sym-stat4-skipscan * +T -sym-trunk * +U dan +Z a77d7df51663ffd17fb07ab0f5d5231a diff --git a/manifest.uuid b/manifest.uuid index 5313f68aef..4422476212 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ca737c0b41f87998d842e7772c3e483bb291c50 \ No newline at end of file +01dc8102592427b71a18c2cb82301d2266dd59c2 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index cc3a030510..7e347c4ee2 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3431,7 +3431,9 @@ void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 void sqlite3AnalyzeFunctions(void); int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*); +int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**); void sqlite3Stat4ProbeFree(UnpackedRecord*); +int sqlite3Stat4Column(sqlite3*, const void*, int, int, sqlite3_value**); #endif /* diff --git a/src/vdbemem.c b/src/vdbemem.c index 2c4aa4ad7f..67dea9c282 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1152,6 +1152,50 @@ void sqlite3AnalyzeFunctions(void){ } } +static int stat4ValueFromExpr( + Parse *pParse, /* Parse context */ + Expr *pExpr, /* The expression to extract a value from */ + u8 affinity, /* Affinity to use */ + struct ValueNewStat4Ctx *pAlloc,/* How to allocate space */ + sqlite3_value **ppVal /* OUT: New value object (or NULL) */ +){ + int rc = SQLITE_OK; + sqlite3_value *pVal = 0; + sqlite3 *db = pParse->db; + + /* Skip over any TK_COLLATE nodes */ + pExpr = sqlite3ExprSkipCollate(pExpr); + + if( !pExpr ){ + pVal = valueNew(db, pAlloc); + if( pVal ){ + sqlite3VdbeMemSetNull((Mem*)pVal); + } + }else if( pExpr->op==TK_VARIABLE + || NEVER(pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) + ){ + Vdbe *v; + int iBindVar = pExpr->iColumn; + sqlite3VdbeSetVarmask(pParse->pVdbe, iBindVar); + if( (v = pParse->pReprepare)!=0 ){ + pVal = valueNew(db, pAlloc); + if( pVal ){ + rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iBindVar-1]); + if( rc==SQLITE_OK ){ + sqlite3ValueApplyAffinity(pVal, affinity, ENC(db)); + } + pVal->db = pParse->db; + } + } + }else{ + rc = valueFromExpr(db, pExpr, ENC(db), affinity, &pVal, pAlloc); + } + + assert( pVal==0 || pVal->db==db ); + *ppVal = pVal; + return rc; +} + /* ** This function is used to allocate and populate UnpackedRecord ** structures intended to be compared against sample index keys stored @@ -1191,47 +1235,76 @@ int sqlite3Stat4ProbeSetValue( int iVal, /* Array element to populate */ int *pbOk /* OUT: True if value was extracted */ ){ - int rc = SQLITE_OK; + int rc; sqlite3_value *pVal = 0; - sqlite3 *db = pParse->db; - - struct ValueNewStat4Ctx alloc; + alloc.pParse = pParse; alloc.pIdx = pIdx; alloc.ppRec = ppRec; alloc.iVal = iVal; - /* Skip over any TK_COLLATE nodes */ - pExpr = sqlite3ExprSkipCollate(pExpr); - - if( !pExpr ){ - pVal = valueNew(db, &alloc); - if( pVal ){ - sqlite3VdbeMemSetNull((Mem*)pVal); - } - }else if( pExpr->op==TK_VARIABLE - || NEVER(pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) - ){ - Vdbe *v; - int iBindVar = pExpr->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iBindVar); - if( (v = pParse->pReprepare)!=0 ){ - pVal = valueNew(db, &alloc); - if( pVal ){ - rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iBindVar-1]); - if( rc==SQLITE_OK ){ - sqlite3ValueApplyAffinity(pVal, affinity, ENC(db)); - } - pVal->db = pParse->db; - } - } - }else{ - rc = valueFromExpr(db, pExpr, ENC(db), affinity, &pVal, &alloc); - } + rc = stat4ValueFromExpr(pParse, pExpr, affinity, &alloc, &pVal); + assert( pVal==0 || pVal->db==pParse->db ); *pbOk = (pVal!=0); + return rc; +} - assert( pVal==0 || pVal->db==db ); +/* +** Attempt to extract a value from expression pExpr using the methods +** as described for sqlite3Stat4ProbeSetValue() above. +** +** If successful, set *ppVal to point to a new value object and return +** SQLITE_OK. If no value can be extracted, but no other error occurs +** (e.g. OOM), return SQLITE_OK and set *ppVal to NULL. Or, if an error +** does occur, return an SQLite error code. The final value of *ppVal +** is undefined in this case. +*/ +int sqlite3Stat4ValueFromExpr( + Parse *pParse, /* Parse context */ + Expr *pExpr, /* The expression to extract a value from */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal /* OUT: New value object (or NULL) */ +){ + return stat4ValueFromExpr(pParse, pExpr, affinity, 0, ppVal); +} + +int sqlite3Stat4Column( + sqlite3 *db, /* Database handle */ + const void *pRec, /* Pointer to buffer containing record */ + int nRec, /* Size of buffer pRec in bytes */ + int iCol, /* Column to extract */ + sqlite3_value **ppVal /* OUT: Extracted value */ +){ + int rc = SQLITE_OK; + Mem *pMem = *ppVal; + if( pMem==0 ){ + pMem = (Mem*)sqlite3ValueNew(db); + if( pMem==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + u32 t; + int nHdr; + int iHdr; + int iField; + int i; + u8 *a = (u8*)pRec; + + iHdr = getVarint32(a, nHdr); + iField = nHdr; + for(i=0; inOut is currently set to the estimated number of rows +** visited for scanning (a=? AND b=?). This function reduces that estimate +** by some factor to account for the (c BETWEEN ? AND ?) expression based +** on the stat4 data for the index. this scan will be peformed multiple +** times (once for each (a,b) combination that matches a=?) is dealt with +** by the caller. +** +** It does this by scanning through all stat4 samples, comparing values +** extracted from pLower and pUpper with the corresponding column in each +** sample. If L and U are the number of samples found to be less than or +** equal to the values extracted from pLower and pUpper respectively, and +** N is the total number of samples, the pLoop->nOut value is adjusted +** as follows: +** +** nOut = nOut * ( min(U - L, 1) / N ) +** +** If pLower is NULL, or a value cannot be extracted from the term, L is +** set to zero. If pUpper is NULL, or a value cannot be extracted from it, +** U is set to N. +** +** Normally, this function sets *pbDone to 1 before returning. However, +** if no value can be extracted from either pLower or pUpper (and so the +** estimate of the number of rows delivered remains unchanged), *pbDone +** is left as is. +** +** If an error occurs, an SQLite error code is returned. Otherwise, +** SQLITE_OK. +*/ +static int whereRangeSkipScanEst( + Parse *pParse, /* Parsing & code generating context */ + WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ + WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ + WhereLoop *pLoop, /* Update the .nOut value of this loop */ + int *pbDone /* Set to true if at least one expr. value extracted */ +){ + Index *p = pLoop->u.btree.pIndex; + int nEq = pLoop->u.btree.nEq; + sqlite3 *db = pParse->db; + int nLower = 0; + int nUpper = 0; + int rc = SQLITE_OK; + u8 aff = p->pTable->aCol[ p->aiColumn[nEq] ].affinity; + CollSeq *pColl; + + sqlite3_value *p1 = 0; /* Value extracted from pLower */ + sqlite3_value *p2 = 0; /* Value extracted from pUpper */ + sqlite3_value *pVal = 0; /* Value extracted from record */ + + pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); + if( pLower ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); + } + if( pUpper && rc==SQLITE_OK ){ + rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); + } + + if( p1 || p2 ){ + int i; + int nDiff; + for(i=0; rc==SQLITE_OK && inSample; i++){ + rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); + if( rc==SQLITE_OK && p1 ){ + int res = sqlite3MemCompare(p1, pVal, pColl); + if( res<=0 ) nLower++; + } + if( rc==SQLITE_OK && p2 ){ + int res = sqlite3MemCompare(p2, pVal, pColl); + if( res<=0 ) nUpper++; + } + } + if( p2==0 ) nUpper = p->nSample; + nDiff = (nUpper - nLower); + if( nDiff<=0 ) nDiff = 1; + pLoop->nOut -= (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); + *pbDone = 1; + }else{ + assert( *pbDone==0 ); + } + + sqlite3ValueFree(p1); + sqlite3ValueFree(p2); + sqlite3ValueFree(pVal); + + return rc; +} + /* ** This function is used to estimate the number of rows that will be visited ** by scanning an index for a range of values. The range may have an upper @@ -2054,95 +2148,100 @@ static int whereRangeScanEst( int nEq = pLoop->u.btree.nEq; if( p->nSample>0 - && nEq==pBuilder->nRecValid && nEqnSampleCol && OptimizationEnabled(pParse->db, SQLITE_Stat3) ){ - UnpackedRecord *pRec = pBuilder->pRec; - tRowcnt a[2]; - u8 aff; + if( nEq==pBuilder->nRecValid ){ + UnpackedRecord *pRec = pBuilder->pRec; + tRowcnt a[2]; + u8 aff; - /* Variable iLower will be set to the estimate of the number of rows in - ** the index that are less than the lower bound of the range query. The - ** lower bound being the concatenation of $P and $L, where $P is the - ** key-prefix formed by the nEq values matched against the nEq left-most - ** columns of the index, and $L is the value in pLower. - ** - ** Or, if pLower is NULL or $L cannot be extracted from it (because it - ** is not a simple variable or literal value), the lower bound of the - ** range is $P. Due to a quirk in the way whereKeyStats() works, even - ** if $L is available, whereKeyStats() is called for both ($P) and - ** ($P:$L) and the larger of the two returned values used. - ** - ** Similarly, iUpper is to be set to the estimate of the number of rows - ** less than the upper bound of the range query. Where the upper bound - ** is either ($P) or ($P:$U). Again, even if $U is available, both values - ** of iUpper are requested of whereKeyStats() and the smaller used. - */ - tRowcnt iLower; - tRowcnt iUpper; + /* Variable iLower will be set to the estimate of the number of rows in + ** the index that are less than the lower bound of the range query. The + ** lower bound being the concatenation of $P and $L, where $P is the + ** key-prefix formed by the nEq values matched against the nEq left-most + ** columns of the index, and $L is the value in pLower. + ** + ** Or, if pLower is NULL or $L cannot be extracted from it (because it + ** is not a simple variable or literal value), the lower bound of the + ** range is $P. Due to a quirk in the way whereKeyStats() works, even + ** if $L is available, whereKeyStats() is called for both ($P) and + ** ($P:$L) and the larger of the two returned values used. + ** + ** Similarly, iUpper is to be set to the estimate of the number of rows + ** less than the upper bound of the range query. Where the upper bound + ** is either ($P) or ($P:$U). Again, even if $U is available, both values + ** of iUpper are requested of whereKeyStats() and the smaller used. + */ + tRowcnt iLower; + tRowcnt iUpper; - if( nEq==p->nKeyCol ){ - aff = SQLITE_AFF_INTEGER; - }else{ - aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; - } - /* Determine iLower and iUpper using ($P) only. */ - if( nEq==0 ){ - iLower = 0; - iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); - }else{ - /* Note: this call could be optimized away - since the same values must - ** have been requested when testing key $P in whereEqualScanEst(). */ - whereKeyStats(pParse, p, pRec, 0, a); - iLower = a[0]; - iUpper = a[0] + a[1]; - } - - /* If possible, improve on the iLower estimate using ($P:$L). */ - if( pLower ){ - int bOk; /* True if value is extracted from pExpr */ - Expr *pExpr = pLower->pExpr->pRight; - assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ - tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 0, a); - iNew = a[0] + ((pLower->eOperator & WO_GT) ? a[1] : 0); - if( iNew>iLower ) iLower = iNew; - nOut--; - } - } - - /* If possible, improve on the iUpper estimate using ($P:$U). */ - if( pUpper ){ - int bOk; /* True if value is extracted from pExpr */ - Expr *pExpr = pUpper->pExpr->pRight; - assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ - tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 1, a); - iNew = a[0] + ((pUpper->eOperator & WO_LE) ? a[1] : 0); - if( iNewpRec = pRec; - if( rc==SQLITE_OK ){ - if( iUpper>iLower ){ - nNew = sqlite3LogEst(iUpper - iLower); + if( nEq==p->nKeyCol ){ + aff = SQLITE_AFF_INTEGER; }else{ - nNew = 10; assert( 10==sqlite3LogEst(2) ); + aff = p->pTable->aCol[p->aiColumn[nEq]].affinity; } - if( nNewaiRowLogEst[0]); + }else{ + /* Note: this call could be optimized away - since the same values must + ** have been requested when testing key $P in whereEqualScanEst(). */ + whereKeyStats(pParse, p, pRec, 0, a); + iLower = a[0]; + iUpper = a[0] + a[1]; } - pLoop->nOut = (LogEst)nOut; - WHERETRACE(0x10, ("range scan regions: %u..%u est=%d\n", - (u32)iLower, (u32)iUpper, nOut)); - return SQLITE_OK; + + /* If possible, improve on the iLower estimate using ($P:$L). */ + if( pLower ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pLower->pExpr->pRight; + assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + whereKeyStats(pParse, p, pRec, 0, a); + iNew = a[0] + ((pLower->eOperator & WO_GT) ? a[1] : 0); + if( iNew>iLower ) iLower = iNew; + nOut--; + } + } + + /* If possible, improve on the iUpper estimate using ($P:$U). */ + if( pUpper ){ + int bOk; /* True if value is extracted from pExpr */ + Expr *pExpr = pUpper->pExpr->pRight; + assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + whereKeyStats(pParse, p, pRec, 1, a); + iNew = a[0] + ((pUpper->eOperator & WO_LE) ? a[1] : 0); + if( iNewpRec = pRec; + if( rc==SQLITE_OK ){ + if( iUpper>iLower ){ + nNew = sqlite3LogEst(iUpper - iLower); + }else{ + nNew = 10; assert( 10==sqlite3LogEst(2) ); + } + if( nNewnOut = (LogEst)nOut; + WHERETRACE(0x10, ("range scan regions: %u..%u est=%d\n", + (u32)iLower, (u32)iUpper, nOut)); + return SQLITE_OK; + } + }else{ + int bDone = 0; + rc = whereRangeSkipScanEst(pParse, pLower, pUpper, pLoop, &bDone); + if( bDone ) return rc; } } #else From 9bec6fb3efce29aebf89a37f2fb134d16868dff9 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 26 Jun 2014 21:28:21 +0000 Subject: [PATCH 083/710] Revise the affinity returned for expressions flagged as 'generic'. Fix for [9a8b09f8e6]. FossilOrigin-Name: 92f7ad43dbfe4e02490df2f932c3c74fb89064d6 --- manifest | 15 +-- manifest.uuid | 2 +- src/expr.c | 2 +- test/tkt-9a8b09f8e6.test | 265 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 275 insertions(+), 9 deletions(-) create mode 100644 test/tkt-9a8b09f8e6.test diff --git a/manifest b/manifest index fea0a6e2cc..31eaa743e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sshowstat4.c\sso\sthat\sit\sdecodes\stypecodes\s8\sand\s9\scorrectly. -D 2014-06-24T20:19:21.030 +C Revise\sthe\saffinity\sreturned\sfor\sexpressions\sflagged\sas\s'generic'.\s\sFix\sfor\s[9a8b09f8e6]. +D 2014-06-26T21:28:21.292 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 4f9e497c66e2f25a4d139357a778c84d5713207c +F src/expr.c 40d06d1543b1355aa02efa9666178f7642a96ed6 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c a549cff9fe8b736cdae21650ea0af6de29b77619 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -904,6 +904,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 +F test/tkt-9a8b09f8e6.test 8b81b1fa5193246dbf5fe876929f71c3712ee755 F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 F test/tkt-9f2eb3abac.test 85bc63e749f050e6a61c8f9207f1eee65c9d3395 F test/tkt-a7b7803e.test 159ef554234fa1f9fb318c751b284bd1cf858da4 @@ -1180,7 +1181,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b4d9f6053d1d95fdc1eab8ce610b51e7df8d896d -R 508dd98757238bd14305734cabeeae37 -U drh -Z 96b01d8567714808bc64d5917b4d4853 +P 9ca737c0b41f87998d842e7772c3e483bb291c50 +R 4121a5e953ac74e556013274c6ab2f95 +U mistachkin +Z 9298eeb693f0be09657ca6e474114119 diff --git a/manifest.uuid b/manifest.uuid index 5313f68aef..1b2cca87bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ca737c0b41f87998d842e7772c3e483bb291c50 \ No newline at end of file +92f7ad43dbfe4e02490df2f932c3c74fb89064d6 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 08c121e386..803d93f30d 100644 --- a/src/expr.c +++ b/src/expr.c @@ -33,7 +33,7 @@ char sqlite3ExprAffinity(Expr *pExpr){ int op; pExpr = sqlite3ExprSkipCollate(pExpr); - if( pExpr->flags & EP_Generic ) return SQLITE_AFF_NONE; + if( pExpr->flags & EP_Generic ) return 0; op = pExpr->op; if( op==TK_SELECT ){ assert( pExpr->flags&EP_xIsSelect ); diff --git a/test/tkt-9a8b09f8e6.test b/test/tkt-9a8b09f8e6.test new file mode 100644 index 0000000000..6bd3db5de0 --- /dev/null +++ b/test/tkt-9a8b09f8e6.test @@ -0,0 +1,265 @@ +# 2014 June 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests to verify that ticket [9a8b09f8e6] has been +# fixed. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix tkt-9a8b09f8e6 + +do_test 1.1 { + execsql { + CREATE TABLE t1(x TEXT); + INSERT INTO t1 VALUES('1'); + } +} {} + +do_test 1.2 { + execsql { + CREATE TABLE t2(x INTEGER); + INSERT INTO t2 VALUES(1); + } +} {} + +do_test 1.3 { + execsql { + CREATE TABLE t3(x REAL); + INSERT INTO t3 VALUES(1.0); + } +} {} + +do_test 1.4 { + execsql { + CREATE TABLE t4(x REAL); + INSERT INTO t4 VALUES(1.11); + } +} {} + +do_test 2.1 { + execsql { + SELECT x FROM t1 WHERE x IN (1); + } +} {1} + +do_test 2.2 { + execsql { + SELECT x FROM t1 WHERE x IN (1.0); + } +} {} + +do_test 2.3 { + execsql { + SELECT x FROM t1 WHERE x IN ('1'); + } +} {1} + +do_test 2.4 { + execsql { + SELECT x FROM t1 WHERE x IN ('1.0'); + } +} {} + +do_test 2.5 { + execsql { + SELECT x FROM t1 WHERE 1 IN (x); + } +} {} + +do_test 2.6 { + execsql { + SELECT x FROM t1 WHERE 1.0 IN (x); + } +} {} + +do_test 2.7 { + execsql { + SELECT x FROM t1 WHERE '1' IN (x); + } +} {1} + +do_test 2.8 { + execsql { + SELECT x FROM t1 WHERE '1.0' IN (x); + } +} {} + +do_test 3.1 { + execsql { + SELECT x FROM t2 WHERE x IN (1); + } +} {1} + +do_test 3.2 { + execsql { + SELECT x FROM t2 WHERE x IN (1.0); + } +} {1} + +do_test 3.3 { + execsql { + SELECT x FROM t2 WHERE x IN ('1'); + } +} {1} + +do_test 3.4 { + execsql { + SELECT x FROM t2 WHERE x IN ('1.0'); + } +} {1} + +do_test 3.5 { + execsql { + SELECT x FROM t2 WHERE 1 IN (x); + } +} {1} + +do_test 3.6 { + execsql { + SELECT x FROM t2 WHERE 1.0 IN (x); + } +} {1} + +do_test 3.7 { + execsql { + SELECT x FROM t2 WHERE '1' IN (x); + } +} {} + +do_test 3.8 { + execsql { + SELECT x FROM t2 WHERE '1.0' IN (x); + } +} {} + +do_test 4.1 { + execsql { + SELECT x FROM t3 WHERE x IN (1); + } +} {1.0} + +do_test 4.2 { + execsql { + SELECT x FROM t3 WHERE x IN (1.0); + } +} {1.0} + +do_test 4.3 { + execsql { + SELECT x FROM t3 WHERE x IN ('1'); + } +} {1.0} + +do_test 4.4 { + execsql { + SELECT x FROM t3 WHERE x IN ('1.0'); + } +} {1.0} + +do_test 4.5 { + execsql { + SELECT x FROM t3 WHERE 1 IN (x); + } +} {1.0} + +do_test 4.6 { + execsql { + SELECT x FROM t3 WHERE 1.0 IN (x); + } +} {1.0} + +do_test 4.7 { + execsql { + SELECT x FROM t3 WHERE '1' IN (x); + } +} {} + +do_test 4.8 { + execsql { + SELECT x FROM t3 WHERE '1.0' IN (x); + } +} {} + +do_test 5.1 { + execsql { + SELECT x FROM t4 WHERE x IN (1); + } +} {} + +do_test 5.2 { + execsql { + SELECT x FROM t4 WHERE x IN (1.0); + } +} {} + +do_test 5.3 { + execsql { + SELECT x FROM t4 WHERE x IN ('1'); + } +} {} + +do_test 5.4 { + execsql { + SELECT x FROM t4 WHERE x IN ('1.0'); + } +} {} + +do_test 5.5 { + execsql { + SELECT x FROM t4 WHERE x IN (1.11); + } +} {1.11} + +do_test 5.6 { + execsql { + SELECT x FROM t4 WHERE x IN ('1.11'); + } +} {1.11} + +do_test 5.7 { + execsql { + SELECT x FROM t4 WHERE 1 IN (x); + } +} {} + +do_test 5.8 { + execsql { + SELECT x FROM t4 WHERE 1.0 IN (x); + } +} {} + +do_test 5.9 { + execsql { + SELECT x FROM t4 WHERE '1' IN (x); + } +} {} + +do_test 5.10 { + execsql { + SELECT x FROM t4 WHERE '1.0' IN (x); + } +} {} + +do_test 5.11 { + execsql { + SELECT x FROM t4 WHERE 1.11 IN (x); + } +} {1.11} + +do_test 5.12 { + execsql { + SELECT x FROM t4 WHERE '1.11' IN (x); + } +} {} + +finish_test From 2d84ac480b2bf35619570b9e17a7e39dd7dd2100 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 26 Jun 2014 21:32:09 +0000 Subject: [PATCH 084/710] Fix compilation issue when STAT4 is not enabled. FossilOrigin-Name: 74a5454a710e1b7d8575cec2f872e6110aefce17 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/where.c | 2 ++ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 5a780681cb..3a8265365f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\suse\ssqlite_stat4\sdata\sto\sestimate\sthe\snumber\sof\srows\svisited\sby\sa\srange\squery\sthat\suses\sa\sskip-scan.\sThis\scode\sis\slargely\suntested. -D 2014-06-26T20:21:46.005 +C Fix\scompilation\sissue\swhen\sSTAT4\sis\snot\senabled. +D 2014-06-26T21:32:09.075 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 643abd2dce6650d537c240c1ecdfc4090271091f +F src/where.c 0396f040a1ffe31b1f63673c6cd6c469c5e4ce57 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1180,10 +1180,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9ca737c0b41f87998d842e7772c3e483bb291c50 -R 48ab9caf5628261ea5cdb35094645ef7 -T *branch * stat4-skipscan -T *sym-stat4-skipscan * -T -sym-trunk * -U dan -Z a77d7df51663ffd17fb07ab0f5d5231a +P 01dc8102592427b71a18c2cb82301d2266dd59c2 +R 1ded9652c09c1b748cd5e7181ce5016c +U mistachkin +Z b89953846d49961fb37fc288882b8874 diff --git a/manifest.uuid b/manifest.uuid index 4422476212..7e63e6d9db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01dc8102592427b71a18c2cb82301d2266dd59c2 \ No newline at end of file +74a5454a710e1b7d8575cec2f872e6110aefce17 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a500b5a024..a8c8bccde4 100644 --- a/src/where.c +++ b/src/where.c @@ -1999,6 +1999,7 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ return nRet; } +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** This function is called to estimate the number of rows visited by a ** range-scan on a skip-scan index. For example: @@ -2091,6 +2092,7 @@ static int whereRangeSkipScanEst( return rc; } +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* ** This function is used to estimate the number of rows that will be visited From 5d249518652c86e9aec977f529da7aa199b4790f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 26 Jun 2014 22:17:21 +0000 Subject: [PATCH 085/710] Add some more IN operator tests. FossilOrigin-Name: fb32e374b75b160e7b535e732ced6c34dbb513eb --- manifest | 12 ++++----- manifest.uuid | 2 +- test/tkt-9a8b09f8e6.test | 58 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 31eaa743e5..27807f1cbc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Revise\sthe\saffinity\sreturned\sfor\sexpressions\sflagged\sas\s'generic'.\s\sFix\sfor\s[9a8b09f8e6]. -D 2014-06-26T21:28:21.292 +C Add\ssome\smore\sIN\soperator\stests. +D 2014-06-26T22:17:21.764 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -904,7 +904,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 -F test/tkt-9a8b09f8e6.test 8b81b1fa5193246dbf5fe876929f71c3712ee755 +F test/tkt-9a8b09f8e6.test b2ef151d0984b2ebf237760dbeaa50724e5a0667 F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 F test/tkt-9f2eb3abac.test 85bc63e749f050e6a61c8f9207f1eee65c9d3395 F test/tkt-a7b7803e.test 159ef554234fa1f9fb318c751b284bd1cf858da4 @@ -1181,7 +1181,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9ca737c0b41f87998d842e7772c3e483bb291c50 -R 4121a5e953ac74e556013274c6ab2f95 +P 92f7ad43dbfe4e02490df2f932c3c74fb89064d6 +R 61cea95999002249f7da20d93ec0602d U mistachkin -Z 9298eeb693f0be09657ca6e474114119 +Z 2021f223ccdab7be5b0b3bf69eadf881 diff --git a/manifest.uuid b/manifest.uuid index 1b2cca87bd..9daab53ded 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92f7ad43dbfe4e02490df2f932c3c74fb89064d6 \ No newline at end of file +fb32e374b75b160e7b535e732ced6c34dbb513eb \ No newline at end of file diff --git a/test/tkt-9a8b09f8e6.test b/test/tkt-9a8b09f8e6.test index 6bd3db5de0..d6b22efb21 100644 --- a/test/tkt-9a8b09f8e6.test +++ b/test/tkt-9a8b09f8e6.test @@ -46,6 +46,16 @@ do_test 1.4 { } } {} +do_test 1.5 { + execsql { + CREATE TABLE t5(x, y); + INSERT INTO t5 VALUES('1', 'one'); + INSERT INTO t5 VALUES(1, 'two'); + INSERT INTO t5 VALUES('1.0', 'three'); + INSERT INTO t5 VALUES(1.0, 'four'); + } +} {} + do_test 2.1 { execsql { SELECT x FROM t1 WHERE x IN (1); @@ -262,4 +272,52 @@ do_test 5.12 { } } {} +do_test 6.1 { + execsql { + SELECT x, y FROM t5 WHERE x IN (1); + } +} {1 two 1.0 four} + +do_test 6.2 { + execsql { + SELECT x, y FROM t5 WHERE x IN (1.0); + } +} {1 two 1.0 four} + +do_test 6.3 { + execsql { + SELECT x, y FROM t5 WHERE x IN ('1'); + } +} {1 one} + +do_test 6.4 { + execsql { + SELECT x, y FROM t5 WHERE x IN ('1.0'); + } +} {1.0 three} + +do_test 6.5 { + execsql { + SELECT x, y FROM t5 WHERE 1 IN (x); + } +} {1 two 1.0 four} + +do_test 6.6 { + execsql { + SELECT x, y FROM t5 WHERE 1.0 IN (x); + } +} {1 two 1.0 four} + +do_test 6.7 { + execsql { + SELECT x, y FROM t5 WHERE '1' IN (x); + } +} {1 one} + +do_test 6.8 { + execsql { + SELECT x, y FROM t5 WHERE '1.0' IN (x); + } +} {1.0 three} + finish_test From 4e42ba4a356f21306a84a2958ee1a66a7f285cfd Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 27 Jun 2014 20:14:25 +0000 Subject: [PATCH 086/710] Fix a couple of problems in estimating the number of rows visited by a range query that uses a skip-scan. FossilOrigin-Name: 219736f54dcd1448af3400e699f1c20755ac6876 --- manifest | 17 ++++---- manifest.uuid | 2 +- src/vdbemem.c | 2 +- src/where.c | 27 +++++++++---- test/skipscan5.test | 95 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 126 insertions(+), 17 deletions(-) create mode 100644 test/skipscan5.test diff --git a/manifest b/manifest index 3a8265365f..14dfe69604 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompilation\sissue\swhen\sSTAT4\sis\snot\senabled. -D 2014-06-26T21:32:09.075 +C Fix\sa\scouple\sof\sproblems\sin\sestimating\sthe\snumber\sof\srows\svisited\sby\sa\srange\squery\sthat\suses\sa\sskip-scan. +D 2014-06-27T20:14:25.215 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,14 +289,14 @@ F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac -F src/vdbemem.c 8f28cb5bdd5b8748dba67aab5a07a47386fe40dc +F src/vdbemem.c 3f191d4113805e3c5e875b732c44c9d98bac0595 F src/vdbesort.c 44441d73b08b3a638dcdb725afffb87c6574ad27 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 0396f040a1ffe31b1f63673c6cd6c469c5e4ce57 +F src/where.c fea270aeef23f978edac94b5591c74b0bf3ce884 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -829,6 +829,7 @@ F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 F test/skipscan1.test 28c7faa41a0d7265040ecb0a0abd90c0904270b2 F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a +F test/skipscan5.test 7eb78ace4e7145fb5005d47a8f837701d7027b85 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 @@ -1180,7 +1181,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 01dc8102592427b71a18c2cb82301d2266dd59c2 -R 1ded9652c09c1b748cd5e7181ce5016c -U mistachkin -Z b89953846d49961fb37fc288882b8874 +P 74a5454a710e1b7d8575cec2f872e6110aefce17 +R 02edb14d4ef805eea40e69e5acdf39b7 +U dan +Z bd7cb892a5b1d67344c0bf9079bc8afa diff --git a/manifest.uuid b/manifest.uuid index 7e63e6d9db..d621593f3a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -74a5454a710e1b7d8575cec2f872e6110aefce17 \ No newline at end of file +219736f54dcd1448af3400e699f1c20755ac6876 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 67dea9c282..634c881f2f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1296,7 +1296,7 @@ int sqlite3Stat4Column( iHdr = getVarint32(a, nHdr); iField = nHdr; for(i=0; iu.btree.pIndex; int nEq = pLoop->u.btree.nEq; sqlite3 *db = pParse->db; - int nLower = 0; - int nUpper = 0; + int nLower = -1; + int nUpper = p->nSample+1; int rc = SQLITE_OK; u8 aff = p->pTable->aCol[ p->aiColumn[nEq] ].affinity; CollSeq *pColl; @@ -2058,9 +2058,11 @@ static int whereRangeSkipScanEst( pColl = sqlite3LocateCollSeq(pParse, p->azColl[nEq]); if( pLower ){ rc = sqlite3Stat4ValueFromExpr(pParse, pLower->pExpr->pRight, aff, &p1); + nLower = 0; } if( pUpper && rc==SQLITE_OK ){ rc = sqlite3Stat4ValueFromExpr(pParse, pUpper->pExpr->pRight, aff, &p2); + nUpper = p2 ? 0 : p->nSample; } if( p1 || p2 ){ @@ -2070,18 +2072,29 @@ static int whereRangeSkipScanEst( rc = sqlite3Stat4Column(db, p->aSample[i].p, p->aSample[i].n, nEq, &pVal); if( rc==SQLITE_OK && p1 ){ int res = sqlite3MemCompare(p1, pVal, pColl); - if( res<=0 ) nLower++; + if( res>=0 ) nLower++; } if( rc==SQLITE_OK && p2 ){ int res = sqlite3MemCompare(p2, pVal, pColl); - if( res<=0 ) nUpper++; + if( res>=0 ) nUpper++; } } - if( p2==0 ) nUpper = p->nSample; nDiff = (nUpper - nLower); if( nDiff<=0 ) nDiff = 1; - pLoop->nOut -= (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); - *pbDone = 1; + + /* If there is both an upper and lower bound specified, and the + ** comparisons indicate that they are close together, use the fallback + ** method (assume that the scan visits 1/64 of the rows) for estimating + ** the number of rows visited. Otherwise, estimate the number of rows + ** using the method described in the header comment for this function. */ + if( nDiff!=1 || pUpper==0 || pLower==0 ){ + int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); + pLoop->nOut -= nAdjust; + *pbDone = 1; + WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + (u32)nLower, (u32)nUpper, nAdjust*-1, pLoop->nOut)); + } + }else{ assert( *pbDone==0 ); } diff --git a/test/skipscan5.test b/test/skipscan5.test new file mode 100644 index 0000000000..b374302f83 --- /dev/null +++ b/test/skipscan5.test @@ -0,0 +1,95 @@ +# 2013-11-13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests of the "skip-scan" query strategy. In +# particular it tests that stat4 data can be used by a range query +# that uses the skip-scan approach. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix skipscan5 + +ifcapable !stat4 { + finish_test + return +} + +do_execsql_test 1.1 { + CREATE TABLE t1(a INT, b INT, c INT); + CREATE INDEX i1 ON t1(a, b); +} {} + +expr srand(4) +do_test 1.2 { + for {set i 0} {$i < 100} {incr i} { + set a [expr int(rand()*4.0) + 1] + set b [expr int(rand()*20.0) + 1] + execsql { INSERT INTO t1 VALUES($a, $b, NULL) } + } + execsql ANALYZE +} {} + +do_eqp_test 1.3 { + SELECT * FROM t1 WHERE b = 5; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b=?)} +} + +do_eqp_test 1.4 { + SELECT * FROM t1 WHERE b > 12 AND b < 16; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>? AND b 2 AND b < 16; +} { + 0 0 0 {SCAN TABLE t1} +} + +do_eqp_test 1.6 { + SELECT * FROM t1 WHERE b > 18 AND b < 25; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>? AND b 18 AND b < 25; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>? AND b 15; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>?)} +} + +do_eqp_test 1.9 { + SELECT * FROM t1 WHERE b > 5; +} { + 0 0 0 {SCAN TABLE t1} +} + +do_eqp_test 1.10 { + SELECT * FROM t1 WHERE b < 5; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b Date: Sat, 28 Jun 2014 14:28:06 +0000 Subject: [PATCH 087/710] Change the VDBE to export the sqlite3MemCompare() routine and thus free where.c from the dependency on vdbeInt.h. FossilOrigin-Name: d186d1ac3c47f0d814636c4b8386a6065a294750 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/vdbe.h | 1 + src/vdbeInt.h | 1 - src/where.c | 1 - 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index fc497fbfa9..620657f6c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sfixes\sfrom\strunk\swith\sthis\sbranch. -D 2014-06-28T14:25:32.570 +C Change\sthe\sVDBE\sto\sexport\sthe\ssqlite3MemCompare()\sroutine\sand\sthus\sfree\nwhere.c\sfrom\sthe\sdependency\son\svdbeInt.h. +D 2014-06-28T14:28:06.405 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,8 +284,8 @@ F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 049fe1d3c0e2209c1bee107aec2fcff6285f909f F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 9bfe6becfc094382ae213656fbe511055ad83a54 -F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 -F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 +F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 +F src/vdbeInt.h 5df5e9afe9b7839cd17256220fc4f7af84b8b1cd F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c fea270aeef23f978edac94b5591c74b0bf3ce884 +F src/where.c 1f448161f1d9ec8cb68e2a8eb2e4a9756a877f69 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 219736f54dcd1448af3400e699f1c20755ac6876 fb32e374b75b160e7b535e732ced6c34dbb513eb -R 638c04374a356a745d9c2d2f0fa602ad -U dan -Z fc19d5103e4b8e0f744c9b3dd9fc467e +P 6af219d1b83ec54b3fb4ad4e6216d9ad9ea0fc70 +R 0d122ee9efb300dc8d78b79e592143bf +U drh +Z 93fe23c37a7fe2b9652da5b4873aa7c3 diff --git a/manifest.uuid b/manifest.uuid index bcfbef8325..172499a547 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6af219d1b83ec54b3fb4ad4e6216d9ad9ea0fc70 \ No newline at end of file +d186d1ac3c47f0d814636c4b8386a6065a294750 \ No newline at end of file diff --git a/src/vdbe.h b/src/vdbe.h index 10a4140ee1..ef91010d80 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -209,6 +209,7 @@ void sqlite3VdbeSetVarmask(Vdbe*, int); #ifndef SQLITE_OMIT_TRACE char *sqlite3VdbeExpandSql(Vdbe*, const char*); #endif +int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*,int); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 043b56da59..d629802a64 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -394,7 +394,6 @@ void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); -int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); int sqlite3VdbeExec(Vdbe*); int sqlite3VdbeList(Vdbe*); int sqlite3VdbeHalt(Vdbe*); diff --git a/src/where.c b/src/where.c index 3cd566c46b..e431e9413e 100644 --- a/src/where.c +++ b/src/where.c @@ -18,7 +18,6 @@ */ #include "sqliteInt.h" #include "whereInt.h" -#include "vdbeInt.h" /* ** Return the estimated number of output rows from a WHERE clause From fa8874542c82254420e483650c9f49d94b413106 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 28 Jun 2014 15:26:10 +0000 Subject: [PATCH 088/710] Fix a problem with using stat4 samples of type text when estimating the rows visited by a range-query/skip-scan loop. FossilOrigin-Name: dfb09db6d412f3bc2a71bda393813783580dbad1 --- manifest | 18 +++++++-------- manifest.uuid | 2 +- src/vdbemem.c | 1 + src/where.c | 2 +- test/skipscan5.test | 56 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 68 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 620657f6c1..624fcba2fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sVDBE\sto\sexport\sthe\ssqlite3MemCompare()\sroutine\sand\sthus\sfree\nwhere.c\sfrom\sthe\sdependency\son\svdbeInt.h. -D 2014-06-28T14:28:06.405 +C Fix\sa\sproblem\swith\susing\sstat4\ssamples\sof\stype\stext\swhen\sestimating\sthe\srows\svisited\sby\sa\srange-query/skip-scan\sloop. +D 2014-06-28T15:26:10.708 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,14 +289,14 @@ F src/vdbeInt.h 5df5e9afe9b7839cd17256220fc4f7af84b8b1cd F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac -F src/vdbemem.c 3f191d4113805e3c5e875b732c44c9d98bac0595 +F src/vdbemem.c 8228bc32622716d26ffd902ceca7b31690744fef F src/vdbesort.c 44441d73b08b3a638dcdb725afffb87c6574ad27 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 1f448161f1d9ec8cb68e2a8eb2e4a9756a877f69 +F src/where.c 84659c308426b20be4efada82a06a26dd6b8545e F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -829,7 +829,7 @@ F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 F test/skipscan1.test 28c7faa41a0d7265040ecb0a0abd90c0904270b2 F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a -F test/skipscan5.test 7eb78ace4e7145fb5005d47a8f837701d7027b85 +F test/skipscan5.test 4b83d803bb17bf922569ce8c3637d0558d6abc83 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6af219d1b83ec54b3fb4ad4e6216d9ad9ea0fc70 -R 0d122ee9efb300dc8d78b79e592143bf -U drh -Z 93fe23c37a7fe2b9652da5b4873aa7c3 +P d186d1ac3c47f0d814636c4b8386a6065a294750 +R 737af1ae1f9d5e706af49b9d72dc98eb +U dan +Z c44a7d177df302c14a246c1d5d1d90b5 diff --git a/manifest.uuid b/manifest.uuid index 172499a547..f498e3b5e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d186d1ac3c47f0d814636c4b8386a6065a294750 \ No newline at end of file +dfb09db6d412f3bc2a71bda393813783580dbad1 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 634c881f2f..7e5c912f39 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1301,6 +1301,7 @@ int sqlite3Stat4Column( } iHdr = getVarint32(&a[iHdr], t); + pMem->enc = ENC(db); sqlite3VdbeSerialGet(&a[iField], t, pMem); } diff --git a/src/where.c b/src/where.c index e431e9413e..7610c20f9b 100644 --- a/src/where.c +++ b/src/where.c @@ -2091,7 +2091,7 @@ static int whereRangeSkipScanEst( pLoop->nOut -= nAdjust; *pbDone = 1; WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", - (u32)nLower, (u32)nUpper, nAdjust*-1, pLoop->nOut)); + nLower, nUpper, nAdjust*-1, pLoop->nOut)); } }else{ diff --git a/test/skipscan5.test b/test/skipscan5.test index b374302f83..6cf35f26a7 100644 --- a/test/skipscan5.test +++ b/test/skipscan5.test @@ -92,4 +92,60 @@ do_eqp_test 1.11 { 0 0 0 {SCAN TABLE t1} } +#------------------------------------------------------------------------- +# Test that range-query/skip-scan estimation works with text values. +# And on UTF-16 databases when there is no UTF-16 collation sequence +# available. + +proc test_collate {enc lhs rhs} { + string compare $lhs $rhs +} + +foreach {tn dbenc coll} { + 1 UTF-8 { add_test_collate db 0 0 1 } + 2 UTF-16 { add_test_collate db 1 0 0 } + 3 UTF-8 { add_test_collate db 0 1 0 } +} { + reset_db + eval $coll + + do_execsql_test 2.$tn.1 " PRAGMA encoding = '$dbenc' " + do_execsql_test 2.$tn.2 { + CREATE TABLE t2(a TEXT, b TEXT, c TEXT COLLATE test_collate, d TEXT); + CREATE INDEX i2 ON t2(a, b, c); + } + + set vocab(d) { :) } + set vocab(c) { a b c d e f g h i j k l m n o p q r s t } + set vocab(b) { one two three } + set vocab(a) { sql } + + do_test 2.$tn.3 { + for {set i 0} {$i < 100} {incr i} { + foreach var {a b c d} { + set $var [lindex $vocab($var) [expr $i % [llength $vocab($var)]]] + } + execsql { INSERT INTO t2 VALUES($a, $b, $c, $d) } + } + execsql ANALYZE + } {} + + foreach {tn2 q res} { + 1 { c BETWEEN 'd' AND 'e' } {/*ANY(a) AND ANY(b) AND c>? AND c 'q' } {/*ANY(a) AND ANY(b) AND c>?*/} + 4 { c > 'e' } {/*SCAN TABLE t2*/} + 5 { c < 'q' } {/*SCAN TABLE t2*/} + 4 { c < 'e' } {/*ANY(a) AND ANY(b) AND c Date: Sat, 28 Jun 2014 16:06:44 +0000 Subject: [PATCH 089/710] Add header comments on new routines. Rework the sqlite3Stat4Column() routine so that is (in theory) able to deal with corrupt samples. FossilOrigin-Name: ef5cdf949bb53a2958fa34e176b4b9eeda269de5 --- manifest | 14 ++++----- manifest.uuid | 2 +- src/vdbemem.c | 86 +++++++++++++++++++++++++++++++++------------------ 3 files changed, 64 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index 624fcba2fe..7600c076fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\susing\sstat4\ssamples\sof\stype\stext\swhen\sestimating\sthe\srows\svisited\sby\sa\srange-query/skip-scan\sloop. -D 2014-06-28T15:26:10.708 +C Add\sheader\scomments\son\snew\sroutines.\s\sRework\sthe\ssqlite3Stat4Column()\sroutine\nso\sthat\sis\s(in\stheory)\sable\sto\sdeal\swith\scorrupt\ssamples. +D 2014-06-28T16:06:44.822 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/vdbeInt.h 5df5e9afe9b7839cd17256220fc4f7af84b8b1cd F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac -F src/vdbemem.c 8228bc32622716d26ffd902ceca7b31690744fef +F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c 44441d73b08b3a638dcdb725afffb87c6574ad27 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d186d1ac3c47f0d814636c4b8386a6065a294750 -R 737af1ae1f9d5e706af49b9d72dc98eb -U dan -Z c44a7d177df302c14a246c1d5d1d90b5 +P dfb09db6d412f3bc2a71bda393813783580dbad1 +R a461409aefe23981a1fbdd4418e17b64 +U drh +Z a2fad482dc40a73ef09e3690c963a5aa diff --git a/manifest.uuid b/manifest.uuid index f498e3b5e9..2e226ad371 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dfb09db6d412f3bc2a71bda393813783580dbad1 \ No newline at end of file +ef5cdf949bb53a2958fa34e176b4b9eeda269de5 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 7e5c912f39..cf44aa7e2d 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1152,11 +1152,29 @@ void sqlite3AnalyzeFunctions(void){ } } +/* +** Attempt to extract a value from pExpr and use it to construct *ppVal. +** +** If pAlloc is not NULL, then an UnpackedRecord object is created for +** pAlloc if one does not exist and the new value is added to the +** UnpackedRecord object. +** +** A value is extracted in the following cases: +** +** * (pExpr==0). In this case the value is assumed to be an SQL NULL, +** +** * The expression is a bound variable, and this is a reprepare, or +** +** * The expression is a literal value. +** +** On success, *ppVal is made to point to the extracted value. The caller +** is responsible for ensuring that the value is eventually freed. +*/ static int stat4ValueFromExpr( Parse *pParse, /* Parse context */ Expr *pExpr, /* The expression to extract a value from */ u8 affinity, /* Affinity to use */ - struct ValueNewStat4Ctx *pAlloc,/* How to allocate space */ + struct ValueNewStat4Ctx *pAlloc,/* How to allocate space. Or NULL */ sqlite3_value **ppVal /* OUT: New value object (or NULL) */ ){ int rc = SQLITE_OK; @@ -1269,6 +1287,14 @@ int sqlite3Stat4ValueFromExpr( return stat4ValueFromExpr(pParse, pExpr, affinity, 0, ppVal); } +/* +** Extract the iCol-th column from the nRec-byte record in pRec. Write +** the column value into *ppVal. If *ppVal is initially NULL then a new +** sqlite3_value object is allocated. +** +** If *ppVal is initially NULL then the caller is responsible for +** ensuring that the value written into *ppVal is eventually freed. +*/ int sqlite3Stat4Column( sqlite3 *db, /* Database handle */ const void *pRec, /* Pointer to buffer containing record */ @@ -1276,37 +1302,37 @@ int sqlite3Stat4Column( int iCol, /* Column to extract */ sqlite3_value **ppVal /* OUT: Extracted value */ ){ - int rc = SQLITE_OK; - Mem *pMem = *ppVal; + u32 t; /* a column type code */ + int nHdr; /* Size of the header in the record */ + int iHdr; /* Next unread header byte */ + int iField; /* Next unread data byte */ + int szField; /* Size of the current data field */ + int i; /* Column index */ + u8 *a = (u8*)pRec; /* Typecast byte array */ + Mem *pMem = *ppVal; /* Write result into this Mem object */ + + assert( iCol>0 ); + iHdr = getVarint32(a, nHdr); + if( nHdr>nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; + iField = nHdr; + for(i=0; i<=iCol; i++){ + iHdr += getVarint32(&a[iHdr], t); + testcase( iHdr==nHdr ); + testcase( iHdr==nHdr+1 ); + if( iHdr>nHdr ) return SQLITE_CORRUPT_BKPT; + szField = sqlite3VdbeSerialTypeLen(t); + iField += szField; + } + testcase( iField==nRec ); + testcase( iField==nRec+1 ); + if( iField>nRec ) return SQLITE_CORRUPT_BKPT; if( pMem==0 ){ - pMem = (Mem*)sqlite3ValueNew(db); - if( pMem==0 ){ - rc = SQLITE_NOMEM; - } + pMem = *ppVal = sqlite3ValueNew(db); + if( pMem==0 ) return SQLITE_NOMEM; } - - if( rc==SQLITE_OK ){ - u32 t; - int nHdr; - int iHdr; - int iField; - int i; - u8 *a = (u8*)pRec; - - iHdr = getVarint32(a, nHdr); - iField = nHdr; - for(i=0; ienc = ENC(db); - sqlite3VdbeSerialGet(&a[iField], t, pMem); - } - - *ppVal = pMem; - return rc; + sqlite3VdbeSerialGet(&a[iField-szField], t, pMem); + pMem->enc = ENC(db); + return SQLITE_OK; } /* From 8e9028d0f862197aacd3f5213680f87275c1c67f Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 28 Jun 2014 17:35:15 +0000 Subject: [PATCH 090/710] Add further tests to skipscan5.test. FossilOrigin-Name: 4b8230e8fe93e73a615a46708aed5fa3557b6228 --- manifest | 14 ++--- manifest.uuid | 2 +- test/skipscan5.test | 135 ++++++++++++++++++++++++++++---------------- 3 files changed, 93 insertions(+), 58 deletions(-) diff --git a/manifest b/manifest index 7600c076fe..c6e5aaa9c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sheader\scomments\son\snew\sroutines.\s\sRework\sthe\ssqlite3Stat4Column()\sroutine\nso\sthat\sis\s(in\stheory)\sable\sto\sdeal\swith\scorrupt\ssamples. -D 2014-06-28T16:06:44.822 +C Add\sfurther\stests\sto\sskipscan5.test. +D 2014-06-28T17:35:15.306 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -829,7 +829,7 @@ F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 F test/skipscan1.test 28c7faa41a0d7265040ecb0a0abd90c0904270b2 F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a -F test/skipscan5.test 4b83d803bb17bf922569ce8c3637d0558d6abc83 +F test/skipscan5.test d8b9692b702745a0e41c23f9da6beac81df01196 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 0e4456e729e5a92a625907c63dcdedfbe72c5dc5 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dfb09db6d412f3bc2a71bda393813783580dbad1 -R a461409aefe23981a1fbdd4418e17b64 -U drh -Z a2fad482dc40a73ef09e3690c963a5aa +P ef5cdf949bb53a2958fa34e176b4b9eeda269de5 +R f17ac3e5a608781f5afc70c3d4b0020c +U dan +Z 86f494cb467655f573b06b5ade5b9c90 diff --git a/manifest.uuid b/manifest.uuid index 2e226ad371..fa3d4db391 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef5cdf949bb53a2958fa34e176b4b9eeda269de5 \ No newline at end of file +4b8230e8fe93e73a615a46708aed5fa3557b6228 \ No newline at end of file diff --git a/test/skipscan5.test b/test/skipscan5.test index 6cf35f26a7..5d6d392998 100644 --- a/test/skipscan5.test +++ b/test/skipscan5.test @@ -38,64 +38,36 @@ do_test 1.2 { execsql ANALYZE } {} -do_eqp_test 1.3 { - SELECT * FROM t1 WHERE b = 5; +foreach {tn q res} { + 1 "b = 5" {/*ANY(a) AND b=?*/} + 2 "b > 12 AND b < 16" {/*ANY(a) AND b>? AND b 2 AND b < 16" {/*SCAN TABLE t1*/} + 4 "b > 18 AND b < 25" {/*ANY(a) AND b>? AND b 15" {/*ANY(a) AND b>?*/} + 6 "b > 5" {/*SCAN TABLE t1*/} + 7 "b < 15" {/*SCAN TABLE t1*/} + 8 "b < 5" {/*ANY(a) AND b b" {/*ANY(a) AND b '12' AND b < '16'" {/*ANY(a) AND b>? AND b '2' AND b < '16'" {/*SCAN TABLE t1*/} + 13 "b > '18' AND b < '25'" {/*ANY(a) AND b>? AND b '15'" {/*ANY(a) AND b>?*/} + 15 "b > '5'" {/*SCAN TABLE t1*/} + 16 "b < '15'" {/*SCAN TABLE t1*/} + 17 "b < '5'" {/*ANY(a) AND b b" {/*ANY(a) AND b 12 AND b < 16; -} { - 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>? AND b 2 AND b < 16; -} { - 0 0 0 {SCAN TABLE t1} -} - -do_eqp_test 1.6 { - SELECT * FROM t1 WHERE b > 18 AND b < 25; -} { - 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>? AND b 18 AND b < 25; -} { - 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>? AND b 15; -} { - 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b>?)} -} - -do_eqp_test 1.9 { - SELECT * FROM t1 WHERE b > 5; -} { - 0 0 0 {SCAN TABLE t1} -} - -do_eqp_test 1.10 { - SELECT * FROM t1 WHERE b < 5; -} { - 0 0 0 {SEARCH TABLE t1 USING INDEX i1 (ANY(a) AND b? AND b X'5555'" {/*ANY(a) AND b>?*/} + 5 "b > 'zzz'" {/*ANY(a) AND b>?*/} + 6 "b < 'zzz'" {/*SCAN TABLE t3*/} +} { + set sql "EXPLAIN QUERY PLAN SELECT * FROM t3 WHERE $q" + do_execsql_test 3.3.$tn $sql $res +} + finish_test From 6696ba3eab6a8a78ca72143aa8ed39809cecfa1b Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 28 Jun 2014 19:06:49 +0000 Subject: [PATCH 091/710] Add an OOM fault injection test for the new code on this branch. FossilOrigin-Name: c96de490ac77dd23c108004b95152cce4922fe27 --- manifest | 14 +++++----- manifest.uuid | 2 +- src/vdbeaux.c | 2 ++ test/mallocK.test | 66 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c6e5aaa9c2..84b9befd7b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sto\sskipscan5.test. -D 2014-06-28T17:35:15.306 +C Add\san\sOOM\sfault\sinjection\stest\sfor\sthe\snew\scode\son\sthis\sbranch. +D 2014-06-28T19:06:49.846 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -287,7 +287,7 @@ F src/vdbe.c 9bfe6becfc094382ae213656fbe511055ad83a54 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h 5df5e9afe9b7839cd17256220fc4f7af84b8b1cd F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c e493f38758c4b8f4ca2007cf6a700bd405d192f3 +F src/vdbeaux.c 8ce7dcdbb8c59e5c2194518ce3099b254ae94c15 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c 44441d73b08b3a638dcdb725afffb87c6574ad27 @@ -693,7 +693,7 @@ F test/mallocG.test 0ff91b65c50bdaba680fb75d87fe4ad35bb7934f F test/mallocH.test 79b65aed612c9b3ed2dcdaa727c85895fd1bfbdb F test/mallocI.test a88c2b9627c8506bf4703d8397420043a786cdb6 F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e -F test/mallocK.test d79968641d1b70d88f6c01bdb9a7eb4a55582cc9 +F test/mallocK.test 3cff7c0f64735f6883bacdd294e45a6ed5714817 F test/malloc_common.tcl 58e54229c4132ef882a11fab6419ec4cd3073589 F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef5cdf949bb53a2958fa34e176b4b9eeda269de5 -R f17ac3e5a608781f5afc70c3d4b0020c +P 4b8230e8fe93e73a615a46708aed5fa3557b6228 +R 626c4988935b78bcb3d2f8f34291bc5b U dan -Z 86f494cb467655f573b06b5ade5b9c90 +Z 19d12a554b06297187fc9a292703f2d4 diff --git a/manifest.uuid b/manifest.uuid index fa3d4db391..d74a6bd2f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4b8230e8fe93e73a615a46708aed5fa3557b6228 \ No newline at end of file +c96de490ac77dd23c108004b95152cce4922fe27 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0a6b536720..de61b55c83 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3589,6 +3589,7 @@ int sqlite3VdbeRecordCompare( ** value. */ assert( CORRUPT_DB || pPKey2->default_rc==vdbeRecordCompareDebug(nKey1, pKey1, pPKey2) + || pKeyInfo->db->mallocFailed ); return pPKey2->default_rc; } @@ -3754,6 +3755,7 @@ static int vdbeRecordCompareString( || (res<0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)<0) || (res>0 && vdbeRecordCompareDebug(nKey1, pKey1, pPKey2)>0) || CORRUPT_DB + || pPKey2->pKeyInfo->db->mallocFailed ); return res; } diff --git a/test/mallocK.test b/test/mallocK.test index 971bd56137..dcf00da9aa 100644 --- a/test/mallocK.test +++ b/test/mallocK.test @@ -16,6 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl +set testprefix mallocK set sql {SELECT * FROM t1, t2 WHERE (a=1 OR a=2)} for {set x 1} {$x<5} {incr x} { @@ -68,5 +69,70 @@ ifcapable vtab { } } +#------------------------------------------------------------------------- +# Test that OOM errors are correctly handled by the code that uses stat4 +# data to estimate the number of rows visited by a skip-scan range query. +# +add_alignment_test_collations db +do_execsql_test 6.0 { + CREATE TABLE t3(a TEXT, b TEXT COLLATE utf16_aligned, c); + INSERT INTO t3 VALUES('one', '.....', 0); + INSERT INTO t3 VALUES('one', '....x', 1); + INSERT INTO t3 VALUES('one', '...x.', 2); + INSERT INTO t3 VALUES('one', '...xx', 3); + INSERT INTO t3 VALUES('one', '..x..', 4); + INSERT INTO t3 VALUES('one', '..x.x', 5); + INSERT INTO t3 VALUES('one', '..xx.', 6); + INSERT INTO t3 VALUES('one', '..xxx', 7); + INSERT INTO t3 VALUES('one', '.x...', 8); + INSERT INTO t3 VALUES('one', '.x..x', 9); + INSERT INTO t3 VALUES('one', '.x.x.', 10); + INSERT INTO t3 VALUES('one', '.x.xx', 11); + INSERT INTO t3 VALUES('one', '.xx..', 12); + INSERT INTO t3 VALUES('one', '.xx.x', 13); + INSERT INTO t3 VALUES('one', '.xxx.', 14); + INSERT INTO t3 VALUES('one', '.xxxx', 15); + + INSERT INTO t3 VALUES('two', 'x....', 16); + INSERT INTO t3 VALUES('two', 'x...x', 17); + INSERT INTO t3 VALUES('two', 'x..x.', 18); + INSERT INTO t3 VALUES('two', 'x..xx', 19); + INSERT INTO t3 VALUES('two', 'x.x..', 20); + INSERT INTO t3 VALUES('two', 'x.x.x', 21); + INSERT INTO t3 VALUES('two', 'x.xx.', 22); + INSERT INTO t3 VALUES('two', 'x.xxx', 23); + INSERT INTO t3 VALUES('two', 'xx...', 24); + INSERT INTO t3 VALUES('two', 'xx..x', 25); + INSERT INTO t3 VALUES('two', 'xx.x.', 26); + INSERT INTO t3 VALUES('two', 'xx.xx', 27); + INSERT INTO t3 VALUES('two', 'xxx..', 28); + INSERT INTO t3 VALUES('two', 'xxx.x', 29); + INSERT INTO t3 VALUES('two', 'xxxx.', 30); + INSERT INTO t3 VALUES('two', 'xxxxx', 31); + + INSERT INTO t3 SELECT * FROM t3; + + CREATE INDEX i3 ON t3(a, b); + ANALYZE; + + SELECT 'x' > '.'; +} {1} + +ifcapable stat4 { + do_eqp_test 6.1 { + SELECT DISTINCT c FROM t3 WHERE b BETWEEN '.xx..' AND '.xxxx'; + } { + 0 0 0 {SEARCH TABLE t3 USING INDEX i3 (ANY(a) AND b>? AND b Date: Mon, 30 Jun 2014 11:14:26 +0000 Subject: [PATCH 092/710] Add makefile targets for various diagnostic tools, such as showstat4. Fix harmless compiler warnings in diagnostic tools. FossilOrigin-Name: 6f86d89b8800c50035da7809bff941d08a33a6a2 --- Makefile.in | 16 ++++++++++++++-- Makefile.msc | 16 ++++++++++++++++ main.mk | 20 ++++++++++++++++++-- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- tool/showdb.c | 4 ++-- tool/showjournal.c | 19 +++++++++---------- 7 files changed, 71 insertions(+), 28 deletions(-) diff --git a/Makefile.in b/Makefile.in index 72f7cfc007..b5768e83a6 100644 --- a/Makefile.in +++ b/Makefile.in @@ -939,8 +939,20 @@ sqlite3_analyzer.c: sqlite3.c $(TOP)/src/test_stat.c $(TOP)/src/tclsqlite.c $(TO sqlite3_analyzer$(TEXE): sqlite3_analyzer.c $(LTLINK) sqlite3_analyzer.c -o $@ $(LIBTCL) $(TLIBS) -showdb$(TEXE): $(TOP)/tool/showdb.c sqlite3.c - $(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.c $(TLIBS) +showdb$(TEXE): $(TOP)/tool/showdb.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/showdb.c sqlite3.lo $(TLIBS) + +showstat4$(TEXE): $(TOP)/tool/showstat4.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/showstat4.c sqlite3.lo $(TLIBS) + +showjournal$(TEXE): $(TOP)/tool/showjournal.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/showjournal.c sqlite3.lo $(TLIBS) + +showwal$(TEXE): $(TOP)/tool/showwal.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/showwal.c sqlite3.lo $(TLIBS) + +rollback-test$(TEXE): $(TOP)/tool/rollback-test.c sqlite3.lo + $(LTLINK) -o $@ $(TOP)/tool/rollback-test.c sqlite3.lo $(TLIBS) LogEst$(TEXE): $(TOP)/tool/logest.c sqlite3.h $(LTLINK) -I. -o $@ $(TOP)/tool/logest.c diff --git a/Makefile.msc b/Makefile.msc index 409f7b3d28..55d26e3053 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1435,6 +1435,22 @@ showdb.exe: $(TOP)\tool\showdb.c $(SQLITE3C) $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showdb.c $(SQLITE3C) +showstat4.exe: $(TOP)\tool\showstat4.c $(SQLITE3C) + $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(TOP)\tool\showstat4.c $(SQLITE3C) + +showjournal.exe: $(TOP)\tool\showjournal.c $(SQLITE3C) + $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(TOP)\tool\showjournal.c $(SQLITE3C) + +showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) + $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(TOP)\tool\showwal.c $(SQLITE3C) + +rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) + $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(TOP)\tool\rollback-test.c $(SQLITE3C) + LogEst.exe: $(TOP)\tool\logest.c sqlite3.h $(LTLINK) -Fe$@ $(TOP)\tool\LogEst.c diff --git a/main.mk b/main.mk index 30149eccab..62b6b22475 100644 --- a/main.mk +++ b/main.mk @@ -628,9 +628,25 @@ $(TEST_EXTENSION): $(TOP)/src/test_loadext.c extensiontest: testfixture$(EXE) $(TEST_EXTENSION) ./testfixture$(EXE) $(TOP)/test/loadext.test -showdb$(EXE): $(TOP)/tool/showdb.c sqlite3.c +showdb$(EXE): $(TOP)/tool/showdb.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showdb$(EXE) \ - $(TOP)/tool/showdb.c sqlite3.c + $(TOP)/tool/showdb.c sqlite3.o $(THREADLIB) + +showstat4$(EXE): $(TOP)/tool/showstat4.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showstat4$(EXE) \ + $(TOP)/tool/showstat4.c sqlite3.o $(THREADLIB) + +showjournal$(EXE): $(TOP)/tool/showjournal.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showjournal$(EXE) \ + $(TOP)/tool/showjournal.c sqlite3.o $(THREADLIB) + +showwal$(EXE): $(TOP)/tool/showwal.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \ + $(TOP)/tool/showwal.c sqlite3.o $(THREADLIB) + +rollback-test$(EXE): $(TOP)/tool/rollback-test.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o rollback-test$(EXE) \ + $(TOP)/tool/rollback-test.c sqlite3.o $(THREADLIB) LogEst$(EXE): $(TOP)/tool/logest.c sqlite3.h $(TCC) -o LogEst$(EXE) $(TOP)/tool/logest.c diff --git a/manifest b/manifest index 27807f1cbc..5cda3a858e 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Add\ssome\smore\sIN\soperator\stests. -D 2014-06-26T22:17:21.764 +C Add\smakefile\stargets\sfor\svarious\sdiagnostic\stools,\ssuch\sas\sshowstat4.\nFix\sharmless\scompiler\swarnings\sin\sdiagnostic\stools. +D 2014-06-30T11:14:26.241 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in b03432313a3aad96c706f8164fb9f5307eaf19f5 +F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc b2d3ac211b479fa5a6c9aaf3b9c49fdea900d228 +F Makefile.msc ad60c4d8be35f98fdcca614088b84a91612b2b4c F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c @@ -146,7 +146,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 7b1d0be0840f213405c977c87917241158126a33 +F main.mk 7850d834ca7f3c772e2b0087631868d5bfdeabb9 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1159,8 +1159,8 @@ F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/showdb.c b018a8a69d07050fc0fe9afcf17313cdef0cc599 -F tool/showjournal.c b62cecaab86a4053d944c276bb5232e4d17ece02 +F tool/showdb.c 213e0288501b2cf67c1b2c72a9e5b8acda4738b3 +F tool/showjournal.c 053eb1cc774710c6890b7dd6293300cc297b16a5 F tool/showstat4.c c39279d6bd37cb999b634f0064f6f86ad7af008f F tool/showwal.c 3f7f7da5ec0cba51b1449a75f700493377da57b5 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe @@ -1181,7 +1181,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 92f7ad43dbfe4e02490df2f932c3c74fb89064d6 -R 61cea95999002249f7da20d93ec0602d -U mistachkin -Z 2021f223ccdab7be5b0b3bf69eadf881 +P fb32e374b75b160e7b535e732ced6c34dbb513eb +R 46fc4bdeeae8d59d2e02be8ec2b2e5ac +U drh +Z cf9f7f35abbae67619fbb464751389eb diff --git a/manifest.uuid b/manifest.uuid index 9daab53ded..2b01869e56 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb32e374b75b160e7b535e732ced6c34dbb513eb \ No newline at end of file +6f86d89b8800c50035da7809bff941d08a33a6a2 \ No newline at end of file diff --git a/tool/showdb.c b/tool/showdb.c index bf4af444b2..8dd387365c 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -66,7 +66,7 @@ static unsigned char *getContent(int ofst, int nByte){ if( aData==0 ) out_of_memory(); memset(aData, 0, nByte+32); lseek(db, ofst, SEEK_SET); - read(db, aData, nByte); + if( read(db, aData, nByte)=nByte ){ @@ -74,7 +73,7 @@ static unsigned print_decode_line( ** in global variables. */ static unsigned decode_journal_header(int iOfst){ - char *pHdr = read_content(64, iOfst); + unsigned char *pHdr = read_content(64, iOfst); unsigned nPage; printf("Header at offset %d:\n", iOfst); print_decode_line(pHdr, 0, 4, "Header part 1 (3654616569)"); @@ -101,12 +100,11 @@ static void print_page(int iOfst){ char zTitle[50]; aData = read_content(pageSize+8, iOfst); sprintf(zTitle, "page number for page at offset %d", iOfst); - print_decode_line(aData, 0, 4, zTitle); + print_decode_line(aData-iOfst, iOfst, 4, zTitle); free(aData); } int main(int argc, char **argv){ - int rc; int nPage, cnt; int iOfst; if( argc!=2 ){ @@ -136,4 +134,5 @@ int main(int argc, char **argv){ iOfst = (iOfst/sectorSize + 1)*sectorSize; } fclose(db); + return 0; } From 48be0c51d4ed30e9a3d9f492870796f4bd13b85f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 30 Jun 2014 13:32:39 +0000 Subject: [PATCH 093/710] Generate complete samples for sqlite_stat4 on WITHOUT ROWID tables. Ticket [b2fa5424e6fcb15b5] FossilOrigin-Name: 8cb43eddab83c68b2163441df70e0e9496d5fa33 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/analyze.c | 7 ++++--- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 5cda3a858e..cbc32f11a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smakefile\stargets\sfor\svarious\sdiagnostic\stools,\ssuch\sas\sshowstat4.\nFix\sharmless\scompiler\swarnings\sin\sdiagnostic\stools. -D 2014-06-30T11:14:26.241 +C Generate\scomplete\ssamples\sfor\ssqlite_stat4\son\sWITHOUT\sROWID\stables.\nTicket\s[b2fa5424e6fcb15b5] +D 2014-06-30T13:32:39.153 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c e8c8a9d20beb2ad156321330e8f4fea002d8deee +F src/analyze.c b53dddb5a1bc4c28877952cd9ea6943aea62215d F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -1181,7 +1181,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fb32e374b75b160e7b535e732ced6c34dbb513eb -R 46fc4bdeeae8d59d2e02be8ec2b2e5ac +P 6f86d89b8800c50035da7809bff941d08a33a6a2 +R 63e5822670c3cf845c9d764a3e96a618 +T *branch * stat4-without-rowid +T *sym-stat4-without-rowid * +T -sym-trunk * U drh -Z cf9f7f35abbae67619fbb464751389eb +Z 2446f0a421b05370ddd23634c032efe0 diff --git a/manifest.uuid b/manifest.uuid index 2b01869e56..3cbe4903ed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f86d89b8800c50035da7809bff941d08a33a6a2 \ No newline at end of file +8cb43eddab83c68b2163441df70e0e9496d5fa33 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 4dcd7e8b8f..e112239a74 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1045,7 +1045,7 @@ static void analyzeOneTable( ** the regPrev array and a trailing rowid (the rowid slot is required ** when building a record to insert into the sample column of ** the sqlite_stat4 table. */ - pParse->nMem = MAX(pParse->nMem, regPrev+nCol); + pParse->nMem = MAX(pParse->nMem, regPrev+pIdx->nColumn); /* Open a read-only cursor on the index being analyzed. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); @@ -1186,11 +1186,12 @@ static void analyzeOneTable( sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pIdx->aiColumn[0], regSample); #else - for(i=0; inColumn-1; + for(; i>=0; i--){ i16 iCol = pIdx->aiColumn[i]; sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol+1, regSample); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, pIdx->nColumn, regSample); #endif sqlite3VdbeAddOp3(v, OP_MakeRecord, regTabname, 6, regTemp); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); From 971fd0713572a7b3c2420e516327bbe8c97711da Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 30 Jun 2014 13:56:34 +0000 Subject: [PATCH 094/710] Omit non-primary-key columns from the STAT4 samples for WITHOUT ROWID tables. Indexes, both rowid and without-rowid, still hold an exact copy of the index entry. FossilOrigin-Name: de826c31589258f0906f87d65796944103e36d5b --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/analyze.c | 11 ++++++++--- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index cbc32f11a2..e5da47ab1f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Generate\scomplete\ssamples\sfor\ssqlite_stat4\son\sWITHOUT\sROWID\stables.\nTicket\s[b2fa5424e6fcb15b5] -D 2014-06-30T13:32:39.153 +C Omit\snon-primary-key\scolumns\sfrom\sthe\sSTAT4\ssamples\sfor\sWITHOUT\sROWID\stables.\nIndexes,\sboth\srowid\sand\swithout-rowid,\sstill\shold\san\sexact\scopy\nof\sthe\sindex\sentry. +D 2014-06-30T13:56:34.968 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c b53dddb5a1bc4c28877952cd9ea6943aea62215d +F src/analyze.c 0f9ee2f88f7c564cedb16f8e0236f2d5ac8429aa F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -1181,10 +1181,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6f86d89b8800c50035da7809bff941d08a33a6a2 -R 63e5822670c3cf845c9d764a3e96a618 -T *branch * stat4-without-rowid -T *sym-stat4-without-rowid * -T -sym-trunk * +P 8cb43eddab83c68b2163441df70e0e9496d5fa33 +R 76a8cf4e27cd1a26fe3722da49c987a3 U drh -Z 2446f0a421b05370ddd23634c032efe0 +Z d18fb0d5c10defb46f6f8aaff06ea5cb diff --git a/manifest.uuid b/manifest.uuid index 3cbe4903ed..96bf983d21 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8cb43eddab83c68b2163441df70e0e9496d5fa33 \ No newline at end of file +de826c31589258f0906f87d65796944103e36d5b \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index e112239a74..9a2b38f2b7 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1166,6 +1166,7 @@ static void analyzeOneTable( int regSampleRowid = regCol + nCol; int addrNext; int addrIsNull; + int nSampleCol; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; pParse->nMem = MAX(pParse->nMem, regCol+nCol+1); @@ -1186,12 +1187,16 @@ static void analyzeOneTable( sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pIdx->aiColumn[0], regSample); #else - i = HasRowid(pTab) ? nCol-1 : pIdx->nColumn-1; - for(; i>=0; i--){ + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIdx) ){ + nSampleCol = pIdx->nKeyCol; + }else{ + nSampleCol = pIdx->nColumn; + } + for(i=nSampleCol-1; i>=0; i--){ i16 iCol = pIdx->aiColumn[i]; sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, pIdx->nColumn, regSample); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nSampleCol, regSample); #endif sqlite3VdbeAddOp3(v, OP_MakeRecord, regTabname, 6, regTemp); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); From 39129ce8d9cb0101bad783fa06365332e0ddd83d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 30 Jun 2014 15:23:57 +0000 Subject: [PATCH 095/710] Fix where.c so that the primary key values appended to every index entry on a WITHOUT ROWID table may be used when useful. FossilOrigin-Name: 6624a61d16e47ad691c4195ca8a1d68b7348118d --- manifest | 16 +++++----- manifest.uuid | 2 +- src/where.c | 20 ++++++------- test/without_rowid1.test | 64 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index e5da47ab1f..91948b0449 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\snon-primary-key\scolumns\sfrom\sthe\sSTAT4\ssamples\sfor\sWITHOUT\sROWID\stables.\nIndexes,\sboth\srowid\sand\swithout-rowid,\sstill\shold\san\sexact\scopy\nof\sthe\sindex\sentry. -D 2014-06-30T13:56:34.968 +C Fix\swhere.c\sso\sthat\sthe\sprimary\skey\svalues\sappended\sto\severy\sindex\sentry\son\sa\sWITHOUT\sROWID\stable\smay\sbe\sused\swhen\suseful. +D 2014-06-30T15:23:57.908 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 7b9e13cff91a2f14ac61e6a1bc3a83b5113e6298 +F src/where.c da15506cd52de8717836921da8ebd9109228f94e F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1122,7 +1122,7 @@ F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/with1.test 268081a6b14817a262ced4d0ee34d4d2a1dd2068 F test/with2.test ee227a663586aa09771cafd4fa269c5217eaf775 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 -F test/without_rowid1.test e00a0a9dc9f0be651f011d61e8a32b7add5afb30 +F test/without_rowid1.test 7862e605753c8d25329f665fa09072e842183151 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 F test/without_rowid3.test eac3d5c8a1924725b58503a368f2cbd24fd6c8a0 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a @@ -1181,7 +1181,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8cb43eddab83c68b2163441df70e0e9496d5fa33 -R 76a8cf4e27cd1a26fe3722da49c987a3 -U drh -Z d18fb0d5c10defb46f6f8aaff06ea5cb +P de826c31589258f0906f87d65796944103e36d5b +R 8442f880f57a242b77225992393b657c +U dan +Z 6c9f8a4d0a3c007f8c2ac1857a8d9913 diff --git a/manifest.uuid b/manifest.uuid index 96bf983d21..cff31f409b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de826c31589258f0906f87d65796944103e36d5b \ No newline at end of file +6624a61d16e47ad691c4195ca8a1d68b7348118d \ No newline at end of file diff --git a/src/where.c b/src/where.c index fd5831872b..2ec5caf99d 100644 --- a/src/where.c +++ b/src/where.c @@ -544,7 +544,7 @@ static WhereTerm *whereScanInit( if( pIdx && iColumn>=0 ){ pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; for(j=0; pIdx->aiColumn[j]!=iColumn; j++){ - if( NEVER(j>=pIdx->nKeyCol) ) return 0; + if( NEVER(j>pIdx->nColumn) ) return 0; } pScan->zCollName = pIdx->azColl[j]; }else{ @@ -2645,7 +2645,7 @@ static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ txt.db = db; sqlite3StrAccumAppend(&txt, " (", 2); for(i=0; inKeyCol ) ? "rowid" : aCol[aiColumn[i]].zName; + char *z = aiColumn[i] < 0 ? "rowid" : aCol[aiColumn[i]].zName; if( i>=nSkip ){ explainAppendTerm(&txt, i, z, "="); }else{ @@ -2658,11 +2658,11 @@ static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ j = i; if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ - char *z = (j==pIndex->nKeyCol ) ? "rowid" : aCol[aiColumn[j]].zName; + char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; explainAppendTerm(&txt, i++, z, ">"); } if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ - char *z = (j==pIndex->nKeyCol ) ? "rowid" : aCol[aiColumn[j]].zName; + char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; explainAppendTerm(&txt, i, z, "<"); } sqlite3StrAccumAppend(&txt, ")", 1); @@ -4164,12 +4164,9 @@ static int whereLoopAddBtreeIndex( } if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); - assert( pNew->u.btree.nEq<=pProbe->nKeyCol ); - if( pNew->u.btree.nEq < pProbe->nKeyCol ){ - iCol = pProbe->aiColumn[pNew->u.btree.nEq]; - }else{ - iCol = -1; - } + assert( pNew->u.btree.nEqnColumn ); + iCol = pProbe->aiColumn[pNew->u.btree.nEq]; + pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); saved_nEq = pNew->u.btree.nEq; @@ -4359,7 +4356,7 @@ static int whereLoopAddBtreeIndex( } if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 - && pNew->u.btree.nEq<(pProbe->nKeyCol + (pProbe->zName!=0)) + && pNew->u.btree.nEqnColumn ){ whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } @@ -4506,6 +4503,7 @@ static int whereLoopAddBtree( Index *pFirst; /* First of real indices on the table */ memset(&sPk, 0, sizeof(Index)); sPk.nKeyCol = 1; + sPk.nColumn = 1; sPk.aiColumn = &aiColumnPk; sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; diff --git a/test/without_rowid1.test b/test/without_rowid1.test index bdd03c57bb..9d7a6430fe 100644 --- a/test/without_rowid1.test +++ b/test/without_rowid1.test @@ -213,5 +213,69 @@ do_execsql_test 4.1 { do_execsql_test 4.2 { SELECT t42.rowid FROM t42, t41; } {1} + + +#-------------------------------------------------------------------------- +# The following tests verify that the trailing PK fields added to each +# entry in an index on a WITHOUT ROWID table are used correctly. +# +do_execsql_test 5.0 { + CREATE TABLE t45(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX i45 ON t45(b); + + INSERT INTO t45 VALUES(2, 'one', 'x'); + INSERT INTO t45 VALUES(4, 'one', 'x'); + INSERT INTO t45 VALUES(6, 'one', 'x'); + INSERT INTO t45 VALUES(8, 'one', 'x'); + INSERT INTO t45 VALUES(10, 'one', 'x'); + + INSERT INTO t45 VALUES(1, 'two', 'x'); + INSERT INTO t45 VALUES(3, 'two', 'x'); + INSERT INTO t45 VALUES(5, 'two', 'x'); + INSERT INTO t45 VALUES(7, 'two', 'x'); + INSERT INTO t45 VALUES(9, 'two', 'x'); +} + +do_eqp_test 5.1 { + SELECT * FROM t45 WHERE b=? AND a>? +} {/*USING INDEX i45 (b=? AND a>?)*/} + +do_execsql_test 5.2 { + SELECT * FROM t45 WHERE b='two' AND a>4 +} {5 two x 7 two x 9 two x} + +do_execsql_test 5.3 { + SELECT * FROM t45 WHERE b='one' AND a<8 +} { 2 one x 4 one x 6 one x } + +do_execsql_test 5.4 { + CREATE TABLE t46(a, b, c, d, PRIMARY KEY(a, b)) WITHOUT ROWID; + WITH r(x) AS ( + SELECT 1 UNION ALL SELECT x+1 FROM r WHERE x<100 + ) + INSERT INTO t46 SELECT x / 20, x % 20, x % 10, x FROM r; +} + +set queries { + 1 2 "c = 5 AND a = 1" {/*i46 (c=? AND a=?)*/} + 2 6 "c = 4 AND a < 3" {/*i46 (c=? AND a= 3" {/*i46 (c=? AND a>?)*/} + 4 1 "c = 2 AND a = 1 AND b<10" {/*i46 (c=? AND a=? AND b5" {/*i46 (c=? AND a=? AND b>?)*/} +} + +foreach {tn cnt where eqp} $queries { + do_execsql_test 5.5.$tn.1 "SELECT count(*) FROM t46 WHERE $where" $cnt +} + +do_execsql_test 5.6 { + CREATE INDEX i46 ON t46(c); +} + +foreach {tn cnt where eqp} $queries { + do_execsql_test 5.7.$tn.1 "SELECT count(*) FROM t46 WHERE $where" $cnt + do_eqp_test 5.7.$tn.2 "SELECT count(*) FROM t46 WHERE $where" $eqp +} + finish_test From ec9e55d35818a428140fd31a6004efb039175311 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 30 Jun 2014 17:07:39 +0000 Subject: [PATCH 096/710] Fix the STAT4 information for WITHOUT ROWID tables. FossilOrigin-Name: 5d8628fdffbcf837313958f0ee1ed8a2043d384e --- manifest | 14 +++++----- manifest.uuid | 2 +- src/analyze.c | 73 +++++++++++++++++++++++++++++---------------------- 3 files changed, 50 insertions(+), 39 deletions(-) diff --git a/manifest b/manifest index 91948b0449..0580d58225 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\swhere.c\sso\sthat\sthe\sprimary\skey\svalues\sappended\sto\severy\sindex\sentry\son\sa\sWITHOUT\sROWID\stable\smay\sbe\sused\swhen\suseful. -D 2014-06-30T15:23:57.908 +C Fix\sthe\sSTAT4\sinformation\sfor\sWITHOUT\sROWID\stables. +D 2014-06-30T17:07:39.466 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 0f9ee2f88f7c564cedb16f8e0236f2d5ac8429aa +F src/analyze.c ec6e0691a6a23e0239dc733109b906ee04b89cc3 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -1181,7 +1181,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P de826c31589258f0906f87d65796944103e36d5b -R 8442f880f57a242b77225992393b657c -U dan -Z 6c9f8a4d0a3c007f8c2ac1857a8d9913 +P 6624a61d16e47ad691c4195ca8a1d68b7348118d +R 54d510c828cc71c8f4dcbc102dfc2a66 +U drh +Z 6aa7249d11aa6d73980f37e6212dc38c diff --git a/manifest.uuid b/manifest.uuid index cff31f409b..6c0650f1c9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6624a61d16e47ad691c4195ca8a1d68b7348118d \ No newline at end of file +5d8628fdffbcf837313958f0ee1ed8a2043d384e \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 9a2b38f2b7..364f3b7131 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -246,6 +246,7 @@ static void openStatTable( assert( i1 ); /* >1 because it includes the rowid column */ + assert( nCol>0 ); nColUp = sizeof(tRowcnt)<8 ? (nCol+1)&~1 : nCol; + nKeyCol = sqlite3_value_int(argv[1]); + assert( nKeyCol<=nCol ); + assert( nKeyCol>0 ); /* Allocate the space required for the Stat4Accum object */ n = sizeof(*p) @@ -415,6 +429,7 @@ static void statInit( p->db = db; p->nRow = 0; p->nCol = nCol; + p->nKeyCol = nKeyCol; p->current.anDLt = (tRowcnt*)&p[1]; p->current.anEq = &p->current.anDLt[nColUp]; @@ -425,9 +440,9 @@ static void statInit( p->iGet = -1; p->mxSample = mxSample; - p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[1])/(mxSample/3+1) + 1); + p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[2])/(mxSample/3+1) + 1); p->current.anLt = &p->current.anEq[nColUp]; - p->iPrn = nCol*0x689e962d ^ sqlite3_value_int(argv[1])*0xd0944565; + p->iPrn = nCol*0x689e962d ^ sqlite3_value_int(argv[2])*0xd0944565; /* Set up the Stat4Accum.a[] and aBest[] arrays */ p->a = (struct Stat4Sample*)&p->current.anLt[nColUp]; @@ -450,7 +465,7 @@ static void statInit( sqlite3_result_blob(context, p, sizeof(p), stat4Destructor); } static const FuncDef statInitFuncdef = { - 1+IsStat34, /* nArg */ + 2+IsStat34, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ @@ -691,7 +706,7 @@ static void statPush( UNUSED_PARAMETER( argc ); UNUSED_PARAMETER( context ); - assert( p->nCol>1 ); /* Includes rowid field */ + assert( p->nCol>0 ); assert( iChngnCol ); if( p->nRow==0 ){ @@ -819,7 +834,7 @@ static void statGet( char *z; int i; - char *zRet = sqlite3MallocZero(p->nCol * 25); + char *zRet = sqlite3MallocZero( (p->nKeyCol+1)*25 ); if( zRet==0 ){ sqlite3_result_error_nomem(context); return; @@ -827,7 +842,7 @@ static void statGet( sqlite3_snprintf(24, zRet, "%llu", (u64)p->nRow); z = zRet + sqlite3Strlen30(zRet); - for(i=0; i<(p->nCol-1); i++){ + for(i=0; inKeyCol; i++){ u64 nDistinct = p->current.anDLt[i] + 1; u64 iVal = (p->nRow + nDistinct - 1) / nDistinct; sqlite3_snprintf(24, z, " %llu", iVal); @@ -996,18 +1011,19 @@ static void analyzeOneTable( if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0; - VdbeNoopComment((v, "Begin analysis of %s", pIdx->zName)); - nCol = pIdx->nKeyCol; + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIdx) ){ + nCol = pIdx->nKeyCol; + zIdxName = pTab->zName; + }else{ + nCol = pIdx->nColumn; + zIdxName = pIdx->zName; + } aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*(nCol+1)); if( aGotoChng==0 ) continue; /* Populate the register containing the index name. */ - if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - zIdxName = pTab->zName; - }else{ - zIdxName = pIdx->zName; - } sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, zIdxName, 0); + VdbeComment((v, "Analysis for %s.%s", pTab->zName, zIdxName)); /* ** Pseudo-code for loop that calls stat_push(): @@ -1045,7 +1061,7 @@ static void analyzeOneTable( ** the regPrev array and a trailing rowid (the rowid slot is required ** when building a record to insert into the sample column of ** the sqlite_stat4 table. */ - pParse->nMem = MAX(pParse->nMem, regPrev+pIdx->nColumn); + pParse->nMem = MAX(pParse->nMem, regPrev+nCol); /* Open a read-only cursor on the index being analyzed. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); @@ -1061,12 +1077,13 @@ static void analyzeOneTable( ** The second argument is only used for STAT3 and STAT4 */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+2); + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); #endif - sqlite3VdbeAddOp2(v, OP_Integer, nCol+1, regStat4+1); + sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 1+IsStat34); + sqlite3VdbeChangeP5(v, 2+IsStat34); /* Implementation of the following: ** @@ -1166,10 +1183,9 @@ static void analyzeOneTable( int regSampleRowid = regCol + nCol; int addrNext; int addrIsNull; - int nSampleCol; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - pParse->nMem = MAX(pParse->nMem, regCol+nCol+1); + pParse->nMem = MAX(pParse->nMem, regCol+nCol); addrNext = sqlite3VdbeCurrentAddr(v); callStatGet(v, regStat4, STAT_GET_ROWID, regSampleRowid); @@ -1187,16 +1203,11 @@ static void analyzeOneTable( sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, pIdx->aiColumn[0], regSample); #else - if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIdx) ){ - nSampleCol = pIdx->nKeyCol; - }else{ - nSampleCol = pIdx->nColumn; - } - for(i=nSampleCol-1; i>=0; i--){ + for(i=0; iaiColumn[i]; sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nSampleCol, regSample); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol, regSample); #endif sqlite3VdbeAddOp3(v, OP_MakeRecord, regTabname, 6, regTemp); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); From fd984b81146f7e909b9e0ecbfad80cbcc6b25343 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 30 Jun 2014 18:02:20 +0000 Subject: [PATCH 097/710] Fix a problem in where.c with using the stat4 sample data of an index on a WITHOUT ROWID table. FossilOrigin-Name: 053a210e3169732c58f84cb54c9b6f6df3a8f4ea --- manifest | 16 +++++------ manifest.uuid | 2 +- src/where.c | 4 +-- test/analyze9.test | 68 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 0580d58225..73bea2a8c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sSTAT4\sinformation\sfor\sWITHOUT\sROWID\stables. -D 2014-06-30T17:07:39.466 +C Fix\sa\sproblem\sin\swhere.c\swith\susing\sthe\sstat4\ssample\sdata\sof\san\sindex\son\sa\sWITHOUT\sROWID\stable. +D 2014-06-30T18:02:20.849 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c da15506cd52de8717836921da8ebd9109228f94e +F src/where.c 2bc0226fced128686c432748324351beb127829b F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -316,7 +316,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test 623e02a99a78fa12fe5def2fd559032d5d887e0f +F test/analyze9.test 93619368fff2db833747a6dfa9b1146a82e5d4d2 F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -1181,7 +1181,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6624a61d16e47ad691c4195ca8a1d68b7348118d -R 54d510c828cc71c8f4dcbc102dfc2a66 -U drh -Z 6aa7249d11aa6d73980f37e6212dc38c +P 5d8628fdffbcf837313958f0ee1ed8a2043d384e +R 48fee5f306f007e5e5bc0779449b568d +U dan +Z 6e1427ceae354dbee93842f8dd267647 diff --git a/manifest.uuid b/manifest.uuid index 6c0650f1c9..e4cd819c16 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d8628fdffbcf837313958f0ee1ed8a2043d384e \ No newline at end of file +053a210e3169732c58f84cb54c9b6f6df3a8f4ea \ No newline at end of file diff --git a/src/where.c b/src/where.c index 2ec5caf99d..57c5a2ad14 100644 --- a/src/where.c +++ b/src/where.c @@ -2201,7 +2201,7 @@ static int whereEqualScanEst( int bOk; assert( nEq>=1 ); - assert( nEq<=(p->nKeyCol+1) ); + assert( nEq<=p->nColumn ); assert( p->aSample!=0 ); assert( p->nSample>0 ); assert( pBuilder->nRecValidp->nKeyCol ){ + if( nEq>=p->nColumn ){ *pnRow = 1; return SQLITE_OK; } diff --git a/test/analyze9.test b/test/analyze9.test index f25e5924e6..d0d3b3524f 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -953,4 +953,72 @@ for {set i 0} {$i<16} {incr i} { } {1} } +#------------------------------------------------------------------------- +# +reset_db + +do_execsql_test 21.0 { + CREATE TABLE t2(a, b); + CREATE INDEX i2 ON t2(a); +} + +do_test 21.1 { + for {set i 1} {$i < 100} {incr i} { + execsql { + INSERT INTO t2 VALUES(CASE WHEN $i < 80 THEN 'one' ELSE 'two' END, $i) + } + } + execsql ANALYZE +} {} + +# Condition (a='one') matches 80% of the table. (rowid<10) reduces this to +# 10%, but (rowid<50) only reduces it to 50%. So in the first case below +# the index is used. In the second, it is not. +# +do_eqp_test 21.2 { + SELECT * FROM t2 WHERE a='one' AND rowid < 10 +} {/*USING INDEX i2 (a=? AND rowid45 AND x<96) THEN 'B' ELSE 'A' END, /* Column "a" */ + x, /* Column "b" */ + CASE WHEN (x<51) THEN 'one' ELSE 'two' END, /* Column "c" */ + x /* Column "d" */ + FROM r; + + CREATE INDEX i3 ON t3(c); + CREATE INDEX i4 ON t3(d); + ANALYZE; +} + +# Expression (c='one' AND a='B') matches 5 table rows. But (c='one' AND a=A') +# matches 45. Expression (d Date: Mon, 30 Jun 2014 19:28:57 +0000 Subject: [PATCH 098/710] Bump the version number to 3.8.6. FossilOrigin-Name: f925e9baafea625f63105f8013abb3807b418379 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 15 +++++++-------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/VERSION b/VERSION index 0cbfaed0d9..2e14a9557d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.5 +3.8.6 diff --git a/configure b/configure index 8be06033f0..8748a98406 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.5. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.6. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.5' -PACKAGE_STRING='sqlite 3.8.5' +PACKAGE_VERSION='3.8.6' +PACKAGE_STRING='sqlite 3.8.6' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.5 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.6 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.5:";; + short | recursive ) echo "Configuration of sqlite 3.8.6:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.5 +sqlite configure 3.8.6 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.5, which was +It was created by sqlite $as_me 3.8.6, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.5, which was +This file was extended by sqlite $as_me 3.8.6, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.5 +sqlite config.status 3.8.6 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index bbfeca06e9..d5d65422e5 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Attempt\sto\suse\sSTAT4\sinformation\sto\sestimate\sthe\sselectivity\sof\sWHERE\sclause\nterms\swhen\susing\sthe\sskip-scan\soptimization. -D 2014-06-30T19:07:58.197 +C Bump\sthe\sversion\snumber\sto\s3.8.6. +D 2014-06-30T19:28:57.595 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc ad60c4d8be35f98fdcca614088b84a91612b2b4c F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c +F VERSION 1c877615a9db323e3cd301e3d57d853f9d5c4a07 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 5de98fac90452f876511fee8a35565a55dfb191f x +F configure 513f1e2464c3673bcdb5471b13d98e7eeb6f5ca2 x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1182,8 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bc2de8095fa9c385db0adf22ca55b0298a33c284 c96de490ac77dd23c108004b95152cce4922fe27 -R 55c9ac9304448ee4805a54515b7a2134 -T +closed c96de490ac77dd23c108004b95152cce4922fe27 +P d09ca6d5efad3e4cfa93a4dc711e6ba6079d4b4b +R 0b9fd011a99704759c36a48aebd29593 U drh -Z bab166b6f1bf8cfb185d9041a405176e +Z 99f7c1257c18120424d63d4c4b9a073c diff --git a/manifest.uuid b/manifest.uuid index 91daee692f..708593f343 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d09ca6d5efad3e4cfa93a4dc711e6ba6079d4b4b \ No newline at end of file +f925e9baafea625f63105f8013abb3807b418379 \ No newline at end of file From 39caccf83d7c8af6f5b2e1a747c4adebe2307031 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 1 Jul 2014 11:54:02 +0000 Subject: [PATCH 099/710] Ensure that all fields are loaded from the stat4 table for records that correspond to indexes on WITHOUT ROWID tables with composite primary keys. FossilOrigin-Name: 21981e35062cc6b30e9576786cbf55265a7a4d41 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/analyze.c | 25 ++++++++++++++++++------- src/where.c | 2 +- test/analyze9.test | 44 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 73 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index d5d65422e5..4b1417a454 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bump\sthe\sversion\snumber\sto\s3.8.6. -D 2014-06-30T19:28:57.595 +C Ensure\sthat\sall\sfields\sare\sloaded\sfrom\sthe\sstat4\stable\sfor\srecords\sthat\scorrespond\sto\sindexes\son\sWITHOUT\sROWID\stables\swith\scomposite\sprimary\skeys. +D 2014-07-01T11:54:02.938 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c ec6e0691a6a23e0239dc733109b906ee04b89cc3 +F src/analyze.c 61bc3467bd7803158d460dd92f5ba5205371bdce F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 6ff6f7e3b272fad66eea601af489777adea3cc1f +F src/where.c 3a451b00516b59f414c59f817508edce51c732f1 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -316,7 +316,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test 93619368fff2db833747a6dfa9b1146a82e5d4d2 +F test/analyze9.test 3129cc8a82c3d6c5dfd340143b18499219faebac F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d09ca6d5efad3e4cfa93a4dc711e6ba6079d4b4b -R 0b9fd011a99704759c36a48aebd29593 -U drh -Z 99f7c1257c18120424d63d4c4b9a073c +P f925e9baafea625f63105f8013abb3807b418379 +R b0479b264ee66056e956ca8c9e5bfcd6 +U dan +Z 2688039829f2bbaff9c051901e2506da diff --git a/manifest.uuid b/manifest.uuid index 708593f343..ec2c1a4071 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f925e9baafea625f63105f8013abb3807b418379 \ No newline at end of file +21981e35062cc6b30e9576786cbf55265a7a4d41 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 364f3b7131..2330bdb342 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1519,7 +1519,16 @@ static void initAvgEq(Index *pIdx){ IndexSample *aSample = pIdx->aSample; IndexSample *pFinal = &aSample[pIdx->nSample-1]; int iCol; - for(iCol=0; iColnKeyCol; iCol++){ + int nCol = 1; + if( pIdx->nSampleCol>1 ){ + /* If this is stat4 data, then calculate aAvgEq[] values for all + ** sample columns except the last. The last is always set to 1, as + ** once the trailing PK fields are considered all index keys are + ** unique. */ + nCol = pIdx->nSampleCol-1; + pIdx->aAvgEq[nCol] = 1; + } + for(iCol=0; iColaAvgEq[iCol] = avgEq; - if( pIdx->nSampleCol==1 ) break; } } } @@ -1601,7 +1609,6 @@ static int loadStatTbl( while( sqlite3_step(pStmt)==SQLITE_ROW ){ int nIdxCol = 1; /* Number of columns in stat4 records */ - int nAvgCol = 1; /* Number of entries in Index.aAvgEq */ char *zIndex; /* Index name */ Index *pIdx; /* Pointer to the index object */ @@ -1619,13 +1626,17 @@ static int loadStatTbl( ** loaded from the stat4 table. In this case ignore stat3 data. */ if( pIdx==0 || pIdx->nSample ) continue; if( bStat3==0 ){ - nIdxCol = pIdx->nKeyCol+1; - nAvgCol = pIdx->nKeyCol; + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nIdxCol = pIdx->nKeyCol; + }else{ + nIdxCol = pIdx->nColumn; + } } pIdx->nSampleCol = nIdxCol; nByte = sizeof(IndexSample) * nSample; nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; - nByte += nAvgCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ pIdx->aSample = sqlite3DbMallocZero(db, nByte); if( pIdx->aSample==0 ){ @@ -1633,7 +1644,7 @@ static int loadStatTbl( return SQLITE_NOMEM; } pSpace = (tRowcnt*)&pIdx->aSample[nSample]; - pIdx->aAvgEq = pSpace; pSpace += nAvgCol; + pIdx->aAvgEq = pSpace; pSpace += nIdxCol; for(i=0; iaSample[i].anEq = pSpace; pSpace += nIdxCol; pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol; diff --git a/src/where.c b/src/where.c index a26a9ea9a1..e52f598e02 100644 --- a/src/where.c +++ b/src/where.c @@ -1959,7 +1959,7 @@ static void whereKeyStats( iUpper = i>=pIdx->nSample ? nRow0 : aSample[i].anLt[iCol]; iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; } - aStat[1] = (pIdx->nKeyCol>iCol ? pIdx->aAvgEq[iCol] : 1); + aStat[1] = pIdx->aAvgEq[iCol]; if( iLower>=iUpper ){ iGap = 0; }else{ diff --git a/test/analyze9.test b/test/analyze9.test index d0d3b3524f..d8bdc768d3 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -1019,6 +1019,50 @@ foreach {tn where res} { do_eqp_test 22.2.$tn "SELECT * FROM t3 WHERE $where" $res } +proc int_to_char {i} { + set ret "" + set char [list a b c d e f g h i j] + foreach {div} {1000 100 10 1} { + append ret [lindex $char [expr ($i / $div) % 10]] + } + set ret +} +db func int_to_char int_to_char + +do_execsql_test 23.0 { + CREATE TABLE t4( + a COLLATE nocase, b, c, + d, e, f, + PRIMARY KEY(c, b, a) + ) WITHOUT ROWID; + CREATE INDEX i41 ON t4(e); + CREATE INDEX i42 ON t4(f); + + WITH data(a, b, c, d, e, f) AS ( + SELECT int_to_char(0), 'xyz', 'zyx', '*', 0, 0 + UNION ALL + SELECT + int_to_char(f+1), b, c, d, (e+1) % 2, f+1 + FROM data WHERE f<1024 + ) + INSERT INTO t4 SELECT a, b, c, d, e, f FROM data; + ANALYZE; +} {} + +do_eqp_test 23.1 { + SELECT * FROM t4 WHERE + (e=1 AND b='xyz' AND c='zyx' AND a<'AEA') AND f<300 +} { + 0 0 0 {SEARCH TABLE t4 USING INDEX i41 (e=? AND c=? AND b=? AND a Date: Tue, 1 Jul 2014 15:22:11 +0000 Subject: [PATCH 100/710] Add another test to verify that SQLite is using stat4 data for composite primary keys on WITHOUT ROWID tables. FossilOrigin-Name: 0df1fe72f8271abc86cd552027d938c910f90967 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/analyze9.test | 27 ++++++++++++++++++++++++++- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4b1417a454..7a52f87c5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sall\sfields\sare\sloaded\sfrom\sthe\sstat4\stable\sfor\srecords\sthat\scorrespond\sto\sindexes\son\sWITHOUT\sROWID\stables\swith\scomposite\sprimary\skeys. -D 2014-07-01T11:54:02.938 +C Add\sanother\stest\sto\sverify\sthat\sSQLite\sis\susing\sstat4\sdata\sfor\scomposite\sprimary\skeys\son\sWITHOUT\sROWID\stables. +D 2014-07-01T15:22:11.950 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -316,7 +316,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test 3129cc8a82c3d6c5dfd340143b18499219faebac +F test/analyze9.test bd5aaf2a8fd2dd774b08251416897185531a8adf F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f925e9baafea625f63105f8013abb3807b418379 -R b0479b264ee66056e956ca8c9e5bfcd6 +P 21981e35062cc6b30e9576786cbf55265a7a4d41 +R e4440e4e4a98b796b4fa8322644ff7eb U dan -Z 2688039829f2bbaff9c051901e2506da +Z 9cb18e75f66a8c4075f21fee1b6ab8d9 diff --git a/manifest.uuid b/manifest.uuid index ec2c1a4071..91d9eea5ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -21981e35062cc6b30e9576786cbf55265a7a4d41 \ No newline at end of file +0df1fe72f8271abc86cd552027d938c910f90967 \ No newline at end of file diff --git a/test/analyze9.test b/test/analyze9.test index d8bdc768d3..fc5a13f43f 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -1055,7 +1055,6 @@ do_eqp_test 23.1 { } { 0 0 0 {SEARCH TABLE t4 USING INDEX i41 (e=? AND c=? AND b=? AND a Date: Thu, 3 Jul 2014 12:18:22 +0000 Subject: [PATCH 101/710] Change fts3/4 so that the "unicode61" is included in builds by default. It may now be excluded by defining SQLITE_DISABLE_FTS3_UNICODE. FossilOrigin-Name: 0cc0230ae9cfc9760fd8ef2c25e82576b052dbeb --- ext/fts3/fts3.c | 8 ++++---- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_unicode.c | 4 ++-- ext/fts3/fts3_unicode2.c | 4 ++-- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/test_config.c | 2 +- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 1676fd41e1..4f4b667430 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3750,7 +3750,7 @@ static void hashDestroy(void *p){ */ void sqlite3Fts3SimpleTokenizerModule(sqlite3_tokenizer_module const**ppModule); void sqlite3Fts3PorterTokenizerModule(sqlite3_tokenizer_module const**ppModule); -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const**ppModule); #endif #ifdef SQLITE_ENABLE_ICU @@ -3768,7 +3768,7 @@ int sqlite3Fts3Init(sqlite3 *db){ Fts3Hash *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE const sqlite3_tokenizer_module *pUnicode = 0; #endif @@ -3777,7 +3777,7 @@ int sqlite3Fts3Init(sqlite3 *db){ sqlite3Fts3IcuTokenizerModule(&pIcu); #endif -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE sqlite3Fts3UnicodeTokenizer(&pUnicode); #endif @@ -3805,7 +3805,7 @@ int sqlite3Fts3Init(sqlite3 *db){ if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) #endif #ifdef SQLITE_ENABLE_ICU diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 1383102f4c..b2827b7352 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -585,7 +585,7 @@ int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); /* fts3_unicode2.c (functions generated by parsing unicode text files) */ -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE int sqlite3FtsUnicodeFold(int, int); int sqlite3FtsUnicodeIsalnum(int); int sqlite3FtsUnicodeIsdiacritic(int); diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c index 188358eade..591cbb8dc9 100644 --- a/ext/fts3/fts3_unicode.c +++ b/ext/fts3/fts3_unicode.c @@ -13,7 +13,7 @@ ** Implementation of the "unicode" full-text-search tokenizer. */ -#ifdef SQLITE_ENABLE_FTS4_UNICODE61 +#ifndef SQLITE_DISABLE_FTS3_UNICODE #include "fts3Int.h" #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) @@ -390,4 +390,4 @@ void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ } #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) */ -#endif /* ifndef SQLITE_ENABLE_FTS4_UNICODE61 */ +#endif /* ifndef SQLITE_DISABLE_FTS3_UNICODE */ diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c index fba688ff9c..4837dbd272 100644 --- a/ext/fts3/fts3_unicode2.c +++ b/ext/fts3/fts3_unicode2.c @@ -15,7 +15,7 @@ ** DO NOT EDIT THIS MACHINE GENERATED FILE. */ -#if defined(SQLITE_ENABLE_FTS4_UNICODE61) +#ifndef SQLITE_DISABLE_FTS3_UNICODE #if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) #include @@ -362,4 +362,4 @@ int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ return ret; } #endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */ -#endif /* !defined(SQLITE_ENABLE_FTS4_UNICODE61) */ +#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ diff --git a/manifest b/manifest index 7a52f87c5e..ee3d1132c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sanother\stest\sto\sverify\sthat\sSQLite\sis\susing\sstat4\sdata\sfor\scomposite\sprimary\skeys\son\sWITHOUT\sROWID\stables. -D 2014-07-01T15:22:11.950 +C Change\sfts3/4\sso\sthat\sthe\s"unicode61"\sis\sincluded\sin\sbuilds\sby\sdefault.\sIt\smay\snow\sbe\sexcluded\sby\sdefining\sSQLITE_DISABLE_FTS3_UNICODE. +D 2014-07-03T12:18:22.515 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,9 +78,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 20bc65862cfcea0a39bb64a819f8fe92a8e144c1 +F ext/fts3/fts3.c 2f5e925bdb9d6d3e488c5a981af60cad4f9cdfe7 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h 16cddf2d7b0e5f3681615ae1d8ca0e45fca44918 +F ext/fts3/fts3Int.h 53d4eca1fb23eab00681fb028fb82eb5705c1e21 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 F ext/fts3/fts3_expr.c 351395fad6fcb16ecfc61db0861008a70101330c F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 @@ -94,8 +94,8 @@ F ext/fts3/fts3_tokenize_vtab.c 011170fe9eba5ff062f1a31d3188e00267716706 F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 -F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d +F ext/fts3/fts3_unicode.c 0f598fa4c96af1188e4d5f5e8e78e25500bd5922 +F ext/fts3/fts3_unicode2.c c8adda75aad0c6c252ef3dd555f811f437485044 F ext/fts3/fts3_write.c 8260388626516a7005d06a9dce94f9e55c6c2a41 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 @@ -245,7 +245,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c dabaa32868974e1ae39770cc17d7e066a9c38e6d +F src/test_config.c 42fb068a038c8684741522f551325228b1389e63 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 21981e35062cc6b30e9576786cbf55265a7a4d41 -R e4440e4e4a98b796b4fa8322644ff7eb +P 0df1fe72f8271abc86cd552027d938c910f90967 +R ef308fba8a4f61e769b2e20d3673cd17 U dan -Z 9cb18e75f66a8c4075f21fee1b6ab8d9 +Z 86452bd7dda1f4219032eac6ac7bc6d3 diff --git a/manifest.uuid b/manifest.uuid index 91d9eea5ec..57da097497 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0df1fe72f8271abc86cd552027d938c910f90967 \ No newline at end of file +0cc0230ae9cfc9760fd8ef2c25e82576b052dbeb \ No newline at end of file diff --git a/src/test_config.c b/src/test_config.c index bf8afd8e67..9ce7f6fdf6 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -330,7 +330,7 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "fts3", "0", TCL_GLOBAL_ONLY); #endif -#if defined(SQLITE_ENABLE_FTS3) && defined(SQLITE_ENABLE_FTS4_UNICODE61) +#if defined(SQLITE_ENABLE_FTS3) && !defined(SQLITE_DISABLE_FTS3_UNICODE) Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "fts3_unicode", "0", TCL_GLOBAL_ONLY); From 42d3d37ad1378dde011c1fa6c21927aa62e60b64 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 7 Jul 2014 16:07:43 +0000 Subject: [PATCH 102/710] Fix a division-by-zero error in the fts3view utility program. Add the fts3view utility program to the "main.mk" makefile. FossilOrigin-Name: 64f02699b4bd747e4c2f0debf72d37ec275137d8 --- ext/fts3/tool/fts3view.c | 2 +- main.mk | 4 ++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c index 479ae9868d..406eb91688 100644 --- a/ext/fts3/tool/fts3view.c +++ b/ext/fts3/tool/fts3view.c @@ -376,7 +376,7 @@ static void showSegmentStats(sqlite3 *db, const char *zTab){ sqlite3_finalize(pStmt); nLeaf = nSeg - nIdx; printf("Leaf segments larger than %5d bytes.... %9d %5.2f%%\n", - pgsz-45, n, n*100.0/nLeaf); + pgsz-45, n, nLeaf>0 ? n*100.0/nLeaf : 0.0); pStmt = prepare(db, "SELECT max(level%%1024) FROM '%q_segdir'", zTab); mxLevel = 0; diff --git a/main.mk b/main.mk index 62b6b22475..865f7a87c2 100644 --- a/main.mk +++ b/main.mk @@ -644,6 +644,10 @@ showwal$(EXE): $(TOP)/tool/showwal.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o showwal$(EXE) \ $(TOP)/tool/showwal.c sqlite3.o $(THREADLIB) +fts3view$(EXE): $(TOP)/ext/fts3/tool/fts3view.c sqlite3.o + $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o fts3view$(EXE) \ + $(TOP)/ext/fts3/tool/fts3view.c sqlite3.o $(THREADLIB) + rollback-test$(EXE): $(TOP)/tool/rollback-test.c sqlite3.o $(TCC) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -o rollback-test$(EXE) \ $(TOP)/tool/rollback-test.c sqlite3.o $(THREADLIB) diff --git a/manifest b/manifest index ee3d1132c1..6f5d54f727 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sfts3/4\sso\sthat\sthe\s"unicode61"\sis\sincluded\sin\sbuilds\sby\sdefault.\sIt\smay\snow\sbe\sexcluded\sby\sdefining\sSQLITE_DISABLE_FTS3_UNICODE. -D 2014-07-03T12:18:22.515 +C Fix\sa\sdivision-by-zero\serror\sin\sthe\sfts3view\sutility\sprogram.\s\sAdd\sthe\nfts3view\sutility\sprogram\sto\sthe\s"main.mk"\smakefile. +D 2014-07-07T16:07:43.091 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -99,7 +99,7 @@ F ext/fts3/fts3_unicode2.c c8adda75aad0c6c252ef3dd555f811f437485044 F ext/fts3/fts3_write.c 8260388626516a7005d06a9dce94f9e55c6c2a41 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 -F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 +F ext/fts3/tool/fts3view.c e2bc4c5c549541b74e367039399ce47320fbae57 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368 @@ -146,7 +146,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 7850d834ca7f3c772e2b0087631868d5bfdeabb9 +F main.mk 0b5baabb84a5d212b40c244e9439329a542e1982 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0df1fe72f8271abc86cd552027d938c910f90967 -R ef308fba8a4f61e769b2e20d3673cd17 -U dan -Z 86452bd7dda1f4219032eac6ac7bc6d3 +P 0cc0230ae9cfc9760fd8ef2c25e82576b052dbeb +R 52eca07cc58a435942000a424a892b43 +U drh +Z 228c9cd7cbfc8e1485536ecff5b51496 diff --git a/manifest.uuid b/manifest.uuid index 57da097497..702645c1b6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0cc0230ae9cfc9760fd8ef2c25e82576b052dbeb \ No newline at end of file +64f02699b4bd747e4c2f0debf72d37ec275137d8 \ No newline at end of file From de54527fb2bb63dfab76236f729340a73c844178 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 7 Jul 2014 17:57:50 +0000 Subject: [PATCH 103/710] Add the fts3view utility program to the MSVC makefile. FossilOrigin-Name: b04751bd59fed513ab615e7f368fe25c64b3607c --- Makefile.msc | 4 ++++ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 55d26e3053..f1fb591553 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1447,6 +1447,10 @@ showwal.exe: $(TOP)\tool\showwal.c $(SQLITE3C) $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\showwal.c $(SQLITE3C) +fts3view.exe: $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) + $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ + $(TOP)\ext\fts3\tool\fts3view.c $(SQLITE3C) + rollback-test.exe: $(TOP)\tool\rollback-test.c $(SQLITE3C) $(LTLINK) -DSQLITE_THREADSAFE=0 -DSQLITE_OMIT_LOAD_EXTENSION -Fe$@ \ $(TOP)\tool\rollback-test.c $(SQLITE3C) diff --git a/manifest b/manifest index 6f5d54f727..71d860ddc9 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Fix\sa\sdivision-by-zero\serror\sin\sthe\sfts3view\sutility\sprogram.\s\sAdd\sthe\nfts3view\sutility\sprogram\sto\sthe\s"main.mk"\smakefile. -D 2014-07-07T16:07:43.091 +C Add\sthe\sfts3view\sutility\sprogram\sto\sthe\sMSVC\smakefile. +D 2014-07-07T17:57:50.539 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc ad60c4d8be35f98fdcca614088b84a91612b2b4c +F Makefile.msc 0fc63e099a4f71fdeef31496b1f4ad04ab419884 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 1c877615a9db323e3cd301e3d57d853f9d5c4a07 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0cc0230ae9cfc9760fd8ef2c25e82576b052dbeb -R 52eca07cc58a435942000a424a892b43 -U drh -Z 228c9cd7cbfc8e1485536ecff5b51496 +P 64f02699b4bd747e4c2f0debf72d37ec275137d8 +R c90797aa6cbf897903a4492efeae675d +U mistachkin +Z 28a255fbc95b5b8b8da59816ed21faba diff --git a/manifest.uuid b/manifest.uuid index 702645c1b6..f2f5008375 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -64f02699b4bd747e4c2f0debf72d37ec275137d8 \ No newline at end of file +b04751bd59fed513ab615e7f368fe25c64b3607c \ No newline at end of file From 166c3737793fe2b0c6fe36ab7f3a040c1e50ac3b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 7 Jul 2014 18:03:38 +0000 Subject: [PATCH 104/710] Fix harmless compiler warnings in the fts3view utility program that can occur with MSVC. FossilOrigin-Name: 1cec1e030035e5253fb7ebbdfe5c1a3029e4e29b --- ext/fts3/tool/fts3view.c | 13 +++++++------ manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ext/fts3/tool/fts3view.c b/ext/fts3/tool/fts3view.c index 406eb91688..3dc1ba80fe 100644 --- a/ext/fts3/tool/fts3view.c +++ b/ext/fts3/tool/fts3view.c @@ -554,7 +554,7 @@ static void decodeSegment( sqlite3_int64 n; sqlite3_int64 iDocsz; int iHeight; - int i = 0; + sqlite3_int64 i = 0; int cnt = 0; char zTerm[1000]; @@ -576,12 +576,12 @@ static void decodeSegment( fprintf(stderr, "term to long\n"); exit(1); } - memcpy(zTerm+iPrefix, aData+i, nTerm); + memcpy(zTerm+iPrefix, aData+i, (size_t)nTerm); zTerm[iPrefix+nTerm] = 0; i += nTerm; if( iHeight==0 ){ i += getVarint(aData+i, &iDocsz); - printf("term: %-25s doclist %7lld bytes offset %d\n", zTerm, iDocsz, i); + printf("term: %-25s doclist %7lld bytes offset %lld\n", zTerm, iDocsz, i); i += iDocsz; }else{ printf("term: %-25s child %lld\n", zTerm, ++iChild); @@ -749,18 +749,19 @@ static void decodeDoclist( */ static void showDoclist(sqlite3 *db, const char *zTab){ const unsigned char *aData; - sqlite3_int64 offset, nData; + sqlite3_int64 offset; + int nData; sqlite3_stmt *pStmt; offset = atoi64(azExtra[1]); - nData = atoi64(azExtra[2]); + nData = atoi(azExtra[2]); pStmt = prepareToGetSegment(db, zTab, azExtra[0]); if( sqlite3_step(pStmt)!=SQLITE_ROW ){ sqlite3_finalize(pStmt); return; } aData = sqlite3_column_blob(pStmt, 0); - printf("Doclist at %s offset %lld of size %lld bytes:\n", + printf("Doclist at %s offset %lld of size %d bytes:\n", azExtra[0], offset, nData); if( findOption("raw", 0, 0)!=0 ){ printBlob(aData+offset, nData); diff --git a/manifest b/manifest index 71d860ddc9..2b4d4ab97f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sfts3view\sutility\sprogram\sto\sthe\sMSVC\smakefile. -D 2014-07-07T17:57:50.539 +C Fix\sharmless\scompiler\swarnings\sin\sthe\sfts3view\sutility\sprogram\sthat\scan\soccur\swith\sMSVC. +D 2014-07-07T18:03:38.442 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -99,7 +99,7 @@ F ext/fts3/fts3_unicode2.c c8adda75aad0c6c252ef3dd555f811f437485044 F ext/fts3/fts3_write.c 8260388626516a7005d06a9dce94f9e55c6c2a41 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 -F ext/fts3/tool/fts3view.c e2bc4c5c549541b74e367039399ce47320fbae57 +F ext/fts3/tool/fts3view.c 3986531f2fc0ceca0c89c31ec7d0589b6adb19d6 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 64f02699b4bd747e4c2f0debf72d37ec275137d8 -R c90797aa6cbf897903a4492efeae675d +P b04751bd59fed513ab615e7f368fe25c64b3607c +R 30e3947db20694bb71560953bb8a13c2 U mistachkin -Z 28a255fbc95b5b8b8da59816ed21faba +Z 58c34cf56858b67c344e911b0fcd0cab diff --git a/manifest.uuid b/manifest.uuid index f2f5008375..afc22bde21 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b04751bd59fed513ab615e7f368fe25c64b3607c \ No newline at end of file +1cec1e030035e5253fb7ebbdfe5c1a3029e4e29b \ No newline at end of file From 56f674c8a0802314ae398b9f6e93fe8a3b6a15f4 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 18 Jul 2014 14:43:29 +0000 Subject: [PATCH 105/710] Improvements to the ".fullschema" command in the command-line shell. FossilOrigin-Name: fa80c64caa573297bf43cb5b11d9e7db58567200 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 43 ++++++++++++++++++++++++++++--------------- 3 files changed, 36 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 2b4d4ab97f..baca280a3a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sthe\sfts3view\sutility\sprogram\sthat\scan\soccur\swith\sMSVC. -D 2014-07-07T18:03:38.442 +C Improvements\sto\sthe\s".fullschema"\scommand\sin\sthe\scommand-line\sshell. +D 2014-07-18T14:43:29.403 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 6762c62e11b504aa014edceab8886495165e3a77 -F src/shell.c 56de2dfa3f25def4bf03098f7e2256fbb42f6e3c +F src/shell.c 566aee8213372a2e81ba0eb34e9759f7b2574009 F src/sqlite.h.in a98eb3e8c86c934ea6f5bcfc6b69653dde2f4ed4 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b04751bd59fed513ab615e7f368fe25c64b3607c -R 30e3947db20694bb71560953bb8a13c2 -U mistachkin -Z 58c34cf56858b67c344e911b0fcd0cab +P 1cec1e030035e5253fb7ebbdfe5c1a3029e4e29b +R 3a5c863b5d4e43cf3d99c01f25a09c91 +U drh +Z f91dd8693f049cec144fbd0b43e0e92a diff --git a/manifest.uuid b/manifest.uuid index afc22bde21..984495a47c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1cec1e030035e5253fb7ebbdfe5c1a3029e4e29b \ No newline at end of file +fa80c64caa573297bf43cb5b11d9e7db58567200 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index ff87d817b9..97167dc877 100644 --- a/src/shell.c +++ b/src/shell.c @@ -2416,6 +2416,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( c=='f' && strncmp(azArg[0], "fullschema", n)==0 ){ struct callback_data data; char *zErrMsg = 0; + int doStats = 0; if( nArg!=1 ){ fprintf(stderr, "Usage: .fullschema\n"); rc = 1; @@ -2434,21 +2435,33 @@ static int do_meta_command(char *zLine, struct callback_data *p){ "ORDER BY rowid", callback, &data, &zErrMsg ); - sqlite3_exec(p->db, "SELECT 'ANALYZE sqlite_master;'", - callback, &data, &zErrMsg); - data.mode = MODE_Insert; - data.zDestTable = "sqlite_stat1"; - shell_exec(p->db, "SELECT * FROM sqlite_stat1", - shell_callback, &data,&zErrMsg); - data.zDestTable = "sqlite_stat3"; - shell_exec(p->db, "SELECT * FROM sqlite_stat3", - shell_callback, &data,&zErrMsg); - data.zDestTable = "sqlite_stat4"; - shell_exec(p->db, "SELECT * FROM sqlite_stat4", - shell_callback, &data, &zErrMsg); - data.mode = MODE_Semi; - shell_exec(p->db, "SELECT 'ANALYZE sqlite_master;'", - shell_callback, &data, &zErrMsg); + if( rc==SQLITE_OK ){ + sqlite3_stmt *pStmt; + rc = sqlite3_prepare_v2(p->db, + "SELECT rowid FROM sqlite_master" + " WHERE name GLOB 'sqlite_stat[134]'", + -1, &pStmt, 0); + doStats = sqlite3_step(pStmt)==SQLITE_ROW; + sqlite3_finalize(pStmt); + } + if( doStats==0 ){ + fprintf(p->out, "/* No STAT tables available */\n"); + }else{ + fprintf(p->out, "ANALYZE sqlite_master;\n"); + sqlite3_exec(p->db, "SELECT 'ANALYZE sqlite_master'", + callback, &data, &zErrMsg); + data.mode = MODE_Insert; + data.zDestTable = "sqlite_stat1"; + shell_exec(p->db, "SELECT * FROM sqlite_stat1", + shell_callback, &data,&zErrMsg); + data.zDestTable = "sqlite_stat3"; + shell_exec(p->db, "SELECT * FROM sqlite_stat3", + shell_callback, &data,&zErrMsg); + data.zDestTable = "sqlite_stat4"; + shell_exec(p->db, "SELECT * FROM sqlite_stat4", + shell_callback, &data, &zErrMsg); + fprintf(p->out, "ANALYZE sqlite_master;\n"); + } }else if( c=='h' && strncmp(azArg[0], "headers", n)==0 ){ From 71607c77c88d70dab468db6cc267a29118163513 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 18 Jul 2014 17:39:48 +0000 Subject: [PATCH 106/710] SQLite has long accepted some unadvertised and non-standard join syntax. Add a test to ensure that future versions continue to accept this non-standard syntax, to avoid breaking legacy applications that use the undefined syntax. FossilOrigin-Name: 824dde7fc48dbca6d6c956c4eb79bbfa2d139ee5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/join.test | 11 +++++++++++ 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index baca280a3a..f047a472d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\s".fullschema"\scommand\sin\sthe\scommand-line\sshell. -D 2014-07-18T14:43:29.403 +C SQLite\shas\slong\saccepted\ssome\sunadvertised\sand\snon-standard\sjoin\ssyntax.\nAdd\sa\stest\sto\sensure\sthat\sfuture\sversions\scontinue\sto\saccept\sthis\snon-standard\nsyntax,\sto\savoid\sbreaking\slegacy\sapplications\sthat\suse\sthe\sundefined\ssyntax. +D 2014-07-18T17:39:48.053 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -643,7 +643,7 @@ F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 F test/ioerr6.test a395a6ab144b26a9e3e21059a1ab6a7149cca65b -F test/join.test 559b81eb56ae350246f9c25986aa52c81c725c7e +F test/join.test 52d4d49f86d0cf46926672878c4eaf0da399104a F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1cec1e030035e5253fb7ebbdfe5c1a3029e4e29b -R 3a5c863b5d4e43cf3d99c01f25a09c91 +P fa80c64caa573297bf43cb5b11d9e7db58567200 +R 98a05e82af602a262cecfbef5a77de2c U drh -Z f91dd8693f049cec144fbd0b43e0e92a +Z 99ce618698b83450b2fdb30e802ca3fb diff --git a/manifest.uuid b/manifest.uuid index 984495a47c..e1252a9375 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa80c64caa573297bf43cb5b11d9e7db58567200 \ No newline at end of file +824dde7fc48dbca6d6c956c4eb79bbfa2d139ee5 \ No newline at end of file diff --git a/test/join.test b/test/join.test index 28d9ddf8b2..4c83fa6b3b 100644 --- a/test/join.test +++ b/test/join.test @@ -36,6 +36,17 @@ do_test join-1.2 { } } {1 2 3 2 3 4 3 4 5} +# A FROM clause of the form: ",
    ON " is not +# allowed by the SQLite syntax diagram, nor by any other SQL database +# engine that we are aware of. Nevertheless, historic versions of +# SQLite have allowed it. We need to continue to support it moving +# forward to prevent breakage of legacy applications. Though, we will +# not advertise it as being supported. +# +do_execsql_test join-1.2.1 { + SELECT t1.rowid, t2.rowid, '|' FROM t1, t2 ON t1.a=t2.b; +} {1 1 | 2 2 | 3 3 |} + do_test join-1.3 { execsql2 { SELECT * FROM t1 NATURAL JOIN t2; From a6f5972c7529bc4b7f4c754278e2b5383126b6d0 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 18 Jul 2014 19:06:39 +0000 Subject: [PATCH 107/710] Improved documentation for checkpoints and the busy handler. No changes to code. FossilOrigin-Name: ca92c0240681b199c4b8ecc557792b8aea71692d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 44 ++++++++++++++++++++++++++++++++------------ 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index f047a472d0..1ef6e6429f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C SQLite\shas\slong\saccepted\ssome\sunadvertised\sand\snon-standard\sjoin\ssyntax.\nAdd\sa\stest\sto\sensure\sthat\sfuture\sversions\scontinue\sto\saccept\sthis\snon-standard\nsyntax,\sto\savoid\sbreaking\slegacy\sapplications\sthat\suse\sthe\sundefined\ssyntax. -D 2014-07-18T17:39:48.053 +C Improved\sdocumentation\sfor\scheckpoints\sand\sthe\sbusy\shandler.\s\sNo\schanges\nto\scode. +D 2014-07-18T19:06:39.261 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -224,7 +224,7 @@ F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 6762c62e11b504aa014edceab8886495165e3a77 F src/shell.c 566aee8213372a2e81ba0eb34e9759f7b2574009 -F src/sqlite.h.in a98eb3e8c86c934ea6f5bcfc6b69653dde2f4ed4 +F src/sqlite.h.in fd8e3a36b0aded082dc93a4b89c1e85324b4cf75 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h e88614d7371b80ff69dbbb5e4b9813ee93dfd890 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fa80c64caa573297bf43cb5b11d9e7db58567200 -R 98a05e82af602a262cecfbef5a77de2c +P 824dde7fc48dbca6d6c956c4eb79bbfa2d139ee5 +R e12aaadd86bb0f0252aac83b76e58ccc U drh -Z 99ce618698b83450b2fdb30e802ca3fb +Z 06586dc6d29c2abfb50a449af797bf0e diff --git a/manifest.uuid b/manifest.uuid index e1252a9375..38385b2f75 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -824dde7fc48dbca6d6c956c4eb79bbfa2d139ee5 \ No newline at end of file +ca92c0240681b199c4b8ecc557792b8aea71692d \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 21ab67a4c2..93cde78340 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2032,9 +2032,13 @@ int sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors ** -** ^This routine sets a callback function that might be invoked whenever -** an attempt is made to open a database table that another thread -** or process has locked. +** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X +** that might be invoked with argument P whenever +** an attempt is made to access a database table associated with +** [database connection] D when another thread +** or process has the table locked. +** The sqlite3_busy_handler() interface is used to implement +** [sqlite3_busy_timeout()] and [PRAGMA busy_timeout]. ** ** ^If the busy callback is NULL, then [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] ** is returned immediately upon encountering the lock. ^If the busy callback @@ -2043,16 +2047,18 @@ int sqlite3_complete16(const void *sql); ** ^The first argument to the busy handler is a copy of the void* pointer which ** is the third argument to sqlite3_busy_handler(). ^The second argument to ** the busy handler callback is the number of times that the busy handler has -** been invoked for this locking event. ^If the +** been invoked for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to -** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned. +** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned +** to the application. ** ^If the callback returns non-zero, then another attempt -** is made to open the database for reading and the cycle repeats. +** is made to access the database and the cycle repeats. ** ** The presence of a busy handler does not guarantee that it will be invoked ** when there is lock contention. ^If SQLite determines that invoking the busy ** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] -** or [SQLITE_IOERR_BLOCKED] instead of invoking the busy handler. +** or [SQLITE_IOERR_BLOCKED] to the application instead of invoking the +** busy handler. ** Consider a scenario where one process is holding a read lock that ** it is trying to promote to a reserved lock and ** a second process is holding a reserved lock that it is trying @@ -2084,10 +2090,12 @@ int sqlite3_complete16(const void *sql); ** ^(There can only be a single busy handler defined for each ** [database connection]. Setting a new busy handler clears any ** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] -** will also set or clear the busy handler. +** or evaluating [PRAGMA busy_timeout=N] will change the +** busy handler and thus clear any previously set busy handler. ** ** The busy callback should not take any actions which modify the -** database connection that invoked the busy handler. Any such actions +** database connection that invoked the busy handler. In other words, +** the busy handler is not reentrant. Any such actions ** result in undefined behavior. ** ** A busy handler must not close the database connection @@ -2112,6 +2120,8 @@ int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); ** [database connection] any any given moment. If another busy handler ** was defined (using [sqlite3_busy_handler()]) prior to calling ** this routine, that other busy handler is cleared.)^ +** +** See also: [PRAGMA busy_timeout] */ int sqlite3_busy_timeout(sqlite3*, int ms); @@ -7139,6 +7149,9 @@ void *sqlite3_wal_hook( ** ^The [wal_autocheckpoint pragma] can be used to invoke this interface ** from SQL. ** +** ^Checkpoints initiated by this mechanism are +** [sqlite3_wal_checkpoint_v2|PASSIVE]. +** ** ^Every new [database connection] defaults to having the auto-checkpoint ** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] ** pages. The use of this interface @@ -7155,6 +7168,10 @@ int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); ** empty string, then a checkpoint is run on all databases of ** connection D. ^If the database connection D is not in ** [WAL | write-ahead log mode] then this interface is a harmless no-op. +** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a +** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. +** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL +** or RESET checkpoint. ** ** ^The [wal_checkpoint pragma] can be used to invoke this interface ** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the @@ -7177,10 +7194,12 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** Checkpoint as many frames as possible without waiting for any database ** readers or writers to finish. Sync the db file if all frames in the log ** are checkpointed. This mode is the same as calling -** sqlite3_wal_checkpoint(). The busy-handler callback is never invoked. +** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] +** is never invoked. ** **
    SQLITE_CHECKPOINT_FULL
    -** This mode blocks (calls the busy-handler callback) until there is no +** This mode blocks (it invokes the +** [sqlite3_busy_handler|busy-handler callback]) until there is no ** database writer and all readers are reading from the most recent database ** snapshot. It then checkpoints all frames in the log file and syncs the ** database file. This call blocks database writers while it is running, @@ -7188,7 +7207,8 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** **
    SQLITE_CHECKPOINT_RESTART
    ** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after -** checkpointing the log file it blocks (calls the busy-handler callback) +** checkpointing the log file it blocks (calls the +** [sqlite3_busy_handler|busy-handler callback]) ** until all readers are reading from the database file only. This ensures ** that the next client to write to the database file restarts the log file ** from the beginning. This call blocks database writers while it is running, From 3a046c6c315788963ddd1f6e16358a5ab4f74ea2 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 18 Jul 2014 21:02:54 +0000 Subject: [PATCH 108/710] Update clean targets with the recently added command-line tools. FossilOrigin-Name: 2beefa68c0c53f663321bebf0ac06f6c936be63f --- Makefile.in | 3 +++ Makefile.msc | 3 +++ main.mk | 10 +++++++++- manifest | 18 +++++++++--------- manifest.uuid | 2 +- 5 files changed, 25 insertions(+), 11 deletions(-) diff --git a/Makefile.in b/Makefile.in index b5768e83a6..cf17b02918 100644 --- a/Makefile.in +++ b/Makefile.in @@ -997,6 +997,9 @@ clean: rm -rf tsrc .target_source rm -f tclsqlite3$(TEXE) rm -f testfixture$(TEXE) test.db + rm -f LogEst$(TEXE) fts3view$(TEXE) rollback-test$(TEXE) showdb$(TEXE) + rm -f showjournal$(TEXE) showstat4$(TEXE) showwal$(TEXE) speedtest1$(TEXE) + rm -f wordcount$(TEXE) rm -f sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def rm -f sqlite3.c rm -f sqlite3rc.h diff --git a/Makefile.msc b/Makefile.msc index f1fb591553..3a9fa29740 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -1483,6 +1483,9 @@ clean: del /Q tclsqlite3.exe tclsqlite3.exp del /Q testloadext.dll testloadext.exp del /Q testfixture.exe testfixture.exp test.db + del /Q LogEst.exe fts3view.exe rollback-test.exe showdb.exe + del /Q showjournal.exe showstat4.exe showwal.exe speedtest1.exe + del /Q wordcount.exe del /Q sqlite3.dll sqlite3.lib sqlite3.exp sqlite3.def del /Q sqlite3.c sqlite3-*.c del /Q sqlite3rc.h diff --git a/main.mk b/main.mk index 865f7a87c2..718cf5cc4d 100644 --- a/main.mk +++ b/main.mk @@ -695,10 +695,18 @@ clean: rm -f fts3-testfixture fts3-testfixture.exe rm -f testfixture testfixture.exe rm -f threadtest3 threadtest3.exe + rm -f LogEst LogEst.exe + rm -f fts3view fts3view.exe + rm -f rollback-test rollback-test.exe + rm -f showdb showdb.exe + rm -f showjournal showjournal.exe + rm -f showstat4 showstat4.exe + rm -f showwal showwal.exe + rm -f speedtest1 speedtest1.exe + rm -f wordcount wordcount.exe rm -f sqlite3.c sqlite3-*.c fts?amal.c tclsqlite3.c rm -f sqlite3rc.h rm -f shell.c sqlite3ext.h rm -f sqlite3_analyzer sqlite3_analyzer.exe sqlite3_analyzer.c rm -f sqlite-*-output.vsix rm -f mptester mptester.exe - rm -f showdb diff --git a/manifest b/manifest index 1ef6e6429f..3fa4f049fa 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Improved\sdocumentation\sfor\scheckpoints\sand\sthe\sbusy\shandler.\s\sNo\schanges\nto\scode. -D 2014-07-18T19:06:39.261 +C Update\sclean\stargets\swith\sthe\srecently\sadded\scommand-line\stools. +D 2014-07-18T21:02:54.697 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 1732320ecac3fee229d560d7ef2afa34681d1815 +F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 0fc63e099a4f71fdeef31496b1f4ad04ab419884 +F Makefile.msc 5b04e657cf08a9ac7fc47d876c5c8be962c47d6b F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION 1c877615a9db323e3cd301e3d57d853f9d5c4a07 @@ -146,7 +146,7 @@ F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 0b5baabb84a5d212b40c244e9439329a542e1982 +F main.mk 036a65c9042b2f5013baf85c82629d700fe031c4 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 824dde7fc48dbca6d6c956c4eb79bbfa2d139ee5 -R e12aaadd86bb0f0252aac83b76e58ccc -U drh -Z 06586dc6d29c2abfb50a449af797bf0e +P ca92c0240681b199c4b8ecc557792b8aea71692d +R 432f9d3a43bd2fc68603a985c53b9df5 +U mistachkin +Z 7459013a6be89cfe73225b448eb2a655 diff --git a/manifest.uuid b/manifest.uuid index 38385b2f75..e37705787e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca92c0240681b199c4b8ecc557792b8aea71692d \ No newline at end of file +2beefa68c0c53f663321bebf0ac06f6c936be63f \ No newline at end of file From 0461cc4795ddcc8fa9d63ef5c956a712bc73dea3 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 18 Jul 2014 21:16:37 +0000 Subject: [PATCH 109/710] Fix harmless compiler warnings for MSVC in the showdb/showwal command line tools. FossilOrigin-Name: 6dc7b2f119eb92da17c9e914bdad30a9ceaebdb5 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- tool/showdb.c | 47 +++++++++++++++++++++++++---------------------- tool/showwal.c | 36 +++++++++++++++++++++--------------- 4 files changed, 57 insertions(+), 45 deletions(-) diff --git a/manifest b/manifest index 3fa4f049fa..685f9ad594 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sclean\stargets\swith\sthe\srecently\sadded\scommand-line\stools. -D 2014-07-18T21:02:54.697 +C Fix\sharmless\scompiler\swarnings\sfor\sMSVC\sin\sthe\sshowdb/showwal\scommand\sline\stools. +D 2014-07-18T21:16:37.850 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1160,10 +1160,10 @@ F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/showdb.c 213e0288501b2cf67c1b2c72a9e5b8acda4738b3 +F tool/showdb.c 3c78d10af9bb7ee667ceaa271e1d1ecbcdc3e69c F tool/showjournal.c 053eb1cc774710c6890b7dd6293300cc297b16a5 F tool/showstat4.c c39279d6bd37cb999b634f0064f6f86ad7af008f -F tool/showwal.c 3f7f7da5ec0cba51b1449a75f700493377da57b5 +F tool/showwal.c 6e98bc0fe0ae75f4b08e39d21ed90d7d78d307c0 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b F tool/spaceanal.tcl 8e50b217c56a6a086a1b47eac9d09c5cd65b996f @@ -1182,7 +1182,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ca92c0240681b199c4b8ecc557792b8aea71692d -R 432f9d3a43bd2fc68603a985c53b9df5 +P 2beefa68c0c53f663321bebf0ac06f6c936be63f +R 7c81b84c85a1fee51213cddde915664f +T *branch * toolWarnings +T *sym-toolWarnings * +T -sym-trunk * U mistachkin -Z 7459013a6be89cfe73225b448eb2a655 +Z 6dc51fde6dee7a6bda251af41648d319 diff --git a/manifest.uuid b/manifest.uuid index e37705787e..e189e09c49 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2beefa68c0c53f663321bebf0ac06f6c936be63f \ No newline at end of file +6dc7b2f119eb92da17c9e914bdad30a9ceaebdb5 \ No newline at end of file diff --git a/tool/showdb.c b/tool/showdb.c index 8dd387365c..3324729184 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -9,6 +9,8 @@ #if !defined(_MSC_VER) #include +#else +#include #endif #include @@ -190,14 +192,14 @@ static void print_db_header(void){ /* ** Describe cell content. */ -static int describeContent( +static i64 describeContent( unsigned char *a, /* Cell content */ - int nLocal, /* Bytes in a[] */ + i64 nLocal, /* Bytes in a[] */ char *zDesc /* Write description here */ ){ - int nDesc = 0; - int n, i, j; - i64 x, v; + i64 nDesc = 0; + int n, j; + i64 i, x, v; const unsigned char *pData; const unsigned char *pLimit; char sep = ' '; @@ -237,7 +239,7 @@ static int describeContent( }else if( x==9 ){ sprintf(zDesc, "1"); }else if( x>=12 ){ - int size = (x-12)/2; + i64 size = (x-12)/2; if( (x&1)==0 ){ sprintf(zDesc, "blob(%d)", size); }else{ @@ -256,11 +258,11 @@ static int describeContent( ** Compute the local payload size given the total payload size and ** the page size. */ -static int localPayload(i64 nPayload, char cType){ - int maxLocal; - int minLocal; - int surplus; - int nLocal; +static i64 localPayload(i64 nPayload, char cType){ + i64 maxLocal; + i64 minLocal; + i64 surplus; + i64 nLocal; if( cType==13 ){ /* Table leaf */ maxLocal = pagesize-35; @@ -288,19 +290,19 @@ static int localPayload(i64 nPayload, char cType){ ** ** The return value is the local cell size. */ -static int describeCell( +static i64 describeCell( unsigned char cType, /* Page type */ unsigned char *a, /* Cell content */ int showCellContent, /* Show cell content if true */ char **pzDesc /* Store description here */ ){ int i; - int nDesc = 0; + i64 nDesc = 0; int n = 0; int leftChild; i64 nPayload; i64 rowid; - int nLocal; + i64 nLocal; static char zDesc[1000]; i = 0; if( cType<=5 ){ @@ -373,13 +375,14 @@ static void decodeCell( int szPgHdr, /* Size of the page header. 0 or 100 */ int ofst /* Cell begins at a[ofst] */ ){ - int i, j, k; + int i, j; int leftChild; + i64 k; i64 nPayload; i64 rowid; i64 nHdr; i64 iType; - int nLocal; + i64 nLocal; unsigned char *x = a + ofst; unsigned char *end; unsigned char cType = a[0]; @@ -452,7 +455,7 @@ static void decodeCell( } printf("%s\n", zTypeName); szCol[nCol] = sz; - ofstCol[nCol] = k; + ofstCol[nCol] = (int)k; typeCol[nCol] = (int)iType; k += sz; nCol++; @@ -585,13 +588,13 @@ static void decode_btree_page( for(i=0; i #include #include + +#if !defined(_MSC_VER) #include +#else +#include +#endif + #include #include @@ -273,14 +279,14 @@ static void print_wal_header(Cksum *pCksum){ /* ** Describe cell content. */ -static int describeContent( +static i64 describeContent( unsigned char *a, /* Cell content */ - int nLocal, /* Bytes in a[] */ + i64 nLocal, /* Bytes in a[] */ char *zDesc /* Write description here */ ){ int nDesc = 0; - int n, i, j; - i64 x, v; + int n, j; + i64 i, x, v; const unsigned char *pData; const unsigned char *pLimit; char sep = ' '; @@ -320,7 +326,7 @@ static int describeContent( }else if( x==9 ){ sprintf(zDesc, "1"); }else if( x>=12 ){ - int size = (x-12)/2; + i64 size = (x-12)/2; if( (x&1)==0 ){ sprintf(zDesc, "blob(%d)", size); }else{ @@ -339,11 +345,11 @@ static int describeContent( ** Compute the local payload size given the total payload size and ** the page size. */ -static int localPayload(i64 nPayload, char cType){ - int maxLocal; - int minLocal; - int surplus; - int nLocal; +static i64 localPayload(i64 nPayload, char cType){ + i64 maxLocal; + i64 minLocal; + i64 surplus; + i64 nLocal; if( cType==13 ){ /* Table leaf */ maxLocal = pagesize-35; @@ -370,19 +376,19 @@ static int localPayload(i64 nPayload, char cType){ ** ** The return value is the local cell size. */ -static int describeCell( +static i64 describeCell( unsigned char cType, /* Page type */ unsigned char *a, /* Cell content */ int showCellContent, /* Show cell content if true */ char **pzDesc /* Store description here */ ){ int i; - int nDesc = 0; + i64 nDesc = 0; int n = 0; int leftChild; i64 nPayload; i64 rowid; - int nLocal; + i64 nLocal; static char zDesc[1000]; i = 0; if( cType<=5 ){ @@ -479,13 +485,13 @@ static void decode_btree_page( for(i=0; i Date: Sat, 19 Jul 2014 15:30:01 +0000 Subject: [PATCH 110/710] Fix warnings caused by the previous commit. FossilOrigin-Name: 89634a419d0c1ef899d0591c7c6ab9290bf665d2 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- tool/showdb.c | 12 ++++++------ tool/showwal.c | 4 ++-- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 685f9ad594..f2ee7060d6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sfor\sMSVC\sin\sthe\sshowdb/showwal\scommand\sline\stools. -D 2014-07-18T21:16:37.850 +C Fix\swarnings\scaused\sby\sthe\sprevious\scommit. +D 2014-07-19T15:30:01.552 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1160,10 +1160,10 @@ F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/showdb.c 3c78d10af9bb7ee667ceaa271e1d1ecbcdc3e69c +F tool/showdb.c d7d354c52f259c06d84938fc2be095656e761b78 F tool/showjournal.c 053eb1cc774710c6890b7dd6293300cc297b16a5 F tool/showstat4.c c39279d6bd37cb999b634f0064f6f86ad7af008f -F tool/showwal.c 6e98bc0fe0ae75f4b08e39d21ed90d7d78d307c0 +F tool/showwal.c 9c299ba842f1ac0364e95f9effbc3a5b3b2349f2 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b F tool/spaceanal.tcl 8e50b217c56a6a086a1b47eac9d09c5cd65b996f @@ -1182,10 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2beefa68c0c53f663321bebf0ac06f6c936be63f -R 7c81b84c85a1fee51213cddde915664f -T *branch * toolWarnings -T *sym-toolWarnings * -T -sym-trunk * +P 6dc7b2f119eb92da17c9e914bdad30a9ceaebdb5 +R 7b842e4424a0effae20a64ee9f307907 U mistachkin -Z 6dc51fde6dee7a6bda251af41648d319 +Z c9d7c6fd91ebbbad7eb3b3a8b17ddccd diff --git a/manifest.uuid b/manifest.uuid index e189e09c49..9e2002f2da 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6dc7b2f119eb92da17c9e914bdad30a9ceaebdb5 \ No newline at end of file +89634a419d0c1ef899d0591c7c6ab9290bf665d2 \ No newline at end of file diff --git a/tool/showdb.c b/tool/showdb.c index 3324729184..1618bdb2fd 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -241,9 +241,9 @@ static i64 describeContent( }else if( x>=12 ){ i64 size = (x-12)/2; if( (x&1)==0 ){ - sprintf(zDesc, "blob(%d)", size); + sprintf(zDesc, "blob(%lld)", size); }else{ - sprintf(zDesc, "txt(%d)", size); + sprintf(zDesc, "txt(%lld)", size); } pData += size; } @@ -403,10 +403,10 @@ static void decodeCell( printBytes(a, x, i); nLocal = localPayload(nPayload, cType); if( nLocal==nPayload ){ - printf("payload-size: %d\n", (int)nPayload); + printf("payload-size: %lld\n", nPayload); }else{ - printf("payload-size: %d (%d local, %d overflow)\n", - (int)nPayload, nLocal, (int)(nPayload-nLocal)); + printf("payload-size: %lld (%lld local, %lld overflow)\n", + nPayload, nLocal, nPayload-nLocal); } x += i; }else{ @@ -509,7 +509,7 @@ static void decodeCell( } if( j=12 ){ i64 size = (x-12)/2; if( (x&1)==0 ){ - sprintf(zDesc, "blob(%d)", size); + sprintf(zDesc, "blob(%lld)", size); }else{ - sprintf(zDesc, "txt(%d)", size); + sprintf(zDesc, "txt(%lld)", size); } pData += size; } From ef60703e8f808b733bb83de3bccfde64a868dcd5 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 19 Jul 2014 15:40:39 +0000 Subject: [PATCH 111/710] Fix warnings related to having a 64-bit size_t. FossilOrigin-Name: 29ac9336dba2b62d13315cecfc5c4a74cc54087a --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/showdb.c | 8 ++++---- tool/showwal.c | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index f2ee7060d6..8160c09155 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\swarnings\scaused\sby\sthe\sprevious\scommit. -D 2014-07-19T15:30:01.552 +C Fix\swarnings\srelated\sto\shaving\sa\s64-bit\ssize_t. +D 2014-07-19T15:40:39.591 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1160,10 +1160,10 @@ F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/showdb.c d7d354c52f259c06d84938fc2be095656e761b78 +F tool/showdb.c b9ee6b6c81a094bf33badbf7e9da34cdbc0cce25 F tool/showjournal.c 053eb1cc774710c6890b7dd6293300cc297b16a5 F tool/showstat4.c c39279d6bd37cb999b634f0064f6f86ad7af008f -F tool/showwal.c 9c299ba842f1ac0364e95f9effbc3a5b3b2349f2 +F tool/showwal.c 3209120269cdf9380f091459e47b776b4f81dfd3 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b F tool/spaceanal.tcl 8e50b217c56a6a086a1b47eac9d09c5cd65b996f @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6dc7b2f119eb92da17c9e914bdad30a9ceaebdb5 -R 7b842e4424a0effae20a64ee9f307907 +P 89634a419d0c1ef899d0591c7c6ab9290bf665d2 +R 7c73c160afe9efe974113734dfcd1ed9 U mistachkin -Z c9d7c6fd91ebbbad7eb3b3a8b17ddccd +Z 6a0ee89ac8bea8d9e5f887e8d36032b1 diff --git a/manifest.uuid b/manifest.uuid index 9e2002f2da..6eee24a63a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89634a419d0c1ef899d0591c7c6ab9290bf665d2 \ No newline at end of file +29ac9336dba2b62d13315cecfc5c4a74cc54087a \ No newline at end of file diff --git a/tool/showdb.c b/tool/showdb.c index 1618bdb2fd..1a51e9d1a2 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -143,7 +143,7 @@ static void print_decode_line( int val = aData[ofst]; char zBuf[100]; sprintf(zBuf, " %03x: %02x", ofst, aData[ofst]); - i = strlen(zBuf); + i = (int)strlen(zBuf); for(j=1; j<4; j++){ if( j>=nByte ){ sprintf(&zBuf[i], " "); @@ -151,7 +151,7 @@ static void print_decode_line( sprintf(&zBuf[i], " %02x", aData[ofst+j]); val = val*256 + aData[ofst+j]; } - i += strlen(&zBuf[i]); + i += (int)strlen(&zBuf[i]); } sprintf(&zBuf[i], " %9d", val); printf("%s %s\n", zBuf, zMsg); @@ -247,7 +247,7 @@ static i64 describeContent( } pData += size; } - j = strlen(zDesc); + j = (int)strlen(zDesc); zDesc += j; nDesc += j; } @@ -598,7 +598,7 @@ static void decode_btree_page( zMap[cofst] = '['; zMap[cofst+n-1] = ']'; sprintf(zBuf, "%d", i); - j = strlen(zBuf); + j = (int)strlen(zBuf); if( j<=n-2 ) memcpy(&zMap[cofst+1], zBuf, j); } if( cellToDecode==(-2) ){ diff --git a/tool/showwal.c b/tool/showwal.c index 8353b8dfc4..6dc1de173f 100644 --- a/tool/showwal.c +++ b/tool/showwal.c @@ -178,7 +178,7 @@ static void print_decode_line( int val = aData[ofst]; char zBuf[100]; sprintf(zBuf, " %03x: %02x", ofst, aData[ofst]); - i = strlen(zBuf); + i = (int)strlen(zBuf); for(j=1; j<4; j++){ if( j>=nByte ){ sprintf(&zBuf[i], " "); @@ -186,7 +186,7 @@ static void print_decode_line( sprintf(&zBuf[i], " %02x", aData[ofst+j]); val = val*256 + aData[ofst+j]; } - i += strlen(&zBuf[i]); + i += (int)strlen(&zBuf[i]); } if( asHex ){ sprintf(&zBuf[i], " 0x%08x", val); @@ -334,7 +334,7 @@ static i64 describeContent( } pData += size; } - j = strlen(zDesc); + j = (int)strlen(zDesc); zDesc += j; nDesc += j; } @@ -495,7 +495,7 @@ static void decode_btree_page( zMap[cofst] = '['; zMap[cofst+n-1] = ']'; sprintf(zBuf, "%d", i); - j = strlen(zBuf); + j = (int)strlen(zBuf); if( j<=n-2 ) memcpy(&zMap[cofst+1], zBuf, j); } printf(" %03x: cell[%d] %s\n", cofst, i, zDesc); From 86ac612e8a8c10504bd4e2864beb23017f7fe94b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 19 Jul 2014 15:44:29 +0000 Subject: [PATCH 112/710] Fix some harmess compiler warnings in the FTS3 Unicode module. FossilOrigin-Name: c01caea5d6ad7a570628b85fb3056ac955ff03dd --- ext/fts3/fts3_unicode.c | 10 +++++----- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c index 591cbb8dc9..57e7a4d3c9 100644 --- a/ext/fts3/fts3_unicode.c +++ b/ext/fts3/fts3_unicode.c @@ -231,7 +231,7 @@ static int unicodeCreate( for(i=0; rc==SQLITE_OK && ibRemoveDiacritic = 1; @@ -363,11 +363,11 @@ static int unicodeNext( ); /* Set the output variables and return. */ - pCsr->iOff = (z - pCsr->aInput); + pCsr->iOff = (int)(z - pCsr->aInput); *paToken = pCsr->zToken; - *pnToken = zOut - pCsr->zToken; - *piStart = (zStart - pCsr->aInput); - *piEnd = (zEnd - pCsr->aInput); + *pnToken = (int)(zOut - pCsr->zToken); + *piStart = (int)(zStart - pCsr->aInput); + *piEnd = (int)(zEnd - pCsr->aInput); *piPos = pCsr->iToken++; return SQLITE_OK; } diff --git a/manifest b/manifest index 8160c09155..59d0b2b601 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\swarnings\srelated\sto\shaving\sa\s64-bit\ssize_t. -D 2014-07-19T15:40:39.591 +C Fix\ssome\sharmess\scompiler\swarnings\sin\sthe\sFTS3\sUnicode\smodule. +D 2014-07-19T15:44:29.116 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -94,7 +94,7 @@ F ext/fts3/fts3_tokenize_vtab.c 011170fe9eba5ff062f1a31d3188e00267716706 F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_unicode.c 0f598fa4c96af1188e4d5f5e8e78e25500bd5922 +F ext/fts3/fts3_unicode.c e80eef8a11f2020dc9c1eb95c5b405b9012f2fbe F ext/fts3/fts3_unicode2.c c8adda75aad0c6c252ef3dd555f811f437485044 F ext/fts3/fts3_write.c 8260388626516a7005d06a9dce94f9e55c6c2a41 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 89634a419d0c1ef899d0591c7c6ab9290bf665d2 -R 7c73c160afe9efe974113734dfcd1ed9 +P 29ac9336dba2b62d13315cecfc5c4a74cc54087a +R 4ac50ab96ea64a1527703075acfe4ff2 U mistachkin -Z 6a0ee89ac8bea8d9e5f887e8d36032b1 +Z ef640c9ada5ada1862dacc56c6ab494a diff --git a/manifest.uuid b/manifest.uuid index 6eee24a63a..e4b4c3f479 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29ac9336dba2b62d13315cecfc5c4a74cc54087a \ No newline at end of file +c01caea5d6ad7a570628b85fb3056ac955ff03dd \ No newline at end of file From 857745c089741831310cbd2071b226dcd4b18bfa Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 19 Jul 2014 17:57:10 +0000 Subject: [PATCH 113/710] Update the sqlite3_stmt_busy() function so that it correctly returns true for "ROLLBACK" statements that have been stepped but not yet reset. FossilOrigin-Name: 61cee3c0678f5abd9131a29ab946a5e71f55643e --- manifest | 19 +++++++++---------- manifest.uuid | 2 +- src/vdbeapi.c | 2 +- src/vdbeaux.c | 2 +- test/capi3d.test | 37 +++++++++++++++++++++++++++++++++++++ 5 files changed, 49 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 52a708bbe8..9c16ebe0d5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sin\sthe\sshowdb\sand\sshowwal\stools\sand\sin\nthe\sunicode\stokenizer\sof\sFTS3. -D 2014-07-19T17:49:40.535 +C Update\sthe\ssqlite3_stmt_busy()\sfunction\sso\sthat\sit\scorrectly\sreturns\strue\sfor\s"ROLLBACK"\sstatements\sthat\shave\sbeen\sstepped\sbut\snot\syet\sreset. +D 2014-07-19T17:57:10.785 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,8 +286,8 @@ F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 9bfe6becfc094382ae213656fbe511055ad83a54 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h 5df5e9afe9b7839cd17256220fc4f7af84b8b1cd -F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 8ce7dcdbb8c59e5c2194518ce3099b254ae94c15 +F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 +F src/vdbeaux.c 8d32b5a68670ccc6c64904924e6a0dddbc3a2fc5 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c 44441d73b08b3a638dcdb725afffb87c6574ad27 @@ -372,7 +372,7 @@ F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 F test/capi3.test 71bcf2fbd36a9732f617766dfd752552c8e491b5 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test a21869e4d50d5dbb7e566e328fc0bc7c2efa6a32 -F test/capi3d.test 6d0fc0a86d73f42dd19a7d8b7761ab9bc02277d0 +F test/capi3d.test c84af0c49267f9c3fbf4c1c46aa647646023811e F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/check.test 5831ddb6f2c687782eaf2e1a07b6e17f24c4f763 @@ -1182,8 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2beefa68c0c53f663321bebf0ac06f6c936be63f c01caea5d6ad7a570628b85fb3056ac955ff03dd -R 4ac50ab96ea64a1527703075acfe4ff2 -T +closed c01caea5d6ad7a570628b85fb3056ac955ff03dd -U drh -Z cd2aeeb09b327b0fcacfc59206b5ba8f +P 574cc8eb1448cff67390f2e16cc9b7c3ddd4658b +R 3e17c7c287ebc0f7c8d4dc519939fd8f +U dan +Z 2190a4b1d8dc0a3facc078eb683578eb diff --git a/manifest.uuid b/manifest.uuid index 1773fd5e15..5e45c34d1e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -574cc8eb1448cff67390f2e16cc9b7c3ddd4658b \ No newline at end of file +61cee3c0678f5abd9131a29ab946a5e71f55643e \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 5d7a0b0608..5e5bb81366 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1323,7 +1323,7 @@ int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ */ int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; - return v!=0 && v->pc>0 && v->magic==VDBE_MAGIC_RUN; + return v!=0 && v->pc>=0 && v->magic==VDBE_MAGIC_RUN; } /* diff --git a/src/vdbeaux.c b/src/vdbeaux.c index de61b55c83..92652de5a7 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2143,7 +2143,7 @@ static void checkActiveVdbeCnt(sqlite3 *db){ int nRead = 0; p = db->pVdbe; while( p ){ - if( p->magic==VDBE_MAGIC_RUN && p->pc>=0 ){ + if( sqlite3_stmt_busy((sqlite3_stmt*)p) ){ cnt++; if( p->readOnly==0 ) nWrite++; if( p->bIsReader ) nRead++; diff --git a/test/capi3d.test b/test/capi3d.test index ed3765b051..fb8abe86d2 100644 --- a/test/capi3d.test +++ b/test/capi3d.test @@ -144,4 +144,41 @@ do_test capi3d-3.99 { sqlite3_stmt_busy 0 } {0} +#-------------------------------------------------------------------------- +# Test the sqlite3_stmt_busy() function with ROLLBACK statements. +# +reset_db + +do_execsql_test capi3d-4.1 { + CREATE TABLE t4(x,y); + BEGIN; +} + +do_test capi3d-4.2.1 { + breakpoint + set ::s1 [sqlite3_prepare_v2 db "ROLLBACK" -1 notused] + sqlite3_step $::s1 +} {SQLITE_DONE} + +do_test capi3d-4.2.2 { + sqlite3_stmt_busy $::s1 +} {1} + +do_catchsql_test capi3d-4.2.3 { + VACUUM +} {1 {cannot VACUUM - SQL statements in progress}} + +do_test capi3d-4.2.4 { + sqlite3_reset $::s1 +} {SQLITE_OK} + +do_catchsql_test capi3d-4.2.5 { + VACUUM +} {0 {}} + +do_test capi3d-4.2.6 { + sqlite3_finalize $::s1 +} {SQLITE_OK} + + finish_test From a7ab6d8165fb25e0e55d72577181399fa2c9692a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 21 Jul 2014 15:44:39 +0000 Subject: [PATCH 114/710] Allow the SQLITE_MAX_ATTACHED compile-time option to be larger than 62. The default limit on the number of attached databases remains 10. FossilOrigin-Name: 1a817ae2f35fa0396148dda2782cd4f919bf5c6f --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/build.c | 34 +++++++++++++++++++++++----------- src/main.c | 4 ++-- src/sqliteInt.h | 15 ++++++++++++++- src/vdbe.c | 22 +++++++++++----------- src/vdbeaux.c | 22 ++++++++++------------ 7 files changed, 72 insertions(+), 49 deletions(-) diff --git a/manifest b/manifest index 9c16ebe0d5..6b6e2f6d22 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\ssqlite3_stmt_busy()\sfunction\sso\sthat\sit\scorrectly\sreturns\strue\sfor\s"ROLLBACK"\sstatements\sthat\shave\sbeen\sstepped\sbut\snot\syet\sreset. -D 2014-07-19T17:57:10.785 +C Allow\sthe\sSQLITE_MAX_ATTACHED\scompile-time\soption\sto\sbe\slarger\sthan\s62.\s\sThe\ndefault\slimit\son\sthe\snumber\sof\sattached\sdatabases\sremains\s10. +D 2014-07-21T15:44:39.054 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -170,7 +170,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 70c60a3807b2072982f184d9614e020d2953f89c F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 927e39b6aaf872c7b28f154f6acfeb9a05a72442 +F src/build.c 48f400fa14fd6add244b954ce7e223ce7ccacf0b F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -189,7 +189,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 7c2c3cafdd6313c8f9319ebec1565782e624372e +F src/main.c bba87834413394f43b1a211c38c282c7d4092d1d F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -227,7 +227,7 @@ F src/shell.c 566aee8213372a2e81ba0eb34e9759f7b2574009 F src/sqlite.h.in fd8e3a36b0aded082dc93a4b89c1e85324b4cf75 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h e88614d7371b80ff69dbbb5e4b9813ee93dfd890 +F src/sqliteInt.h 5dbcdcf81cb15e77df16f88699a15f7af491c580 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -283,11 +283,11 @@ F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 049fe1d3c0e2209c1bee107aec2fcff6285f909f F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 9bfe6becfc094382ae213656fbe511055ad83a54 +F src/vdbe.c 514adcc9478e461d01b9b064565d2c99604ed537 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h 5df5e9afe9b7839cd17256220fc4f7af84b8b1cd F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 -F src/vdbeaux.c 8d32b5a68670ccc6c64904924e6a0dddbc3a2fc5 +F src/vdbeaux.c 1ffe0bbc3a2c8aedd95622de9be6b6d07b3d3c6b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c 44441d73b08b3a638dcdb725afffb87c6574ad27 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 574cc8eb1448cff67390f2e16cc9b7c3ddd4658b -R 3e17c7c287ebc0f7c8d4dc519939fd8f -U dan -Z 2190a4b1d8dc0a3facc078eb683578eb +P 61cee3c0678f5abd9131a29ab946a5e71f55643e +R 03bc11d876fd7682716b5fcb2f08b30c +U drh +Z 1b802be3c637c9b3363a3a4e9caf7d2a diff --git a/manifest.uuid b/manifest.uuid index 5e45c34d1e..e7fd3c7be7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -61cee3c0678f5abd9131a29ab946a5e71f55643e \ No newline at end of file +1a817ae2f35fa0396148dda2782cd4f919bf5c6f \ No newline at end of file diff --git a/src/build.c b/src/build.c index 677ba326c6..27c9671da6 100644 --- a/src/build.c +++ b/src/build.c @@ -113,6 +113,19 @@ static void codeTableLocks(Parse *pParse){ #define codeTableLocks(x) #endif +/* +** Return TRUE if the given yDbMask object is empty - if it contains no +** 1 bits. This routine is used by the DbMaskAllZero() and DbMaskNotZero() +** macros when SQLITE_MAX_ATTACHED is greater than 30. +*/ +#if SQLITE_MAX_ATTACHED>30 +int sqlite3DbMaskAllZero(yDbMask m){ + int i; + for(i=0; imallocFailed==0 && (pParse->cookieMask || pParse->pConstExpr) ){ - yDbMask mask; + if( db->mallocFailed==0 + && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr) + ){ int iDb, i; assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); sqlite3VdbeJumpHere(v, 0); - for(iDb=0, mask=1; iDbnDb; mask<<=1, iDb++){ - if( (mask & pParse->cookieMask)==0 ) continue; + for(iDb=0; iDbnDb; iDb++){ + if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; sqlite3VdbeUsesBtree(v, iDb); sqlite3VdbeAddOp4Int(v, OP_Transaction, /* Opcode */ iDb, /* P1 */ - (mask & pParse->writeMask)!=0, /* P2 */ + DbMaskTest(pParse->writeMask,iDb), /* P2 */ pParse->cookieValue[iDb], /* P3 */ db->aDb[iDb].pSchema->iGeneration /* P4 */ ); @@ -216,7 +230,7 @@ void sqlite3FinishCoding(Parse *pParse){ pParse->nMem = 0; pParse->nSet = 0; pParse->nVar = 0; - pParse->cookieMask = 0; + DbMaskZero(pParse->cookieMask); } /* @@ -3843,15 +3857,13 @@ int sqlite3OpenTempDatabase(Parse *pParse){ void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); sqlite3 *db = pToplevel->db; - yDbMask mask; assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 || iDb==1 ); assert( iDbcookieMask & mask)==0 ){ - pToplevel->cookieMask |= mask; + if( DbMaskTest(pToplevel->cookieMask, iDb)==0 ){ + DbMaskSet(pToplevel->cookieMask, iDb); pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; if( !OMIT_TEMPDB && iDb==1 ){ sqlite3OpenTempDatabase(pToplevel); @@ -3890,7 +3902,7 @@ void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb){ void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); sqlite3CodeVerifySchema(pParse, iDb); - pToplevel->writeMask |= ((yDbMask)1)<writeMask, iDb); pToplevel->isMultiWrite |= setStatement; } diff --git a/src/main.c b/src/main.c index 564c012472..c1d59976a6 100644 --- a/src/main.c +++ b/src/main.c @@ -2101,8 +2101,8 @@ static const int aHardLimit[] = { #if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 # error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 #endif -#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>62 -# error SQLITE_MAX_ATTACHED must be between 0 and 62 +#if SQLITE_MAX_ATTACHED<0 +# error SQLITE_MAX_ATTACHED must be greater than 0 #endif #if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 # error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7e347c4ee2..8f256d52fc 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2392,9 +2392,19 @@ struct TriggerPrg { ** The yDbMask datatype for the bitmask of all attached databases. */ #if SQLITE_MAX_ATTACHED>30 - typedef sqlite3_uint64 yDbMask; + typedef unsigned char yDbMask[(SQLITE_MAX_ATTACHED+9)/8]; +# define DbMaskTest(M,I) (((M)[(I)/8]&(1<<((I)&7)))!=0) +# define DbMaskZero(M) memset((M),0,sizeof(M)) +# define DbMaskSet(M,I) (M)[(I)/8]|=(1<<((I)&7)) +# define DbMaskAllZero(M) sqlite3DbMaskAllZero(M) +# define DbMaskNonZero(M) (sqlite3DbMaskAllZero(M)==0) #else typedef unsigned int yDbMask; +# define DbMaskTest(M,I) (((M)&(((yDbMask)1)<<(I)))!=0) +# define DbMaskZero(M) (M)=0 +# define DbMaskSet(M,I) (M)|=(((yDbMask)1)<<(I)) +# define DbMaskAllZero(M) (M)==0 +# define DbMaskNonZero(M) (M)!=0 #endif /* @@ -3067,6 +3077,9 @@ void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int); # define sqlite3ViewGetColumnNames(A,B) 0 #endif +#if SQLITE_MAX_ATTACHED>30 + int sqlite3DbMaskAllZero(yDbMask); +#endif void sqlite3DropTable(Parse*, SrcList*, int, int); void sqlite3CodeDropTable(Parse*, Table*, int, int); void sqlite3DeleteTable(sqlite3*, Table*); diff --git a/src/vdbe.c b/src/vdbe.c index 11aa035026..ecea3782d4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3034,7 +3034,7 @@ case OP_Transaction: { assert( p->bIsReader ); assert( p->readOnly==0 || pOp->p2==0 ); assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); if( pOp->p2 && (db->flags & SQLITE_QueryOnly)!=0 ){ rc = SQLITE_READONLY; goto abort_due_to_error; @@ -3129,7 +3129,7 @@ case OP_ReadCookie: { /* out2-prerelease */ assert( pOp->p3=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 ); - assert( (p->btreeMask & (((yDbMask)1)<btreeMask, iDb) ); sqlite3BtreeGetMeta(db->aDb[iDb].pBt, iCookie, (u32 *)&iMeta); pOut->u.i = iMeta; @@ -3150,7 +3150,7 @@ case OP_SetCookie: { /* in3 */ Db *pDb; assert( pOp->p2p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); assert( p->readOnly==0 ); pDb = &db->aDb[pOp->p1]; assert( pDb->pBt!=0 ); @@ -3253,7 +3253,7 @@ case OP_OpenWrite: { p2 = pOp->p2; iDb = pOp->p3; assert( iDb>=0 && iDbnDb ); - assert( (p->btreeMask & (((yDbMask)1)<btreeMask, iDb) ); pDb = &db->aDb[iDb]; pX = pDb->pBt; assert( pX!=0 ); @@ -4844,7 +4844,7 @@ case OP_Destroy: { /* out2-prerelease */ }else{ iDb = pOp->p3; assert( iCnt==1 ); - assert( (p->btreeMask & (((yDbMask)1)<btreeMask, iDb) ); iMoved = 0; /* Not needed. Only to silence a warning. */ rc = sqlite3BtreeDropTable(db->aDb[iDb].pBt, pOp->p1, &iMoved); pOut->flags = MEM_Int; @@ -4884,7 +4884,7 @@ case OP_Clear: { nChange = 0; assert( p->readOnly==0 ); - assert( (p->btreeMask & (((yDbMask)1)<p2))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p2) ); rc = sqlite3BtreeClearTable( db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &nChange : 0) ); @@ -4954,7 +4954,7 @@ case OP_CreateTable: { /* out2-prerelease */ pgno = 0; assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); assert( p->readOnly==0 ); pDb = &db->aDb[pOp->p1]; assert( pDb->pBt!=0 ); @@ -5119,7 +5119,7 @@ case OP_IntegrityCk: { } aRoot[j] = 0; assert( pOp->p5nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p5))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p5) ); z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot, (int)pnErr->u.i, &nErr); sqlite3DbFree(db, aRoot); @@ -5779,7 +5779,7 @@ case OP_IncrVacuum: { /* jump */ Btree *pBt; assert( pOp->p1>=0 && pOp->p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<p1))!=0 ); + assert( DbMaskTest(p->btreeMask, pOp->p1) ); assert( p->readOnly==0 ); pBt = db->aDb[pOp->p1].pBt; rc = sqlite3BtreeIncrVacuum(pBt); @@ -5831,7 +5831,7 @@ case OP_TableLock: { if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommitted) ){ int p1 = pOp->p1; assert( p1>=0 && p1nDb ); - assert( (p->btreeMask & (((yDbMask)1)<btreeMask, p1) ); assert( isWriteLock==0 || isWriteLock==1 ); rc = sqlite3BtreeLockTable(db->aDb[p1].pBt, pOp->p2, isWriteLock); if( (rc&0xFF)==SQLITE_LOCKED ){ @@ -6281,7 +6281,7 @@ case OP_Init: { /* jump */ if( zTrace ){ int i; for(i=0; inDb; i++){ - if( (MASKBIT(i) & p->btreeMask)==0 ) continue; + if( DbMaskTest(p->btreeMask, i)==0 ) continue; sqlite3_file_control(db, db->aDb[i].zName, SQLITE_FCNTL_TRACE, zTrace); } } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 92652de5a7..ac945df59e 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -499,7 +499,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ pParse->aLabel = 0; pParse->nLabel = 0; *pMaxFuncArgs = nMaxArgs; - assert( p->bIsReader!=0 || p->btreeMask==0 ); + assert( p->bIsReader!=0 || DbMaskAllZero(p->btreeMask) ); } /* @@ -526,7 +526,7 @@ VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg){ assert( aOp && !p->db->mallocFailed ); /* Check that sqlite3VdbeUsesBtree() was not called on this VM */ - assert( p->btreeMask==0 ); + assert( DbMaskAllZero(p->btreeMask) ); resolveP2Values(p, pnMaxArg); *pnOp = p->nOp; @@ -1111,9 +1111,9 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ void sqlite3VdbeUsesBtree(Vdbe *p, int i){ assert( i>=0 && idb->nDb && i<(int)sizeof(yDbMask)*8 ); assert( i<(int)sizeof(p->btreeMask)*8 ); - p->btreeMask |= ((yDbMask)1)<btreeMask, i); if( i!=1 && sqlite3BtreeSharable(p->db->aDb[i].pBt) ){ - p->lockMask |= ((yDbMask)1)<lockMask, i); } } @@ -1141,16 +1141,15 @@ void sqlite3VdbeUsesBtree(Vdbe *p, int i){ */ void sqlite3VdbeEnter(Vdbe *p){ int i; - yDbMask mask; sqlite3 *db; Db *aDb; int nDb; - if( p->lockMask==0 ) return; /* The common case */ + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ db = p->db; aDb = db->aDb; nDb = db->nDb; - for(i=0, mask=1; ilockMask)!=0 && ALWAYS(aDb[i].pBt!=0) ){ + for(i=0; ilockMask,i) && ALWAYS(aDb[i].pBt!=0) ){ sqlite3BtreeEnter(aDb[i].pBt); } } @@ -1163,16 +1162,15 @@ void sqlite3VdbeEnter(Vdbe *p){ */ void sqlite3VdbeLeave(Vdbe *p){ int i; - yDbMask mask; sqlite3 *db; Db *aDb; int nDb; - if( p->lockMask==0 ) return; /* The common case */ + if( DbMaskAllZero(p->lockMask) ) return; /* The common case */ db = p->db; aDb = db->aDb; nDb = db->nDb; - for(i=0, mask=1; ilockMask)!=0 && ALWAYS(aDb[i].pBt!=0) ){ + for(i=0; ilockMask,i) && ALWAYS(aDb[i].pBt!=0) ){ sqlite3BtreeLeave(aDb[i].pBt); } } From 215c0c82fa08e224a747ac22ac0c34318fe422ff Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 21 Jul 2014 20:07:17 +0000 Subject: [PATCH 115/710] Limit the maximum SQLITE_MAX_ATTACHED to 127, since a signed character is sometimes used to store the database index. FossilOrigin-Name: f1c76c7c4c7d16855db60d16e23ecdf8d7ca862a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6b6e2f6d22..907c781f6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sthe\sSQLITE_MAX_ATTACHED\scompile-time\soption\sto\sbe\slarger\sthan\s62.\s\sThe\ndefault\slimit\son\sthe\snumber\sof\sattached\sdatabases\sremains\s10. -D 2014-07-21T15:44:39.054 +C Limit\sthe\smaximum\sSQLITE_MAX_ATTACHED\sto\s127,\ssince\sa\ssigned\scharacter\nis\ssometimes\sused\sto\sstore\sthe\sdatabase\sindex. +D 2014-07-21T20:07:17.299 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -189,7 +189,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c bba87834413394f43b1a211c38c282c7d4092d1d +F src/main.c d0fa35a8582923730ab2113d34dddc061f59d211 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 61cee3c0678f5abd9131a29ab946a5e71f55643e -R 03bc11d876fd7682716b5fcb2f08b30c +P 1a817ae2f35fa0396148dda2782cd4f919bf5c6f +R 848516fae6f90d131f9449e42d77b0c0 U drh -Z 1b802be3c637c9b3363a3a4e9caf7d2a +Z 0a273a2cc8050e8888f6349db4185a5f diff --git a/manifest.uuid b/manifest.uuid index e7fd3c7be7..f108a708b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a817ae2f35fa0396148dda2782cd4f919bf5c6f \ No newline at end of file +f1c76c7c4c7d16855db60d16e23ecdf8d7ca862a \ No newline at end of file diff --git a/src/main.c b/src/main.c index c1d59976a6..1e05dd8486 100644 --- a/src/main.c +++ b/src/main.c @@ -2101,8 +2101,8 @@ static const int aHardLimit[] = { #if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 # error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 #endif -#if SQLITE_MAX_ATTACHED<0 -# error SQLITE_MAX_ATTACHED must be greater than 0 +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>127 +# error SQLITE_MAX_ATTACHED must be between 0 and 127 #endif #if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 # error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 From 53ad7e48ab72d07728e4db6875c12561f3748d98 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 22 Jul 2014 00:40:45 +0000 Subject: [PATCH 116/710] For the OR-optimization, avoid generating OP_OpenRead opcodes that reopen exactly the same index. FossilOrigin-Name: b67a6e33f23ce5f5d9a545fa9d6700a7ed636901 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 1 + src/where.c | 19 +++++++++++++++++-- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 907c781f6f..df25ada5a5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Limit\sthe\smaximum\sSQLITE_MAX_ATTACHED\sto\s127,\ssince\sa\ssigned\scharacter\nis\ssometimes\sused\sto\sstore\sthe\sdatabase\sindex. -D 2014-07-21T20:07:17.299 +C For\sthe\sOR-optimization,\savoid\sgenerating\sOP_OpenRead\sopcodes\sthat\sreopen\nexactly\sthe\ssame\sindex. +D 2014-07-22T00:40:45.480 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -227,7 +227,7 @@ F src/shell.c 566aee8213372a2e81ba0eb34e9759f7b2574009 F src/sqlite.h.in fd8e3a36b0aded082dc93a4b89c1e85324b4cf75 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 5dbcdcf81cb15e77df16f88699a15f7af491c580 +F src/sqliteInt.h 0991b31a98c6ce072d255954af617baf50735eb1 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 3a451b00516b59f414c59f817508edce51c732f1 +F src/where.c 4f8dce8f4e50b3b3675e8af7811e48526d6b4be2 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1a817ae2f35fa0396148dda2782cd4f919bf5c6f -R 848516fae6f90d131f9449e42d77b0c0 +P f1c76c7c4c7d16855db60d16e23ecdf8d7ca862a +R ff4e333e5c258e291fb4ff256a0eb1e1 U drh -Z 0a273a2cc8050e8888f6349db4185a5f +Z 9049ac2491ab8361811966ab022cf8d0 diff --git a/manifest.uuid b/manifest.uuid index f108a708b4..9e127a138a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1c76c7c4c7d16855db60d16e23ecdf8d7ca862a \ No newline at end of file +b67a6e33f23ce5f5d9a545fa9d6700a7ed636901 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8f256d52fc..6b0e031b30 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2136,6 +2136,7 @@ struct SrcList { #define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ #define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ #define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ +#define WHERE_OR_INDEX_OPEN 0x1000 /* OP_OpenRead for the OR index exists */ /* Allowed return values from sqlite3WhereIsDistinct() */ diff --git a/src/where.c b/src/where.c index e52f598e02..84a8d9a03f 100644 --- a/src/where.c +++ b/src/where.c @@ -3422,6 +3422,7 @@ static Bitmask codeOneLoopStart( int iRetInit; /* Address of regReturn init */ int untestedTerms = 0; /* Some terms not completely tested */ int ii; /* Loop counter */ + int subWctrlFlags; /* wctrlFlags for sub-queries */ Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ Table *pTab = pTabItem->pTab; @@ -3517,6 +3518,8 @@ static Bitmask codeOneLoopStart( ** eliminating duplicates from other WHERE clauses, the action for each ** sub-WHERE clause is to to invoke the main loop body as a subroutine. */ + subWctrlFlags = WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | + WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY; for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ @@ -3529,8 +3532,7 @@ static Bitmask codeOneLoopStart( } /* Loop through table entries that match term pOrTerm. */ pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, - WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | - WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY, iCovCur); + subWctrlFlags, iCovCur); assert( pSubWInfo || pParse->nErr || db->mallocFailed ); if( pSubWInfo ){ WhereLoop *pSubLoop; @@ -3621,6 +3623,7 @@ static Bitmask codeOneLoopStart( ){ assert( pSubWInfo->a[0].iIdxCur==iCovCur ); pCov = pSubLoop->u.btree.pIndex; + subWctrlFlags |= WHERE_OR_INDEX_OPEN; }else{ pCov = 0; } @@ -6212,6 +6215,18 @@ WhereInfo *sqlite3WhereBegin( pWInfo->aiCurOnePass[1] = iIndexCur; }else if( iIdxCur && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ iIndexCur = iIdxCur; + if( (wctrlFlags & WHERE_OR_INDEX_OPEN)!=0 ){ + /* For 2nd and subsequent subqueries for processing OR terms, + ** try to reuse the previous OP_OpenRead, if there is one. The + ** WHERE_OR_INDEX_OPEN bit will only be set if there is a prior + ** OP_OpenRead opcode on cursor iIndexCur, so the while() loop + ** below is guaranteed to terminate. + */ + VdbeOp *pOp = sqlite3VdbeGetOp(v, -1); + while( pOp->opcode!=OP_OpenRead || pOp->p1!=iIndexCur ) pOp--; + assert( pOp->p3==iDb ); + if( pOp->p2==pIx->tnum ) op = 0; + } }else{ iIndexCur = pParse->nTab++; } From 83a305f20e69101b7439d0e20d00c72aacbcfdca Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 22 Jul 2014 12:05:32 +0000 Subject: [PATCH 117/710] Enhance the comment on whereLoopAddBtree(). No changes to code. FossilOrigin-Name: b22dd165da227a52d88b17a91e80a1701dce61ad --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 8 ++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index df25ada5a5..0b57ed3d1b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sthe\sOR-optimization,\savoid\sgenerating\sOP_OpenRead\sopcodes\sthat\sreopen\nexactly\sthe\ssame\sindex. -D 2014-07-22T00:40:45.480 +C Enhance\sthe\scomment\son\swhereLoopAddBtree().\s\sNo\schanges\sto\scode. +D 2014-07-22T12:05:32.008 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 4f8dce8f4e50b3b3675e8af7811e48526d6b4be2 +F src/where.c 00c4b52d973f5176297ab3b633c8f5d8f9475af6 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f1c76c7c4c7d16855db60d16e23ecdf8d7ca862a -R ff4e333e5c258e291fb4ff256a0eb1e1 +P b67a6e33f23ce5f5d9a545fa9d6700a7ed636901 +R 7e3c00e4bb7f0f9e6e95bc158e12c00a U drh -Z 9049ac2491ab8361811966ab022cf8d0 +Z bea5212013e532ea4358a5003d399c52 diff --git a/manifest.uuid b/manifest.uuid index 9e127a138a..6d720e3002 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b67a6e33f23ce5f5d9a545fa9d6700a7ed636901 \ No newline at end of file +b22dd165da227a52d88b17a91e80a1701dce61ad \ No newline at end of file diff --git a/src/where.c b/src/where.c index 84a8d9a03f..753284bc32 100644 --- a/src/where.c +++ b/src/where.c @@ -4577,6 +4577,14 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ ** Normally, nSeek is 1. nSeek values greater than 1 come about if the ** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when ** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. +** +** The estimated values (nRow, nVisit, nSeek) often contain a large amount +** of uncertainty. For this reason, scoring is designed to pick plans that +** "do the least harm" if the estimates are inaccurate. For example, a +** log(nRow) factor is omitted from a non-covering index scan in order to +** bias the scoring in favor of using an index, since the worst-case +** performance of using an index is far better than the worst-case performance +** of a full table scan. */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ From 358406fc573b047a95150319a417e93a967e5107 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 22 Jul 2014 14:42:16 +0000 Subject: [PATCH 118/710] Expire prepared statements after running ANALYZE. FossilOrigin-Name: b083a961f8ac3a6158e822574d73cd63e3103b09 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 3 +++ src/where.c | 1 - test/analyze6.test | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 0b57ed3d1b..5a933c3991 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\scomment\son\swhereLoopAddBtree().\s\sNo\schanges\sto\scode. -D 2014-07-22T12:05:32.008 +C Expire\sprepared\sstatements\safter\srunning\sANALYZE. +D 2014-07-22T14:42:16.806 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 61bc3467bd7803158d460dd92f5ba5205371bdce +F src/analyze.c 9f4e473c83a582d696ce57dc8cb7a40ad1314d02 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 00c4b52d973f5176297ab3b633c8f5d8f9475af6 +F src/where.c 52edd7f5df58d8f73230e5f9883ce867ca2f4a7d F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -313,7 +313,7 @@ F test/analyze.test 1772936d66471c65221e437b6d1999c3a03166c4 F test/analyze3.test bf41f0f680dd1e0d44eed5e769531e93a5320275 F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 -F test/analyze6.test d31defa011a561b938b4608d3538c1b4e0b5e92c +F test/analyze6.test f1c552ce39cca4ec922a7e4e0e5d0203d6b3281f F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 F test/analyze9.test bd5aaf2a8fd2dd774b08251416897185531a8adf @@ -1182,7 +1182,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b67a6e33f23ce5f5d9a545fa9d6700a7ed636901 -R 7e3c00e4bb7f0f9e6e95bc158e12c00a +P b22dd165da227a52d88b17a91e80a1701dce61ad +R 9fe107c1276b2eda80832b2d6848bc6f U drh -Z bea5212013e532ea4358a5003d399c52 +Z 54d8743399e09513763a4a3a09f86610 diff --git a/manifest.uuid b/manifest.uuid index 6d720e3002..300e0983ce 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b22dd165da227a52d88b17a91e80a1701dce61ad \ No newline at end of file +b083a961f8ac3a6158e822574d73cd63e3103b09 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 2330bdb342..7d2bed8c7f 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1320,6 +1320,7 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ Table *pTab; Index *pIdx; Token *pTableName; + Vdbe *v; /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ @@ -1367,6 +1368,8 @@ void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } } } + v = sqlite3GetVdbe(pParse); + if( v ) sqlite3VdbeAddOp0(v, OP_Expire); } /* diff --git a/src/where.c b/src/where.c index 753284bc32..4ecf02f0c1 100644 --- a/src/where.c +++ b/src/where.c @@ -4426,7 +4426,6 @@ static int whereLoopAddBtreeIndex( }else{ rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); } - assert( rc!=SQLITE_OK || nOut>0 ); if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ if( nOut ){ diff --git a/test/analyze6.test b/test/analyze6.test index f77f05324a..31ace8eda5 100644 --- a/test/analyze6.test +++ b/test/analyze6.test @@ -91,7 +91,7 @@ do_test analyze6-2.3 { } {0 0 0 {SEARCH TABLE t201 USING INTEGER PRIMARY KEY (rowid=?)}} do_test analyze6-2.4 { execsql { - INSERT INTO t201 VALUES(1,2,3); + INSERT INTO t201 VALUES(1,2,3),(2,3,4),(3,4,5); ANALYZE t201; } eqp {SELECT * FROM t201 WHERE z=5} From 25df48d967207568584016d7d922f01e77f36cef Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 22 Jul 2014 14:58:12 +0000 Subject: [PATCH 119/710] Enhance the sqlite_stat1.stat parsing to allow additional text parameters at the end. Unrecognized parameters are silently ignored. FossilOrigin-Name: ca2a5a2c770fa94cd8db1b1b241ede058a7c58e2 --- manifest | 17 ++--- manifest.uuid | 2 +- src/analyze.c | 7 +- src/vdbe.c | 9 +-- test/analyze9.test | 2 - test/analyzeC.test | 167 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 187 insertions(+), 17 deletions(-) create mode 100644 test/analyzeC.test diff --git a/manifest b/manifest index 5a933c3991..de21246669 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Expire\sprepared\sstatements\safter\srunning\sANALYZE. -D 2014-07-22T14:42:16.806 +C Enhance\sthe\ssqlite_stat1.stat\sparsing\sto\sallow\sadditional\stext\sparameters\nat\sthe\send.\s\sUnrecognized\sparameters\sare\ssilently\signored. +D 2014-07-22T14:58:12.596 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 9f4e473c83a582d696ce57dc8cb7a40ad1314d02 +F src/analyze.c 56799121c72753f40d56f3c17c66c3961af6acb7 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -283,7 +283,7 @@ F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 049fe1d3c0e2209c1bee107aec2fcff6285f909f F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 514adcc9478e461d01b9b064565d2c99604ed537 +F src/vdbe.c 2f3f0582dfacce292d3d6f22d129fe1abe3ddd00 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h 5df5e9afe9b7839cd17256220fc4f7af84b8b1cd F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 @@ -316,9 +316,10 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test f1c552ce39cca4ec922a7e4e0e5d0203d6b3281f F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test bd5aaf2a8fd2dd774b08251416897185531a8adf +F test/analyze9.test 3ef1b471247308e710a794b6e50a6ab536c5604b F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d +F test/analyzeC.test 555a6cc388b9818b6eda6df816f01ce0a75d3a93 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -1182,7 +1183,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b22dd165da227a52d88b17a91e80a1701dce61ad -R 9fe107c1276b2eda80832b2d6848bc6f +P b083a961f8ac3a6158e822574d73cd63e3103b09 +R 5b14cb35846d5c4a5d2461874ff76990 U drh -Z 54d8743399e09513763a4a3a09f86610 +Z 267c7ac0e4de2238ef8d9515f94ccafd diff --git a/manifest.uuid b/manifest.uuid index 300e0983ce..cc3037dca8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b083a961f8ac3a6158e822574d73cd63e3103b09 \ No newline at end of file +ca2a5a2c770fa94cd8db1b1b241ede058a7c58e2 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 7d2bed8c7f..97305f0dc1 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1428,14 +1428,16 @@ static void decodeIntArray( #else if( pIndex ) #endif - { - if( strcmp(z, "unordered")==0 ){ + while( z[0] ){ + if( sqlite3_strglob("unordered*", z)==0 ){ pIndex->bUnordered = 1; }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ int v32 = 0; sqlite3GetInt32(z+3, &v32); pIndex->szIdxRow = sqlite3LogEst(v32); } + while( z[0]!=0 && z[0]!=' ' ) z++; + while( z[0]==' ' ) z++; } } @@ -1476,6 +1478,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ z = argv[2]; if( pIndex ){ + pIndex->bUnordered = 0; decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex); if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; }else{ diff --git a/src/vdbe.c b/src/vdbe.c index ecea3782d4..86cf8b0484 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5794,12 +5794,13 @@ case OP_IncrVacuum: { /* jump */ /* Opcode: Expire P1 * * * * ** -** Cause precompiled statements to become expired. An expired statement -** fails with an error code of SQLITE_SCHEMA if it is ever executed -** (via sqlite3_step()). +** Cause precompiled statements to expire. When an expired statement +** is executed using sqlite3_step() it will either automatically +** reprepare itself (if it was originally created using sqlite3_prepare_v2()) +** or it will fail with SQLITE_SCHEMA. ** ** If P1 is 0, then all SQL statements become expired. If P1 is non-zero, -** then only the currently executing statement is affected. +** then only the currently executing statement is expired. */ case OP_Expire: { if( !pOp->p1 ){ diff --git a/test/analyze9.test b/test/analyze9.test index fc5a13f43f..0d72658df6 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -1089,5 +1089,3 @@ foreach {tn where eqp} { } finish_test - - diff --git a/test/analyzeC.test b/test/analyzeC.test new file mode 100644 index 0000000000..02faa9c7e9 --- /dev/null +++ b/test/analyzeC.test @@ -0,0 +1,167 @@ +# 2014-07-22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains automated tests used to verify that the text terms +# at the end of sqlite_stat1.stat are processed correctly. +# +# (1) "unordered" means that the index cannot be used for ORDER BY +# or for range queries +# +# (2) "sz=NNN" sets the relative size of the index entries +# +# (3) All other fields are silently ignored +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix analyzeC + +# Baseline case. Range queries work OK. Indexes can be used for +# ORDER BY. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a,b,c); + INSERT INTO t1(a,b,c) + VALUES(1,2,3),(7,8,9),(4,5,6),(10,11,12),(4,8,12),(1,11,111); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + ANALYZE; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl,idx,stat) + VALUES('t1','t1a','12345 2'),('t1','t1b','12345 4'); + ANALYZE sqlite_master; + SELECT *, '#' FROM t1 WHERE a BETWEEN 3 AND 8 ORDER BY c; +} {4 5 6 # 7 8 9 # 4 8 12 #} +do_execsql_test 1.1 { + EXPLAIN QUERY PLAN + SELECT *, '#' FROM t1 WHERE a BETWEEN 3 AND 8 ORDER BY c; +} {/.* USING INDEX t1a .a>. AND a<...*/} +do_execsql_test 1.2 { + SELECT c FROM t1 ORDER BY a; +} {3 111 6 12 9 12} +do_execsql_test 1.3 { + EXPLAIN QUERY PLAN + SELECT c FROM t1 ORDER BY a; +} {/.*SCAN TABLE t1 USING INDEX t1a.*/} +do_execsql_test 1.3x { + EXPLAIN QUERY PLAN + SELECT c FROM t1 ORDER BY a; +} {~/.*B-TREE FOR ORDER BY.*/} + +# Now mark the t1a index as "unordered". Range queries and ORDER BY no +# longer use the index, but equality queries do. +# +do_execsql_test 2.0 { + UPDATE sqlite_stat1 SET stat='12345 2 unordered' WHERE idx='t1a'; + ANALYZE sqlite_master; + SELECT *, '#' FROM t1 WHERE a BETWEEN 3 AND 8 ORDER BY c; +} {4 5 6 # 7 8 9 # 4 8 12 #} +do_execsql_test 2.1 { + EXPLAIN QUERY PLAN + SELECT *, '#' FROM t1 WHERE a BETWEEN 3 AND 8 ORDER BY c; +} {~/.*USING INDEX.*/} +do_execsql_test 2.2 { + SELECT c FROM t1 ORDER BY a; +} {3 111 6 12 9 12} +do_execsql_test 2.3 { + EXPLAIN QUERY PLAN + SELECT c FROM t1 ORDER BY a; +} {~/.*USING INDEX.*/} +do_execsql_test 2.3x { + EXPLAIN QUERY PLAN + SELECT c FROM t1 ORDER BY a; +} {/.*B-TREE FOR ORDER BY.*/} + +# Ignore extraneous text parameters in the sqlite_stat1.stat field. +# +do_execsql_test 3.0 { + UPDATE sqlite_stat1 SET stat='12345 2 whatever=5 unordered xyzzy=11' + WHERE idx='t1a'; + ANALYZE sqlite_master; + SELECT *, '#' FROM t1 WHERE a BETWEEN 3 AND 8 ORDER BY c; +} {4 5 6 # 7 8 9 # 4 8 12 #} +do_execsql_test 3.1 { + EXPLAIN QUERY PLAN + SELECT *, '#' FROM t1 WHERE a BETWEEN 3 AND 8 ORDER BY c; +} {~/.*USING INDEX.*/} +do_execsql_test 3.2 { + SELECT c FROM t1 ORDER BY a; +} {3 111 6 12 9 12} +do_execsql_test 3.3 { + EXPLAIN QUERY PLAN + SELECT c FROM t1 ORDER BY a; +} {~/.*USING INDEX.*/} +do_execsql_test 3.3x { + EXPLAIN QUERY PLAN + SELECT c FROM t1 ORDER BY a; +} {/.*B-TREE FOR ORDER BY.*/} + +# The sz=NNN parameter determines which index to scan +# +do_execsql_test 4.0 { + DROP INDEX t1a; + CREATE INDEX t1ab ON t1(a,b); + CREATE INDEX t1ca ON t1(c,a); + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl,idx,stat) + VALUES('t1','t1ab','12345 3 2 sz=10'),('t1','t1ca','12345 3 2 sz=20'); + ANALYZE sqlite_master; + SELECT count(a) FROM t1; +} {6} +do_execsql_test 4.1 { + EXPLAIN QUERY PLAN + SELECT count(a) FROM t1; +} {/.*INDEX t1ab.*/} +do_execsql_test 4.2 { + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl,idx,stat) + VALUES('t1','t1ab','12345 3 2 sz=20'),('t1','t1ca','12345 3 2 sz=10'); + ANALYZE sqlite_master; + SELECT count(a) FROM t1; +} {6} +do_execsql_test 4.3 { + EXPLAIN QUERY PLAN + SELECT count(a) FROM t1; +} {/.*INDEX t1ca.*/} + + +# The sz=NNN parameter works even if there is other extraneous text +# in the sqlite_stat1.stat column. +# +do_execsql_test 5.0 { + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl,idx,stat) + VALUES('t1','t1ab','12345 3 2 x=5 sz=10 y=10'), + ('t1','t1ca','12345 3 2 whatever sz=20 junk'); + ANALYZE sqlite_master; + SELECT count(a) FROM t1; +} {6} +do_execsql_test 5.1 { + EXPLAIN QUERY PLAN + SELECT count(a) FROM t1; +} {/.*INDEX t1ab.*/} +do_execsql_test 5.2 { + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl,idx,stat) + VALUES('t1','t1ca','12345 3 2 x=5 sz=10 y=10'), + ('t1','t1ab','12345 3 2 whatever sz=20 junk'); + ANALYZE sqlite_master; + SELECT count(a) FROM t1; +} {6} +do_execsql_test 5.3 { + EXPLAIN QUERY PLAN + SELECT count(a) FROM t1; +} {/.*INDEX t1ca.*/} + + + + +finish_test From d08b27988c9467740a89951e100462f7d57f226c Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 22 Jul 2014 15:33:31 +0000 Subject: [PATCH 120/710] Correction: The maximum SQLITE_MAX_ATTACHED value to avoid overflowing a signed 8-bit integer is 125, not 127. FossilOrigin-Name: 48e378029528dac33d29866016128d236a7995d9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index de21246669..8bed619c3d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\ssqlite_stat1.stat\sparsing\sto\sallow\sadditional\stext\sparameters\nat\sthe\send.\s\sUnrecognized\sparameters\sare\ssilently\signored. -D 2014-07-22T14:58:12.596 +C Correction:\s\sThe\smaximum\sSQLITE_MAX_ATTACHED\svalue\sto\savoid\soverflowing\sa\nsigned\s8-bit\sinteger\sis\s125,\snot\s127. +D 2014-07-22T15:33:31.603 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -189,7 +189,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c d0fa35a8582923730ab2113d34dddc061f59d211 +F src/main.c 1a420efa9a34e8603c4807886408cba5546b87f9 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -1183,7 +1183,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b083a961f8ac3a6158e822574d73cd63e3103b09 -R 5b14cb35846d5c4a5d2461874ff76990 +P ca2a5a2c770fa94cd8db1b1b241ede058a7c58e2 +R c3cff698f37a9f53c6d75b84d0e942cf U drh -Z 267c7ac0e4de2238ef8d9515f94ccafd +Z 54df2fe47e5bf7bb724052e4ddee7e12 diff --git a/manifest.uuid b/manifest.uuid index cc3037dca8..29f2036f2d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca2a5a2c770fa94cd8db1b1b241ede058a7c58e2 \ No newline at end of file +48e378029528dac33d29866016128d236a7995d9 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 1e05dd8486..5e976e3397 100644 --- a/src/main.c +++ b/src/main.c @@ -2101,8 +2101,8 @@ static const int aHardLimit[] = { #if SQLITE_MAX_FUNCTION_ARG<0 || SQLITE_MAX_FUNCTION_ARG>1000 # error SQLITE_MAX_FUNCTION_ARG must be between 0 and 1000 #endif -#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>127 -# error SQLITE_MAX_ATTACHED must be between 0 and 127 +#if SQLITE_MAX_ATTACHED<0 || SQLITE_MAX_ATTACHED>125 +# error SQLITE_MAX_ATTACHED must be between 0 and 125 #endif #if SQLITE_MAX_LIKE_PATTERN_LENGTH<1 # error SQLITE_MAX_LIKE_PATTERN_LENGTH must be at least 1 From f5d87f77e2a3d1b34cb67539ea0c172cb811120a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 22 Jul 2014 16:00:49 +0000 Subject: [PATCH 121/710] Fix the index name for the shadow tables in the spellfix1 extension so that multiple instances of the spellfix1 virtual table can each have their own index. FossilOrigin-Name: 438c348a40383796a710499e4e1b7a6b08f75778 --- ext/misc/spellfix.c | 4 ++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index a62d24c592..2e6743e4f7 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -1893,7 +1893,7 @@ static int spellfix1Init( char **pzErr ){ spellfix1_vtab *pNew = 0; - const char *zModule = argv[0]; + /* const char *zModule = argv[0]; // not used */ const char *zDbName = argv[1]; const char *zTableName = argv[2]; int nDbName; @@ -1947,7 +1947,7 @@ static int spellfix1Init( spellfix1DbExec(&rc, db, "CREATE INDEX IF NOT EXISTS \"%w\".\"%w_vocab_index_langid_k2\" " "ON \"%w_vocab\"(langid,k2);", - zDbName, zModule, zTableName + zDbName, zTableName, zTableName ); } for(i=3; rc==SQLITE_OK && i Date: Tue, 22 Jul 2014 19:14:42 +0000 Subject: [PATCH 122/710] The optimization of check-in [b67a6e33f2] does not work (it generates incorrect VDBE code) if an OR term is AND-ed with a constant expression. So back that optimization out and add a test case to make sure it does not get added back in. FossilOrigin-Name: eed754fe93644f4df082eac0c0a7ffc5c78ccd10 --- manifest | 17 +++++++++-------- manifest.uuid | 2 +- src/sqliteInt.h | 1 - src/where.c | 19 ++----------------- test/where2.test | 8 ++++++++ 5 files changed, 20 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index cdf41d31ba..39a7dab1e8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sindex\sname\sfor\sthe\sshadow\stables\sin\sthe\sspellfix1\sextension\sso\sthat\nmultiple\sinstances\sof\sthe\sspellfix1\svirtual\stable\scan\seach\shave\stheir\sown\nindex. -D 2014-07-22T16:00:49.088 +C The\soptimization\sof\scheck-in\s[b67a6e33f2]\sdoes\snot\swork\s(it\sgenerates\nincorrect\sVDBE\scode)\sif\san\sOR\sterm\sis\sAND-ed\swith\sa\sconstant\sexpression.\nSo\sback\sthat\soptimization\sout\sand\sadd\sa\stest\scase\sto\smake\ssure\sit\sdoes\snot\nget\sadded\sback\sin. +D 2014-07-22T19:14:42.618 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -227,7 +227,7 @@ F src/shell.c 566aee8213372a2e81ba0eb34e9759f7b2574009 F src/sqlite.h.in fd8e3a36b0aded082dc93a4b89c1e85324b4cf75 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 0991b31a98c6ce072d255954af617baf50735eb1 +F src/sqliteInt.h 5dbcdcf81cb15e77df16f88699a15f7af491c580 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 52edd7f5df58d8f73230e5f9883ce867ca2f4a7d +F src/where.c 4e7e6bc639a938eb35c433bdca2aad80478fece4 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1097,7 +1097,7 @@ F test/walshared.test 0befc811dcf0b287efae21612304d15576e35417 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e F test/where.test 28b64e93428961b07b0d486778d63fd672948f6b -F test/where2.test 455a2eb2666e66c1e84e2cb5815173a85e6237db +F test/where2.test 23fdb5d8e756554aad4ca7ae03de9dd8367a2c6e F test/where3.test 1ad55ba900bd7747f98b6082e65bd3e442c5004e F test/where4.test d8420ceeb8323a41ceff1f1841fc528e824e1ecf F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2 @@ -1183,7 +1183,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 48e378029528dac33d29866016128d236a7995d9 -R ef5e99a62dffbd3ce7a7196840380036 +P 438c348a40383796a710499e4e1b7a6b08f75778 +Q -b67a6e33f23ce5f5d9a545fa9d6700a7ed636901 +R 086bb10698e6defd83e7aae53f79809b U drh -Z f16a3660aabbbb2778203c851a571a70 +Z 6c73d4c61d83e0b2dc5ff58e1c82716d diff --git a/manifest.uuid b/manifest.uuid index e6d3a6b9d4..730f453e7f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -438c348a40383796a710499e4e1b7a6b08f75778 \ No newline at end of file +eed754fe93644f4df082eac0c0a7ffc5c78ccd10 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6b0e031b30..8f256d52fc 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2136,7 +2136,6 @@ struct SrcList { #define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ #define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ #define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ -#define WHERE_OR_INDEX_OPEN 0x1000 /* OP_OpenRead for the OR index exists */ /* Allowed return values from sqlite3WhereIsDistinct() */ diff --git a/src/where.c b/src/where.c index 4ecf02f0c1..284f141ff9 100644 --- a/src/where.c +++ b/src/where.c @@ -3422,7 +3422,6 @@ static Bitmask codeOneLoopStart( int iRetInit; /* Address of regReturn init */ int untestedTerms = 0; /* Some terms not completely tested */ int ii; /* Loop counter */ - int subWctrlFlags; /* wctrlFlags for sub-queries */ Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ Table *pTab = pTabItem->pTab; @@ -3518,8 +3517,6 @@ static Bitmask codeOneLoopStart( ** eliminating duplicates from other WHERE clauses, the action for each ** sub-WHERE clause is to to invoke the main loop body as a subroutine. */ - subWctrlFlags = WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | - WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY; for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ @@ -3532,7 +3529,8 @@ static Bitmask codeOneLoopStart( } /* Loop through table entries that match term pOrTerm. */ pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, - subWctrlFlags, iCovCur); + WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | + WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY, iCovCur); assert( pSubWInfo || pParse->nErr || db->mallocFailed ); if( pSubWInfo ){ WhereLoop *pSubLoop; @@ -3623,7 +3621,6 @@ static Bitmask codeOneLoopStart( ){ assert( pSubWInfo->a[0].iIdxCur==iCovCur ); pCov = pSubLoop->u.btree.pIndex; - subWctrlFlags |= WHERE_OR_INDEX_OPEN; }else{ pCov = 0; } @@ -6222,18 +6219,6 @@ WhereInfo *sqlite3WhereBegin( pWInfo->aiCurOnePass[1] = iIndexCur; }else if( iIdxCur && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ iIndexCur = iIdxCur; - if( (wctrlFlags & WHERE_OR_INDEX_OPEN)!=0 ){ - /* For 2nd and subsequent subqueries for processing OR terms, - ** try to reuse the previous OP_OpenRead, if there is one. The - ** WHERE_OR_INDEX_OPEN bit will only be set if there is a prior - ** OP_OpenRead opcode on cursor iIndexCur, so the while() loop - ** below is guaranteed to terminate. - */ - VdbeOp *pOp = sqlite3VdbeGetOp(v, -1); - while( pOp->opcode!=OP_OpenRead || pOp->p1!=iIndexCur ) pOp--; - assert( pOp->p3==iDb ); - if( pOp->p2==pIx->tnum ) op = 0; - } }else{ iIndexCur = pParse->nTab++; } diff --git a/test/where2.test b/test/where2.test index d9b2b23de9..367eb0dfea 100644 --- a/test/where2.test +++ b/test/where2.test @@ -751,5 +751,13 @@ do_execsql_test where2-12.1 { } {/.*SEARCH TABLE t12 AS b .*SEARCH TABLE t12 AS b .*/} } +# Verify that all necessary OP_OpenRead opcodes occur in the OR optimization. +# +do_execsql_test where2-13.1 { + CREATE TABLE t13(a,b); + CREATE INDEX t13a ON t13(a); + INSERT INTO t13 VALUES(4,5); + SELECT * FROM t13 WHERE (1=2 AND a=3) OR a=4; +} {4 5} finish_test From 3526319b483fac1aba4dd19c93ebd61aa2663b97 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 22 Jul 2014 20:02:19 +0000 Subject: [PATCH 123/710] Add the OP_ReopenIdx opcode that works like OP_OpenRead except that it becomes a no-op if the cursor is already open on the same index. Update the OR-optimization logic to make use of OP_ReopenIdx in order to avoid unnecessary cursor open requests sent to the B-Tree layer. FossilOrigin-Name: 77f412caf0192d3e7fecb377d6d72123d8b64424 --- manifest | 19 +++++++++---------- manifest.uuid | 2 +- src/sqliteInt.h | 1 + src/vdbe.c | 33 +++++++++++++++++++++++++++++++-- src/vdbeInt.h | 1 + src/where.c | 8 ++++++-- 6 files changed, 49 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 39a7dab1e8..4ac400af64 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\soptimization\sof\scheck-in\s[b67a6e33f2]\sdoes\snot\swork\s(it\sgenerates\nincorrect\sVDBE\scode)\sif\san\sOR\sterm\sis\sAND-ed\swith\sa\sconstant\sexpression.\nSo\sback\sthat\soptimization\sout\sand\sadd\sa\stest\scase\sto\smake\ssure\sit\sdoes\snot\nget\sadded\sback\sin. -D 2014-07-22T19:14:42.618 +C Add\sthe\sOP_ReopenIdx\sopcode\sthat\sworks\slike\sOP_OpenRead\sexcept\sthat\sit\sbecomes\na\sno-op\sif\sthe\scursor\sis\salready\sopen\son\sthe\ssame\sindex.\s\sUpdate\sthe\nOR-optimization\slogic\sto\smake\suse\sof\sOP_ReopenIdx\sin\sorder\sto\savoid\nunnecessary\scursor\sopen\srequests\ssent\sto\sthe\sB-Tree\slayer. +D 2014-07-22T20:02:19.743 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -227,7 +227,7 @@ F src/shell.c 566aee8213372a2e81ba0eb34e9759f7b2574009 F src/sqlite.h.in fd8e3a36b0aded082dc93a4b89c1e85324b4cf75 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 5dbcdcf81cb15e77df16f88699a15f7af491c580 +F src/sqliteInt.h b72a09326d7cbd8375ec3d9a04ea5e0cf476beb3 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -283,9 +283,9 @@ F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 049fe1d3c0e2209c1bee107aec2fcff6285f909f F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 2f3f0582dfacce292d3d6f22d129fe1abe3ddd00 +F src/vdbe.c fa74c6563486022920db4d73897bd9b837c7441d F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 -F src/vdbeInt.h 5df5e9afe9b7839cd17256220fc4f7af84b8b1cd +F src/vdbeInt.h f41a7cf5a40e7abd994d5fe3a04589ae3e1154e7 F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 1ffe0bbc3a2c8aedd95622de9be6b6d07b3d3c6b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 4e7e6bc639a938eb35c433bdca2aad80478fece4 +F src/where.c 0ffd586e04cd069ed6ce7f820194ccaa6cbd669e F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1183,8 +1183,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 438c348a40383796a710499e4e1b7a6b08f75778 -Q -b67a6e33f23ce5f5d9a545fa9d6700a7ed636901 -R 086bb10698e6defd83e7aae53f79809b +P eed754fe93644f4df082eac0c0a7ffc5c78ccd10 +R 09dedf30e4922c66cabf472458c9ab58 U drh -Z 6c73d4c61d83e0b2dc5ff58e1c82716d +Z 96627cd265c0a3b75631b3111ef23fd4 diff --git a/manifest.uuid b/manifest.uuid index 730f453e7f..6783218a06 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eed754fe93644f4df082eac0c0a7ffc5c78ccd10 \ No newline at end of file +77f412caf0192d3e7fecb377d6d72123d8b64424 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8f256d52fc..7edc04ca42 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2136,6 +2136,7 @@ struct SrcList { #define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ #define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ #define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ +#define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ /* Allowed return values from sqlite3WhereIsDistinct() */ diff --git a/src/vdbe.c b/src/vdbe.c index 86cf8b0484..093669a82d 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3205,7 +3205,21 @@ case OP_SetCookie: { /* in3 */ ** sequence of the index being opened. Otherwise, if P4 is an integer ** value, it is set to the number of columns in the table. ** -** See also OpenWrite. +** See also: OpenWrite, ReopenIdx +*/ +/* Opcode: ReopenIdx P1 P2 P3 P4 P5 +** Synopsis: root=P2 iDb=P3 +** +** The ReopenIdx opcode works exactly like ReadOpen except that it first +** checks to see if the cursor on P1 is already open with a root page +** number of P2 and if it is this opcode becomes a no-op. In other words, +** if the cursor is already open, do not reopen it. +** +** The ReopenIdx opcode may only be used with P5==0 and with P4 being +** a P4_KEYINFO object. Furthermore, the P3 value must be the same as +** every other ReopenIdx or OpenRead for the same cursor number. +** +** See the OpenRead opcode documentation for additional information. */ /* Opcode: OpenWrite P1 P2 P3 P4 P5 ** Synopsis: root=P2 iDb=P3 @@ -3227,6 +3241,19 @@ case OP_SetCookie: { /* in3 */ ** ** See also OpenRead. */ +case OP_ReopenIdx: { + VdbeCursor *pCur; + + assert( pOp->p5==0 ); + assert( pOp->p4type==P4_KEYINFO ); + pCur = p->apCsr[pOp->p1]; + if( pCur && pCur->pgnoRoot==pOp->p2 ){ + assert( pCur->iDb==pOp->p3 ); /* Guaranteed by the code generator */ + break; + } + /* If the cursor is not currently open or is open on a different + ** index, then fall through into OP_OpenRead to force a reopen */ +} case OP_OpenRead: case OP_OpenWrite: { int nField; @@ -3241,7 +3268,8 @@ case OP_OpenWrite: { assert( (pOp->p5&(OPFLAG_P2ISREG|OPFLAG_BULKCSR))==pOp->p5 ); assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 ); assert( p->bIsReader ); - assert( pOp->opcode==OP_OpenRead || p->readOnly==0 ); + assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx + || p->readOnly==0 ); if( p->expired ){ rc = SQLITE_ABORT; @@ -3298,6 +3326,7 @@ case OP_OpenWrite: { if( pCur==0 ) goto no_mem; pCur->nullRow = 1; pCur->isOrdered = 1; + pCur->pgnoRoot = p2; rc = sqlite3BtreeCursor(pX, p2, wrFlag, pKeyInfo, pCur->pCursor); pCur->pKeyInfo = pKeyInfo; assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index d629802a64..f541aa7170 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -76,6 +76,7 @@ struct VdbeCursor { Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */ Bool isTable:1; /* True if a table requiring integer keys */ Bool isOrdered:1; /* True if the underlying table is BTREE_UNORDERED */ + Pgno pgnoRoot; /* Root page of the open btree cursor */ sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ diff --git a/src/where.c b/src/where.c index 284f141ff9..ff18d5d4e2 100644 --- a/src/where.c +++ b/src/where.c @@ -3422,6 +3422,7 @@ static Bitmask codeOneLoopStart( int iRetInit; /* Address of regReturn init */ int untestedTerms = 0; /* Some terms not completely tested */ int ii; /* Loop counter */ + u16 wctrlFlags; /* Flags for sub-WHERE clause */ Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ Table *pTab = pTabItem->pTab; @@ -3517,6 +3518,8 @@ static Bitmask codeOneLoopStart( ** eliminating duplicates from other WHERE clauses, the action for each ** sub-WHERE clause is to to invoke the main loop body as a subroutine. */ + wctrlFlags = WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | + WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY; for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ @@ -3529,8 +3532,7 @@ static Bitmask codeOneLoopStart( } /* Loop through table entries that match term pOrTerm. */ pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, - WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | - WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY, iCovCur); + wctrlFlags, iCovCur); assert( pSubWInfo || pParse->nErr || db->mallocFailed ); if( pSubWInfo ){ WhereLoop *pSubLoop; @@ -3621,6 +3623,7 @@ static Bitmask codeOneLoopStart( ){ assert( pSubWInfo->a[0].iIdxCur==iCovCur ); pCov = pSubLoop->u.btree.pIndex; + wctrlFlags |= WHERE_REOPEN_IDX; }else{ pCov = 0; } @@ -6219,6 +6222,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->aiCurOnePass[1] = iIndexCur; }else if( iIdxCur && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ iIndexCur = iIdxCur; + if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; } From 0f65cce82580619ba3498e3b2ef7bd338d73b6e5 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 22 Jul 2014 22:46:54 +0000 Subject: [PATCH 124/710] When running ANALYZE, it is not necessary to check the right-most key column for changes since that column will always change if none of the previous columns have. FossilOrigin-Name: 48f40861db4fbd10725a2b8b606d44fe16d5bd27 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 4ac400af64..f57975f5d3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sOP_ReopenIdx\sopcode\sthat\sworks\slike\sOP_OpenRead\sexcept\sthat\sit\sbecomes\na\sno-op\sif\sthe\scursor\sis\salready\sopen\son\sthe\ssame\sindex.\s\sUpdate\sthe\nOR-optimization\slogic\sto\smake\suse\sof\sOP_ReopenIdx\sin\sorder\sto\savoid\nunnecessary\scursor\sopen\srequests\ssent\sto\sthe\sB-Tree\slayer. -D 2014-07-22T20:02:19.743 +C When\srunning\sANALYZE,\sit\sis\snot\snecessary\sto\scheck\sthe\sright-most\skey\scolumn\nfor\schanges\ssince\sthat\scolumn\swill\salways\schange\sif\snone\sof\sthe\sprevious\ncolumns\shave. +D 2014-07-22T22:46:54.689 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 56799121c72753f40d56f3c17c66c3961af6acb7 +F src/analyze.c ab1e44158c25490bfda557f3d690217b2bb509e2 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -1183,7 +1183,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eed754fe93644f4df082eac0c0a7ffc5c78ccd10 -R 09dedf30e4922c66cabf472458c9ab58 +P 77f412caf0192d3e7fecb377d6d72123d8b64424 +R e38e472b4e4b7503ed49bdb3ca667f5c U drh -Z 96627cd265c0a3b75631b3111ef23fd4 +Z e390ae6bf708e4580525fb01df14fd1e diff --git a/manifest.uuid b/manifest.uuid index 6783218a06..691fcc01d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -77f412caf0192d3e7fecb377d6d72123d8b64424 \ No newline at end of file +48f40861db4fbd10725a2b8b606d44fe16d5bd27 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 97305f0dc1..c614350ae3 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1109,7 +1109,7 @@ static void analyzeOneTable( ** goto chng_addr_N */ addrNextRow = sqlite3VdbeCurrentAddr(v); - for(i=0; iazColl[i]); sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); @@ -1118,7 +1118,7 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); VdbeCoverage(v); } - sqlite3VdbeAddOp2(v, OP_Integer, nCol, regChng); + sqlite3VdbeAddOp2(v, OP_Integer, nCol-1, regChng); aGotoChng[nCol] = sqlite3VdbeAddOp0(v, OP_Goto); /* @@ -1129,7 +1129,7 @@ static void analyzeOneTable( ** ... */ sqlite3VdbeJumpHere(v, addrGotoChng0); - for(i=0; i Date: Wed, 23 Jul 2014 01:26:51 +0000 Subject: [PATCH 125/710] Add support for parsing C-style hexadecimal literals. FossilOrigin-Name: 34a1f38b7a23c64f5c6e5b34c19a20480be53961 --- manifest | 17 +++++----- manifest.uuid | 2 +- src/tokenize.c | 6 ++++ src/util.c | 84 ++++++++++++++++++++++++++++++++++++++------------ 4 files changed, 81 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index f57975f5d3..1ebf10efcd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\srunning\sANALYZE,\sit\sis\snot\snecessary\sto\scheck\sthe\sright-most\skey\scolumn\nfor\schanges\ssince\sthat\scolumn\swill\salways\schange\sif\snone\sof\sthe\sprevious\ncolumns\shave. -D 2014-07-22T22:46:54.689 +C Add\ssupport\sfor\sparsing\sC-style\shexadecimal\sliterals. +D 2014-07-23T01:26:51.616 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,11 +277,11 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 +F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 -F src/util.c 049fe1d3c0e2209c1bee107aec2fcff6285f909f +F src/util.c 9d87de90e59d78e69ab944f34a03a4228b05de6e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c fa74c6563486022920db4d73897bd9b837c7441d F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 @@ -1183,7 +1183,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 77f412caf0192d3e7fecb377d6d72123d8b64424 -R e38e472b4e4b7503ed49bdb3ca667f5c +P 48f40861db4fbd10725a2b8b606d44fe16d5bd27 +R bc60c63a2067ce5c2bcc7a7ef85e26f4 +T *branch * hex-literal +T *sym-hex-literal * +T -sym-trunk * U drh -Z e390ae6bf708e4580525fb01df14fd1e +Z 425e58328b59810233115ebd1d9053d0 diff --git a/manifest.uuid b/manifest.uuid index 691fcc01d6..6e3e9df36d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -48f40861db4fbd10725a2b8b606d44fe16d5bd27 \ No newline at end of file +34a1f38b7a23c64f5c6e5b34c19a20480be53961 \ No newline at end of file diff --git a/src/tokenize.c b/src/tokenize.c index 87553e25b0..4017c3b816 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -270,6 +270,12 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); testcase( z[0]=='9' ); *tokenType = TK_INTEGER; +#ifndef SQLITE_OMIT_HEX_INTEGER + if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ + for(i=3; sqlite3Isxdigit(z[i]); i++){} + return i; + } +#endif for(i=0; sqlite3Isdigit(z[i]); i++){} #ifndef SQLITE_OMIT_FLOATING_POINT if( z[i]=='.' ){ diff --git a/src/util.c b/src/util.c index 4fe07a862a..1686d4342d 100644 --- a/src/util.c +++ b/src/util.c @@ -255,6 +255,22 @@ int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; } +/* +** Translate a single byte of Hex into an integer. +** This routine only works if h really is a valid hexadecimal +** character: 0..9a..fA..F +*/ +u8 sqlite3HexToInt(int h){ + assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); +#ifdef SQLITE_ASCII + h += 9*(1&(h>>6)); +#endif +#ifdef SQLITE_EBCDIC + h += 9*(1&~(h>>4)); +#endif + return (u8)(h & 0xf); +} + /* ** The string z[] is an text representation of a real number. ** Convert this string to a double and write it into *pResult. @@ -318,6 +334,20 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ }else if( *z=='+' ){ z+=incr; } +#ifndef SQLITE_OMIT_HEX_INTEGER + else if( *z==0 + && &z[incr*2]=zEnd && nonNum==0; + } +#endif /* skip leading zeroes */ while( z16*incr || nonNum; + } +#endif } zStart = zNum; while( zNum=0 && c<=9; i++){ v = v*10 + c; } @@ -1029,24 +1091,6 @@ void sqlite3Put4byte(unsigned char *p, u32 v){ p[3] = (u8)v; } - - -/* -** Translate a single byte of Hex into an integer. -** This routine only works if h really is a valid hexadecimal -** character: 0..9a..fA..F -*/ -u8 sqlite3HexToInt(int h){ - assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); -#ifdef SQLITE_ASCII - h += 9*(1&(h>>6)); -#endif -#ifdef SQLITE_EBCDIC - h += 9*(1&~(h>>4)); -#endif - return (u8)(h & 0xf); -} - #if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) /* ** Convert a BLOB literal of the form "x'hhhhhh'" into its binary From fce4da1b737b34432ca06432dd6d71101a89f00a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 23 Jul 2014 01:56:32 +0000 Subject: [PATCH 126/710] Test cases for hex literals. FossilOrigin-Name: 19054339c47448bcdfd1f7be35daa3826c409077 --- manifest | 16 +++-- manifest.uuid | 2 +- src/util.c | 2 +- test/hexlit.test | 150 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 159 insertions(+), 11 deletions(-) create mode 100644 test/hexlit.test diff --git a/manifest b/manifest index 1ebf10efcd..133cabf19e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sparsing\sC-style\shexadecimal\sliterals. -D 2014-07-23T01:26:51.616 +C Test\scases\sfor\shex\sliterals. +D 2014-07-23T01:56:32.016 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -281,7 +281,7 @@ F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 -F src/util.c 9d87de90e59d78e69ab944f34a03a4228b05de6e +F src/util.c eff2c1e5a49a3c64af0fe9f2fb32cada3436a167 F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c fa74c6563486022920db4d73897bd9b837c7441d F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 @@ -601,6 +601,7 @@ F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 F test/fuzzer1.test d4c52aaf3ef923da293a2653cfab33d02f718a36 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 +F test/hexlit.test c7f30348053065b75c189356ffcaf3d09824d57f F test/hook.test 162d7cef7a2d2b04839fe14402934e6a1b79442f F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e @@ -1183,10 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 48f40861db4fbd10725a2b8b606d44fe16d5bd27 -R bc60c63a2067ce5c2bcc7a7ef85e26f4 -T *branch * hex-literal -T *sym-hex-literal * -T -sym-trunk * +P 34a1f38b7a23c64f5c6e5b34c19a20480be53961 +R 31f7dbbcb963767edb50861a2259f157 U drh -Z 425e58328b59810233115ebd1d9053d0 +Z fe6debea68030ad6a2bb2c17bc2c014e diff --git a/manifest.uuid b/manifest.uuid index 6e3e9df36d..8a951ec1f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -34a1f38b7a23c64f5c6e5b34c19a20480be53961 \ No newline at end of file +19054339c47448bcdfd1f7be35daa3826c409077 \ No newline at end of file diff --git a/src/util.c b/src/util.c index 1686d4342d..f7f3f58b2b 100644 --- a/src/util.c +++ b/src/util.c @@ -335,7 +335,7 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ z+=incr; } #ifndef SQLITE_OMIT_HEX_INTEGER - else if( *z==0 + else if( *z=='0' && &z[incr*2] Date: Wed, 23 Jul 2014 01:59:47 +0000 Subject: [PATCH 127/710] Reformatting a few test cases for clarity. FossilOrigin-Name: 7e1bbacb11a4689b69856450125cae3d045307af --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/hexlit.test | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 133cabf19e..7608ebf7fa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scases\sfor\shex\sliterals. -D 2014-07-23T01:56:32.016 +C Reformatting\sa\sfew\stest\scases\sfor\sclarity. +D 2014-07-23T01:59:47.721 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -601,7 +601,7 @@ F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 F test/fuzzer1.test d4c52aaf3ef923da293a2653cfab33d02f718a36 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 -F test/hexlit.test c7f30348053065b75c189356ffcaf3d09824d57f +F test/hexlit.test 499de0bfdfd16e8d406f3a4104d275c1a473b2d5 F test/hook.test 162d7cef7a2d2b04839fe14402934e6a1b79442f F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e @@ -1184,7 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 34a1f38b7a23c64f5c6e5b34c19a20480be53961 -R 31f7dbbcb963767edb50861a2259f157 +P 19054339c47448bcdfd1f7be35daa3826c409077 +R 52ea51dcf5a265235552cc5fcbffc473 U drh -Z fe6debea68030ad6a2bb2c17bc2c014e +Z 7e4424638caf7e83118206706ce53455 diff --git a/manifest.uuid b/manifest.uuid index 8a951ec1f6..fc8e55fbc5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19054339c47448bcdfd1f7be35daa3826c409077 \ No newline at end of file +7e1bbacb11a4689b69856450125cae3d045307af \ No newline at end of file diff --git a/test/hexlit.test b/test/hexlit.test index 79bd7e4e6e..bfb4682571 100644 --- a/test/hexlit.test +++ b/test/hexlit.test @@ -142,9 +142,9 @@ hexlit2 327 0x1000000000000000000000000000 3.245186e+32 hexlit2 328 0x10000000000000000000000000000 5.192297e+33 hexlit2 329 0x100000000000000000000000000000 8.307675e+34 -hexlit2 400 0x7fffffffffffffff 9.223372e+18 -hexlit2 401 0x8000000000000000 -9.223372e+18 -hexlit2 402 0xffffffffffffffff -1.000000e+00 -hexlit2 403 0x10000000000000000 1.844674e+19 +hexlit2 400 0x07fffffffffffffff 9.223372e+18 +hexlit2 401 0x08000000000000000 -9.223372e+18 +hexlit2 402 0x0ffffffffffffffff -1.000000e+00 +hexlit2 403 0x10000000000000000 1.844674e+19 finish_test From 5ec659cfcb030f8aa47eac97edf74046605c3bfb Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 23 Jul 2014 02:07:11 +0000 Subject: [PATCH 128/710] Casting hex literals directly from string to float always results in a positive number. FossilOrigin-Name: 4b86ccdf4f4eb4339a5706e10ad24f01b6c3939e --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/hexlit.test | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 7608ebf7fa..656415a599 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reformatting\sa\sfew\stest\scases\sfor\sclarity. -D 2014-07-23T01:59:47.721 +C Casting\shex\sliterals\sdirectly\sfrom\sstring\sto\sfloat\salways\sresults\sin\sa\npositive\snumber. +D 2014-07-23T02:07:11.054 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -601,7 +601,7 @@ F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 F test/fuzzer1.test d4c52aaf3ef923da293a2653cfab33d02f718a36 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 -F test/hexlit.test 499de0bfdfd16e8d406f3a4104d275c1a473b2d5 +F test/hexlit.test 58b653845b60da52161b5f451db9a89c569187d1 F test/hook.test 162d7cef7a2d2b04839fe14402934e6a1b79442f F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e @@ -1184,7 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 19054339c47448bcdfd1f7be35daa3826c409077 -R 52ea51dcf5a265235552cc5fcbffc473 +P 7e1bbacb11a4689b69856450125cae3d045307af +R 8a73a1b568b990d7d0a1847902994baf U drh -Z 7e4424638caf7e83118206706ce53455 +Z d9d4d7765f779882d47568941cd5db2a diff --git a/manifest.uuid b/manifest.uuid index fc8e55fbc5..4442ea274c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e1bbacb11a4689b69856450125cae3d045307af \ No newline at end of file +4b86ccdf4f4eb4339a5706e10ad24f01b6c3939e \ No newline at end of file diff --git a/test/hexlit.test b/test/hexlit.test index bfb4682571..39ddf96e0b 100644 --- a/test/hexlit.test +++ b/test/hexlit.test @@ -108,7 +108,7 @@ hexlit1 219 0x00f 15 hexlit1 220 0x00F 15 proc hexlit2 {tnum hex ans} { - do_execsql_test hexlit-$tnum "SELECT printf('%7e',CAST($hex AS real))" $ans + do_execsql_test hexlit-$tnum "SELECT printf('%7e',CAST('$hex' AS real))" $ans } hexlit2 300 0x1 1.000000e+00 @@ -142,9 +142,9 @@ hexlit2 327 0x1000000000000000000000000000 3.245186e+32 hexlit2 328 0x10000000000000000000000000000 5.192297e+33 hexlit2 329 0x100000000000000000000000000000 8.307675e+34 -hexlit2 400 0x07fffffffffffffff 9.223372e+18 -hexlit2 401 0x08000000000000000 -9.223372e+18 -hexlit2 402 0x0ffffffffffffffff -1.000000e+00 -hexlit2 403 0x10000000000000000 1.844674e+19 +hexlit2 400 0x07fffffffffffffff 9.223372e+18 +hexlit2 401 0x08000000000000000 9.223372e+18 +hexlit2 402 0x0ffffffffffffffff 1.844674e+19 +hexlit2 403 0x10000000000000000 1.844674e+19 finish_test From 9296c18a509e32dac34e0ac9560946cee7f560e0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 23 Jul 2014 13:40:49 +0000 Subject: [PATCH 129/710] Change the hex literal processing so that only the SQL parser understands hex literals. Casting and coercing string literals into numeric values does not understand hexadecimal integers. This preserves backwards compatibility. Also: Throw an error on any hex literal that is too big to fit into 64 bits. FossilOrigin-Name: 6c6f0de59bf96b79c8ace8c9bfe48c7a6a306a50 --- manifest | 22 +++++------ manifest.uuid | 2 +- src/expr.c | 8 +++- src/main.c | 2 +- src/pragma.c | 6 +-- src/sqliteInt.h | 2 + src/util.c | 100 +++++++++++++++++++++++++---------------------- test/hexlit.test | 52 +++++++----------------- 8 files changed, 93 insertions(+), 101 deletions(-) diff --git a/manifest b/manifest index 656415a599..8a6aa1f099 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Casting\shex\sliterals\sdirectly\sfrom\sstring\sto\sfloat\salways\sresults\sin\sa\npositive\snumber. -D 2014-07-23T02:07:11.054 +C Change\sthe\shex\sliteral\sprocessing\sso\sthat\sonly\sthe\sSQL\sparser\sunderstands\nhex\sliterals.\s\sCasting\sand\scoercing\sstring\sliterals\sinto\snumeric\svalues\sdoes\nnot\sunderstand\shexadecimal\sintegers.\s\sThis\spreserves\sbackwards\scompatibility.\nAlso:\s\sThrow\san\serror\son\sany\shex\sliteral\sthat\sis\stoo\sbig\sto\sfit\sinto\s64\sbits. +D 2014-07-23T13:40:49.410 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 40d06d1543b1355aa02efa9666178f7642a96ed6 +F src/expr.c b1ffac76b69ae005ca38eba6dcdefa5d65eb5c95 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c a549cff9fe8b736cdae21650ea0af6de29b77619 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -189,7 +189,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 1a420efa9a34e8603c4807886408cba5546b87f9 +F src/main.c cfdb2aa5d248ff1af60227cc3f6d485ba86f92dc F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -216,7 +216,7 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c 810ef31ccfaa233201dcf100637a9777cc24e897 +F src/pragma.c e17c5ea1cb9eb9d93c41bbb7c3a17e747d5e0335 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c af06f66927919730f03479fed6ae9854f73419f4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -227,7 +227,7 @@ F src/shell.c 566aee8213372a2e81ba0eb34e9759f7b2574009 F src/sqlite.h.in fd8e3a36b0aded082dc93a4b89c1e85324b4cf75 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h b72a09326d7cbd8375ec3d9a04ea5e0cf476beb3 +F src/sqliteInt.h 783e77ab498ac05a3fab396dfc1c2d6c2083dced F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -281,7 +281,7 @@ F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 -F src/util.c eff2c1e5a49a3c64af0fe9f2fb32cada3436a167 +F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c fa74c6563486022920db4d73897bd9b837c7441d F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 @@ -601,7 +601,7 @@ F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 F test/fuzzer1.test d4c52aaf3ef923da293a2653cfab33d02f718a36 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 -F test/hexlit.test 58b653845b60da52161b5f451db9a89c569187d1 +F test/hexlit.test 1dc49cfd1c8938a8f028e392775bc3e61623ec46 F test/hook.test 162d7cef7a2d2b04839fe14402934e6a1b79442f F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e @@ -1184,7 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7e1bbacb11a4689b69856450125cae3d045307af -R 8a73a1b568b990d7d0a1847902994baf +P 4b86ccdf4f4eb4339a5706e10ad24f01b6c3939e +R a36ec645688e0ae2f526ebda77270f17 U drh -Z d9d4d7765f779882d47568941cd5db2a +Z 32ccd1ec50cb642ea84793b61793e5b3 diff --git a/manifest.uuid b/manifest.uuid index 4442ea274c..f47a66e629 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4b86ccdf4f4eb4339a5706e10ad24f01b6c3939e \ No newline at end of file +6c6f0de59bf96b79c8ace8c9bfe48c7a6a306a50 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 803d93f30d..a60c0cba43 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2075,7 +2075,7 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ i64 value; const char *z = pExpr->u.zToken; assert( z!=0 ); - c = sqlite3Atoi64(z, &value, sqlite3Strlen30(z), SQLITE_UTF8); + c = sqlite3DecOrHexToI64(z, &value); if( c==0 || (c==2 && negFlag) ){ char *zV; if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } @@ -2085,7 +2085,11 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ #ifdef SQLITE_OMIT_FLOATING_POINT sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); #else - codeReal(v, z, negFlag, iMem); + if( sqlite3_strnicmp(z,"0x",2)==0 ){ + sqlite3ErrorMsg(pParse, "hex literal too big: %s", z); + }else{ + codeReal(v, z, negFlag, iMem); + } #endif } } diff --git a/src/main.c b/src/main.c index 5e976e3397..904e0a4fc1 100644 --- a/src/main.c +++ b/src/main.c @@ -3409,7 +3409,7 @@ sqlite3_int64 sqlite3_uri_int64( ){ const char *z = sqlite3_uri_parameter(zFilename, zParam); sqlite3_int64 v; - if( z && sqlite3Atoi64(z, &v, sqlite3Strlen30(z), SQLITE_UTF8)==SQLITE_OK ){ + if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){ bDflt = v; } return bDflt; diff --git a/src/pragma.c b/src/pragma.c index 4c69ceb4fd..709662c989 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1048,7 +1048,7 @@ void sqlite3Pragma( Pager *pPager = sqlite3BtreePager(pDb->pBt); i64 iLimit = -2; if( zRight ){ - sqlite3Atoi64(zRight, &iLimit, sqlite3Strlen30(zRight), SQLITE_UTF8); + sqlite3DecOrHexToI64(zRight, &iLimit); if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); @@ -1176,7 +1176,7 @@ void sqlite3Pragma( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( zRight ){ int ii; - sqlite3Atoi64(zRight, &sz, sqlite3Strlen30(zRight), SQLITE_UTF8); + sqlite3DecOrHexToI64(zRight, &sz); if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; if( pId2->n==0 ) db->szMmap = sz; for(ii=db->nDb-1; ii>=0; ii--){ @@ -2219,7 +2219,7 @@ void sqlite3Pragma( */ case PragTyp_SOFT_HEAP_LIMIT: { sqlite3_int64 N; - if( zRight && sqlite3Atoi64(zRight, &N, 1000000, SQLITE_UTF8)==SQLITE_OK ){ + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ sqlite3_soft_heap_limit64(N); } returnSingleInt(pParse, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7edc04ca42..0bbf9945ce 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3331,9 +3331,11 @@ char sqlite3CompareAffinity(Expr *pExpr, char aff2); int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); char sqlite3ExprAffinity(Expr *pExpr); int sqlite3Atoi64(const char*, i64*, int, u8); +int sqlite3DecOrHexToI64(const char*, i64*); void sqlite3Error(sqlite3*, int, const char*,...); void *sqlite3HexToBlob(sqlite3*, const char *z, int n); u8 sqlite3HexToInt(int h); +int sqlite3HexToI64(const char*,i64*); int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); #if defined(SQLITE_TEST) diff --git a/src/util.c b/src/util.c index f7f3f58b2b..619af7f758 100644 --- a/src/util.c +++ b/src/util.c @@ -255,22 +255,6 @@ int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; } -/* -** Translate a single byte of Hex into an integer. -** This routine only works if h really is a valid hexadecimal -** character: 0..9a..fA..F -*/ -u8 sqlite3HexToInt(int h){ - assert( (h>='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); -#ifdef SQLITE_ASCII - h += 9*(1&(h>>6)); -#endif -#ifdef SQLITE_EBCDIC - h += 9*(1&~(h>>4)); -#endif - return (u8)(h & 0xf); -} - /* ** The string z[] is an text representation of a real number. ** Convert this string to a double and write it into *pResult. @@ -334,20 +318,6 @@ int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ }else if( *z=='+' ){ z+=incr; } -#ifndef SQLITE_OMIT_HEX_INTEGER - else if( *z=='0' - && &z[incr*2]=zEnd && nonNum==0; - } -#endif /* skip leading zeroes */ while( z16*incr || nonNum; - } -#endif } zStart = zNum; while( zNum='0' && h<='9') || (h>='a' && h<='f') || (h>='A' && h<='F') ); +#ifdef SQLITE_ASCII + h += 9*(1&(h>>6)); +#endif +#ifdef SQLITE_EBCDIC + h += 9*(1&~(h>>4)); +#endif + return (u8)(h & 0xf); +} + #if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) /* ** Convert a BLOB literal of the form "x'hhhhhh'" into its binary diff --git a/test/hexlit.test b/test/hexlit.test index 39ddf96e0b..41133aba0f 100644 --- a/test/hexlit.test +++ b/test/hexlit.test @@ -107,44 +107,22 @@ hexlit1 218 0x00E 14 hexlit1 219 0x00f 15 hexlit1 220 0x00F 15 -proc hexlit2 {tnum hex ans} { - do_execsql_test hexlit-$tnum "SELECT printf('%7e',CAST('$hex' AS real))" $ans -} +# String literals that look like hex do not get cast or coerced. +# +do_execsql_test hexlit-300 { + CREATE TABLE t1(x INT, y REAL); + INSERT INTO t1 VALUES('1234','4567'),('0x1234','0x4567'); + SELECT typeof(x), x, typeof(y), y, '#' FROM t1 ORDER BY rowid; +} {integer 1234 real 4567.0 # text 0x1234 text 0x4567 #} +do_execsql_test hexlit-301 { + SELECT CAST('0x1234' AS INTEGER); +} {0} -hexlit2 300 0x1 1.000000e+00 -hexlit2 301 0x10 1.600000e+01 -hexlit2 302 0x100 2.560000e+02 -hexlit2 303 0x1000 4.096000e+03 -hexlit2 304 0x10000 6.553600e+04 -hexlit2 305 0x100000 1.048576e+06 -hexlit2 306 0x1000000 1.677722e+07 -hexlit2 307 0x10000000 2.684355e+08 -hexlit2 308 0x100000000 4.294967e+09 -hexlit2 309 0x1000000000 6.871948e+10 -hexlit2 310 0x10000000000 1.099512e+12 -hexlit2 311 0x100000000000 1.759219e+13 -hexlit2 312 0x1000000000000 2.814750e+14 -hexlit2 313 0x10000000000000 4.503600e+15 -hexlit2 314 0x100000000000000 7.205759e+16 -hexlit2 315 0x1000000000000000 1.152922e+18 -hexlit2 316 0x10000000000000000 1.844674e+19 -hexlit2 317 0x100000000000000000 2.951479e+20 -hexlit2 318 0x1000000000000000000 4.722366e+21 -hexlit2 319 0x10000000000000000000 7.555786e+22 -hexlit2 320 0x100000000000000000000 1.208926e+24 -hexlit2 321 0x1000000000000000000000 1.934281e+25 -hexlit2 322 0x10000000000000000000000 3.094850e+26 -hexlit2 323 0x100000000000000000000000 4.951760e+27 -hexlit2 324 0x1000000000000000000000000 7.922816e+28 -hexlit2 325 0x10000000000000000000000000 1.267651e+30 -hexlit2 326 0x100000000000000000000000000 2.028241e+31 -hexlit2 327 0x1000000000000000000000000000 3.245186e+32 -hexlit2 328 0x10000000000000000000000000000 5.192297e+33 -hexlit2 329 0x100000000000000000000000000000 8.307675e+34 +# Oversized hex literals are rejected +# +do_catchsql_test hexlist-400 { + SELECT 0x10000000000000000; +} {1 {hex literal too big: 0x10000000000000000}} -hexlit2 400 0x07fffffffffffffff 9.223372e+18 -hexlit2 401 0x08000000000000000 9.223372e+18 -hexlit2 402 0x0ffffffffffffffff 1.844674e+19 -hexlit2 403 0x10000000000000000 1.844674e+19 finish_test From 1b7ddc59158b4676e4382a5b39a0b40697c5f91d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 23 Jul 2014 14:52:05 +0000 Subject: [PATCH 130/710] Remove a surplus function prototype. #ifdef code that is not used when hex integers are omitted at compile time. FossilOrigin-Name: a5b383e077e4b277a65920f7cc202f32f07aa9ee --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 5 ++++- src/sqliteInt.h | 1 - 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 8a6aa1f099..70612a16ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\shex\sliteral\sprocessing\sso\sthat\sonly\sthe\sSQL\sparser\sunderstands\nhex\sliterals.\s\sCasting\sand\scoercing\sstring\sliterals\sinto\snumeric\svalues\sdoes\nnot\sunderstand\shexadecimal\sintegers.\s\sThis\spreserves\sbackwards\scompatibility.\nAlso:\s\sThrow\san\serror\son\sany\shex\sliteral\sthat\sis\stoo\sbig\sto\sfit\sinto\s64\sbits. -D 2014-07-23T13:40:49.410 +C Remove\sa\ssurplus\sfunction\sprototype.\s\s#ifdef\scode\sthat\sis\snot\sused\swhen\nhex\sintegers\sare\somitted\sat\scompile\stime. +D 2014-07-23T14:52:05.196 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c b1ffac76b69ae005ca38eba6dcdefa5d65eb5c95 +F src/expr.c b989d07fc7c8780fff77365a4fc59881223e340c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c a549cff9fe8b736cdae21650ea0af6de29b77619 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -227,7 +227,7 @@ F src/shell.c 566aee8213372a2e81ba0eb34e9759f7b2574009 F src/sqlite.h.in fd8e3a36b0aded082dc93a4b89c1e85324b4cf75 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 783e77ab498ac05a3fab396dfc1c2d6c2083dced +F src/sqliteInt.h e8ff8fdb515774d330af4a4800e032ed00cc9adb F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1184,7 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4b86ccdf4f4eb4339a5706e10ad24f01b6c3939e -R a36ec645688e0ae2f526ebda77270f17 +P 6c6f0de59bf96b79c8ace8c9bfe48c7a6a306a50 +R 9da9b225ebea231badb4246df3d3d1b3 U drh -Z 32ccd1ec50cb642ea84793b61793e5b3 +Z 6e030ade89af7e36cbcb367d7755ee7b diff --git a/manifest.uuid b/manifest.uuid index f47a66e629..af55c0e7ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c6f0de59bf96b79c8ace8c9bfe48c7a6a306a50 \ No newline at end of file +a5b383e077e4b277a65920f7cc202f32f07aa9ee \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index a60c0cba43..72286dfdf9 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2085,9 +2085,12 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ #ifdef SQLITE_OMIT_FLOATING_POINT sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); #else +#ifndef SQLITE_OMIT_HEX_INTEGER if( sqlite3_strnicmp(z,"0x",2)==0 ){ sqlite3ErrorMsg(pParse, "hex literal too big: %s", z); - }else{ + }else +#endif + { codeReal(v, z, negFlag, iMem); } #endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0bbf9945ce..ca075307eb 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3335,7 +3335,6 @@ int sqlite3DecOrHexToI64(const char*, i64*); void sqlite3Error(sqlite3*, int, const char*,...); void *sqlite3HexToBlob(sqlite3*, const char *z, int n); u8 sqlite3HexToInt(int h); -int sqlite3HexToI64(const char*,i64*); int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); #if defined(SQLITE_TEST) From 11d451eb8a8b01610106b0792be64cfad5b85a45 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 23 Jul 2014 15:51:29 +0000 Subject: [PATCH 131/710] Updated documentation on sqlite3_temp_directory. No changes to code. FossilOrigin-Name: e6225a7bf77a700b318563b1a854b4b3a9e031e1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 12 ++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index f57975f5d3..449ddf282d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\srunning\sANALYZE,\sit\sis\snot\snecessary\sto\scheck\sthe\sright-most\skey\scolumn\nfor\schanges\ssince\sthat\scolumn\swill\salways\schange\sif\snone\sof\sthe\sprevious\ncolumns\shave. -D 2014-07-22T22:46:54.689 +C Updated\sdocumentation\son\ssqlite3_temp_directory.\s\sNo\schanges\sto\scode. +D 2014-07-23T15:51:29.276 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -224,7 +224,7 @@ F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 6762c62e11b504aa014edceab8886495165e3a77 F src/shell.c 566aee8213372a2e81ba0eb34e9759f7b2574009 -F src/sqlite.h.in fd8e3a36b0aded082dc93a4b89c1e85324b4cf75 +F src/sqlite.h.in ac4451c9da2771d2f4d702ef89722407242906d9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h b72a09326d7cbd8375ec3d9a04ea5e0cf476beb3 @@ -1183,7 +1183,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 77f412caf0192d3e7fecb377d6d72123d8b64424 -R e38e472b4e4b7503ed49bdb3ca667f5c +P 48f40861db4fbd10725a2b8b606d44fe16d5bd27 +R c881f4a63c48850e8a9fb96aa00a5db1 U drh -Z e390ae6bf708e4580525fb01df14fd1e +Z cd8aaf55159956edb23d2b840f6a6165 diff --git a/manifest.uuid b/manifest.uuid index 691fcc01d6..470adaeeca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -48f40861db4fbd10725a2b8b606d44fe16d5bd27 \ No newline at end of file +e6225a7bf77a700b318563b1a854b4b3a9e031e1 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 93cde78340..67113dc418 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4711,6 +4711,13 @@ int sqlite3_sleep(int); ** is a NULL pointer, then SQLite performs a search for an appropriate ** temporary file directory. ** +** Applications are strongly discouraged from using this global variable. +** It is required to set a temporary folder on Windows Runtime (WinRT). +** But for all other platforms, it is highly recommended that applications +** neither read nor write this variable. This global variable is a relic +** that exists for backwards compatibility of legacy applications and should +** be avoided in new projects. +** ** It is not safe to read or modify this variable in more than one ** thread at a time. It is not safe to read or modify this variable ** if a [database connection] is being used at the same time in a separate @@ -4729,6 +4736,11 @@ int sqlite3_sleep(int); ** Hence, if this variable is modified directly, either it should be ** made NULL or made to point to memory obtained from [sqlite3_malloc] ** or else the use of the [temp_store_directory pragma] should be avoided. +** Except when requested by the [temp_store_directory pragma], SQLite +** does not free the memory that sqlite3_temp_directory points to. If +** the application wants that memory to be freed, it must do +** so itself, taking care to only do so after all [database connection] +** objects have been destroyed. ** ** Note to Windows Runtime users: The temporary directory must be set ** prior to calling [sqlite3_open] or [sqlite3_open_v2]. Otherwise, various From 553818a0aaffdbf62fd038eb0a7d3dd3e830cf59 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 23 Jul 2014 18:36:55 +0000 Subject: [PATCH 132/710] Improve the performance of the ANALYZE command by taking advantage of the fact that every row of a UNIQUE index is distinct. FossilOrigin-Name: 3e1e79e1335f7ad33cd35f384f2a063c4aa2253b --- manifest | 12 ++--- manifest.uuid | 2 +- src/analyze.c | 124 +++++++++++++++++++++++++++++--------------------- 3 files changed, 79 insertions(+), 59 deletions(-) diff --git a/manifest b/manifest index 449ddf282d..f572551082 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updated\sdocumentation\son\ssqlite3_temp_directory.\s\sNo\schanges\sto\scode. -D 2014-07-23T15:51:29.276 +C Improve\sthe\sperformance\sof\sthe\sANALYZE\scommand\sby\staking\sadvantage\sof\sthe\nfact\sthat\severy\srow\sof\sa\sUNIQUE\sindex\sis\sdistinct. +D 2014-07-23T18:36:55.821 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c ab1e44158c25490bfda557f3d690217b2bb509e2 +F src/analyze.c c72bb66997ebdbc76936cfe96d7cc9da6ffe2b22 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -1183,7 +1183,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 48f40861db4fbd10725a2b8b606d44fe16d5bd27 -R c881f4a63c48850e8a9fb96aa00a5db1 +P e6225a7bf77a700b318563b1a854b4b3a9e031e1 +R fa098d0ebca9c403761ac8a7cdd64221 U drh -Z cd8aaf55159956edb23d2b840f6a6165 +Z 71590bde4580a255d675842a920968d0 diff --git a/manifest.uuid b/manifest.uuid index 470adaeeca..2ecff9239b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6225a7bf77a700b318563b1a854b4b3a9e031e1 \ No newline at end of file +3e1e79e1335f7ad33cd35f384f2a063c4aa2253b \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index c614350ae3..2c31f81f3a 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -371,15 +371,20 @@ static void stat4Destructor(void *pOld){ /* ** Implementation of the stat_init(N,K,C) SQL function. The three parameters ** are: -** N: The number of columns in the index including the rowid/pk -** K: The number of columns in the index excluding the rowid/pk -** C: The number of rows in the index +** N: The number of columns in the index including the rowid/pk (note 1) +** K: The number of columns in the index excluding the rowid/pk. +** C: The number of rows in the index (note 2) ** -** C is only used for STAT3 and STAT4. +** Note 1: In the special case of the covering index that implements a +** WITHOUT ROWID table, N is the number of PRIMARY KEY columns, not the +** total number of columns in the table. ** -** For ordinary rowid tables, N==K+1. But for WITHOUT ROWID tables, -** N=K+P where P is the number of columns in the primary key. For the -** covering index that implements the original WITHOUT ROWID table, N==K. +** Note 2: C is only used for STAT3 and STAT4. +** +** For indexes on ordinary rowid tables, N==K+1. But for indexes on +** WITHOUT ROWID tables, N=K+P where P is the number of columns in the +** PRIMARY KEY of the table. The covering index that implements the +** original WITHOUT ROWID table as N==K as a special case. ** ** This routine allocates the Stat4Accum object in heap memory. The return ** value is a pointer to the the Stat4Accum object encoded as a blob (i.e. @@ -689,7 +694,10 @@ static void samplePushPrevious(Stat4Accum *p, int iChng){ ** R Rowid for the current row. Might be a key record for ** WITHOUT ROWID tables. ** -** The SQL function always returns NULL. +** This SQL function always returns NULL. It's purpose it to accumulate +** statistical data and/or samples in the Stat4Accum object about the +** index being analyzed. The stat_get() SQL function will later be used to +** extract relevant information for constructing the sqlite_statN tables. ** ** The R parameter is only used for STAT3 and STAT4 */ @@ -783,7 +791,10 @@ static const FuncDef statPushFuncdef = { /* ** Implementation of the stat_get(P,J) SQL function. This routine is -** used to query the results. Content is returned for parameter J +** used to query statistical information that has been gathered into +** the Stat4Accum object by prior calls to stat_push(). The P parameter +** is a BLOB which is decoded into a pointer to the Stat4Accum objects. +** The content to returned is determined by the parameter J ** which is one of the STAT_GET_xxxx values defined above. ** ** If neither STAT3 nor STAT4 are enabled, then J is always @@ -1002,23 +1013,25 @@ static void analyzeOneTable( sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - int nCol; /* Number of columns indexed by pIdx */ + int nCol; /* Number of columns in pIdx. "N" */ int *aGotoChng; /* Array of jump instruction addresses */ int addrRewind; /* Address of "OP_Rewind iIdxCur" */ - int addrGotoChng0; /* Address of "Goto addr_chng_0" */ int addrNextRow; /* Address of "next_row:" */ const char *zIdxName; /* Name of the index */ + int nColTest; /* Number of columns to test for changes */ if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0; if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIdx) ){ nCol = pIdx->nKeyCol; zIdxName = pTab->zName; + nColTest = nCol - 1; }else{ nCol = pIdx->nColumn; zIdxName = pIdx->zName; + nColTest = pIdx->onError==OE_None ? nCol-1 : pIdx->nKeyCol-1; } - aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*(nCol+1)); + aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*(nColTest+1)); if( aGotoChng==0 ) continue; /* Populate the register containing the index name. */ @@ -1061,7 +1074,7 @@ static void analyzeOneTable( ** the regPrev array and a trailing rowid (the rowid slot is required ** when building a record to insert into the sample column of ** the sqlite_stat4 table. */ - pParse->nMem = MAX(pParse->nMem, regPrev+nCol); + pParse->nMem = MAX(pParse->nMem, regPrev+nColTest); /* Open a read-only cursor on the index being analyzed. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); @@ -1071,10 +1084,13 @@ static void analyzeOneTable( /* Invoke the stat_init() function. The arguments are: ** - ** (1) the number of columns in the index including the rowid, - ** (2) the number of rows in the index, + ** (1) the number of columns in the index including the rowid + ** (or for a WITHOUT ROWID table, the number of PK columns), + ** (2) the number of columns in the key without the rowid/pk + ** (3) the number of rows in the index, ** - ** The second argument is only used for STAT3 and STAT4 + ** + ** The third argument is only used for STAT3 and STAT4 */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); @@ -1096,44 +1112,49 @@ static void analyzeOneTable( addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); - addrGotoChng0 = sqlite3VdbeAddOp0(v, OP_Goto); - - /* - ** next_row: - ** regChng = 0 - ** if( idx(0) != regPrev(0) ) goto chng_addr_0 - ** regChng = 1 - ** if( idx(1) != regPrev(1) ) goto chng_addr_1 - ** ... - ** regChng = N - ** goto chng_addr_N - */ addrNextRow = sqlite3VdbeCurrentAddr(v); - for(i=0; iazColl[i]); - sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); - aGotoChng[i] = - sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); - sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); - VdbeCoverage(v); - } - sqlite3VdbeAddOp2(v, OP_Integer, nCol-1, regChng); - aGotoChng[nCol] = sqlite3VdbeAddOp0(v, OP_Goto); - /* - ** chng_addr_0: - ** regPrev(0) = idx(0) - ** chng_addr_1: - ** regPrev(1) = idx(1) - ** ... - */ - sqlite3VdbeJumpHere(v, addrGotoChng0); - for(i=0; i0 ){ + /* + ** next_row: + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto chng_addr_N + */ + sqlite3VdbeAddOp0(v, OP_Goto); + addrNextRow = sqlite3VdbeCurrentAddr(v); + for(i=0; iazColl[i]); + sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); + aGotoChng[i] = + sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + VdbeCoverage(v); + } + sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); + aGotoChng[nColTest] = sqlite3VdbeAddOp0(v, OP_Goto); + + + /* + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... + */ + sqlite3VdbeJumpHere(v, addrNextRow-1); + for(i=0; i Date: Wed, 23 Jul 2014 19:04:25 +0000 Subject: [PATCH 133/710] Enhancements to the hex literal tests. FossilOrigin-Name: a3cc027fa7ca41da23ecd0770a075a48416af020 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/hexlit.test | 28 +++++++--------------------- 3 files changed, 15 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 70612a16ab..eafe791868 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\ssurplus\sfunction\sprototype.\s\s#ifdef\scode\sthat\sis\snot\sused\swhen\nhex\sintegers\sare\somitted\sat\scompile\stime. -D 2014-07-23T14:52:05.196 +C Enhancements\sto\sthe\shex\sliteral\stests. +D 2014-07-23T19:04:25.426 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -601,7 +601,7 @@ F test/fuzz_malloc.test 328f70aaca63adf29b4c6f06505ed0cf57ca7c26 F test/fuzzer1.test d4c52aaf3ef923da293a2653cfab33d02f718a36 F test/fuzzerfault.test 8792cd77fd5bce765b05d0c8e01b9edcf8af8536 F test/genesis.tcl 1e2e2e8e5cc4058549a154ff1892fe5c9de19f98 -F test/hexlit.test 1dc49cfd1c8938a8f028e392775bc3e61623ec46 +F test/hexlit.test f9ecde8145bfc2341573473256c74ae37a200497 F test/hook.test 162d7cef7a2d2b04839fe14402934e6a1b79442f F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e @@ -1184,7 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6c6f0de59bf96b79c8ace8c9bfe48c7a6a306a50 -R 9da9b225ebea231badb4246df3d3d1b3 -U drh -Z 6e030ade89af7e36cbcb367d7755ee7b +P a5b383e077e4b277a65920f7cc202f32f07aa9ee +R 623c41f29572bef99f944ba802bce57d +U mistachkin +Z 3da04c4e52c1109f378a2e2e3819b02f diff --git a/manifest.uuid b/manifest.uuid index af55c0e7ef..af3d91e158 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a5b383e077e4b277a65920f7cc202f32f07aa9ee \ No newline at end of file +a3cc027fa7ca41da23ecd0770a075a48416af020 \ No newline at end of file diff --git a/test/hexlit.test b/test/hexlit.test index 41133aba0f..10909e6f4c 100644 --- a/test/hexlit.test +++ b/test/hexlit.test @@ -84,28 +84,14 @@ hexlit1 160 0X1000000000000000 1152921504606846976 hexlit1 161 0x2000000000000000 2305843009213693952 hexlit1 162 0X4000000000000000 4611686018427387904 hexlit1 163 0x8000000000000000 -9223372036854775808 +hexlit1 164 0XFFFFFFFFFFFFFFFF -1 -hexlit1 200 0x001 1 -hexlit1 201 0X002 2 -hexlit1 202 0x003 3 -hexlit1 203 0X004 4 -hexlit1 204 0x005 5 -hexlit1 205 0X006 6 -hexlit1 206 0x007 7 -hexlit1 207 0X008 8 -hexlit1 208 0x009 9 -hexlit1 209 0x00a 10 -hexlit1 210 0x00A 10 -hexlit1 211 0x00b 11 -hexlit1 212 0x00B 11 -hexlit1 213 0x00c 12 -hexlit1 214 0x00C 12 -hexlit1 215 0x00d 13 -hexlit1 216 0x00D 13 -hexlit1 217 0x00e 14 -hexlit1 218 0x00E 14 -hexlit1 219 0x00f 15 -hexlit1 220 0x00F 15 +for {set n 1} {$n < 0x10} {incr n} { + hexlit1 200.$n.1 0X[format %03X $n] $n + hexlit1 200.$n.2 0x[format %03X $n] $n + hexlit1 200.$n.3 0X[format %03x $n] $n + hexlit1 200.$n.4 0x[format %03x $n] $n +} # String literals that look like hex do not get cast or coerced. # From 6861b8a13fb0ab906c2ab41e6bb6f5d557969365 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 23 Jul 2014 19:37:21 +0000 Subject: [PATCH 134/710] Ugh. Consecutive UNIQUE index entries are only distinct if the index is on NOT NULL columns. So the previous version was not quite right. This check-in fixes the problem. FossilOrigin-Name: 30033f965030a015fad15e532bcaba1314c8cc0f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f572551082..c9cc80f306 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\sperformance\sof\sthe\sANALYZE\scommand\sby\staking\sadvantage\sof\sthe\nfact\sthat\severy\srow\sof\sa\sUNIQUE\sindex\sis\sdistinct. -D 2014-07-23T18:36:55.821 +C Ugh.\s\sConsecutive\sUNIQUE\sindex\sentries\sare\sonly\sdistinct\sif\sthe\sindex\sis\non\sNOT\sNULL\scolumns.\s\sSo\sthe\sprevious\sversion\swas\snot\squite\sright.\s\sThis\ncheck-in\sfixes\sthe\sproblem. +D 2014-07-23T19:37:21.314 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c c72bb66997ebdbc76936cfe96d7cc9da6ffe2b22 +F src/analyze.c 7f893416d1e4375e31e497107e92b28c2f8f82d1 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -1183,7 +1183,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e6225a7bf77a700b318563b1a854b4b3a9e031e1 -R fa098d0ebca9c403761ac8a7cdd64221 +P 3e1e79e1335f7ad33cd35f384f2a063c4aa2253b +R 8df862d446d339da769e0f44096b055d U drh -Z 71590bde4580a255d675842a920968d0 +Z cc08889d86841af01a0687614b9dbde8 diff --git a/manifest.uuid b/manifest.uuid index 2ecff9239b..b9ae50e83b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e1e79e1335f7ad33cd35f384f2a063c4aa2253b \ No newline at end of file +30033f965030a015fad15e532bcaba1314c8cc0f \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 2c31f81f3a..e235389e2d 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1029,7 +1029,7 @@ static void analyzeOneTable( }else{ nCol = pIdx->nColumn; zIdxName = pIdx->zName; - nColTest = pIdx->onError==OE_None ? nCol-1 : pIdx->nKeyCol-1; + nColTest = pIdx->uniqNotNull ? pIdx->nKeyCol-1 : nCol-1; } aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*(nColTest+1)); if( aGotoChng==0 ) continue; From dbd9486d5bada969d2d5abd89edb158eee1d0ec0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 23 Jul 2014 23:57:42 +0000 Subject: [PATCH 135/710] Add experimental "costmult" logic. Only enabled when compiled with -DSQLITE_ENABLE_COSTMULT. FossilOrigin-Name: 729ece40885ed7f52c5981364833fc62281a388b --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 12 +++++++++--- src/sqliteInt.h | 3 +++ src/where.c | 15 ++++++++++++++- 5 files changed, 35 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 449ddf282d..5b3d8c340d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updated\sdocumentation\son\ssqlite3_temp_directory.\s\sNo\schanges\sto\scode. -D 2014-07-23T15:51:29.276 +C Add\sexperimental\s"costmult"\slogic.\s\sOnly\senabled\swhen\scompiled\swith\n-DSQLITE_ENABLE_COSTMULT. +D 2014-07-23T23:57:42.403 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c ab1e44158c25490bfda557f3d690217b2bb509e2 +F src/analyze.c 1c9831015e8c575796a97692d1493ba720d16f27 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -227,7 +227,7 @@ F src/shell.c 566aee8213372a2e81ba0eb34e9759f7b2574009 F src/sqlite.h.in ac4451c9da2771d2f4d702ef89722407242906d9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h b72a09326d7cbd8375ec3d9a04ea5e0cf476beb3 +F src/sqliteInt.h deee99a62cd9fe4f4d094dad45ce7e85c475603e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 0ffd586e04cd069ed6ce7f820194ccaa6cbd669e +F src/where.c 9454af8e5e18d6e91e5169eadfb61878e2cb42b1 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1183,7 +1183,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 48f40861db4fbd10725a2b8b606d44fe16d5bd27 -R c881f4a63c48850e8a9fb96aa00a5db1 +P e6225a7bf77a700b318563b1a854b4b3a9e031e1 +R 07ca3465b9cbcf38e018d96848d50309 U drh -Z cd8aaf55159956edb23d2b840f6a6165 +Z 5f4178ba82b72065a3c11af477d1e9bb diff --git a/manifest.uuid b/manifest.uuid index 470adaeeca..ee7e54f8b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6225a7bf77a700b318563b1a854b4b3a9e031e1 \ No newline at end of file +729ece40885ed7f52c5981364833fc62281a388b \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index c614350ae3..3a6823c107 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1432,10 +1432,13 @@ static void decodeIntArray( if( sqlite3_strglob("unordered*", z)==0 ){ pIndex->bUnordered = 1; }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ - int v32 = 0; - sqlite3GetInt32(z+3, &v32); - pIndex->szIdxRow = sqlite3LogEst(v32); + pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); } +#ifdef SQLITE_ENABLE_COSTMULT + else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ + pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); + } +#endif while( z[0]!=0 && z[0]!=' ' ) z++; while( z[0]==' ' ) z++; } @@ -1484,6 +1487,9 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; +#ifdef SQLITE_ENABLE_COSTMULT + fakeIdx.pTable = pTable; +#endif decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7edc04ca42..03f443f47c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1477,6 +1477,9 @@ struct Table { i16 nCol; /* Number of columns in this table */ u16 nRef; /* Number of pointers to this Table */ LogEst szTabRow; /* Estimated size of each table row in bytes */ +#ifdef SQLITE_ENABLE_COSTMULT + LogEst costMult; /* Cost multiplier for using this table */ +#endif u8 tabFlags; /* Mask of TF_* values */ u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ #ifndef SQLITE_OMIT_ALTERTABLE diff --git a/src/where.c b/src/where.c index ff18d5d4e2..3901dd468f 100644 --- a/src/where.c +++ b/src/where.c @@ -4230,6 +4230,16 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ } } +/* +** Adjust the cost C by the costMult facter T. This only occurs if +** compiled with -DSQLITE_ENABLE_COSTMULT +*/ +#ifdef SQLITE_ENABLE_COSTMULT +# define ApplyCostMultiplier(C,T) C += T +#else +# define ApplyCostMultiplier(C,T) +#endif + /* ** We have so far matched pBuilder->pNew->u.btree.nEq terms of the ** index pIndex. Try to match one more. @@ -4457,6 +4467,7 @@ static int whereLoopAddBtreeIndex( if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } + ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); nOutUnadjusted = pNew->nOut; pNew->rRun += nInMul + nIn; @@ -4671,6 +4682,7 @@ static int whereLoopAddBtree( ** approximately 7*N*log2(N) where N is the number of rows in ** the table being indexed. */ pNew->rSetup = rLogSize + rSize + 28; assert( 28==sqlite3LogEst(7) ); + ApplyCostMultiplier(pNew->rSetup, pTab->costMult); /* TUNING: Each index lookup yields 20 rows in the table. This ** is more than the usual guess of 10 rows, since we have no way ** of knowning how selective the index will ultimately be. It would @@ -4712,6 +4724,7 @@ static int whereLoopAddBtree( pNew->iSortIdx = b ? iSortIdx : 0; /* TUNING: Cost of full table scan is (N*3.0). */ pNew->rRun = rSize + 16; + ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; @@ -4747,7 +4760,7 @@ static int whereLoopAddBtree( if( m!=0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); } - + ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; From 6976c2123b6498e6e5744a8be4caa4fa62de8906 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 24 Jul 2014 12:09:47 +0000 Subject: [PATCH 136/710] In the command-line shell, in CSV output mode, terminate rows with CRNL but do not expand NL characters in data into CRNL. Provide the extra -newline command-line option and the extra argument to .separator to designate an alternative newline character sequence for CSV output. FossilOrigin-Name: 16c8ce10e1530731441e6c4538691b71564684ed --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/shell.c | 49 ++++++++++++++++++++++++++++++++++++------------ test/shell1.test | 9 ++++++--- test/shell5.test | 11 +++++++---- 5 files changed, 59 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 5b3d8c340d..3bd01efe72 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sexperimental\s"costmult"\slogic.\s\sOnly\senabled\swhen\scompiled\swith\n-DSQLITE_ENABLE_COSTMULT. -D 2014-07-23T23:57:42.403 +C In\sthe\scommand-line\sshell,\sin\sCSV\soutput\smode,\sterminate\srows\swith\sCRNL\sbut\ndo\snot\sexpand\sNL\scharacters\sin\sdata\sinto\sCRNL.\s\sProvide\sthe\sextra\s-newline\ncommand-line\soption\sand\sthe\sextra\sargument\sto\s.separator\sto\sdesignate\san\nalternative\snewline\scharacter\ssequence\sfor\sCSV\soutput. +D 2014-07-24T12:09:47.370 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 6762c62e11b504aa014edceab8886495165e3a77 -F src/shell.c 566aee8213372a2e81ba0eb34e9759f7b2574009 +F src/shell.c cca6ea15719f2a3f41b8a1e0030d0b67a8aae3ca F src/sqlite.h.in ac4451c9da2771d2f4d702ef89722407242906d9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -819,11 +819,11 @@ F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21 F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 F test/shared_err.test 0079c05c97d88cfa03989b7c20a8b266983087aa F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test fb080d67c81e8a80a79ea04b36f127209b5bd112 +F test/shell1.test d60946b5fde4d85fe06db7331dfe89011f564350 F test/shell2.test c57da3a381c099b02c813ba156298d5c2f5c93a3 F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29 F test/shell4.test 8a9c08976291e6c6c808b4d718f4a8b299f339f5 -F test/shell5.test ef0c52952a4a96dc1d9ec3b1fa81ec897ca48154 +F test/shell5.test 15a419cc1df21c892ed64f5596ae7a501f2816f2 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 @@ -1183,7 +1183,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e6225a7bf77a700b318563b1a854b4b3a9e031e1 -R 07ca3465b9cbcf38e018d96848d50309 +P 729ece40885ed7f52c5981364833fc62281a388b +R d4352ef214ae9bc2c731800627aed3f1 U drh -Z 5f4178ba82b72065a3c11af477d1e9bb +Z ddfb7b81152d55231f21f3e3700ae2bf diff --git a/manifest.uuid b/manifest.uuid index ee7e54f8b2..c78547e7f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -729ece40885ed7f52c5981364833fc62281a388b \ No newline at end of file +16c8ce10e1530731441e6c4538691b71564684ed \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 97167dc877..ea92b07e37 100644 --- a/src/shell.c +++ b/src/shell.c @@ -64,6 +64,7 @@ #if defined(_WIN32) || defined(WIN32) # include +# include #define isatty(h) _isatty(h) #ifndef access # define access(f,m) _access((f),(m)) @@ -458,6 +459,7 @@ struct callback_data { int showHeader; /* True to show column names in List or Column mode */ char *zDestTable; /* Name of destination table when MODE_Insert */ char separator[20]; /* Separator character for MODE_List */ + char newline[20]; /* Record separator in MODE_Csv */ int colWidth[100]; /* Requested width of each column when in column mode*/ int actualWidth[100]; /* Actual width of each column */ char nullvalue[20]; /* The text to print when a NULL comes back from @@ -659,7 +661,8 @@ static const char needCsvQuote[] = { /* ** Output a single term of CSV. Actually, p->separator is used for ** the separator, which may or may not be a comma. p->nullvalue is -** the null value. Strings are quoted if necessary. +** the null value. Strings are quoted if necessary. The separator +** is only issued if bSep is true. */ static void output_csv(struct callback_data *p, const char *z, int bSep){ FILE *out = p->out; @@ -855,17 +858,26 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int break; } case MODE_Csv: { +#if defined(WIN32) || defined(_WIN32) + fflush(p->out); + _setmode(_fileno(p->out), _O_BINARY); +#endif if( p->cnt++==0 && p->showHeader ){ for(i=0; iout,"\n"); + fprintf(p->out,"%s",p->newline); } - if( azArg==0 ) break; - for(i=0; i0 ){ + for(i=0; iout,"%s",p->newline); } - fprintf(p->out,"\n"); +#if defined(WIN32) || defined(_WIN32) + fflush(p->out); + _setmode(_fileno(p->out), _O_TEXT); +#endif break; } case MODE_Insert: { @@ -1619,7 +1631,8 @@ static char zHelp[] = ".schema ?TABLE? Show the CREATE statements\n" " If TABLE specified, only show tables matching\n" " LIKE pattern TABLE.\n" - ".separator STRING Change separator used by output mode and .import\n" + ".separator STRING ?NL? Change separator used by output mode and .import\n" + " NL is the end-of-line mark for CSV\n" ".shell CMD ARGS... Run CMD ARGS... in a system shell\n" ".show Show the current values for various settings\n" ".stats on|off Turn stats on or off\n" @@ -2751,6 +2764,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c2=='c' && strncmp(azArg[1],"csv",n2)==0 ){ p->mode = MODE_Csv; sqlite3_snprintf(sizeof(p->separator), p->separator, ","); + sqlite3_snprintf(sizeof(p->newline), p->newline, "\r\n"); }else if( c2=='t' && strncmp(azArg[1],"tabs",n2)==0 ){ p->mode = MODE_List; sqlite3_snprintf(sizeof(p->separator), p->separator, "\t"); @@ -3029,13 +3043,16 @@ static int do_meta_command(char *zLine, struct callback_data *p){ #endif if( c=='s' && strncmp(azArg[0], "separator", n)==0 ){ - if( nArg==2 ){ - sqlite3_snprintf(sizeof(p->separator), p->separator, - "%.*s", (int)sizeof(p->separator)-1, azArg[1]); - }else{ - fprintf(stderr, "Usage: .separator STRING\n"); + if( nArg<2 || nArg>3 ){ + fprintf(stderr, "Usage: .separator SEPARATOR ?NEWLINE?\n"); rc = 1; } + if( nArg>=2 ){ + sqlite3_snprintf(sizeof(p->separator), p->separator, azArg[1]); + } + if( nArg>=3 ){ + sqlite3_snprintf(sizeof(p->newline), p->newline, azArg[2]); + } }else if( c=='s' @@ -3076,6 +3093,8 @@ static int do_meta_command(char *zLine, struct callback_data *p){ strlen30(p->outfile) ? p->outfile : "stdout"); fprintf(p->out,"%9.9s: ", "separator"); output_c_string(p->out, p->separator); + fprintf(p->out," "); + output_c_string(p->out, p->newline); fprintf(p->out, "\n"); fprintf(p->out,"%9.9s: %s\n","stats", p->statsOn ? "on" : "off"); fprintf(p->out,"%9.9s: ","width"); @@ -3692,6 +3711,7 @@ static const char zOptions[] = #ifdef SQLITE_ENABLE_MULTIPLEX " -multiplex enable the multiplexor VFS\n" #endif + " -newline SEP set newline character(s) for CSV\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" " -separator SEP set output field separator. Default: '|'\n" " -stats print memory stats before each finalize\n" @@ -3721,6 +3741,7 @@ static void main_init(struct callback_data *data) { memset(data, 0, sizeof(*data)); data->mode = MODE_List; memcpy(data->separator,"|", 2); + memcpy(data->newline,"\r\n", 3); data->showHeader = 0; sqlite3_config(SQLITE_CONFIG_URI, 1); sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); @@ -3813,6 +3834,7 @@ int main(int argc, char **argv){ if( z[1]=='-' ) z++; if( strcmp(z,"-separator")==0 || strcmp(z,"-nullvalue")==0 + || strcmp(z,"-newline")==0 || strcmp(z,"-cmd")==0 ){ (void)cmdline_option_value(argc, argv, ++i); @@ -3922,6 +3944,9 @@ int main(int argc, char **argv){ }else if( strcmp(z,"-separator")==0 ){ sqlite3_snprintf(sizeof(data.separator), data.separator, "%s",cmdline_option_value(argc,argv,++i)); + }else if( strcmp(z,"-newline")==0 ){ + sqlite3_snprintf(sizeof(data.newline), data.newline, + "%s",cmdline_option_value(argc,argv,++i)); }else if( strcmp(z,"-nullvalue")==0 ){ sqlite3_snprintf(sizeof(data.nullvalue), data.nullvalue, "%s",cmdline_option_value(argc,argv,++i)); diff --git a/test/shell1.test b/test/shell1.test index e6fb0c28d2..ab382e74cb 100644 --- a/test/shell1.test +++ b/test/shell1.test @@ -588,14 +588,17 @@ db eval {DROP VIEW v1; DROP VIEW v2; DROP TABLE t1;} # .separator STRING Change separator used by output mode and .import do_test shell1-3.22.1 { catchcmd "test.db" ".separator" -} {1 {Usage: .separator STRING}} +} {1 {Usage: .separator SEPARATOR ?NEWLINE?}} do_test shell1-3.22.2 { catchcmd "test.db" ".separator FOO" } {0 {}} do_test shell1-3.22.3 { + catchcmd "test.db" ".separator ABC XYZ" +} {0 {}} +do_test shell1-3.22.4 { # too many arguments - catchcmd "test.db" ".separator FOO BAD" -} {1 {Usage: .separator STRING}} + catchcmd "test.db" ".separator FOO BAD BAD2" +} {1 {Usage: .separator SEPARATOR ?NEWLINE?}} # .show Show the current values for various settings do_test shell1-3.23.1 { diff --git a/test/shell5.test b/test/shell5.test index 6e9dd20639..8d740cb980 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -55,14 +55,17 @@ do_test shell5-1.1.3 { # .separator STRING Change separator used by output mode and .import do_test shell5-1.2.1 { catchcmd "test.db" ".separator" -} {1 {Usage: .separator STRING}} +} {1 {Usage: .separator SEPARATOR ?NEWLINE?}} do_test shell5-1.2.2 { - catchcmd "test.db" ".separator FOO" + catchcmd "test.db" ".separator ONE" } {0 {}} do_test shell5-1.2.3 { + catchcmd "test.db" ".separator ONE TWO" +} {0 {}} +do_test shell5-1.2.4 { # too many arguments - catchcmd "test.db" ".separator FOO BAD" -} {1 {Usage: .separator STRING}} + catchcmd "test.db" ".separator ONE TWO THREE" +} {1 {Usage: .separator SEPARATOR ?NEWLINE?}} # separator should default to "|" do_test shell5-1.3.1 { From ba5b09319e2f79707bcc55a84a5f059ea0949334 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 24 Jul 2014 12:39:59 +0000 Subject: [PATCH 137/710] Add the readfile(FILENAME) and writefile(FILENAME,CONTENT) SQL functions to the command-line shell. FossilOrigin-Name: fb1048cb2b613a0dbfe625a5df05e9dcd736a433 --- manifest | 13 +++++----- manifest.uuid | 2 +- src/shell.c | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 987d285913..c9c66de972 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\shexadecimal\sinteger\sliterals\sin\sthe\sparser. -D 2014-07-24T12:19:41.241 +C Add\sthe\sreadfile(FILENAME)\sand\swritefile(FILENAME,CONTENT)\sSQL\sfunctions\sto\nthe\scommand-line\sshell. +D 2014-07-24T12:39:59.049 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 6762c62e11b504aa014edceab8886495165e3a77 -F src/shell.c cca6ea15719f2a3f41b8a1e0030d0b67a8aae3ca +F src/shell.c 191129c3f7a9cf241aea90ff6a6be3e74d3767f0 F src/sqlite.h.in ac4451c9da2771d2f4d702ef89722407242906d9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1184,8 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 16c8ce10e1530731441e6c4538691b71564684ed a3cc027fa7ca41da23ecd0770a075a48416af020 -R 800080a0f835a3b88d7f1d5c8ee5f8c1 -T +closed a3cc027fa7ca41da23ecd0770a075a48416af020 +P f8f79f28785db716b10c3bc9d6652b98253fd125 +R 3cb7c1591be4171f813333f040f9142c U drh -Z c1345ae836570db977d9f440b563b830 +Z d6bf09c8f21a2381ecdf6b5a91f85057 diff --git a/manifest.uuid b/manifest.uuid index 566e373026..c704a64b44 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f8f79f28785db716b10c3bc9d6652b98253fd125 \ No newline at end of file +fb1048cb2b613a0dbfe625a5df05e9dcd736a433 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index ea92b07e37..371efa024e 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1650,6 +1650,69 @@ static char zHelp[] = /* Forward reference */ static int process_input(struct callback_data *p, FILE *in); +/* +** Implementation of the "readfile(X)" SQL function. The entire content +** of the file named X is read and returned as a BLOB. NULL is returned +** if the file does not exist or is unreadable. +*/ +static void readfileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName; + FILE *in; + long nIn; + void *pBuf; + + zName = (const char*)sqlite3_value_text(argv[0]); + if( zName==0 ) return; + in = fopen(zName, "rb"); + if( in==0 ) return; + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + pBuf = sqlite3_malloc( nIn ); + if( pBuf && 1==fread(pBuf, nIn, 1, in) ){ + sqlite3_result_blob(context, pBuf, nIn, sqlite3_free); + }else{ + sqlite3_free(pBuf); + } + fclose(in); +} + +/* +** Implementation of the "writefile(X,Y)" SQL function. The argument Y +** is written into file X. The number of bytes written is returned. Or +** NULL is returned if something goes wrong, such as being unable to open +** file X for writing. +*/ +static void writefileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + FILE *out; + const char *z; + int n; + sqlite3_int64 rc; + const char *zFile; + + zFile = (const char*)sqlite3_value_text(argv[0]); + if( zFile==0 ) return; + out = fopen(zFile, "wb"); + if( out==0 ) return; + z = (const char*)sqlite3_value_blob(argv[1]); + if( z==0 ){ + n = 0; + rc = 0; + }else{ + n = sqlite3_value_bytes(argv[1]); + rc = fwrite(z, 1, n, out); + } + fclose(out); + sqlite3_result_int64(context, rc); +} /* ** Make sure the database is open. If it is not, then open it. If @@ -1673,6 +1736,10 @@ static void open_db(struct callback_data *p, int keepAlive){ #ifndef SQLITE_OMIT_LOAD_EXTENSION sqlite3_enable_load_extension(p->db, 1); #endif + sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0, + readfileFunc, 0, 0); + sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0, + writefileFunc, 0, 0); } } From 9d793325b0efb94a4a2395087959e3147aa9c339 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 24 Jul 2014 19:54:20 +0000 Subject: [PATCH 138/710] Avoid change tests when analyzing single-column unique indexes after getting past the initial NULL entries. FossilOrigin-Name: 4690e99c07024f40fafba1db8e4487b287b788a9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 18 +++++++++++++----- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index c9cc80f306..f33fbbea51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ugh.\s\sConsecutive\sUNIQUE\sindex\sentries\sare\sonly\sdistinct\sif\sthe\sindex\sis\non\sNOT\sNULL\scolumns.\s\sSo\sthe\sprevious\sversion\swas\snot\squite\sright.\s\sThis\ncheck-in\sfixes\sthe\sproblem. -D 2014-07-23T19:37:21.314 +C Avoid\schange\stests\swhen\sanalyzing\ssingle-column\sunique\sindexes\safter\ngetting\spast\sthe\sinitial\sNULL\sentries. +D 2014-07-24T19:54:20.150 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 7f893416d1e4375e31e497107e92b28c2f8f82d1 +F src/analyze.c 45216a14e7e71084859d2899cd323b2f597f8527 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -1183,7 +1183,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3e1e79e1335f7ad33cd35f384f2a063c4aa2253b -R 8df862d446d339da769e0f44096b055d +P 30033f965030a015fad15e532bcaba1314c8cc0f +R e98a107e1f5891c465c70c3fe5e38f27 U drh -Z cc08889d86841af01a0687614b9dbde8 +Z 7cfd64b5f36044a7bdee3831c72ee79f diff --git a/manifest.uuid b/manifest.uuid index b9ae50e83b..bfe6770cbb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30033f965030a015fad15e532bcaba1314c8cc0f \ No newline at end of file +4690e99c07024f40fafba1db8e4487b287b788a9 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index e235389e2d..9a64108384 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1031,7 +1031,7 @@ static void analyzeOneTable( zIdxName = pIdx->zName; nColTest = pIdx->uniqNotNull ? pIdx->nKeyCol-1 : nCol-1; } - aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*(nColTest+1)); + aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*nColTest); if( aGotoChng==0 ) continue; /* Populate the register containing the index name. */ @@ -1061,7 +1061,7 @@ static void analyzeOneTable( ** regPrev(1) = idx(1) ** ... ** - ** chng_addr_N: + ** endDistinctTest: ** regRowid = idx(rowid) ** stat_push(P, regChng, regRowid) ** Next csr @@ -1115,6 +1115,7 @@ static void analyzeOneTable( addrNextRow = sqlite3VdbeCurrentAddr(v); if( nColTest>0 ){ + int endDistinctTest = sqlite3VdbeMakeLabel(v); /* ** next_row: ** regChng = 0 @@ -1123,10 +1124,17 @@ static void analyzeOneTable( ** if( idx(1) != regPrev(1) ) goto chng_addr_1 ** ... ** regChng = N - ** goto chng_addr_N + ** goto endDistinctTest */ sqlite3VdbeAddOp0(v, OP_Goto); addrNextRow = sqlite3VdbeCurrentAddr(v); + if( nColTest==1 && pIdx->nKeyCol==1 && pIdx->onError!=OE_None ){ + /* For a single-column UNIQUE index, once we have found a non-NULL + ** row, we know that all the rest will be distinct, so skip + ** subsequent distinctness tests. */ + sqlite3VdbeAddOp2(v, OP_NotNull, regPrev, endDistinctTest); + VdbeCoverage(v); + } for(i=0; iazColl[i]); sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); @@ -1137,7 +1145,7 @@ static void analyzeOneTable( VdbeCoverage(v); } sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); - aGotoChng[nColTest] = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeAddOp2(v, OP_Goto, 0, endDistinctTest); /* @@ -1152,7 +1160,7 @@ static void analyzeOneTable( sqlite3VdbeJumpHere(v, aGotoChng[i]); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regPrev+i); } - sqlite3VdbeJumpHere(v, aGotoChng[nColTest]); + sqlite3VdbeResolveLabel(v, endDistinctTest); } /* From b12879fd1af673bcb41eec70c50ca3726196f693 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 24 Jul 2014 20:25:16 +0000 Subject: [PATCH 139/710] Avoid trying to allocation zero bytes when analyzing a unique non-null index. FossilOrigin-Name: 85e2badeeb7f7599eb6fd35512f9bd524f0b1b3f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 9 +++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f33fbbea51..6c8e62a3ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\schange\stests\swhen\sanalyzing\ssingle-column\sunique\sindexes\safter\ngetting\spast\sthe\sinitial\sNULL\sentries. -D 2014-07-24T19:54:20.150 +C Avoid\strying\sto\sallocation\szero\sbytes\swhen\sanalyzing\sa\sunique\snon-null\sindex. +D 2014-07-24T20:25:16.037 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 45216a14e7e71084859d2899cd323b2f597f8527 +F src/analyze.c 9799a65d8e8892e6eb96d0094a1d4fa6d7aebd0e F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -1183,7 +1183,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 30033f965030a015fad15e532bcaba1314c8cc0f -R e98a107e1f5891c465c70c3fe5e38f27 +P 4690e99c07024f40fafba1db8e4487b287b788a9 +R 665515c1fc4ab29e7e80bbd8e675bbe2 U drh -Z 7cfd64b5f36044a7bdee3831c72ee79f +Z 06b30e9b7410b92f6a1122ec1ae06529 diff --git a/manifest.uuid b/manifest.uuid index bfe6770cbb..e165c09bcc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4690e99c07024f40fafba1db8e4487b287b788a9 \ No newline at end of file +85e2badeeb7f7599eb6fd35512f9bd524f0b1b3f \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 9a64108384..3931b2ff8c 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1014,7 +1014,6 @@ static void analyzeOneTable( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ - int *aGotoChng; /* Array of jump instruction addresses */ int addrRewind; /* Address of "OP_Rewind iIdxCur" */ int addrNextRow; /* Address of "next_row:" */ const char *zIdxName; /* Name of the index */ @@ -1031,8 +1030,6 @@ static void analyzeOneTable( zIdxName = pIdx->zName; nColTest = pIdx->uniqNotNull ? pIdx->nKeyCol-1 : nCol-1; } - aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*nColTest); - if( aGotoChng==0 ) continue; /* Populate the register containing the index name. */ sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, zIdxName, 0); @@ -1116,6 +1113,10 @@ static void analyzeOneTable( if( nColTest>0 ){ int endDistinctTest = sqlite3VdbeMakeLabel(v); + int *aGotoChng; /* Array of jump instruction addresses */ + aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*nColTest); + if( aGotoChng==0 ) continue; + /* ** next_row: ** regChng = 0 @@ -1161,6 +1162,7 @@ static void analyzeOneTable( sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regPrev+i); } sqlite3VdbeResolveLabel(v, endDistinctTest); + sqlite3DbFree(db, aGotoChng); } /* @@ -1247,7 +1249,6 @@ static void analyzeOneTable( /* End of analysis */ sqlite3VdbeJumpHere(v, addrRewind); - sqlite3DbFree(db, aGotoChng); } From d15f87ecfdf0f1bf12d58da5284c9792f966f4d4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 24 Jul 2014 22:41:20 +0000 Subject: [PATCH 140/710] Fix a bug in the whereRangeSkipScanEst() procedure (added by check-in [d09ca6d5efad3e4cfa]) where it fails to consider the possibility of a ROWID column when computing the affinity of a table column. FossilOrigin-Name: 6aea2258dd7d3967900cefe4cd0af48a7ee7e169 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c9c66de972..87c15adf11 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sreadfile(FILENAME)\sand\swritefile(FILENAME,CONTENT)\sSQL\sfunctions\sto\nthe\scommand-line\sshell. -D 2014-07-24T12:39:59.049 +C Fix\sa\sbug\sin\sthe\swhereRangeSkipScanEst()\sprocedure\s(added\sby\ncheck-in\s[d09ca6d5efad3e4cfa])\swhere\sit\sfails\sto\sconsider\sthe\spossibility\nof\sa\sROWID\scolumn\swhen\scomputing\sthe\saffinity\sof\sa\stable\scolumn. +D 2014-07-24T22:41:20.186 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 9454af8e5e18d6e91e5169eadfb61878e2cb42b1 +F src/where.c 4dfcd80380d154be434c4b51e890e17ce9754b3e F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1184,7 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f8f79f28785db716b10c3bc9d6652b98253fd125 -R 3cb7c1591be4171f813333f040f9142c +P fb1048cb2b613a0dbfe625a5df05e9dcd736a433 +R 7154486a7ce11d45d2aed19cdac8eb64 U drh -Z d6bf09c8f21a2381ecdf6b5a91f85057 +Z 3de5689fcbb67ba500bf3c297659c386 diff --git a/manifest.uuid b/manifest.uuid index c704a64b44..8f811c9bc6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb1048cb2b613a0dbfe625a5df05e9dcd736a433 \ No newline at end of file +6aea2258dd7d3967900cefe4cd0af48a7ee7e169 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 3901dd468f..38d1014ac0 100644 --- a/src/where.c +++ b/src/where.c @@ -2047,7 +2047,8 @@ static int whereRangeSkipScanEst( int nLower = -1; int nUpper = p->nSample+1; int rc = SQLITE_OK; - u8 aff = p->pTable->aCol[ p->aiColumn[nEq] ].affinity; + int iCol = p->aiColumn[nEq]; + u8 aff = iCol>=0 ? p->pTable->aCol[iCol].affinity : SQLITE_AFF_INTEGER; CollSeq *pColl; sqlite3_value *p1 = 0; /* Value extracted from pLower */ From 8af3f772c10e55d473e0cbe08e879101e66877ce Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 25 Jul 2014 18:01:06 +0000 Subject: [PATCH 141/710] Add constraints (enforced only when SQLITE_DEBUG is enabled) on the use of OP_Next and OP_Prev. FossilOrigin-Name: 2230c74f1efa591770176c9b40e920724a3c39e1 --- manifest | 15 +++++---- manifest.uuid | 2 +- src/vdbe.c | 84 ++++++++++++++++++++++++++++++++++++++++++++++++--- src/vdbeInt.h | 3 ++ 4 files changed, 90 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index a378b1468d..30ae97fa3c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\sperformance\sof\sthe\sANALYZE\scommand\sby\staking\sadvantage\sof\nUNIQUE\sconstraints\son\sindices. -D 2014-07-24T23:23:26.687 +C Add\sconstraints\s(enforced\sonly\swhen\sSQLITE_DEBUG\sis\senabled)\son\sthe\suse\sof\nOP_Next\sand\sOP_Prev. +D 2014-07-25T18:01:06.435 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,9 +283,9 @@ F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c fa74c6563486022920db4d73897bd9b837c7441d +F src/vdbe.c c6a9094a56b3dd4769243051a512f89763af7de5 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 -F src/vdbeInt.h f41a7cf5a40e7abd994d5fe3a04589ae3e1154e7 +F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 1ffe0bbc3a2c8aedd95622de9be6b6d07b3d3c6b F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac @@ -1184,8 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6aea2258dd7d3967900cefe4cd0af48a7ee7e169 85e2badeeb7f7599eb6fd35512f9bd524f0b1b3f -R ac28d1873bd62cdf71787453262a0966 -T +closed 85e2badeeb7f7599eb6fd35512f9bd524f0b1b3f +P 114dcf33670fd98a1ebbac0e44f66b2d8bcccddf +R f27032fe9d592a52e4ec63c28d270176 U drh -Z 72b5de0db489041a156923cd906bfefb +Z 96714d48e7c72e5d439e8f51e5a0d88e diff --git a/manifest.uuid b/manifest.uuid index a9e63c9b42..9d6edd3229 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -114dcf33670fd98a1ebbac0e44f66b2d8bcccddf \ No newline at end of file +2230c74f1efa591770176c9b40e920724a3c39e1 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 093669a82d..60d8ebf3e5 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3481,7 +3481,7 @@ case OP_Close: { break; } -/* Opcode: SeekGe P1 P2 P3 P4 * +/* Opcode: SeekGE P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** ** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), @@ -3493,9 +3493,13 @@ case OP_Close: { ** is greater than or equal to the key value. If there are no records ** greater than or equal to the key and P2 is not zero, then jump to P2. ** +** This opcode leaves the cursor configured to move in forward order, +** from the begining toward the end. In other words, the cursor is +** configured to use OP_Next, not OP_Prev. +** ** See also: Found, NotFound, SeekLt, SeekGt, SeekLe */ -/* Opcode: SeekGt P1 P2 P3 P4 * +/* Opcode: SeekGT P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** ** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), @@ -3507,9 +3511,13 @@ case OP_Close: { ** is greater than the key value. If there are no records greater than ** the key and P2 is not zero, then jump to P2. ** +** This opcode leaves the cursor configured to move in forward order, +** from the begining toward the end. In other words, the cursor is +** configured to use OP_Next, not OP_Prev. +** ** See also: Found, NotFound, SeekLt, SeekGe, SeekLe */ -/* Opcode: SeekLt P1 P2 P3 P4 * +/* Opcode: SeekLT P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** ** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), @@ -3521,9 +3529,13 @@ case OP_Close: { ** is less than the key value. If there are no records less than ** the key and P2 is not zero, then jump to P2. ** +** This opcode leaves the cursor configured to move in reverse order, +** from the end toward the beginning. In other words, the cursor is +** configured to use OP_Prev, not OP_Next. +** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLe */ -/* Opcode: SeekLe P1 P2 P3 P4 * +/* Opcode: SeekLE P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** ** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), @@ -3535,6 +3547,10 @@ case OP_Close: { ** is less than or equal to the key value. If there are no records ** less than or equal to the key and P2 is not zero, then jump to P2. ** +** This opcode leaves the cursor configured to move in reverse order, +** from the end toward the beginning. In other words, the cursor is +** configured to use OP_Prev, not OP_Next. +** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ case OP_SeekLT: /* jump, in3 */ @@ -3560,6 +3576,9 @@ case OP_SeekGT: { /* jump, in3 */ assert( pC->pCursor!=0 ); oc = pOp->opcode; pC->nullRow = 0; +#ifdef SQLITE_DEBUG + pC->seekOp = pOp->opcode; +#endif if( pC->isTable ){ /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do @@ -3715,6 +3734,10 @@ case OP_Seek: { /* in2 */ ** is a prefix of any entry in P1 then a jump is made to P2 and ** P1 is left pointing at the matching entry. ** +** This operation leaves the cursor in a state where it cannot be +** advanced in either direction. In other words, the Next and Prev +** opcodes do not work after this operation. +** ** See also: NotFound, NoConflict, NotExists. SeekGe */ /* Opcode: NotFound P1 P2 P3 P4 * @@ -3730,6 +3753,10 @@ case OP_Seek: { /* in2 */ ** falls through to the next instruction and P1 is left pointing at the ** matching entry. ** +** This operation leaves the cursor in a state where it cannot be +** advanced in either direction. In other words, the Next and Prev +** opcodes do not work after this operation. +** ** See also: Found, NotExists, NoConflict */ /* Opcode: NoConflict P1 P2 P3 P4 * @@ -3749,6 +3776,10 @@ case OP_Seek: { /* in2 */ ** This opcode is similar to OP_NotFound with the exceptions that the ** branch is always taken if any part of the search key input is NULL. ** +** This operation leaves the cursor in a state where it cannot be +** advanced in either direction. In other words, the Next and Prev +** opcodes do not work after this operation. +** ** See also: NotFound, Found, NotExists */ case OP_NoConflict: /* jump, in3 */ @@ -3771,6 +3802,9 @@ case OP_Found: { /* jump, in3 */ assert( pOp->p4type==P4_INT32 ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); +#ifdef SQLITE_DEBUG + pC->seekOp = 0; +#endif pIn3 = &aMem[pOp->p3]; assert( pC->pCursor!=0 ); assert( pC->isTable==0 ); @@ -3842,6 +3876,10 @@ case OP_Found: { /* jump, in3 */ ** The OP_NotFound opcode performs the same operation on index btrees ** (with arbitrary multi-value keys). ** +** This opcode leaves the cursor in a state where it cannot be advanced +** in either direction. In other words, the Next and Prev opcodes will +** not work following this opcode. +** ** See also: Found, NotFound, NoConflict */ case OP_NotExists: { /* jump, in3 */ @@ -3855,6 +3893,9 @@ case OP_NotExists: { /* jump, in3 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); +#ifdef SQLITE_DEBUG + pC->seekOp = 0; +#endif assert( pC->isTable ); assert( pC->pseudoTableReg==0 ); pCrsr = pC->pCursor; @@ -4421,11 +4462,15 @@ case OP_NullRow: { /* Opcode: Last P1 P2 * * * ** -** The next use of the Rowid or Column or Next instruction for P1 +** The next use of the Rowid or Column or Prev instruction for P1 ** will refer to the last entry in the database table or index. ** If the table or index is empty and P2>0, then jump immediately to P2. ** If P2 is 0 or if the table or index is not empty, fall through ** to the following instruction. +** +** This opcode leaves the cursor configured to move in reverse order, +** from the end toward the beginning. In other words, the cursor is +** configured to use OP_Prev, not OP_Next. */ case OP_Last: { /* jump */ VdbeCursor *pC; @@ -4443,6 +4488,9 @@ case OP_Last: { /* jump */ pC->deferredMoveto = 0; pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; +#ifdef SQLITE_DEBUG + pC->seekOp = OP_Last; +#endif if( pOp->p2>0 ){ VdbeBranchTaken(res!=0,2); if( res ) pc = pOp->p2 - 1; @@ -4479,6 +4527,10 @@ case OP_Sort: { /* jump */ ** If the table or index is empty and P2>0, then jump immediately to P2. ** If P2 is 0 or if the table or index is not empty, fall through ** to the following instruction. +** +** This opcode leaves the cursor configured to move in forward order, +** from the begining toward the end. In other words, the cursor is +** configured to use OP_Next, not OP_Prev. */ case OP_Rewind: { /* jump */ VdbeCursor *pC; @@ -4490,6 +4542,9 @@ case OP_Rewind: { /* jump */ assert( pC!=0 ); assert( isSorter(pC)==(pOp->opcode==OP_SorterSort) ); res = 1; +#ifdef SQLITE_DEBUG + pC->seekOp = OP_Rewind; +#endif if( isSorter(pC) ){ rc = sqlite3VdbeSorterRewind(db, pC, &res); }else{ @@ -4516,6 +4571,10 @@ case OP_Rewind: { /* jump */ ** to the following instruction. But if the cursor advance was successful, ** jump immediately to P2. ** +** The OP_Next opcode is only valid following an OP_SeekGT, OP_SeekGE, or +** OP_Rewind opcode used to position the cursor. OP_Next is not allowed +** to follow OP_SeekLT, OP_SeekLE, or OP_Last. +** ** The P1 cursor must be for a real table, not a pseudo-table. P1 must have ** been opened prior to this opcode or the program will segfault. ** @@ -4544,6 +4603,11 @@ case OP_Rewind: { /* jump */ ** to the following instruction. But if the cursor backup was successful, ** jump immediately to P2. ** +** +** The OP_Prev opcode is only valid following an OP_SeekLT, OP_SeekLE, or +** OP_Last opcode used to position the cursor. OP_Prev is not allowed +** to follow OP_SeekGT, OP_SeekGE, or OP_Rewind. +** ** The P1 cursor must be for a real table, not a pseudo-table. If P1 is ** not open then the behavior is undefined. ** @@ -4591,6 +4655,16 @@ case OP_Next: /* jump */ assert( pOp->opcode!=OP_Prev || pOp->p4.xAdvance==sqlite3BtreePrevious ); assert( pOp->opcode!=OP_NextIfOpen || pOp->p4.xAdvance==sqlite3BtreeNext ); assert( pOp->opcode!=OP_PrevIfOpen || pOp->p4.xAdvance==sqlite3BtreePrevious); + + /* The Next opcode is only used after SeekGT, SeekGE, and Rewind. + ** The Prev opcode is only used after SeekLT, SeekLE, and Last. */ + assert( pOp->opcode!=OP_Next || pOp->opcode!=OP_NextIfOpen + || pC->seekOp==OP_SeekGT || pC->seekOp==OP_SeekGE + || pC->seekOp==OP_Rewind ); + assert( pOp->opcode!=OP_Prev || pOp->opcode!=OP_PrevIfOpen + || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE + || pC->seekOp==OP_Last ); + rc = pOp->p4.xAdvance(pC->pCursor, &res); next_tail: pC->cacheStatus = CACHE_STALE; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index f541aa7170..141573eef4 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -68,6 +68,9 @@ struct VdbeCursor { int pseudoTableReg; /* Register holding pseudotable content. */ i16 nField; /* Number of fields in the header */ u16 nHdrParsed; /* Number of header fields parsed so far */ +#ifdef SQLITE_DEBUG + u8 seekOp; /* Most recent seek operation on this cursor */ +#endif i8 iDb; /* Index of cursor database in db->aDb[] (or -1) */ u8 nullRow; /* True if pointing to a row with no data */ u8 rowidIsValid; /* True if lastRowid is valid */ From 5dad9a3497f47f9277c8d1562b814e9292371d0c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 25 Jul 2014 18:37:42 +0000 Subject: [PATCH 142/710] Improved comments on VDBE opcodes, for better documentation. No code or logic changes. FossilOrigin-Name: 2d32e4876e0b162730f81e5c2658be12d64a9a99 --- manifest | 12 ++++---- manifest.uuid | 2 +- src/vdbe.c | 76 +++++++++++++++++++++++++++++++-------------------- 3 files changed, 53 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index 30ae97fa3c..34fbf9cae2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sconstraints\s(enforced\sonly\swhen\sSQLITE_DEBUG\sis\senabled)\son\sthe\suse\sof\nOP_Next\sand\sOP_Prev. -D 2014-07-25T18:01:06.435 +C Improved\scomments\son\sVDBE\sopcodes,\sfor\sbetter\sdocumentation.\s\sNo\scode\nor\slogic\schanges. +D 2014-07-25T18:37:42.754 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c c6a9094a56b3dd4769243051a512f89763af7de5 +F src/vdbe.c b7af861342a215c7f27d3f20d92fe0b70f54fb3e F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 @@ -1184,7 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 114dcf33670fd98a1ebbac0e44f66b2d8bcccddf -R f27032fe9d592a52e4ec63c28d270176 +P 2230c74f1efa591770176c9b40e920724a3c39e1 +R 29600123a95daa5177cd67bf8abb597c U drh -Z 96714d48e7c72e5d439e8f51e5a0d88e +Z 36e2e79e107a063c258f47507608b039 diff --git a/manifest.uuid b/manifest.uuid index 9d6edd3229..a16837f979 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2230c74f1efa591770176c9b40e920724a3c39e1 \ No newline at end of file +2d32e4876e0b162730f81e5c2658be12d64a9a99 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 60d8ebf3e5..e53b9251a3 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -767,12 +767,14 @@ case OP_Return: { /* in1 */ /* Opcode: InitCoroutine P1 P2 P3 * * ** -** Set up register P1 so that it will OP_Yield to the co-routine +** Set up register P1 so that it will Yield to the coroutine ** located at address P3. ** -** If P2!=0 then the co-routine implementation immediately follows -** this opcode. So jump over the co-routine implementation to +** If P2!=0 then the coroutine implementation immediately follows +** this opcode. So jump over the coroutine implementation to ** address P2. +** +** See also: EndCoroutine */ case OP_InitCoroutine: { /* jump */ assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); @@ -788,9 +790,11 @@ case OP_InitCoroutine: { /* jump */ /* Opcode: EndCoroutine P1 * * * * ** -** The instruction at the address in register P1 is an OP_Yield. -** Jump to the P2 parameter of that OP_Yield. +** The instruction at the address in register P1 is an Yield. +** Jump to the P2 parameter of that Yield. ** After the jump, register P1 becomes undefined. +** +** See also: InitCoroutine */ case OP_EndCoroutine: { /* in1 */ VdbeOp *pCaller; @@ -807,11 +811,16 @@ case OP_EndCoroutine: { /* in1 */ /* Opcode: Yield P1 P2 * * * ** -** Swap the program counter with the value in register P1. +** Swap the program counter with the value in register P1. This +** has the effect of yielding to a coroutine. ** -** If the co-routine ends with OP_Yield or OP_Return then continue -** to the next instruction. But if the co-routine ends with -** OP_EndCoroutine, jump immediately to P2. +** If the coroutine that is launched by this instruction ends with +** Yield or Return then continue to the next instruction. But if +** the coroutine launched by this instruction ends with +** EndCoroutine, then jump to P2 rather than continuing with the +** next instruction. +** +** See also: InitCoroutine */ case OP_Yield: { /* in1, jump */ int pcDest; @@ -2196,10 +2205,14 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ /* Opcode: Once P1 P2 * * * ** -** Check if OP_Once flag P1 is set. If so, jump to instruction P2. Otherwise, -** set the flag and fall through to the next instruction. In other words, -** this opcode causes all following opcodes up through P2 (but not including -** P2) to run just once and to be skipped on subsequent times through the loop. +** Check the "once" flag number P1. If it is set, jump to instruction P2. +** Otherwise, set the flag and fall through to the next instruction. +** In other words, this opcode causes all following opcodes up through P2 +** (but not including P2) to run just once and to be skipped on subsequent +** times through the loop. +** +** All "once" flags are initially cleared whenever a prepared statement +** first begins to run. */ case OP_Once: { /* jump */ assert( pOp->p1nOnceFlag ); @@ -3495,7 +3508,7 @@ case OP_Close: { ** ** This opcode leaves the cursor configured to move in forward order, ** from the begining toward the end. In other words, the cursor is -** configured to use OP_Next, not OP_Prev. +** configured to use Next, not Prev. ** ** See also: Found, NotFound, SeekLt, SeekGt, SeekLe */ @@ -3513,7 +3526,7 @@ case OP_Close: { ** ** This opcode leaves the cursor configured to move in forward order, ** from the begining toward the end. In other words, the cursor is -** configured to use OP_Next, not OP_Prev. +** configured to use Next, not Prev. ** ** See also: Found, NotFound, SeekLt, SeekGe, SeekLe */ @@ -3531,7 +3544,7 @@ case OP_Close: { ** ** This opcode leaves the cursor configured to move in reverse order, ** from the end toward the beginning. In other words, the cursor is -** configured to use OP_Prev, not OP_Next. +** configured to use Prev, not Next. ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLe */ @@ -3549,7 +3562,7 @@ case OP_Close: { ** ** This opcode leaves the cursor configured to move in reverse order, ** from the end toward the beginning. In other words, the cursor is -** configured to use OP_Prev, not OP_Next. +** configured to use Prev, not Next. ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ @@ -4470,7 +4483,7 @@ case OP_NullRow: { ** ** This opcode leaves the cursor configured to move in reverse order, ** from the end toward the beginning. In other words, the cursor is -** configured to use OP_Prev, not OP_Next. +** configured to use Prev, not Next. */ case OP_Last: { /* jump */ VdbeCursor *pC; @@ -4530,7 +4543,7 @@ case OP_Sort: { /* jump */ ** ** This opcode leaves the cursor configured to move in forward order, ** from the begining toward the end. In other words, the cursor is -** configured to use OP_Next, not OP_Prev. +** configured to use Next, not Prev. */ case OP_Rewind: { /* jump */ VdbeCursor *pC; @@ -4571,9 +4584,9 @@ case OP_Rewind: { /* jump */ ** to the following instruction. But if the cursor advance was successful, ** jump immediately to P2. ** -** The OP_Next opcode is only valid following an OP_SeekGT, OP_SeekGE, or -** OP_Rewind opcode used to position the cursor. OP_Next is not allowed -** to follow OP_SeekLT, OP_SeekLE, or OP_Last. +** The Next opcode is only valid following an SeekGT, SeekGE, or +** OP_Rewind opcode used to position the cursor. Next is not allowed +** to follow SeekLT, SeekLE, or OP_Last. ** ** The P1 cursor must be for a real table, not a pseudo-table. P1 must have ** been opened prior to this opcode or the program will segfault. @@ -4593,7 +4606,7 @@ case OP_Rewind: { /* jump */ */ /* Opcode: NextIfOpen P1 P2 P3 P4 P5 ** -** This opcode works just like OP_Next except that if cursor P1 is not +** This opcode works just like Next except that if cursor P1 is not ** open it behaves a no-op. */ /* Opcode: Prev P1 P2 P3 P4 P5 @@ -4604,9 +4617,9 @@ case OP_Rewind: { /* jump */ ** jump immediately to P2. ** ** -** The OP_Prev opcode is only valid following an OP_SeekLT, OP_SeekLE, or -** OP_Last opcode used to position the cursor. OP_Prev is not allowed -** to follow OP_SeekGT, OP_SeekGE, or OP_Rewind. +** The Prev opcode is only valid following an SeekLT, SeekLE, or +** OP_Last opcode used to position the cursor. Prev is not allowed +** to follow SeekGT, SeekGE, or OP_Rewind. ** ** The P1 cursor must be for a real table, not a pseudo-table. If P1 is ** not open then the behavior is undefined. @@ -4624,7 +4637,7 @@ case OP_Rewind: { /* jump */ */ /* Opcode: PrevIfOpen P1 P2 P3 P4 P5 ** -** This opcode works just like OP_Prev except that if cursor P1 is not +** This opcode works just like Prev except that if cursor P1 is not ** open it behaves a no-op. */ case OP_SorterNext: { /* jump */ @@ -5145,7 +5158,8 @@ case OP_LoadAnalysis: { ** ** Remove the internal (in-memory) data structures that describe ** the table named P4 in database P1. This is called after a table -** is dropped in order to keep the internal representation of the +** is dropped from disk (using the Destroy opcode) in order to keep +** the internal representation of the ** schema consistent with what is on disk. */ case OP_DropTable: { @@ -5157,7 +5171,8 @@ case OP_DropTable: { ** ** Remove the internal (in-memory) data structures that describe ** the index named P4 in database P1. This is called after an index -** is dropped in order to keep the internal representation of the +** is dropped from disk (using the Destroy opcode) +** in order to keep the internal representation of the ** schema consistent with what is on disk. */ case OP_DropIndex: { @@ -5169,7 +5184,8 @@ case OP_DropIndex: { ** ** Remove the internal (in-memory) data structures that describe ** the trigger named P4 in database P1. This is called after a trigger -** is dropped in order to keep the internal representation of the +** is dropped from disk (using the Destroy opcode) in order to keep +** the internal representation of the ** schema consistent with what is on disk. */ case OP_DropTrigger: { From eac5bd78704006937a9bb5bee2dd724c7682b5e9 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 25 Jul 2014 21:35:39 +0000 Subject: [PATCH 143/710] Try to fix harmless compiler warnings reported by Fortify. FossilOrigin-Name: e0fa6fdc14ac5458f9200cbae124f8025ea534ea --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/btree.c | 2 +- src/pragma.c | 4 ++-- src/sqliteInt.h | 2 +- src/vdbeaux.c | 6 +++--- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 34fbf9cae2..1097f4e5de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\son\sVDBE\sopcodes,\sfor\sbetter\sdocumentation.\s\sNo\scode\nor\slogic\schanges. -D 2014-07-25T18:37:42.754 +C Try\sto\sfix\sharmless\scompiler\swarnings\sreported\sby\sFortify. +D 2014-07-25T21:35:39.767 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 70c60a3807b2072982f184d9614e020d2953f89c +F src/btree.c b5531339cd826af46b9621e4a9323971a9380e12 F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 48f400fa14fd6add244b954ce7e223ce7ccacf0b @@ -216,7 +216,7 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c e17c5ea1cb9eb9d93c41bbb7c3a17e747d5e0335 +F src/pragma.c 30f3b2ac09fef58320375d78e0e18b976198fc69 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c af06f66927919730f03479fed6ae9854f73419f4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -227,7 +227,7 @@ F src/shell.c 191129c3f7a9cf241aea90ff6a6be3e74d3767f0 F src/sqlite.h.in ac4451c9da2771d2f4d702ef89722407242906d9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 152134c6223369e2e1acb7b9cc2c0db052dae28a +F src/sqliteInt.h 068e42f41a09ce6b9edbe194ac8a470ab53145df F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -287,7 +287,7 @@ F src/vdbe.c b7af861342a215c7f27d3f20d92fe0b70f54fb3e F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 -F src/vdbeaux.c 1ffe0bbc3a2c8aedd95622de9be6b6d07b3d3c6b +F src/vdbeaux.c ac063f36c929f88bf6cecdbcc413000e272265bb F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c 44441d73b08b3a638dcdb725afffb87c6574ad27 @@ -1184,7 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2230c74f1efa591770176c9b40e920724a3c39e1 -R 29600123a95daa5177cd67bf8abb597c +P 2d32e4876e0b162730f81e5c2658be12d64a9a99 +R 863e6abd5a7272b515477418397f96e6 U drh -Z 36e2e79e107a063c258f47507608b039 +Z f462357bfa110cd51b2e07287181f811 diff --git a/manifest.uuid b/manifest.uuid index a16837f979..908cd7712f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d32e4876e0b162730f81e5c2658be12d64a9a99 \ No newline at end of file +e0fa6fdc14ac5458f9200cbae124f8025ea534ea \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9032a9304c..b72a2ab01d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1632,7 +1632,7 @@ static Pgno btreePagecount(BtShared *pBt){ u32 sqlite3BtreeLastPage(Btree *p){ assert( sqlite3BtreeHoldsMutex(p) ); assert( ((p->pBt->nPage)&0x8000000)==0 ); - return (int)btreePagecount(p->pBt); + return btreePagecount(p->pBt); } /* diff --git a/src/pragma.c b/src/pragma.c index 709662c989..c4374cc71c 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -480,7 +480,7 @@ static const struct sPragmaNames { ** to support legacy SQL code. The safety level used to be boolean ** and older scripts may have used numbers 0 for OFF and 1 for ON. */ -static u8 getSafetyLevel(const char *z, int omitFull, int dflt){ +static u8 getSafetyLevel(const char *z, int omitFull, u8 dflt){ /* 123456789 123456789 */ static const char zText[] = "onoffalseyestruefull"; static const u8 iOffset[] = {0, 1, 2, 4, 9, 12, 16}; @@ -502,7 +502,7 @@ static u8 getSafetyLevel(const char *z, int omitFull, int dflt){ /* ** Interpret the given string as a boolean value. */ -u8 sqlite3GetBoolean(const char *z, int dflt){ +u8 sqlite3GetBoolean(const char *z, u8 dflt){ return getSafetyLevel(z,1,dflt)!=0; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7a5d225b84..5d72295d66 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3364,7 +3364,7 @@ void sqlite3FileSuffix3(const char*, char*); #else # define sqlite3FileSuffix3(X,Y) #endif -u8 sqlite3GetBoolean(const char *z,int); +u8 sqlite3GetBoolean(const char *z,u8); const void *sqlite3ValueText(sqlite3_value*, u8); int sqlite3ValueBytes(sqlite3_value*, u8); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index ac945df59e..240085bfbf 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2786,7 +2786,7 @@ int sqlite3VdbeCursorMoveto(VdbeCursor *p){ */ u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ int flags = pMem->flags; - int n; + u32 n; if( flags&MEM_Null ){ return 0; @@ -2816,11 +2816,11 @@ u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ return 7; } assert( pMem->db->mallocFailed || flags&(MEM_Str|MEM_Blob) ); - n = pMem->n; + assert( pMem->n>=0 ); + n = (u32)pMem->n; if( flags & MEM_Zero ){ n += pMem->u.nZero; } - assert( n>=0 ); return ((n*2) + 12 + ((flags&MEM_Str)!=0)); } From 975b4c6e8bc70bfb0d8f29107d393f692d162a69 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 26 Jul 2014 16:47:23 +0000 Subject: [PATCH 144/710] Avoid unnecessary no-op calls to applyNumericAffinity() for a small performance improvement. FossilOrigin-Name: 413d7287977702fa651c0140bd5cf29021fe3e79 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 32 ++++++++++++++++---------------- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 1097f4e5de..480568c8f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Try\sto\sfix\sharmless\scompiler\swarnings\sreported\sby\sFortify. -D 2014-07-25T21:35:39.767 +C Avoid\sunnecessary\sno-op\scalls\sto\sapplyNumericAffinity()\sfor\sa\ssmall\nperformance\simprovement. +D 2014-07-26T16:47:23.387 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c b7af861342a215c7f27d3f20d92fe0b70f54fb3e +F src/vdbe.c 145f909c1c7b16b0899bcab8b241b811e4af9534 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 @@ -1184,7 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2d32e4876e0b162730f81e5c2658be12d64a9a99 -R 863e6abd5a7272b515477418397f96e6 +P e0fa6fdc14ac5458f9200cbae124f8025ea534ea +R 497e2677c499db4d4a797e5c1acd577b U drh -Z f462357bfa110cd51b2e07287181f811 +Z ee817a86e8cc0ab5547d2241830d36e2 diff --git a/manifest.uuid b/manifest.uuid index 908cd7712f..06b44f1d10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e0fa6fdc14ac5458f9200cbae124f8025ea534ea \ No newline at end of file +413d7287977702fa651c0140bd5cf29021fe3e79 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index e53b9251a3..49183129aa 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -224,21 +224,21 @@ static VdbeCursor *allocateCursor( ** look like a number, leave it alone. */ static void applyNumericAffinity(Mem *pRec){ - if( (pRec->flags & (MEM_Real|MEM_Int))==0 ){ - double rValue; - i64 iValue; - u8 enc = pRec->enc; - if( (pRec->flags&MEM_Str)==0 ) return; - if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; - if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ - pRec->u.i = iValue; - pRec->flags |= MEM_Int; - }else{ - pRec->r = rValue; - pRec->flags |= MEM_Real; - } + double rValue; + i64 iValue; + u8 enc = pRec->enc; + if( (pRec->flags&MEM_Str)==0 ) return; + if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; + if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ + pRec->u.i = iValue; + pRec->flags |= MEM_Int; + }else{ + pRec->r = rValue; + pRec->flags |= MEM_Real; } } +#define ApplyNumericAffinity(X) \ + if(((X)->flags&(MEM_Real|MEM_Int))==0){applyNumericAffinity(X);} /* ** Processing is determine by the affinity parameter: @@ -275,7 +275,7 @@ static void applyAffinity( }else if( affinity!=SQLITE_AFF_NONE ){ assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL || affinity==SQLITE_AFF_NUMERIC ); - applyNumericAffinity(pRec); + ApplyNumericAffinity(pRec); if( pRec->flags & MEM_Real ){ sqlite3VdbeIntegerAffinity(pRec); } @@ -292,7 +292,7 @@ int sqlite3_value_numeric_type(sqlite3_value *pVal){ int eType = sqlite3_value_type(pVal); if( eType==SQLITE_TEXT ){ Mem *pMem = (Mem*)pVal; - applyNumericAffinity(pMem); + ApplyNumericAffinity(pMem); eType = sqlite3_value_type(pVal); } return eType; @@ -3597,7 +3597,7 @@ case OP_SeekGT: { /* jump, in3 */ ** blob, or NULL. But it needs to be an integer before we can do ** the seek, so covert it. */ pIn3 = &aMem[pOp->p3]; - applyNumericAffinity(pIn3); + ApplyNumericAffinity(pIn3); iKey = sqlite3VdbeIntValue(pIn3); pC->rowidIsValid = 0; From e6bc1ef68f17e438e91169d4c97cc03cb03aac2e Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 26 Jul 2014 20:12:56 +0000 Subject: [PATCH 145/710] Remove an unreachable branch from the sqlite3_value_numeric_type() interface. FossilOrigin-Name: 5350229b52b18a4961858a30538c5c75e5bd3048 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 480568c8f0..4de9f86a27 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sunnecessary\sno-op\scalls\sto\sapplyNumericAffinity()\sfor\sa\ssmall\nperformance\simprovement. -D 2014-07-26T16:47:23.387 +C Remove\san\sunreachable\sbranch\sfrom\sthe\ssqlite3_value_numeric_type()\sinterface. +D 2014-07-26T20:12:56.006 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 145f909c1c7b16b0899bcab8b241b811e4af9534 +F src/vdbe.c b127f05ff110c1f640b5aff98fa57d5028c8f2ee F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 @@ -1184,7 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e0fa6fdc14ac5458f9200cbae124f8025ea534ea -R 497e2677c499db4d4a797e5c1acd577b +P 413d7287977702fa651c0140bd5cf29021fe3e79 +R 882befeb8b61995267c9bc4311daa690 U drh -Z ee817a86e8cc0ab5547d2241830d36e2 +Z 28fa8e103cbfa69958d1f03600e24fde diff --git a/manifest.uuid b/manifest.uuid index 06b44f1d10..58f6f007db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -413d7287977702fa651c0140bd5cf29021fe3e79 \ No newline at end of file +5350229b52b18a4961858a30538c5c75e5bd3048 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 49183129aa..a00a1da836 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -292,7 +292,7 @@ int sqlite3_value_numeric_type(sqlite3_value *pVal){ int eType = sqlite3_value_type(pVal); if( eType==SQLITE_TEXT ){ Mem *pMem = (Mem*)pVal; - ApplyNumericAffinity(pMem); + applyNumericAffinity(pMem); eType = sqlite3_value_type(pVal); } return eType; From ac65196e4d002ae36ea82871134f445728342b5b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 28 Jul 2014 14:54:50 +0000 Subject: [PATCH 146/710] Improvements to comments in the multi-threaded sorter. Also include a function name change for clarity. And add a test to help show that the MergeEngine object is only used by a single thread. FossilOrigin-Name: 9af50a878f67c1c2a4f1520160cc989650d7196a --- manifest | 12 +++++------ manifest.uuid | 2 +- src/vdbesort.c | 55 ++++++++++++++++++++++++++++++++++---------------- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index ab912afdb5..1af171baaa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sall\srecent\strunk\schanges\sinto\sthe\sthreads\sbranch. -D 2014-07-24T16:54:28.599 +C Improvements\sto\scomments\sin\sthe\smulti-threaded\ssorter.\s\sAlso\sinclude\sa\nfunction\sname\schange\sfor\sclarity.\s\sAnd\sadd\sa\stest\sto\shelp\sshow\sthat\sthe\nMergeEngine\sobject\sis\sonly\sused\sby\sa\ssingle\sthread. +D 2014-07-28T14:54:50.442 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 68ef480fa75b27d5860fb96ca4f5a9af98ba102f F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c e2784e2e1f1819a55ce6f22c6ab22eca576ae6d8 +F src/vdbesort.c ef998096c8b2a1a85fbd730183a9b62f652e1af3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ae23a65eb1547fbe8b86ab71477071990a22d31d fb1048cb2b613a0dbfe625a5df05e9dcd736a433 -R 5b949bc8daa338f7894b4848fcc3e5ce +P 770685892c8f09b9cddb2fbb2877cfb291e19425 +R d6f4c6811acb8e4555290f0ceab890b8 U drh -Z 1a0a2770bde58d6f05cf30741962cf0a +Z 8438d80edf2a72a2fe3b270fb06c810d diff --git a/manifest.uuid b/manifest.uuid index d06af4d0d3..64a549188e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -770685892c8f09b9cddb2fbb2877cfb291e19425 \ No newline at end of file +9af50a878f67c1c2a4f1520160cc989650d7196a \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 8065f6d982..96d9e2976e 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -85,7 +85,7 @@ ** The threshold for the amount of main memory to use before flushing ** records to a PMA is roughly the same as the limit configured for the ** page-cache of the main database. Specifically, the threshold is set to -** the value returned multiplied by "PRAGMA main.page_size" multipled by +** the value returned by "PRAGMA main.page_size" multipled by ** that returned by "PRAGMA main.cache_size", in bytes. ** ** If the sorter is running in single-threaded mode, then all PMAs generated @@ -190,6 +190,7 @@ struct SorterList { ** ** The aReadr[] array contains a PmaReader object for each of the PMAs being ** merged. An aReadr[] object either points to a valid key or else is at EOF. +** ("EOF" means "End Of File". When aReadr[] is at EOF there is no more data.) ** For the purposes of the paragraphs below, we assume that the array is ** actually N elements in size, where N is the smallest power of 2 greater ** to or equal to the number of PMAs being merged. The extra aReadr[] elements @@ -247,15 +248,19 @@ struct SorterList { */ struct MergeEngine { int nTree; /* Used size of aTree/aReadr (power of 2) */ + SortSubtask *pTask; /* Used by this thread only */ int *aTree; /* Current state of incremental merge */ PmaReader *aReadr; /* Array of PmaReaders to merge data from */ }; /* +** This object represents a single thread of control in a sort operation. ** Exactly VdbeSorter.nTask instances of this object are allocated ** as part of each VdbeSorter object. Instances are never allocated any ** other way. VdbeSorter.nTask is set to the number of worker threads allowed -** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). +** (see SQLITE_CONFIG_WORKER_THREADS) plus one (the main thread). Thus for +** single-threaded operation, there is exactly one instance of this object +** and for multi-threaded operation there are two or more instances. ** ** Essentially, this structure contains all those fields of the VdbeSorter ** structure for which each thread requires a separate instance. For example, @@ -443,7 +448,7 @@ static int vdbeIncrSwap(IncrMerger*); static void vdbeIncrFree(IncrMerger *); /* -** Free all memory belonging to the PmaReader object passed as the second +** Free all memory belonging to the PmaReader object passed as the ** argument. All structure fields are set to zero before returning. */ static void vdbePmaReaderClear(PmaReader *pReadr){ @@ -455,12 +460,12 @@ static void vdbePmaReaderClear(PmaReader *pReadr){ } /* -** Read nByte bytes of data from the stream of data iterated by object p. +** Read the next nByte bytes of data from the PMA p. ** If successful, set *ppOut to point to a buffer containing the data ** and return SQLITE_OK. Otherwise, if an error occurs, return an SQLite ** error code. ** -** The buffer indicated by *ppOut may only be considered valid until the +** The buffer returned in *ppOut is only valid until the ** next call to this function. */ static int vdbePmaReadBlob( @@ -594,6 +599,7 @@ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ int rc = SQLITE_OK; if( pFile->iEof<=(i64)(pTask->pSorter->db->nMaxSorterMmap) ){ rc = sqlite3OsFetch(pFile->pFd, 0, (int)pFile->iEof, (void**)pp); + testcase( rc!=SQLITE_OK ); } return rc; } @@ -604,7 +610,7 @@ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ */ static int vdbePmaReaderSeek( SortSubtask *pTask, /* Task context */ - PmaReader *pReadr, /* Iterate to populate */ + PmaReader *pReadr, /* Reader whose cursor is to be moved */ SorterFile *pFile, /* Sorter file to read from */ i64 iOff /* Offset in pFile */ ){ @@ -637,6 +643,7 @@ static int vdbePmaReaderSeek( rc = sqlite3OsRead( pReadr->pFile, &pReadr->aBuffer[iBuf], nRead, pReadr->iReadOff ); + testcase( rc!=SQLITE_OK ); } } @@ -668,6 +675,7 @@ static int vdbePmaReaderNext(PmaReader *pReadr){ if( bEof ){ /* This is an EOF condition */ vdbePmaReaderClear(pReadr); + testcase( rc!=SQLITE_OK ); return rc; } } @@ -678,6 +686,7 @@ static int vdbePmaReaderNext(PmaReader *pReadr){ if( rc==SQLITE_OK ){ pReadr->nKey = (int)nRec; rc = vdbePmaReadBlob(pReadr, (int)nRec, &pReadr->aKey); + testcase( rc!=SQLITE_OK ); } return rc; @@ -1026,7 +1035,11 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ #endif /* -** Allocate a new MergeEngine object with space for nReader PmaReaders. +** Allocate a new MergeEngine object capable of handling up to +** nReader PmaReader inputs. +** +** nReader is automatically rounded up to the next power of two. +** nReader may not exceed SORTER_MAX_MERGE_COUNT even after rounding up. */ static MergeEngine *vdbeMergeEngineNew(int nReader){ int N = 2; /* Smallest power of two >= nReader */ @@ -1041,6 +1054,7 @@ static MergeEngine *vdbeMergeEngineNew(int nReader){ pNew = sqlite3FaultSim(100) ? 0 : (MergeEngine*)sqlite3MallocZero(nByte); if( pNew ){ pNew->nTree = N; + pNew->pTask = 0; pNew->aReadr = (PmaReader*)&pNew[1]; pNew->aTree = (int*)&pNew->aReadr[N]; } @@ -1438,20 +1452,24 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ } /* -** Advance the MergeEngine PmaReader passed as the second argument to -** the next entry. Set *pbEof to true if this means the PmaReader has -** reached EOF. +** Advance the MergeEngine pMerge (passed as the second argument) to +** its next entry. Set *pbEof to true there is no next entry because +** the MergeEngine has reached the end of all its inputs. ** ** Return SQLITE_OK if successful or an error code if an error occurs. */ -static int vdbeSorterNext( - SortSubtask *pTask, - MergeEngine *pMerger, - int *pbEof +static int vdbeMergeEngineStep( + SortSubtask *pTask, /* The thread in which this MergeEngine runs */ + MergeEngine *pMerger, /* The merge engine to advance to the next row */ + int *pbEof /* Set TRUE at EOF. Set false for more content */ ){ int rc; int iPrev = pMerger->aTree[1];/* Index of PmaReader to advance */ + /* A MergeEngine object is only used by a single thread */ + assert( pMerger->pTask==0 || pMerger->pTask==pTask ); + pMerger->pTask = pTask; + /* Advance the current PmaReader */ rc = vdbePmaReaderNext(&pMerger->aReadr[iPrev]); @@ -1720,7 +1738,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ /* Write the next key to the output. */ vdbePmaWriteVarint(&writer, nKey); vdbePmaWriteBlob(&writer, pReader->aKey, nKey); - rc = vdbeSorterNext(pTask, pIncr->pMerger, &dummy); + rc = vdbeMergeEngineStep(pTask, pIncr->pMerger, &dummy); } rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); @@ -2128,7 +2146,10 @@ static int vdbeSorterAddToTree( ** error occurs, an SQLite error code is returned and the final value ** of *ppOut is undefined. */ -static int vdbeSorterMergeTreeBuild(VdbeSorter *pSorter, MergeEngine **ppOut){ +static int vdbeSorterMergeTreeBuild( + VdbeSorter *pSorter, /* The VDBE cursor that implements the sort */ + MergeEngine **ppOut /* Write the MergeEngine here */ +){ MergeEngine *pMain = 0; int rc = SQLITE_OK; int iTask; @@ -2336,7 +2357,7 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ }else #endif /*if( !pSorter->bUseThreads )*/ { - rc = vdbeSorterNext(&pSorter->aTask[0], pSorter->pMerger, pbEof); + rc = vdbeMergeEngineStep(&pSorter->aTask[0], pSorter->pMerger, pbEof); } }else{ SorterRecord *pFree = pSorter->list.pList; From a4c8ca049e7833e78df61e0f866056b6f39afea5 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 28 Jul 2014 17:18:28 +0000 Subject: [PATCH 147/710] In vdbesort.c, rename all pointers to sqlite3_file objects "pFd" and use the name "pFile" only for pointers to SortFile objects. Other comment enhancements. FossilOrigin-Name: 518290a7fc0994f9593c5c3ba5b2610a1b86dae1 --- manifest | 12 ++--- manifest.uuid | 2 +- src/vdbesort.c | 116 ++++++++++++++++++++++++++----------------------- 3 files changed, 69 insertions(+), 61 deletions(-) diff --git a/manifest b/manifest index 006de9f272..cd56be770f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\strunk\schanges\sinto\sthe\sthreads\sbranch. -D 2014-07-28T15:01:37.313 +C In\svdbesort.c,\srename\sall\spointers\sto\ssqlite3_file\sobjects\s"pFd"\sand\suse\sthe\nname\s"pFile"\sonly\sfor\spointers\sto\sSortFile\sobjects.\s\sOther\scomment\senhancements. +D 2014-07-28T17:18:28.406 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c ef998096c8b2a1a85fbd730183a9b62f652e1af3 +F src/vdbesort.c cbd9f27e3521737ad87149550c6d30994f74df7a F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9af50a878f67c1c2a4f1520160cc989650d7196a 5350229b52b18a4961858a30538c5c75e5bd3048 -R c247ffbe0be6e5ba20860955ecb933f2 +P 163c247bd8280ab14fe577329c631c8bd884707f +R 502fd7906a0bc29e23e7a03a06d6bf2a U drh -Z eec00caa889efc5f1f3b6e693b6885aa +Z e1002bd8dfbb36530151cc4f8c9e6ac6 diff --git a/manifest.uuid b/manifest.uuid index 2637b6c6f7..1b6e273eef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -163c247bd8280ab14fe577329c631c8bd884707f \ No newline at end of file +518290a7fc0994f9593c5c3ba5b2610a1b86dae1 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 96d9e2976e..ce26e9acc8 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -327,20 +327,24 @@ struct VdbeSorter { /* ** An instance of the following object is used to read records out of a ** PMA, in sorted order. The next key to be read is cached in nKey/aKey. -** pFile==0 at EOF. +** aKey might point into aMap or into aBuffer. If neither of those locations +** contain a contiguous representation of the key, then aAlloc is allocated +** and the key is copied into aAlloc and aKey is made to poitn to aAlloc. +** +** pFd==0 at EOF. */ struct PmaReader { - i64 iReadOff; /* Current read offset */ - i64 iEof; /* 1 byte past EOF for this PmaReader */ - int nAlloc; /* Bytes of space at aAlloc */ - int nKey; /* Number of bytes in key */ - sqlite3_file *pFile; /* File we are reading from */ - u8 *aAlloc; /* Allocated space */ - u8 *aKey; /* Pointer to current key */ - u8 *aBuffer; /* Current read buffer */ - int nBuffer; /* Size of read buffer in bytes */ - u8 *aMap; /* Pointer to mapping of entire file */ - IncrMerger *pIncr; /* Incremental merger */ + i64 iReadOff; /* Current read offset */ + i64 iEof; /* 1 byte past EOF for this PmaReader */ + int nAlloc; /* Bytes of space at aAlloc */ + int nKey; /* Number of bytes in key */ + sqlite3_file *pFd; /* File handle we are reading from */ + u8 *aAlloc; /* Space for aKey if aBuffer and pMap wont work */ + u8 *aKey; /* Pointer to current key */ + u8 *aBuffer; /* Current read buffer */ + int nBuffer; /* Size of read buffer in bytes */ + u8 *aMap; /* Pointer to mapping of entire file */ + IncrMerger *pIncr; /* Incremental merger */ }; /* @@ -400,7 +404,7 @@ struct PmaWriter { int iBufStart; /* First byte of buffer to write */ int iBufEnd; /* Last byte of buffer to write */ i64 iWriteOff; /* Offset of start of buffer in file */ - sqlite3_file *pFile; /* File to write to */ + sqlite3_file *pFd; /* File handle to write to */ }; /* @@ -454,7 +458,7 @@ static void vdbeIncrFree(IncrMerger *); static void vdbePmaReaderClear(PmaReader *pReadr){ sqlite3_free(pReadr->aAlloc); sqlite3_free(pReadr->aBuffer); - if( pReadr->aMap ) sqlite3OsUnfetch(pReadr->pFile, 0, pReadr->aMap); + if( pReadr->aMap ) sqlite3OsUnfetch(pReadr->pFd, 0, pReadr->aMap); vdbeIncrFree(pReadr->pIncr); memset(pReadr, 0, sizeof(PmaReader)); } @@ -501,7 +505,7 @@ static int vdbePmaReadBlob( assert( nRead>0 ); /* Readr data from the file. Return early if an error occurs. */ - rc = sqlite3OsRead(p->pFile, p->aBuffer, nRead, p->iReadOff); + rc = sqlite3OsRead(p->pFd, p->aBuffer, nRead, p->iReadOff); assert( rc!=SQLITE_IOERR_SHORT_READ ); if( rc!=SQLITE_OK ) return rc; } @@ -619,12 +623,12 @@ static int vdbePmaReaderSeek( assert( pReadr->pIncr==0 || pReadr->pIncr->bEof==0 ); if( pReadr->aMap ){ - sqlite3OsUnfetch(pReadr->pFile, 0, pReadr->aMap); + sqlite3OsUnfetch(pReadr->pFd, 0, pReadr->aMap); pReadr->aMap = 0; } pReadr->iReadOff = iOff; pReadr->iEof = pFile->iEof; - pReadr->pFile = pFile->pFd; + pReadr->pFd = pFile->pFd; rc = vdbeSorterMapFile(pTask, pFile, &pReadr->aMap); if( rc==SQLITE_OK && pReadr->aMap==0 ){ @@ -641,7 +645,7 @@ static int vdbePmaReaderSeek( nRead = (int)(pReadr->iEof - pReadr->iReadOff); } rc = sqlite3OsRead( - pReadr->pFile, &pReadr->aBuffer[iBuf], nRead, pReadr->iReadOff + pReadr->pFd, &pReadr->aBuffer[iBuf], nRead, pReadr->iReadOff ); testcase( rc!=SQLITE_OK ); } @@ -784,9 +788,9 @@ static int vdbeSorterDoCompare( p1 = &pMerger->aReadr[i1]; p2 = &pMerger->aReadr[i2]; - if( p1->pFile==0 ){ + if( p1->pFd==0 ){ iRes = i2; - }else if( p2->pFile==0 ){ + }else if( p2->pFd==0 ){ iRes = i1; }else{ int res; @@ -1147,13 +1151,13 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ ** Whether or not the file does end up memory mapped of course depends on ** the specific VFS implementation. */ -static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ +static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ if( nByte<=(i64)(db->nMaxSorterMmap) ){ - int rc = sqlite3OsTruncate(pFile, nByte); + int rc = sqlite3OsTruncate(pFd, nByte); if( rc==SQLITE_OK ){ void *p = 0; - sqlite3OsFetch(pFile, 0, (int)nByte, &p); - sqlite3OsUnfetch(pFile, 0, p); + sqlite3OsFetch(pFd, 0, (int)nByte, &p); + sqlite3OsUnfetch(pFd, 0, p); } } } @@ -1163,25 +1167,25 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFile, i64 nByte){ /* ** Allocate space for a file-handle and open a temporary file. If successful, -** set *ppFile to point to the malloc'd file-handle and return SQLITE_OK. -** Otherwise, set *ppFile to 0 and return an SQLite error code. +** set *ppFd to point to the malloc'd file-handle and return SQLITE_OK. +** Otherwise, set *ppFd to 0 and return an SQLite error code. */ static int vdbeSorterOpenTempFile( sqlite3 *db, /* Database handle doing sort */ i64 nExtend, /* Attempt to extend file to this size */ - sqlite3_file **ppFile + sqlite3_file **ppFd ){ int rc; - rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFile, + rc = sqlite3OsOpenMalloc(db->pVfs, 0, ppFd, SQLITE_OPEN_TEMP_JOURNAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE, &rc ); if( rc==SQLITE_OK ){ i64 max = SQLITE_MAX_MMAP_SIZE; - sqlite3OsFileControlHint(*ppFile, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); + sqlite3OsFileControlHint(*ppFd, SQLITE_FCNTL_MMAP_SIZE, (void*)&max); if( nExtend>0 ){ - vdbeSorterExtendFile(db, *ppFile, nExtend); + vdbeSorterExtendFile(db, *ppFd, nExtend); } } return rc; @@ -1300,10 +1304,10 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ ** Initialize a PMA-writer object. */ static void vdbePmaWriterInit( - sqlite3_file *pFile, /* File to write to */ + sqlite3_file *pFd, /* File handle to write to */ PmaWriter *p, /* Object to populate */ int nBuf, /* Buffer size */ - i64 iStart /* Offset of pFile to begin writing at */ + i64 iStart /* Offset of pFd to begin writing at */ ){ memset(p, 0, sizeof(PmaWriter)); p->aBuffer = (u8*)sqlite3Malloc(nBuf); @@ -1313,7 +1317,7 @@ static void vdbePmaWriterInit( p->iBufEnd = p->iBufStart = (iStart % nBuf); p->iWriteOff = iStart - p->iBufStart; p->nBuffer = nBuf; - p->pFile = pFile; + p->pFd = pFd; } } @@ -1332,7 +1336,7 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ memcpy(&p->aBuffer[p->iBufEnd], &pData[nData-nRem], nCopy); p->iBufEnd += nCopy; if( p->iBufEnd==p->nBuffer ){ - p->eFWErr = sqlite3OsWrite(p->pFile, + p->eFWErr = sqlite3OsWrite(p->pFd, &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); @@ -1357,7 +1361,7 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ - p->eFWErr = sqlite3OsWrite(p->pFile, + p->eFWErr = sqlite3OsWrite(p->pFd, &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); @@ -1489,9 +1493,9 @@ static int vdbeMergeEngineStep( for(i=(pMerger->nTree+iPrev)/2; i>0; i=i/2){ /* Compare pReadr1 and pReadr2. Store the result in variable iRes. */ int iRes; - if( pReadr1->pFile==0 ){ + if( pReadr1->pFd==0 ){ iRes = +1; - }else if( pReadr2->pFile==0 ){ + }else if( pReadr2->pFd==0 ){ iRes = -1; }else{ iRes = vdbeSorterCompare(pTask, @@ -1519,12 +1523,12 @@ static int vdbeMergeEngineStep( pReadr2 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; pKey2 = pReadr2->aKey; }else{ - if( pReadr1->pFile ) pKey2 = 0; + if( pReadr1->pFd ) pKey2 = 0; pMerger->aTree[i] = (int)(pReadr2 - pMerger->aReadr); pReadr1 = &pMerger->aReadr[ pMerger->aTree[i ^ 0x0001] ]; } } - *pbEof = (pMerger->aReadr[pMerger->aTree[1]].pFile==0); + *pbEof = (pMerger->aReadr[pMerger->aTree[1]].pFd==0); } return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); @@ -1732,7 +1736,7 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ /* Check if the output file is full or if the input has been exhausted. ** In either case exit the loop. */ - if( pReader->pFile==0 ) break; + if( pReader->pFd==0 ) break; if( (iEof + nKey + sqlite3VarintLen(nKey))>(iStart + pIncr->mxSz) ) break; /* Write the next key to the output. */ @@ -1826,9 +1830,9 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ ** pMerger argument before returning. */ static int vdbeIncrNew( - SortSubtask *pTask, - MergeEngine *pMerger, - IncrMerger **ppOut + SortSubtask *pTask, /* The thread that will be using the new IncrMerger */ + MergeEngine *pMerger, /* The MergeEngine that the IncrMerger will control */ + IncrMerger **ppOut /* Write the new IncrMerger here */ ){ int rc = SQLITE_OK; IncrMerger *pIncr = *ppOut = (IncrMerger*) @@ -1861,9 +1865,9 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr){ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); /* -** Initialize the merger argument passed as the second argument. Once this -** function returns, the first key of merged data may be read from the merger -** object in the usual fashion. +** Initialize the MergeEngine object passed as the second argument. Once this +** function returns, the first key of merged data may be read from the +** MergeEngine object in the usual fashion. ** ** If argument eMode is INCRINIT_ROOT, then it is assumed that any IncrMerge ** objects attached to the PmaReader objects that the merger reads from have @@ -1879,14 +1883,18 @@ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ static int vdbeIncrInitMerger( - SortSubtask *pTask, - MergeEngine *pMerger, + SortSubtask *pTask, /* Thread that will run pMerger */ + MergeEngine *pMerger, /* MergeEngine to initialize */ int eMode /* One of the INCRINIT_XXX constants */ ){ int rc = SQLITE_OK; /* Return code */ int i; /* For looping over PmaReader objects */ int nTree = pMerger->nTree; + /* Verify that the MergeEngine is assigned to a single thread */ + assert( pMerger->pTask==0 || pMerger->pTask==pTask ); + pMerger->pTask = pTask; + for(i=0; rc==SQLITE_OK && ipIncr). ** -** If argument eMode is set to INCRINIT_NORMAL, then PmaReader PmaReaders +** If argument eMode is set to INCRINIT_NORMAL, then all PmaReaders ** in the sub-tree headed by pReadr are also initialized. Data is then loaded -** into the buffers belonging to this PmaReader, pReadr, and it is set to +** into the buffers belonging to pReadr and it is set to ** point to the first key in its range. ** -** If argument eMode is set to INCRINIT_TASK, then PmaReader is guaranteed +** If argument eMode is set to INCRINIT_TASK, then pReadr is guaranteed ** to be a multi-threaded PmaReader and this function is being called in a ** background thread. In this case all PmaReaders in the sub-tree are ** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to -** pReadr is populated. However, the PmaReader itself is not set up to point +** pReadr is populated. However, pReadr itself is not set up to point ** to its first key. A call to vdbePmaReaderNext() is still required to do ** that. ** ** The reason this function does not call vdbePmaReaderNext() immediately -** in the INCRINIT_TASK case is that vdbePmaReaderNext() assumes that has +** in the INCRINIT_TASK case is that vdbePmaReaderNext() assumes that it has ** to block on thread (pTask->thread) before accessing aFile[1]. But, since ** this entire function is being run by thread (pTask->thread), that will ** lead to the current background thread attempting to join itself. @@ -2353,7 +2361,7 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ #if SQLITE_MAX_WORKER_THREADS>0 if( pSorter->bUseThreads ){ rc = vdbePmaReaderNext(pSorter->pReader); - *pbEof = (pSorter->pReader->pFile==0); + *pbEof = (pSorter->pReader->pFd==0); }else #endif /*if( !pSorter->bUseThreads )*/ { From 8a4865f139e210ea92d41171b7f39a424d981f8e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 28 Jul 2014 18:57:40 +0000 Subject: [PATCH 148/710] In vdbesort.c, rename vdbeSorterDoCompare() to vdbeMergeEngineCompare() and move it closer to the one place where it is called. Other minor comment changes. FossilOrigin-Name: 09d50d9f0fe7df26dadb0a332731683a07a89fde --- manifest | 12 ++-- manifest.uuid | 2 +- src/vdbesort.c | 150 +++++++++++++++++++++++++++---------------------- 3 files changed, 89 insertions(+), 75 deletions(-) diff --git a/manifest b/manifest index cd56be770f..9b892425af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\svdbesort.c,\srename\sall\spointers\sto\ssqlite3_file\sobjects\s"pFd"\sand\suse\sthe\nname\s"pFile"\sonly\sfor\spointers\sto\sSortFile\sobjects.\s\sOther\scomment\senhancements. -D 2014-07-28T17:18:28.406 +C In\svdbesort.c,\srename\svdbeSorterDoCompare()\sto\svdbeMergeEngineCompare()\sand\nmove\sit\scloser\sto\sthe\sone\splace\swhere\sit\sis\scalled.\s\sOther\sminor\scomment\nchanges. +D 2014-07-28T18:57:40.217 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c cbd9f27e3521737ad87149550c6d30994f74df7a +F src/vdbesort.c 2ee0867189cd40199e7d4ed944190b9453e82eec F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 163c247bd8280ab14fe577329c631c8bd884707f -R 502fd7906a0bc29e23e7a03a06d6bf2a +P 518290a7fc0994f9593c5c3ba5b2610a1b86dae1 +R c47502cced56ced8e0a3892ddc45fd18 U drh -Z e1002bd8dfbb36530151cc4f8c9e6ac6 +Z 1aa9f494eefb9d25759f33de8d16a270 diff --git a/manifest.uuid b/manifest.uuid index 1b6e273eef..e77bb124e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -518290a7fc0994f9593c5c3ba5b2610a1b86dae1 \ No newline at end of file +09d50d9f0fe7df26dadb0a332731683a07a89fde \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index ce26e9acc8..d69a48618a 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -609,7 +609,8 @@ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ } /* -** Seek PmaReader pReadr to offset iOff within file pFile. Return SQLITE_OK +** Attach PmaReader pReadr to file pFile (if it is not already attached to +** that file) and seek it to offset iOff within the file. Return SQLITE_OK ** if successful, or an SQLite error code if an error occurs. */ static int vdbePmaReaderSeek( @@ -759,56 +760,6 @@ static int vdbeSorterCompare( return sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); } -/* -** This function is called to compare two PmaReader keys when merging -** multiple b-tree segments. Parameter iOut is the index of the aTree[] -** value to recalculate. -*/ -static int vdbeSorterDoCompare( - SortSubtask *pTask, - MergeEngine *pMerger, - int iOut -){ - int i1; - int i2; - int iRes; - PmaReader *p1; - PmaReader *p2; - - assert( iOutnTree && iOut>0 ); - - if( iOut>=(pMerger->nTree/2) ){ - i1 = (iOut - pMerger->nTree/2) * 2; - i2 = i1 + 1; - }else{ - i1 = pMerger->aTree[iOut*2]; - i2 = pMerger->aTree[iOut*2+1]; - } - - p1 = &pMerger->aReadr[i1]; - p2 = &pMerger->aReadr[i2]; - - if( p1->pFd==0 ){ - iRes = i2; - }else if( p2->pFd==0 ){ - iRes = i1; - }else{ - int res; - assert( pTask->pUnpacked!=0 ); /* allocated in vdbeSortSubtaskMain() */ - res = vdbeSorterCompare( - pTask, p1->aKey, p1->nKey, p2->aKey, p2->nKey - ); - if( res<=0 ){ - iRes = i1; - }else{ - iRes = i2; - } - } - - pMerger->aTree[iOut] = iRes; - return SQLITE_OK; -} - /* ** Initialize the temporary index cursor just opened as a sorter cursor. ** @@ -1859,10 +1810,69 @@ static void vdbeIncrSetThreads(IncrMerger *pIncr){ } #endif /* SQLITE_MAX_WORKER_THREADS>0 */ + + +/* +** Recompute pMerger->aTree[iOut] by comparing the next keys on the +** two PmaReaders that feed that entry. Neither of the PmaReaders +** are advanced. This routine merely does the comparison. +*/ +static void vdbeMergeEngineCompare( + MergeEngine *pMerger, /* Merge engine containing PmaReaders to compare */ + int iOut /* Store the result in pMerger->aTree[iOut] */ +){ + int i1; + int i2; + int iRes; + PmaReader *p1; + PmaReader *p2; + + assert( iOutnTree && iOut>0 ); + + if( iOut>=(pMerger->nTree/2) ){ + i1 = (iOut - pMerger->nTree/2) * 2; + i2 = i1 + 1; + }else{ + i1 = pMerger->aTree[iOut*2]; + i2 = pMerger->aTree[iOut*2+1]; + } + + p1 = &pMerger->aReadr[i1]; + p2 = &pMerger->aReadr[i2]; + + if( p1->pFd==0 ){ + iRes = i2; + }else if( p2->pFd==0 ){ + iRes = i1; + }else{ + int res; + assert( pMerger->pTask->pUnpacked!=0 ); /* from vdbeSortSubtaskMain() */ + res = vdbeSorterCompare( + pMerger->pTask, p1->aKey, p1->nKey, p2->aKey, p2->nKey + ); + if( res<=0 ){ + iRes = i1; + }else{ + iRes = i2; + } + } + + pMerger->aTree[iOut] = iRes; +} + +/* +** Allowed values for the eMode parameter to vdbeIncrMergerInit() +** and vdbePmaReaderIncrMergeInit(). +*/ #define INCRINIT_NORMAL 0 #define INCRINIT_TASK 1 #define INCRINIT_ROOT 2 -static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); + +/* Forward reference. +** The vdbeIncrMergeInit() and vdbePmaReaderIncrMergeInit() routines call each +** other (when building a merge tree). +*/ +static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode); /* ** Initialize the MergeEngine object passed as the second argument. Once this @@ -1877,12 +1887,12 @@ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); ** its first key. ** ** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use -** vdbePmaReaderIncrInit() to initialize each PmaReader that feeds data +** vdbePmaReaderIncrMergeInit() to initialize each PmaReader that feeds data ** to pMerger. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int vdbeIncrInitMerger( +static int vdbeIncrMergerInit( SortSubtask *pTask, /* Thread that will run pMerger */ MergeEngine *pMerger, /* MergeEngine to initialize */ int eMode /* One of the INCRINIT_XXX constants */ @@ -1895,7 +1905,7 @@ static int vdbeIncrInitMerger( assert( pMerger->pTask==0 || pMerger->pTask==pTask ); pMerger->pTask = pTask; - for(i=0; rc==SQLITE_OK && iaReadr[nTree-i-1]); }else{ - rc = vdbePmaReaderIncrInit(&pMerger->aReadr[i], INCRINIT_NORMAL); + rc = vdbePmaReaderIncrMergeInit(&pMerger->aReadr[i], INCRINIT_NORMAL); } + if( rc!=SQLITE_OK ) return rc; } - for(i=pMerger->nTree-1; rc==SQLITE_OK && i>0; i--){ - rc = vdbeSorterDoCompare(pTask, pMerger, i); + for(i=pMerger->nTree-1; i>0; i--){ + vdbeMergeEngineCompare(pMerger, i); } - - return (rc==SQLITE_OK ? pTask->pUnpacked->errCode : rc); + return pTask->pUnpacked->errCode; } /* +** Initialize the IncrMerge field of a PmaReader. +** ** If the PmaReader passed as the first argument is not an incremental-reader ** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it serves ** to open and/or initialize the temp file related fields of the IncrMerge @@ -1950,14 +1962,14 @@ static int vdbeIncrInitMerger( ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode){ +static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pReadr->pIncr; if( pIncr ){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; - rc = vdbeIncrInitMerger(pTask, pIncr->pMerger, eMode); + rc = vdbeIncrMergerInit(pTask, pIncr->pMerger, eMode); /* Set up the required files for pIncr. A multi-theaded IncrMerge object ** requires two temp files to itself, whereas a single-threaded object @@ -2005,18 +2017,20 @@ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode){ #if SQLITE_MAX_WORKER_THREADS>0 /* -** The main routine for vdbePmaReaderIncrInit() operations run in +** The main routine for vdbePmaReaderIncrMergeInit() operations run in ** background threads. */ static void *vdbePmaReaderBgInit(void *pCtx){ PmaReader *pReader = (PmaReader*)pCtx; - void *pRet = SQLITE_INT_TO_PTR(vdbePmaReaderIncrInit(pReader,INCRINIT_TASK)); + void *pRet = SQLITE_INT_TO_PTR( + vdbePmaReaderIncrMergeInit(pReader,INCRINIT_TASK) + ); pReader->pIncr->pTask->bDone = 1; return pRet; } /* -** Use a background thread to invoke vdbePmaReaderIncrInit(INCRINIT_TASK) +** Use a background thread to invoke vdbePmaReaderIncrMergeInit(INCRINIT_TASK) ** on the the PmaReader object passed as the first argument. ** ** This call will initialize the various fields of the pReadr->pIncr @@ -2270,7 +2284,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ assert( p->pIncr==0 || p->pIncr->pTask==&pSorter->aTask[iTask] ); if( p->pIncr ){ if( iTask==pSorter->nTask-1 ){ - rc = vdbePmaReaderIncrInit(p, INCRINIT_TASK); + rc = vdbePmaReaderIncrMergeInit(p, INCRINIT_TASK); }else{ rc = vdbePmaReaderBgIncrInit(p); } @@ -2280,12 +2294,12 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ pMain = 0; } if( rc==SQLITE_OK ){ - rc = vdbePmaReaderIncrInit(pReadr, INCRINIT_ROOT); + rc = vdbePmaReaderIncrMergeInit(pReadr, INCRINIT_ROOT); } }else #endif { - rc = vdbeIncrInitMerger(pTask0, pMain, INCRINIT_NORMAL); + rc = vdbeIncrMergerInit(pTask0, pMain, INCRINIT_NORMAL); pSorter->pMerger = pMain; pMain = 0; } From d906514d98822ce5eaaca164fdbaa0cf955f53c6 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 28 Jul 2014 19:58:41 +0000 Subject: [PATCH 149/710] Rename vdbeIncrMergerInit() to vdbeMergeEngineInit() - a much more accurate name. FossilOrigin-Name: 5b084a2dd581141b2d0cd9d1a5975625f65ec34d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 9b892425af..061768f24d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\svdbesort.c,\srename\svdbeSorterDoCompare()\sto\svdbeMergeEngineCompare()\sand\nmove\sit\scloser\sto\sthe\sone\splace\swhere\sit\sis\scalled.\s\sOther\sminor\scomment\nchanges. -D 2014-07-28T18:57:40.217 +C Rename\svdbeIncrMergerInit()\sto\svdbeMergeEngineInit()\s-\sa\smuch\smore\saccurate\nname. +D 2014-07-28T19:58:41.375 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c 2ee0867189cd40199e7d4ed944190b9453e82eec +F src/vdbesort.c 143de4b9da4e42a4270cb615dddfc9e8d5224373 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 518290a7fc0994f9593c5c3ba5b2610a1b86dae1 -R c47502cced56ced8e0a3892ddc45fd18 +P 09d50d9f0fe7df26dadb0a332731683a07a89fde +R 174b1c88fd5de2d3dafe85889f7f84b2 U drh -Z 1aa9f494eefb9d25759f33de8d16a270 +Z dbe81e54bc35480cf09bd2aae1bd678b diff --git a/manifest.uuid b/manifest.uuid index e77bb124e6..565619f60c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -09d50d9f0fe7df26dadb0a332731683a07a89fde \ No newline at end of file +5b084a2dd581141b2d0cd9d1a5975625f65ec34d \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index d69a48618a..5f83493061 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1861,7 +1861,7 @@ static void vdbeMergeEngineCompare( } /* -** Allowed values for the eMode parameter to vdbeIncrMergerInit() +** Allowed values for the eMode parameter to vdbeMergeEngineInit() ** and vdbePmaReaderIncrMergeInit(). */ #define INCRINIT_NORMAL 0 @@ -1892,7 +1892,7 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode); ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -static int vdbeIncrMergerInit( +static int vdbeMergeEngineInit( SortSubtask *pTask, /* Thread that will run pMerger */ MergeEngine *pMerger, /* MergeEngine to initialize */ int eMode /* One of the INCRINIT_XXX constants */ @@ -1969,7 +1969,7 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; - rc = vdbeIncrMergerInit(pTask, pIncr->pMerger, eMode); + rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode); /* Set up the required files for pIncr. A multi-theaded IncrMerge object ** requires two temp files to itself, whereas a single-threaded object @@ -2299,7 +2299,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ }else #endif { - rc = vdbeIncrMergerInit(pTask0, pMain, INCRINIT_NORMAL); + rc = vdbeMergeEngineInit(pTask0, pMain, INCRINIT_NORMAL); pSorter->pMerger = pMain; pMain = 0; } From bde27aaa5a439a1d91dbcbcd5ae9d4e60933145d Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 28 Jul 2014 20:16:41 +0000 Subject: [PATCH 150/710] Remove an unnecessary parameter from vdbeMergeEngineStep(). Rename a couple other routines to be more descriptive of what they do. FossilOrigin-Name: f2407a40f339fa6c2cec194f78ae7c93655b1ec1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 34 ++++++++++++++++------------------ 3 files changed, 23 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 061768f24d..5ef711c807 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rename\svdbeIncrMergerInit()\sto\svdbeMergeEngineInit()\s-\sa\smuch\smore\saccurate\nname. -D 2014-07-28T19:58:41.375 +C Remove\san\sunnecessary\sparameter\sfrom\svdbeMergeEngineStep().\s\sRename\sa\scouple\nother\sroutines\sto\sbe\smore\sdescriptive\sof\swhat\sthey\sdo. +D 2014-07-28T20:16:41.054 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c 143de4b9da4e42a4270cb615dddfc9e8d5224373 +F src/vdbesort.c e78efeed9c87d47d7a09df7d176f3fbf6dcb77e2 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 09d50d9f0fe7df26dadb0a332731683a07a89fde -R 174b1c88fd5de2d3dafe85889f7f84b2 +P 5b084a2dd581141b2d0cd9d1a5975625f65ec34d +R 73873dcfe41a2b7643bb45186ddcf360 U drh -Z dbe81e54bc35480cf09bd2aae1bd678b +Z 40695df9717fb1b9f124285f8dbce47e diff --git a/manifest.uuid b/manifest.uuid index 565619f60c..ec9fe1279a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b084a2dd581141b2d0cd9d1a5975625f65ec34d \ No newline at end of file +f2407a40f339fa6c2cec194f78ae7c93655b1ec1 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 5f83493061..e9fa6b456f 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1407,23 +1407,19 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ } /* -** Advance the MergeEngine pMerge (passed as the second argument) to -** its next entry. Set *pbEof to true there is no next entry because +** Advance the MergeEngine to its next entry. +** Set *pbEof to true there is no next entry because ** the MergeEngine has reached the end of all its inputs. ** ** Return SQLITE_OK if successful or an error code if an error occurs. */ static int vdbeMergeEngineStep( - SortSubtask *pTask, /* The thread in which this MergeEngine runs */ MergeEngine *pMerger, /* The merge engine to advance to the next row */ int *pbEof /* Set TRUE at EOF. Set false for more content */ ){ int rc; int iPrev = pMerger->aTree[1];/* Index of PmaReader to advance */ - - /* A MergeEngine object is only used by a single thread */ - assert( pMerger->pTask==0 || pMerger->pTask==pTask ); - pMerger->pTask = pTask; + SortSubtask *pTask = pMerger->pTask; /* Advance the current PmaReader */ rc = vdbePmaReaderNext(&pMerger->aReadr[iPrev]); @@ -1693,7 +1689,8 @@ static int vdbeIncrPopulate(IncrMerger *pIncr){ /* Write the next key to the output. */ vdbePmaWriteVarint(&writer, nKey); vdbePmaWriteBlob(&writer, pReader->aKey, nKey); - rc = vdbeMergeEngineStep(pTask, pIncr->pMerger, &dummy); + assert( pIncr->pMerger->pTask==pTask ); + rc = vdbeMergeEngineStep(pIncr->pMerger, &dummy); } rc2 = vdbePmaWriterFinish(&writer, &pOut->iEof); @@ -1780,7 +1777,7 @@ static int vdbeIncrSwap(IncrMerger *pIncr){ ** If an OOM condition is encountered, return NULL. In this case free the ** pMerger argument before returning. */ -static int vdbeIncrNew( +static int vdbeIncrMergerNew( SortSubtask *pTask, /* The thread that will be using the new IncrMerger */ MergeEngine *pMerger, /* The MergeEngine that the IncrMerger will control */ IncrMerger **ppOut /* Write the new IncrMerger here */ @@ -1804,7 +1801,7 @@ static int vdbeIncrNew( /* ** Set the "use-threads" flag on object pIncr. */ -static void vdbeIncrSetThreads(IncrMerger *pIncr){ +static void vdbeIncrMergerSetThreads(IncrMerger *pIncr){ pIncr->bUseThread = 1; pIncr->pTask->file2.iEof -= pIncr->mxSz; } @@ -1902,7 +1899,7 @@ static int vdbeMergeEngineInit( int nTree = pMerger->nTree; /* Verify that the MergeEngine is assigned to a single thread */ - assert( pMerger->pTask==0 || pMerger->pTask==pTask ); + assert( pMerger->pTask==0 ); // || pMerger->pTask==pTask ); pMerger->pTask = pTask; for(i=0; ipIncr); + rc = vdbeIncrMergerNew(pTask, pNew, &pReadr->pIncr); } } if( rc==SQLITE_OK ){ @@ -2216,7 +2213,7 @@ static int vdbeSorterMergeTreeBuild( if( rc==SQLITE_OK ){ #if SQLITE_MAX_WORKER_THREADS>0 if( pMain!=0 ){ - rc = vdbeIncrNew(pTask, pRoot, &pMain->aReadr[iTask].pIncr); + rc = vdbeIncrMergerNew(pTask, pRoot, &pMain->aReadr[iTask].pIncr); }else #endif { @@ -2269,13 +2266,13 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ if( pReadr==0 ) rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ - rc = vdbeIncrNew(pLast, pMain, &pReadr->pIncr); + rc = vdbeIncrMergerNew(pLast, pMain, &pReadr->pIncr); if( rc==SQLITE_OK ){ - vdbeIncrSetThreads(pReadr->pIncr); + vdbeIncrMergerSetThreads(pReadr->pIncr); for(iTask=0; iTask<(pSorter->nTask-1); iTask++){ IncrMerger *pIncr; if( (pIncr = pMain->aReadr[iTask].pIncr) ){ - vdbeIncrSetThreads(pIncr); + vdbeIncrMergerSetThreads(pIncr); assert( pIncr->pTask!=pLast ); } } @@ -2379,7 +2376,8 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ }else #endif /*if( !pSorter->bUseThreads )*/ { - rc = vdbeMergeEngineStep(&pSorter->aTask[0], pSorter->pMerger, pbEof); + assert( pSorter->pMerger->pTask==(&pSorter->aTask[0]) ); + rc = vdbeMergeEngineStep(pSorter->pMerger, pbEof); } }else{ SorterRecord *pFree = pSorter->list.pList; From 2b49327d0845e375571bb8dce0f511706acd197a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 29 Jul 2014 00:23:08 +0000 Subject: [PATCH 151/710] Fix the build on windows. FossilOrigin-Name: 2773a5f9879a106a89a3d0bc3c5bfdcb2fe43c7c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_win.c | 7 ------- src/sqliteInt.h | 7 ------- src/threads.c | 8 ++++++++ 5 files changed, 17 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 5ef711c807..dd136942c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sparameter\sfrom\svdbeMergeEngineStep().\s\sRename\sa\scouple\nother\sroutines\sto\sbe\smore\sdescriptive\sof\swhat\sthey\sdo. -D 2014-07-28T20:16:41.054 +C Fix\sthe\sbuild\son\swindows. +D 2014-07-29T00:23:08.770 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c a7baf1b30f3c58ba20b813e01aab23b18ae44f85 -F src/os_win.c 5f8c5568cc749d6ab44006124e7701f463559223 +F src/os_win.c 047e903174b018c50f425be793eafa8d849737a9 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -227,7 +227,7 @@ F src/shell.c 05e9e7f667a6340643b647c4be0db15dd7627d92 F src/sqlite.h.in a30af69fcbc8fab8b4a00032f9f1d24ba2f01c2c F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h f2b28ce01099fdeb75a222aeddeece7384c9d56a +F src/sqliteInt.h 9c1e5b965191c54157b155e321fa473b9734e062 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 3c63f60ce0aae4c40ed4b8dc745490ff42a05308 +F src/threads.c afdab9acbda8b96dd0e55052de59b87d972cadc1 F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5b084a2dd581141b2d0cd9d1a5975625f65ec34d -R 73873dcfe41a2b7643bb45186ddcf360 +P f2407a40f339fa6c2cec194f78ae7c93655b1ec1 +R bc66c7b33352e0660fc93d07e7c8b3a0 U drh -Z 40695df9717fb1b9f124285f8dbce47e +Z c1a505884ff19c661f20155161198823 diff --git a/manifest.uuid b/manifest.uuid index ec9fe1279a..2e59c45d3e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f2407a40f339fa6c2cec194f78ae7c93655b1ec1 \ No newline at end of file +2773a5f9879a106a89a3d0bc3c5bfdcb2fe43c7c \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 5c246a8586..fa485978d3 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1275,13 +1275,6 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #endif } -DWORD sqlite3Win32Wait(HANDLE hObject){ - DWORD rc; - while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, - TRUE))==WAIT_IO_COMPLETION ){} - return rc; -} - /* ** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, ** or WinCE. Return false (zero) for Win95, Win98, or WinME. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4ad80e4373..8761556ae2 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3723,11 +3723,4 @@ int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); int sqlite3ThreadJoin(SQLiteThread*, void**); #endif -/* -** Win32 interface -*/ -#if SQLITE_OS_WIN - DWORD sqlite3Win32Wait(HANDLE hObject); -#endif - #endif /* _SQLITEINT_H_ */ diff --git a/src/threads.c b/src/threads.c index 1cfe2cc6ce..634dda3b7d 100644 --- a/src/threads.c +++ b/src/threads.c @@ -147,6 +147,14 @@ int sqlite3ThreadCreate( return SQLITE_OK; } +/* Wait on an object */ +DWORD sqlite3Win32Wait(HANDLE hObject){ + DWORD rc; + while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, + TRUE))==WAIT_IO_COMPLETION ){} + return rc; +} + /* Get the results of the thread */ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ DWORD rc; From fc129698e046e6c2341aa8aa2d07fe6c30ad8608 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 29 Jul 2014 00:42:39 +0000 Subject: [PATCH 152/710] Add some asserts to the Win32 mutex subsystem. FossilOrigin-Name: e8f2dc5fadae96252649875c234fcdef1108bd48 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex_w32.c | 4 ++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index dd136942c2..04cafa5227 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sbuild\son\swindows. -D 2014-07-29T00:23:08.770 +C Add\ssome\sasserts\sto\sthe\sWin32\smutex\ssubsystem. +D 2014-07-29T00:42:39.711 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -201,7 +201,7 @@ F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 -F src/mutex_w32.c 13a6b8a3902f3750a3f251ee6640d983e7ca4d29 +F src/mutex_w32.c 92e8ead41598a289c93ac64dfd5e78820c4dabc1 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f2407a40f339fa6c2cec194f78ae7c93655b1ec1 -R bc66c7b33352e0660fc93d07e7c8b3a0 -U drh -Z c1a505884ff19c661f20155161198823 +P 2773a5f9879a106a89a3d0bc3c5bfdcb2fe43c7c +R 9e10c19fd0ddd3066b284eac0abe3eac +U mistachkin +Z 477d633de18b2c200b7eed3df7c1c412 diff --git a/manifest.uuid b/manifest.uuid index 2e59c45d3e..7bff472a40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2773a5f9879a106a89a3d0bc3c5bfdcb2fe43c7c \ No newline at end of file +e8f2dc5fadae96252649875c234fcdef1108bd48 \ No newline at end of file diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 1f5775ccec..78d1a6fba4 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -245,6 +245,7 @@ static void winMutexFree(sqlite3_mutex *p){ assert( p ); assert( p->nRef==0 && p->owner==0 ); assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); + assert( winMutex_isInit==1 ); DeleteCriticalSection(&p->mutex); sqlite3_free(p); } @@ -265,6 +266,7 @@ static void winMutexEnter(sqlite3_mutex *p){ DWORD tid = GetCurrentThreadId(); assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); #endif + assert( winMutex_isInit==1 ); EnterCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG assert( p->nRef>0 || p->owner==0 ); @@ -293,6 +295,7 @@ static int winMutexTry(sqlite3_mutex *p){ ** ticket #2685. */ #if 0 + assert( winMutex_isInit==1 ); if( mutexIsNT() && TryEnterCriticalSection(&p->mutex) ){ p->owner = tid; p->nRef++; @@ -324,6 +327,7 @@ static void winMutexLeave(sqlite3_mutex *p){ if( p->nRef==0 ) p->owner = 0; assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); #endif + assert( winMutex_isInit==1 ); LeaveCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG if( p->trace ){ From d6918657d8d50b7bec81f1c430709586ebc0c30f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 29 Jul 2014 05:49:02 +0000 Subject: [PATCH 153/710] Enhancements and updates to the Win32 mutex subsystem. FossilOrigin-Name: 18984c321049a759f6619cfa17fb3f4e7b3e08ea --- manifest | 21 ++++---- manifest.uuid | 2 +- src/main.c | 2 +- src/mutex_w32.c | 139 +++++++++++++++++++++++++----------------------- src/os_win.c | 30 ++++++----- 5 files changed, 105 insertions(+), 89 deletions(-) diff --git a/manifest b/manifest index 4de9f86a27..632f5523e3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunreachable\sbranch\sfrom\sthe\ssqlite3_value_numeric_type()\sinterface. -D 2014-07-26T20:12:56.006 +C Enhancements\sand\supdates\sto\sthe\sWin32\smutex\ssubsystem. +D 2014-07-29T05:49:02.604 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -189,7 +189,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c cfdb2aa5d248ff1af60227cc3f6d485ba86f92dc +F src/main.c 2c7647fd706cbb048e2b70429e552a7bf732d298 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -201,14 +201,14 @@ F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc -F src/mutex_w32.c 9bcab6e699fdb20eae784237c36b299830fb3316 +F src/mutex_w32.c 45020ed78735a202ff14efcf19415d29f556f16d F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c a7baf1b30f3c58ba20b813e01aab23b18ae44f85 -F src/os_win.c 8dbf6c11780fe2eb96c1f289e664d0c7b2911d37 +F src/os_win.c c29e3a80b47ebdbabd61fc3d4015e52d2654d8c5 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1184,7 +1184,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 413d7287977702fa651c0140bd5cf29021fe3e79 -R 882befeb8b61995267c9bc4311daa690 -U drh -Z 28fa8e103cbfa69958d1f03600e24fde +P 5350229b52b18a4961858a30538c5c75e5bd3048 +R de509ee2752907682d663ce7b389c5a6 +T *branch * winMutex +T *sym-winMutex * +T -sym-trunk * +U mistachkin +Z 9395c7aaf9a6a0a0988325332158758a diff --git a/manifest.uuid b/manifest.uuid index 58f6f007db..24c9f3e6fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5350229b52b18a4961858a30538c5c75e5bd3048 \ No newline at end of file +18984c321049a759f6619cfa17fb3f4e7b3e08ea \ No newline at end of file diff --git a/src/main.c b/src/main.c index 904e0a4fc1..438ae7aeb5 100644 --- a/src/main.c +++ b/src/main.c @@ -1056,7 +1056,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** Return a static string containing the name corresponding to the error code ** specified in the argument. */ -#if defined(SQLITE_TEST) +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) const char *sqlite3ErrName(int rc){ const char *zName = 0; int i, origRc = rc; diff --git a/src/mutex_w32.c b/src/mutex_w32.c index f43a152389..48f3bc6969 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -9,11 +9,16 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains the C functions that implement mutexes for win32 +** This file contains the C functions that implement mutexes for Win32. */ #include "sqliteInt.h" #if SQLITE_OS_WIN +/* +** Include code that is common to all os_*.c files +*/ +#include "os_common.h" + /* ** Include the header file for the Windows VFS. */ @@ -22,7 +27,7 @@ /* ** The code in this file is only used if we are compiling multithreaded -** on a win32 system. +** on a Win32 system. */ #ifdef SQLITE_MUTEX_W32 @@ -35,48 +40,22 @@ struct sqlite3_mutex { #ifdef SQLITE_DEBUG volatile int nRef; /* Number of enterances */ volatile DWORD owner; /* Thread holding this mutex */ - int trace; /* True to trace changes */ + volatile int trace; /* True to trace changes */ #endif }; -#define SQLITE_W32_MUTEX_INITIALIZER { 0 } -#ifdef SQLITE_DEBUG -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, 0L, (DWORD)0, 0 } -#else -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 } -#endif /* -** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, -** or WinCE. Return false (zero) for Win95, Win98, or WinME. -** -** Here is an interesting observation: Win95, Win98, and WinME lack -** the LockFileEx() API. But we can still statically link against that -** API as long as we don't call it win running Win95/98/ME. A call to -** this routine is used to determine if the host is Win95/98/ME or -** WinNT/2K/XP so that we will know whether or not we can safely call -** the LockFileEx() API. -** -** mutexIsNT() is only used for the TryEnterCriticalSection() API call, -** which is only available if your application was compiled with -** _WIN32_WINNT defined to a value >= 0x0400. Currently, the only -** call to TryEnterCriticalSection() is #ifdef'ed out, so #ifdef -** this out as well. +** These are the initializer values used when declaring a "static" mutex +** on Win32. It should be noted that all mutexes require initialization +** on the Win32 platform. */ -#if 0 -#if SQLITE_OS_WINCE || SQLITE_OS_WINRT -# define mutexIsNT() (1) +#define SQLITE_W32_MUTEX_INITIALIZER { 0 } + +#ifdef SQLITE_DEBUG +#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, \ + 0L, (DWORD)0, 0 } #else - static int mutexIsNT(void){ - static int osType = 0; - if( osType==0 ){ - OSVERSIONINFO sInfo; - sInfo.dwOSVersionInfoSize = sizeof(sInfo); - GetVersionEx(&sInfo); - osType = sInfo.dwPlatformId==VER_PLATFORM_WIN32_NT ? 2 : 1; - } - return osType==2; - } -#endif /* SQLITE_OS_WINCE || SQLITE_OS_WINRT */ +#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 } #endif #ifdef SQLITE_DEBUG @@ -87,16 +66,17 @@ struct sqlite3_mutex { static int winMutexHeld(sqlite3_mutex *p){ return p->nRef!=0 && p->owner==GetCurrentThreadId(); } + static int winMutexNotheld2(sqlite3_mutex *p, DWORD tid){ return p->nRef==0 || p->owner!=tid; } + static int winMutexNotheld(sqlite3_mutex *p){ - DWORD tid = GetCurrentThreadId(); + DWORD tid = GetCurrentThreadId(); return winMutexNotheld2(p, tid); } #endif - /* ** Initialize and deinitialize the mutex subsystem. */ @@ -108,17 +88,20 @@ static sqlite3_mutex winMutex_staticMutexes[6] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; + static int winMutex_isInit = 0; -/* As winMutexInit() and winMutexEnd() are called as part -** of the sqlite3_initialize and sqlite3_shutdown() -** processing, the "interlocked" magic is probably not -** strictly necessary. + +/* As winMutexInit() and winMutexEnd() are called as part of the +** sqlite3_initialize() and sqlite3_shutdown() processing, the +** "interlocked" magic used in this section may not strictly +** necessary. */ static LONG winMutex_lock = 0; +int sqlite3_win32_is_nt(void); /* os_win.c */ void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ -static int winMutexInit(void){ +static int winMutexInit(void){ /* The first to increment to 1 does actual initialization */ if( InterlockedCompareExchange(&winMutex_lock, 1, 0)==0 ){ int i; @@ -131,16 +114,17 @@ static int winMutexInit(void){ } winMutex_isInit = 1; }else{ - /* Someone else is in the process of initing the static mutexes */ + /* Another thread is (in the process of) initializing the static + ** mutexes */ while( !winMutex_isInit ){ sqlite3_win32_sleep(1); } } - return SQLITE_OK; + return SQLITE_OK; } -static int winMutexEnd(void){ - /* The first to decrement to 0 does actual shutdown +static int winMutexEnd(void){ + /* The first to decrement to 0 does actual shutdown ** (which should be the last to shutdown.) */ if( InterlockedCompareExchange(&winMutex_lock, 0, 1)==1 ){ if( winMutex_isInit==1 ){ @@ -151,7 +135,7 @@ static int winMutexEnd(void){ winMutex_isInit = 0; } } - return SQLITE_OK; + return SQLITE_OK; } /* @@ -192,7 +176,7 @@ static int winMutexEnd(void){ ** ** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. But for the static +** returns a different mutex on every call. But for the static ** mutex types, the same mutex is returned on every call that has ** the same type number. */ @@ -203,9 +187,12 @@ static sqlite3_mutex *winMutexAlloc(int iType){ case SQLITE_MUTEX_FAST: case SQLITE_MUTEX_RECURSIVE: { p = sqlite3MallocZero( sizeof(*p) ); - if( p ){ + if( p ){ #ifdef SQLITE_DEBUG p->id = iType; +#ifdef SQLITE_WIN32_MUTEX_TRACE_DYNAMIC + p->trace = 1; +#endif #endif #if SQLITE_OS_WINRT InitializeCriticalSectionEx(&p->mutex, 0, 0); @@ -216,12 +203,15 @@ static sqlite3_mutex *winMutexAlloc(int iType){ break; } default: { - assert( winMutex_isInit==1 ); assert( iType-2 >= 0 ); assert( iType-2 < ArraySize(winMutex_staticMutexes) ); + assert( winMutex_isInit==1 ); p = &winMutex_staticMutexes[iType-2]; #ifdef SQLITE_DEBUG p->id = iType; +#ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC + p->trace = 1; +#endif #endif break; } @@ -237,8 +227,10 @@ static sqlite3_mutex *winMutexAlloc(int iType){ */ static void winMutexFree(sqlite3_mutex *p){ assert( p ); +#ifdef SQLITE_DEBUG assert( p->nRef==0 && p->owner==0 ); assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); +#endif DeleteCriticalSection(&p->mutex); sqlite3_free(p); } @@ -255,30 +247,38 @@ static void winMutexFree(sqlite3_mutex *p){ ** more than once, the behavior is undefined. */ static void winMutexEnter(sqlite3_mutex *p){ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + DWORD tid = GetCurrentThreadId(); +#endif #ifdef SQLITE_DEBUG - DWORD tid = GetCurrentThreadId(); + assert( p ); assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); +#else + assert( p ); #endif EnterCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG assert( p->nRef>0 || p->owner==0 ); - p->owner = tid; + p->owner = tid; p->nRef++; if( p->trace ){ - printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + OSTRACE(("ENTER-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", + tid, p, p->trace, p->nRef)); } #endif } + static int winMutexTry(sqlite3_mutex *p){ -#ifndef NDEBUG - DWORD tid = GetCurrentThreadId(); +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + DWORD tid = GetCurrentThreadId(); #endif int rc = SQLITE_BUSY; + assert( p ); assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); /* ** The sqlite3_mutex_try() routine is very rarely used, and when it ** is used it is merely an optimization. So it is OK for it to always - ** fail. + ** fail. ** ** The TryEnterCriticalSection() interface is only available on WinNT. ** And some windows compilers complain if you try to use it without @@ -286,18 +286,21 @@ static int winMutexTry(sqlite3_mutex *p){ ** For that reason, we will omit this optimization for now. See ** ticket #2685. */ -#if 0 - if( mutexIsNT() && TryEnterCriticalSection(&p->mutex) ){ +#if defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x0400 + if( sqlite3_win32_is_nt() && TryEnterCriticalSection(&p->mutex) ){ +#ifdef SQLITE_DEBUG p->owner = tid; p->nRef++; +#endif rc = SQLITE_OK; } #else UNUSED_PARAMETER(p); #endif #ifdef SQLITE_DEBUG - if( rc==SQLITE_OK && p->trace ){ - printf("try mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + if( p->trace ){ + OSTRACE(("TRY-MUTEX tid=%lu, mutex=%p (%d), owner=%lu, nRef=%d, rc=%s\n", + tid, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); } #endif return rc; @@ -310,8 +313,11 @@ static int winMutexTry(sqlite3_mutex *p){ ** is not currently allocated. SQLite will never do either. */ static void winMutexLeave(sqlite3_mutex *p){ -#ifndef NDEBUG +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) DWORD tid = GetCurrentThreadId(); +#endif + assert( p ); +#ifdef SQLITE_DEBUG assert( p->nRef>0 ); assert( p->owner==tid ); p->nRef--; @@ -321,7 +327,8 @@ static void winMutexLeave(sqlite3_mutex *p){ LeaveCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG if( p->trace ){ - printf("leave mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + OSTRACE(("LEAVE-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", + tid, p, p->trace, p->nRef)); } #endif } @@ -343,7 +350,7 @@ sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ 0 #endif }; - return &sMutex; } + #endif /* SQLITE_MUTEX_W32 */ diff --git a/src/os_win.c b/src/os_win.c index 015b6c61da..842a4ad7cf 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1298,22 +1298,28 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #elif !defined(SQLITE_WIN32_HAS_WIDE) # define osIsNT() (0) #else - static int osIsNT(void){ - if( sqlite3_os_type==0 ){ +# define osIsNT() (sqlite3_win32_is_nt()) +#endif + +/* +** This function determines if the machine is running a version of Windows +** based on the NT kernel. +*/ +int sqlite3_win32_is_nt(void){ + if( sqlite3_os_type==0 ){ #if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WIN8 - OSVERSIONINFOW sInfo; - sInfo.dwOSVersionInfoSize = sizeof(sInfo); - osGetVersionExW(&sInfo); + OSVERSIONINFOW sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + osGetVersionExW(&sInfo); #else - OSVERSIONINFOA sInfo; - sInfo.dwOSVersionInfoSize = sizeof(sInfo); - osGetVersionExA(&sInfo); + OSVERSIONINFOA sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + osGetVersionExA(&sInfo); #endif - sqlite3_os_type = sInfo.dwPlatformId==VER_PLATFORM_WIN32_NT ? 2 : 1; - } - return sqlite3_os_type==2; + sqlite3_os_type = (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1; } -#endif + return (sqlite3_os_type == 2); +} #ifdef SQLITE_WIN32_MALLOC /* From 57ff60b19b7596ad8259694d77f7861a314a5189 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 29 Jul 2014 11:54:18 +0000 Subject: [PATCH 154/710] Have calls to the xFilter() method of rtree virtual tables ensure that cursor is initialized before proceeding. Fix for [d2889096e7bdeac]. FossilOrigin-Name: 8cc41b0bf365af47c2061ffe44c86018945dd239 --- ext/rtree/rtree.c | 6 +++++- ext/rtree/rtree1.test | 22 ++++++++++++++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 7b540b4be1..8150538d45 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -1533,9 +1533,13 @@ static int rtreeFilter( rtreeReference(pRtree); + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ freeCursorConstraints(pCsr); - pCsr->iStrategy = idxNum; + sqlite3_free(pCsr->aPoint); + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = (sqlite3_vtab*)pRtree; + pCsr->iStrategy = idxNum; if( idxNum==1 ){ /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ diff --git a/ext/rtree/rtree1.test b/ext/rtree/rtree1.test index 9de5362781..0beb16cc94 100644 --- a/ext/rtree/rtree1.test +++ b/ext/rtree/rtree1.test @@ -33,6 +33,7 @@ set testprefix rtree1 # rtree-8.*: Test constrained scans of r-tree data. # # rtree-12.*: Test that on-conflict clauses are supported. +# rtree-13.*: Test that bug [d2889096e7bdeac6d] has been fixed. # ifcapable !rtree { @@ -513,4 +514,25 @@ foreach {tn sql_template testdata} { db close } } + +#------------------------------------------------------------------------- +# Test that bug [d2889096e7bdeac6d] has been fixed. +# +reset_db +do_execsql_test 13.1 { + CREATE VIRTUAL TABLE t9 USING rtree(id, xmin, xmax); + INSERT INTO t9 VALUES(1,0,0); + INSERT INTO t9 VALUES(2,0,0); + SELECT * FROM t9 WHERE id IN (1, 2); +} {1 0.0 0.0 2 0.0 0.0} + +do_execsql_test 13.2 { + WITH r(x) AS ( + SELECT 1 UNION ALL + SELECT 2 UNION ALL + SELECT 3 + ) + SELECT * FROM r CROSS JOIN t9 WHERE id=x; +} {1 1 0.0 0.0 2 2 0.0 0.0} + finish_test diff --git a/manifest b/manifest index 4de9f86a27..d1716a2069 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunreachable\sbranch\sfrom\sthe\ssqlite3_value_numeric_type()\sinterface. -D 2014-07-26T20:12:56.006 +C Have\scalls\sto\sthe\sxFilter()\smethod\sof\srtree\svirtual\stables\sensure\sthat\scursor\sis\sinitialized\sbefore\sproceeding.\sFix\sfor\s[d2889096e7bdeac]. +D 2014-07-29T11:54:18.063 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -122,9 +122,9 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c 6f70db93e0e42c369325c5cddcf2024c5a87ca43 +F ext/rtree/rtree.c 57bec53e1a677ab74217fe1f20a58c3a47261d6b F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e -F ext/rtree/rtree1.test e2da4aaa426918d27122d1a1066c6ecf8409a514 +F ext/rtree/rtree1.test 541bbcab74613907fea08b2ecdcdd5b7aa724cc9 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 @@ -1184,7 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 413d7287977702fa651c0140bd5cf29021fe3e79 -R 882befeb8b61995267c9bc4311daa690 -U drh -Z 28fa8e103cbfa69958d1f03600e24fde +P 5350229b52b18a4961858a30538c5c75e5bd3048 +R 0da0b70a772b464fda61490d5328a931 +U dan +Z f2b0f473c1ab5a411c1b4bc1349722f0 diff --git a/manifest.uuid b/manifest.uuid index 58f6f007db..b934c9dde5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5350229b52b18a4961858a30538c5c75e5bd3048 \ No newline at end of file +8cc41b0bf365af47c2061ffe44c86018945dd239 \ No newline at end of file From 43cfc230abd3b6e87323682cff2fa3d526478422 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 29 Jul 2014 14:09:21 +0000 Subject: [PATCH 155/710] Add the SQLITE_TESTCTRL_ISINIT file control. FossilOrigin-Name: 8b651d4d6cde7efbc0cc7155948f477477be100e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/main.c | 10 ++++++++++ src/sqlite.h.in | 3 ++- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d1716a2069..7def92e6e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\scalls\sto\sthe\sxFilter()\smethod\sof\srtree\svirtual\stables\sensure\sthat\scursor\sis\sinitialized\sbefore\sproceeding.\sFix\sfor\s[d2889096e7bdeac]. -D 2014-07-29T11:54:18.063 +C Add\sthe\sSQLITE_TESTCTRL_ISINIT\sfile\scontrol. +D 2014-07-29T14:09:21.164 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -189,7 +189,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c cfdb2aa5d248ff1af60227cc3f6d485ba86f92dc +F src/main.c fbe7c756f8974c547f82396253dd507e877f411a F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -224,7 +224,7 @@ F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 6762c62e11b504aa014edceab8886495165e3a77 F src/shell.c 191129c3f7a9cf241aea90ff6a6be3e74d3767f0 -F src/sqlite.h.in ac4451c9da2771d2f4d702ef89722407242906d9 +F src/sqlite.h.in 5967719136e14183b2573ba619993438e2b9856e F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 068e42f41a09ce6b9edbe194ac8a470ab53145df @@ -1184,7 +1184,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5350229b52b18a4961858a30538c5c75e5bd3048 -R 0da0b70a772b464fda61490d5328a931 -U dan -Z f2b0f473c1ab5a411c1b4bc1349722f0 +P 8cc41b0bf365af47c2061ffe44c86018945dd239 +R 0a47475429662a9995bbe963fdd69743 +U drh +Z e526885cd3c9012daec258cf388db74a diff --git a/manifest.uuid b/manifest.uuid index b934c9dde5..60b259b932 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8cc41b0bf365af47c2061ffe44c86018945dd239 \ No newline at end of file +8b651d4d6cde7efbc0cc7155948f477477be100e \ No newline at end of file diff --git a/src/main.c b/src/main.c index 904e0a4fc1..6c29602f47 100644 --- a/src/main.c +++ b/src/main.c @@ -3361,6 +3361,16 @@ int sqlite3_test_control(int op, ...){ break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_ISINIT); + ** + ** Return SQLITE_OK if SQLite has been initialized and SQLITE_ERROR if + ** not. + */ + case SQLITE_TESTCTRL_ISINIT: { + if( sqlite3GlobalConfig.isInit==0 ) rc = SQLITE_ERROR; + break; + } + } va_end(ap); #endif /* SQLITE_OMIT_BUILTIN_TEST */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 67113dc418..8d51632285 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6177,7 +6177,8 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 #define SQLITE_TESTCTRL_VDBE_COVERAGE 21 #define SQLITE_TESTCTRL_BYTEORDER 22 -#define SQLITE_TESTCTRL_LAST 22 +#define SQLITE_TESTCTRL_ISINIT 23 +#define SQLITE_TESTCTRL_LAST 23 /* ** CAPI3REF: SQLite Runtime Status From 19f30dc171b8addb457e1032dba0f126cce61d85 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 29 Jul 2014 15:18:00 +0000 Subject: [PATCH 156/710] Fix the threads build on Windows when SQLITE_MAX_WORKER_THREADS is greater than 0. FossilOrigin-Name: f37db3a03d95b508066745613029b7dd1240b31c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/threads.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 114e1759be..6f77fc7896 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sR-Tree\sfix\sand\sthe\snew\sSQLITE_TESTCTRL_ISINIT\stest\scontrol\sfrom\ntrunk. -D 2014-07-29T14:16:42.855 +C Fix\sthe\sthreads\sbuild\son\sWindows\swhen\sSQLITE_MAX_WORKER_THREADS\sis\sgreater\nthan\s0. +D 2014-07-29T15:18:00.197 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c afdab9acbda8b96dd0e55052de59b87d972cadc1 +F src/threads.c 28c72cb7818fea8df83116c54d774e9541e60121 F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e8f2dc5fadae96252649875c234fcdef1108bd48 8b651d4d6cde7efbc0cc7155948f477477be100e -R cf6ae596a6404c515cc34a8685610ab8 +P b2f7eb3cc27ecf9a6a88907991148bca1a7d54b3 +R d111ee75a275c87deb7a4599506773e2 U drh -Z da4425a950252cc837b6f80952633a27 +Z e24fba9313c2b62a7de12b4c1acec1ff diff --git a/manifest.uuid b/manifest.uuid index c91039efc5..9e436c5c79 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b2f7eb3cc27ecf9a6a88907991148bca1a7d54b3 \ No newline at end of file +f37db3a03d95b508066745613029b7dd1240b31c \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 634dda3b7d..0f7f8a83bf 100644 --- a/src/threads.c +++ b/src/threads.c @@ -150,7 +150,7 @@ int sqlite3ThreadCreate( /* Wait on an object */ DWORD sqlite3Win32Wait(HANDLE hObject){ DWORD rc; - while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, + while( (rc = WaitForSingleObjectEx(hObject, INFINITE, TRUE))==WAIT_IO_COMPLETION ){} return rc; } From b1ac2bc8da60b01ecd639a2aa35aba953431da3e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 29 Jul 2014 16:37:53 +0000 Subject: [PATCH 157/710] Make the Win32 thread handles are available after the threads exit. FossilOrigin-Name: 565c5af7a75ad5c759ce1a61dab3a61c42819644 --- manifest | 17 +++++++++-------- manifest.uuid | 2 +- src/os_win.c | 7 +++++++ src/threads.c | 23 +++++++++++++---------- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 6f77fc7896..d98cd45916 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sthreads\sbuild\son\sWindows\swhen\sSQLITE_MAX_WORKER_THREADS\sis\sgreater\nthan\s0. -D 2014-07-29T15:18:00.197 +C Make\sthe\sWin32\sthread\shandles\sare\savailable\safter\sthe\sthreads\sexit. +D 2014-07-29T16:37:53.918 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c a7baf1b30f3c58ba20b813e01aab23b18ae44f85 -F src/os_win.c 047e903174b018c50f425be793eafa8d849737a9 +F src/os_win.c 5f8c5568cc749d6ab44006124e7701f463559223 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 28c72cb7818fea8df83116c54d774e9541e60121 +F src/threads.c dfc566f8b5744914bb3e6fe77b5ed63037b9b35d F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -1189,7 +1189,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b2f7eb3cc27ecf9a6a88907991148bca1a7d54b3 -R d111ee75a275c87deb7a4599506773e2 -U drh -Z e24fba9313c2b62a7de12b4c1acec1ff +P f37db3a03d95b508066745613029b7dd1240b31c +Q -2773a5f9879a106a89a3d0bc3c5bfdcb2fe43c7c +R 3e55555e18d084a6f84ef756240b91ed +U mistachkin +Z e9ccfd5ccb7ae9590b75500531d4b897 diff --git a/manifest.uuid b/manifest.uuid index 9e436c5c79..ed3eb9f7ff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f37db3a03d95b508066745613029b7dd1240b31c \ No newline at end of file +565c5af7a75ad5c759ce1a61dab3a61c42819644 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index fa485978d3..5c246a8586 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1275,6 +1275,13 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #endif } +DWORD sqlite3Win32Wait(HANDLE hObject){ + DWORD rc; + while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, + TRUE))==WAIT_IO_COMPLETION ){} + return rc; +} + /* ** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, ** or WinCE. Return false (zero) for Win95, Win98, or WinME. diff --git a/src/threads.c b/src/threads.c index 0f7f8a83bf..5813b34893 100644 --- a/src/threads.c +++ b/src/threads.c @@ -100,21 +100,25 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ /* A running thread */ struct SQLiteThread { uintptr_t tid; /* The thread handle */ + unsigned id; /* The thread identifier */ void *(*xTask)(void*); /* The routine to run as a thread */ void *pIn; /* Argument to xTask */ void *pResult; /* Result of xTask */ }; /* Thread procedure Win32 compatibility shim */ -static void sqlite3ThreadProc( +static unsigned __stdcall sqlite3ThreadProc( void *pArg /* IN: Pointer to the SQLiteThread structure */ ){ SQLiteThread *p = (SQLiteThread *)pArg; assert( p!=0 ); + assert( p->id==GetCurrentThreadId() ); assert( p->xTask!=0 ); p->pResult = p->xTask(p->pIn); - _endthread(); + + _endthreadex(0); + return 0; /* NOT REACHED */ } /* Create a new thread */ @@ -135,37 +139,36 @@ int sqlite3ThreadCreate( }else{ p->xTask = xTask; p->pIn = pIn; - p->tid = _beginthread(sqlite3ThreadProc, 0, p); + p->tid = _beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id); if( p->tid==(uintptr_t)-1 ){ memset(p, 0, sizeof(*p)); } } if( p->xTask==0 ){ + p->id = GetCurrentThreadId(); p->pResult = xTask(pIn); } *ppThread = p; return SQLITE_OK; } -/* Wait on an object */ -DWORD sqlite3Win32Wait(HANDLE hObject){ - DWORD rc; - while( (rc = WaitForSingleObjectEx(hObject, INFINITE, - TRUE))==WAIT_IO_COMPLETION ){} - return rc; -} +DWORD sqlite3Win32Wait(HANDLE hObject); /* os_win.c */ /* Get the results of the thread */ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ DWORD rc; + BOOL bRc; assert( ppOut!=0 ); if( p==0 ) return SQLITE_NOMEM; if( p->xTask==0 ){ rc = WAIT_OBJECT_0; }else{ + assert( p->id!=0 && p->id!=GetCurrentThreadId() ); rc = sqlite3Win32Wait((HANDLE)p->tid); assert( rc!=WAIT_IO_COMPLETION ); + bRc = CloseHandle((HANDLE)p->tid); + assert( bRc ); } if( rc==WAIT_OBJECT_0 ) *ppOut = p->pResult; sqlite3_free(p); From cd4b637760f66cfab3c4c8a9ae3bdadadbaa609e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 29 Jul 2014 17:22:12 +0000 Subject: [PATCH 158/710] Fix a harmless compiler warning. FossilOrigin-Name: 216d21d0e62b3c0ad49f3cb395c845bf4f17ac61 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/vdbesort.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d98cd45916..de6ec8a279 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\sWin32\sthread\shandles\sare\savailable\safter\sthe\sthreads\sexit. -D 2014-07-29T16:37:53.918 +C Fix\sa\sharmless\scompiler\swarning. +D 2014-07-29T17:22:12.808 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c e78efeed9c87d47d7a09df7d176f3fbf6dcb77e2 +F src/vdbesort.c 48b99b15ac27229a67ed601a711aad1a36cc8631 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,8 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f37db3a03d95b508066745613029b7dd1240b31c -Q -2773a5f9879a106a89a3d0bc3c5bfdcb2fe43c7c -R 3e55555e18d084a6f84ef756240b91ed -U mistachkin -Z e9ccfd5ccb7ae9590b75500531d4b897 +P 565c5af7a75ad5c759ce1a61dab3a61c42819644 +R c113c051fb0a0f7ee541f07d792965ad +U drh +Z 1016c6f05046b2d8d5d597484d5273e9 diff --git a/manifest.uuid b/manifest.uuid index ed3eb9f7ff..f715ad6084 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -565c5af7a75ad5c759ce1a61dab3a61c42819644 \ No newline at end of file +216d21d0e62b3c0ad49f3cb395c845bf4f17ac61 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index e9fa6b456f..96f5b64d57 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1113,7 +1113,7 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ } } #else -# define vdbeSorterExtendFile(x,y,z) SQLITE_OK +# define vdbeSorterExtendFile(x,y,z) #endif /* From b92284de1167f5114614e3be9e2dba88048ec7d5 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 29 Jul 2014 18:46:30 +0000 Subject: [PATCH 159/710] Fix unreachable branches in the threads.c module. FossilOrigin-Name: 3175e366bbf7579ec9ab27214b0a4e5cd27ea204 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/threads.c | 14 ++++++++++---- src/vdbesort.c | 6 +++--- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index de6ec8a279..064bb2589f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning. -D 2014-07-29T17:22:12.808 +C Fix\sunreachable\sbranches\sin\sthe\sthreads.c\smodule. +D 2014-07-29T18:46:30.709 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c dfc566f8b5744914bb3e6fe77b5ed63037b9b35d +F src/threads.c b4152ced8515d11a2293c036109069a4e23d78a8 F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c 48b99b15ac27229a67ed601a711aad1a36cc8631 +F src/vdbesort.c 2198e33de4c2c32bdb3fb4f06dbe2ae90a0fce33 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 565c5af7a75ad5c759ce1a61dab3a61c42819644 -R c113c051fb0a0f7ee541f07d792965ad +P 216d21d0e62b3c0ad49f3cb395c845bf4f17ac61 +R ccb9d21b3a566e207d222a64f9b56de4 U drh -Z 1016c6f05046b2d8d5d597484d5273e9 +Z 93aca20bb7a5a87f1d3c643bafb97dd4 diff --git a/manifest.uuid b/manifest.uuid index f715ad6084..cdfed7b950 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -216d21d0e62b3c0ad49f3cb395c845bf4f17ac61 \ No newline at end of file +3175e366bbf7579ec9ab27214b0a4e5cd27ea204 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 5813b34893..7cd2256b49 100644 --- a/src/threads.c +++ b/src/threads.c @@ -51,6 +51,7 @@ int sqlite3ThreadCreate( void *pIn /* Argument passed into xTask() */ ){ SQLiteThread *p; + int rc; assert( ppThread!=0 ); assert( xTask!=0 ); @@ -63,7 +64,12 @@ int sqlite3ThreadCreate( memset(p, 0, sizeof(*p)); p->xTask = xTask; p->pIn = pIn; - if( sqlite3FaultSim(200) ? 1 : pthread_create(&p->tid, 0, xTask, pIn) ){ + if( sqlite3FaultSim(200) ){ + rc = 1; + }else{ + rc = pthread_create(&p->tid, 0, xTask, pIn); + } + if( rc ){ p->done = 1; p->pOut = xTask(pIn); } @@ -76,7 +82,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ int rc; assert( ppOut!=0 ); - if( p==0 ) return SQLITE_NOMEM; + if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->done ){ *ppOut = p->pOut; rc = SQLITE_OK; @@ -160,7 +166,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ BOOL bRc; assert( ppOut!=0 ); - if( p==0 ) return SQLITE_NOMEM; + if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->xTask==0 ){ rc = WAIT_OBJECT_0; }else{ @@ -222,7 +228,7 @@ int sqlite3ThreadCreate( int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); - if( p==0 ) return SQLITE_NOMEM; + if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->xTask ){ *ppOut = p->xTask(p->pIn); }else{ diff --git a/src/vdbesort.c b/src/vdbesort.c index 96f5b64d57..d4a2cc757e 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -938,11 +938,11 @@ static int vdbeSorterJoinThread(SortSubtask *pTask){ #ifdef SQLITE_DEBUG_SORTER_THREADS int bDone = pTask->bDone; #endif - void *pRet; + void *pRet = SQLITE_INT_TO_PTR(SQLITE_ERROR); vdbeSorterBlockDebug(pTask, !bDone, "enter"); - rc = sqlite3ThreadJoin(pTask->pThread, &pRet); + (void)sqlite3ThreadJoin(pTask->pThread, &pRet); vdbeSorterBlockDebug(pTask, !bDone, "exit"); - if( rc==SQLITE_OK ) rc = SQLITE_PTR_TO_INT(pRet); + rc = SQLITE_PTR_TO_INT(pRet); assert( pTask->bDone==1 ); pTask->bDone = 0; pTask->pThread = 0; From 7c2231cf3b9400f1dca70974b3cbbb0d3e660203 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 29 Jul 2014 18:53:01 +0000 Subject: [PATCH 160/710] Add a couple more assert statements. FossilOrigin-Name: 4e816db235025c7998c649fddabfd807290a08b9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/threads.c | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 064bb2589f..968ac9514b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sunreachable\sbranches\sin\sthe\sthreads.c\smodule. -D 2014-07-29T18:46:30.709 +C Add\sa\scouple\smore\sassert\sstatements. +D 2014-07-29T18:53:01.887 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c b4152ced8515d11a2293c036109069a4e23d78a8 +F src/threads.c c0c04b9c8beed10327412819126edbc6f3b3d4d7 F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 216d21d0e62b3c0ad49f3cb395c845bf4f17ac61 -R ccb9d21b3a566e207d222a64f9b56de4 -U drh -Z 93aca20bb7a5a87f1d3c643bafb97dd4 +P 3175e366bbf7579ec9ab27214b0a4e5cd27ea204 +R f89aa137e4caaa8d5d9c03d775d53084 +U mistachkin +Z 86b91bc53f3caa97895c33b5247f45da diff --git a/manifest.uuid b/manifest.uuid index cdfed7b950..bb2eb8265b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3175e366bbf7579ec9ab27214b0a4e5cd27ea204 \ No newline at end of file +4e816db235025c7998c649fddabfd807290a08b9 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 7cd2256b49..8bb24003b7 100644 --- a/src/threads.c +++ b/src/threads.c @@ -168,7 +168,9 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ assert( ppOut!=0 ); if( NEVER(p==0) ) return SQLITE_NOMEM; if( p->xTask==0 ){ + assert( p->id==GetCurrentThreadId() ); rc = WAIT_OBJECT_0; + assert( p->tid==0 ); }else{ assert( p->id!=0 && p->id!=GetCurrentThreadId() ); rc = sqlite3Win32Wait((HANDLE)p->tid); From acd57ea0c2bbd41cbb1d49a2ac6d97fda9854094 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 29 Jul 2014 19:00:43 +0000 Subject: [PATCH 161/710] Update return value checking to conform to the beginthreadex() specs. FossilOrigin-Name: 3144a16f91364b455e54e6030714e1c12ab0bdb7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/threads.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 968ac9514b..7c55c884b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\scouple\smore\sassert\sstatements. -D 2014-07-29T18:53:01.887 +C Update\sreturn\svalue\schecking\sto\sconform\sto\sthe\sbeginthreadex()\sspecs. +D 2014-07-29T19:00:43.216 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c c0c04b9c8beed10327412819126edbc6f3b3d4d7 +F src/threads.c d5812ef982cbd9f1a211142dba1a756a632adeea F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3175e366bbf7579ec9ab27214b0a4e5cd27ea204 -R f89aa137e4caaa8d5d9c03d775d53084 +P 4e816db235025c7998c649fddabfd807290a08b9 +R 5dbe393ae5ff45dc682e2cd252c64ea7 U mistachkin -Z 86b91bc53f3caa97895c33b5247f45da +Z dbee73cad348dd0ed17a62d1c858d9f3 diff --git a/manifest.uuid b/manifest.uuid index bb2eb8265b..383d189592 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e816db235025c7998c649fddabfd807290a08b9 \ No newline at end of file +3144a16f91364b455e54e6030714e1c12ab0bdb7 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 8bb24003b7..908ad784ae 100644 --- a/src/threads.c +++ b/src/threads.c @@ -146,7 +146,7 @@ int sqlite3ThreadCreate( p->xTask = xTask; p->pIn = pIn; p->tid = _beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id); - if( p->tid==(uintptr_t)-1 ){ + if( p->tid==0 ){ memset(p, 0, sizeof(*p)); } } From 0479c6ab9b9dbd354f4ad0344d18b5356807d137 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 29 Jul 2014 21:44:13 +0000 Subject: [PATCH 162/710] Disable an assert that is sometimes generated spuriously. FossilOrigin-Name: bd9ee0ea69181526cfc3cadac33a5ec5190112b0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/threads.c | 8 ++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 7c55c884b5..1054831700 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sreturn\svalue\schecking\sto\sconform\sto\sthe\sbeginthreadex()\sspecs. -D 2014-07-29T19:00:43.216 +C Disable\san\sassert\sthat\sis\ssometimes\sgenerated\sspuriously. +D 2014-07-29T21:44:13.328 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,7 +277,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c d5812ef982cbd9f1a211142dba1a756a632adeea +F src/threads.c 22dded4283dc4b25422f6444cdcb8d6b1ea0b5ff F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 @@ -1189,7 +1189,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4e816db235025c7998c649fddabfd807290a08b9 -R 5dbe393ae5ff45dc682e2cd252c64ea7 +P 3144a16f91364b455e54e6030714e1c12ab0bdb7 +R 9e96844dee61c8b0e69a9d02798bc497 U mistachkin -Z dbee73cad348dd0ed17a62d1c858d9f3 +Z d96e323c9f24218fc7129fc10b94c004 diff --git a/manifest.uuid b/manifest.uuid index 383d189592..157c4b0987 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3144a16f91364b455e54e6030714e1c12ab0bdb7 \ No newline at end of file +bd9ee0ea69181526cfc3cadac33a5ec5190112b0 \ No newline at end of file diff --git a/src/threads.c b/src/threads.c index 908ad784ae..213a129c99 100644 --- a/src/threads.c +++ b/src/threads.c @@ -119,7 +119,15 @@ static unsigned __stdcall sqlite3ThreadProc( SQLiteThread *p = (SQLiteThread *)pArg; assert( p!=0 ); +#if 0 + /* + ** This assert appears to trigger spuriously on certain + ** versions of Windows, possibly due to _beginthreadex() + ** and/or CreateThread() not fully setting their thread + ** ID parameter before starting the thread. + */ assert( p->id==GetCurrentThreadId() ); +#endif assert( p->xTask!=0 ); p->pResult = p->xTask(p->pIn); From ac50232d0fb86ef6733ee697686aed3527ff693a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Jul 2014 13:56:48 +0000 Subject: [PATCH 163/710] Ensure that the correct number of columns in a UNIQUE index are checked for uniqueness, regardless of whether or not the original table has a ROWID or if the columns are NOT NULL, etc. Ticket [9a6daf340df99ba93c]. FossilOrigin-Name: 6b785e92f279cb65746834d5cd25594fd3333342 --- manifest | 19 ++++++------ manifest.uuid | 2 +- src/build.c | 2 +- src/vdbe.c | 12 ++++---- src/vdbesort.c | 13 ++++---- test/unique2.test | 78 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 102 insertions(+), 24 deletions(-) create mode 100644 test/unique2.test diff --git a/manifest b/manifest index 36b66e55d7..551a1332b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhancements\sand\supdates\sto\sthe\sWin32\smutex\ssubsystem. -D 2014-07-29T19:54:03.929 +C Ensure\sthat\sthe\scorrect\snumber\sof\scolumns\sin\sa\sUNIQUE\sindex\sare\schecked\sfor\nuniqueness,\sregardless\sof\swhether\sor\snot\sthe\soriginal\stable\shas\sa\sROWID\sor\nif\sthe\scolumns\sare\sNOT\sNULL,\setc.\s\sTicket\s[9a6daf340df99ba93c]. +D 2014-07-30T13:56:48.415 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -170,7 +170,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c b5531339cd826af46b9621e4a9323971a9380e12 F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 48f400fa14fd6add244b954ce7e223ce7ccacf0b +F src/build.c 7ba21d8f0f5f1e8b5a0ed21aab9be2b39d1af516 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -283,14 +283,14 @@ F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c b127f05ff110c1f640b5aff98fa57d5028c8f2ee +F src/vdbe.c f87f77b0049cbef1fa68b331c3551d82b4d9fba4 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c ac063f36c929f88bf6cecdbcc413000e272265bb F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c 44441d73b08b3a638dcdb725afffb87c6574ad27 +F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1041,6 +1041,7 @@ F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 +F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb @@ -1184,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8b651d4d6cde7efbc0cc7155948f477477be100e 08c9a4ea6353900095a471365a8611a443f5f399 -R b75cbb8ba71bd17b820832cbd89e629a -U mistachkin -Z f55203ab4632e071cc6adc34ad558bc2 +P ca9868cdae19045dc522490b34dee0f14d928ebe +R aa3d0aadabff4023861d0c947293977e +U drh +Z 0e144877cde1d96e38c3dc4f9894f6d4 diff --git a/manifest.uuid b/manifest.uuid index 2526bc68aa..194b99221e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca9868cdae19045dc522490b34dee0f14d928ebe \ No newline at end of file +6b785e92f279cb65746834d5cd25594fd3333342 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 27c9671da6..384802c533 100644 --- a/src/build.c +++ b/src/build.c @@ -2712,7 +2712,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); addr2 = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, - pKey->nField - pIndex->nKeyCol); VdbeCoverage(v); + pIndex->nKeyCol); VdbeCoverage(v); sqlite3UniqueConstraint(pParse, OE_Abort, pIndex); }else{ addr2 = sqlite3VdbeCurrentAddr(v); diff --git a/src/vdbe.c b/src/vdbe.c index a00a1da836..52f0f3aa74 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4271,12 +4271,12 @@ case OP_ResetCount: { } /* Opcode: SorterCompare P1 P2 P3 P4 -** Synopsis: if key(P1)!=rtrim(r[P3],P4) goto P2 +** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2 ** ** P1 is a sorter cursor. This instruction compares a prefix of the ** the record blob in register P3 against a prefix of the entry that -** the sorter cursor currently points to. The final P4 fields of both -** the P3 and sorter record are ignored. +** the sorter cursor currently points to. Only the first P4 fields +** of r[P3] and the sorter record are compared. ** ** If either P3 or the sorter contains a NULL in one of their significant ** fields (not counting the P4 fields at the end which are ignored) then @@ -4288,14 +4288,14 @@ case OP_ResetCount: { case OP_SorterCompare: { VdbeCursor *pC; int res; - int nIgnore; + int nKeyCol; pC = p->apCsr[pOp->p1]; assert( isSorter(pC) ); assert( pOp->p4type==P4_INT32 ); pIn3 = &aMem[pOp->p3]; - nIgnore = pOp->p4.i; - rc = sqlite3VdbeSorterCompare(pC, pIn3, nIgnore, &res); + nKeyCol = pOp->p4.i; + rc = sqlite3VdbeSorterCompare(pC, pIn3, nKeyCol, &res); VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2-1; diff --git a/src/vdbesort.c b/src/vdbesort.c index d3690244c4..6a5855f2ef 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -385,7 +385,7 @@ static int vdbeSorterIterInit( */ static void vdbeSorterCompare( const VdbeCursor *pCsr, /* Cursor object (for pKeyInfo) */ - int nIgnore, /* Ignore the last nIgnore fields */ + int nKeyCol, /* Num of columns. 0 means "all" */ const void *pKey1, int nKey1, /* Left side of comparison */ const void *pKey2, int nKey2, /* Right side of comparison */ int *pRes /* OUT: Result of comparison */ @@ -399,10 +399,9 @@ static void vdbeSorterCompare( sqlite3VdbeRecordUnpack(pKeyInfo, nKey2, pKey2, r2); } - if( nIgnore ){ - r2->nField = pKeyInfo->nField - nIgnore; - assert( r2->nField>0 ); - for(i=0; inField; i++){ + if( nKeyCol ){ + r2->nField = nKeyCol; + for(i=0; iaMem[i].flags & MEM_Null ){ *pRes = -1; return; @@ -1084,13 +1083,13 @@ int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ int sqlite3VdbeSorterCompare( const VdbeCursor *pCsr, /* Sorter cursor */ Mem *pVal, /* Value to compare to current sorter key */ - int nIgnore, /* Ignore this many fields at the end */ + int nKeyCol, /* Only compare this many fields */ int *pRes /* OUT: Result of comparison */ ){ VdbeSorter *pSorter = pCsr->pSorter; void *pKey; int nKey; /* Sorter key to compare pVal with */ pKey = vdbeSorterRowkey(pSorter, &nKey); - vdbeSorterCompare(pCsr, nIgnore, pVal->z, pVal->n, pKey, nKey, pRes); + vdbeSorterCompare(pCsr, nKeyCol, pVal->z, pVal->n, pKey, nKey, pRes); return SQLITE_OK; } diff --git a/test/unique2.test b/test/unique2.test new file mode 100644 index 0000000000..6a320d4d7d --- /dev/null +++ b/test/unique2.test @@ -0,0 +1,78 @@ +# 2014-07-30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the CREATE UNIQUE INDEX statement +# to verify that ticket 9a6daf340df99ba93c53bcf8fa83d9f28040d2a8 +# has been fixed: +# +# drh added on 2014-07-30 12:33:04: +# +# The CREATE UNIQUE INDEX on the third line below does not fail even +# though the x column values are not all unique. +# +# CREATE TABLE t1(x NOT NULL); +# INSERT INTO t1 VALUES(1),(2),(2),(3); +# CREATE UNIQUE INDEX t1x ON t1(x); +# +# If the index is created before the INSERT, then uniqueness is enforced +# at the point of the INSERT. Note that the NOT NULL on the indexed column +# seems to be required in order to exhibit this bug. +# +# "PRAGMA integrity_check" does not detect the resulting malformed database. +# That might be considered a separate issue. +# +# Bisecting shows that this problem was introduced by the addition of +# WITHOUT ROWID support in version 3.8.2, specifically in check-in +# [c80e229dd9c1230] on 2013-11-07. This problem was reported on the mailing +# list by Pavel Pimenov. and primary keys, and the UNIQUE constraint +# on table columns +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +foreach {id sql} { + 1 {CREATE TABLE t1(x TEXT PRIMARY KEY, y NOT NULL) WITHOUT ROWID} + 2 {CREATE TABLE t1(x TEXT PRIMARY KEY, y NOT NULL)} + 3 {CREATE TABLE t1(x TEXT PRIMARY KEY, y) WITHOUT ROWID} + 4 {CREATE TABLE t1(x TEXT PRIMARY KEY, y)} +} { + do_test $id.1 { + db eval {DROP TABLE IF EXISTS t1} + db eval $sql + db eval {INSERT INTO t1(x,y) VALUES(1,1),(2,2),(3,2),(4,3)} + } {} + do_test $id.2 { + catchsql {CREATE UNIQUE INDEX t1y ON t1(y)} + } {1 {UNIQUE constraint failed: t1.y}} +} + +foreach {id sql} { + 5 {CREATE TABLE t1(w,x,y NOT NULL,z NOT NULL,PRIMARY KEY(w,x)) WITHOUT ROWID} + 6 {CREATE TABLE t1(w,x,y NOT NULL,z NOT NULL,PRIMARY KEY(w,x))} + 7 {CREATE TABLE t1(w,x,y NOT NULL,z,PRIMARY KEY(w,x)) WITHOUT ROWID} + 8 {CREATE TABLE t1(w,x,y NOT NULL,z,PRIMARY KEY(w,x))} + 9 {CREATE TABLE t1(w,x,y,z NOT NULL,PRIMARY KEY(w,x)) WITHOUT ROWID} + 10 {CREATE TABLE t1(w,x,y,z NOT NULL,PRIMARY KEY(w,x))} + 11 {CREATE TABLE t1(w,x,y,z,PRIMARY KEY(w,x)) WITHOUT ROWID} + 12 {CREATE TABLE t1(w,x,y,z,PRIMARY KEY(w,x))} +} { + do_test $id.1 { + db eval {DROP TABLE IF EXISTS t1} + db eval $sql + db eval {INSERT INTO t1(w,x,y,z) VALUES(1,2,3,4),(2,3,3,4)} + } {} + do_test $id.2 { + catchsql {CREATE UNIQUE INDEX t1yz ON t1(y,z)} + } {1 {UNIQUE constraint failed: t1.y, t1.z}} +} + +finish_test From a1a8298c2e89d45be0865b3bead1faf83b926705 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Jul 2014 15:43:05 +0000 Subject: [PATCH 164/710] Add the "eForce" parameter to the sqlite3_multiplex_shutdown() entry point in test_multiplex.c. Shutdown is forced if true. Shutdown is not done if there are pending database connections and eForce is false, but an error log entry is made instead. FossilOrigin-Name: c7303d0139f7e7f4fa7060b52942e6c6c6d4b622 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/test_multiplex.c | 21 +++++++++++++++------ src/test_multiplex.h | 2 +- test/multiplex.test | 10 ++++++++++ 5 files changed, 35 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 551a1332b7..ac3ca46634 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\scorrect\snumber\sof\scolumns\sin\sa\sUNIQUE\sindex\sare\schecked\sfor\nuniqueness,\sregardless\sof\swhether\sor\snot\sthe\soriginal\stable\shas\sa\sROWID\sor\nif\sthe\scolumns\sare\sNOT\sNULL,\setc.\s\sTicket\s[9a6daf340df99ba93c]. -D 2014-07-30T13:56:48.415 +C Add\sthe\s"eForce"\sparameter\sto\sthe\ssqlite3_multiplex_shutdown()\sentry\spoint\nin\stest_multiplex.c.\s\sShutdown\sis\sforced\sif\strue.\s\sShutdown\sis\snot\sdone\sif\nthere\sare\spending\sdatabase\sconnections\sand\seForce\sis\sfalse,\sbut\san\serror\slog\nentry\sis\smade\sinstead. +D 2014-07-30T15:43:05.568 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -257,8 +257,8 @@ F src/test_intarray.h 2ece66438cfd177b78d1bfda7a4180cd3a10844d F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 F src/test_malloc.c 1ff5b1243d96124c9a180f3b89424820a1f337f3 -F src/test_multiplex.c 9f304bf04170c91c0318238d512df2da039eb1c8 -F src/test_multiplex.h 110a8c4d356e0aa464ca8730375608a9a0b61ae1 +F src/test_multiplex.c ca90057438b63bf0840ebb84d0ef050624519a76 +F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f F src/test_onefile.c 0396f220561f3b4eedc450cef26d40c593c69a25 F src/test_osinst.c 3d0340bc31a9f3d8a3547e0272373e80f78dde25 @@ -719,7 +719,7 @@ F test/mmap1.test 93d167b328255cbe6679fe1e1a23be1b1197d07b F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 F test/mmap3.test c92273e16eb8d23c1d55c9815b446bb72ef0512e F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 -F test/multiplex.test e08cc7177bd6d85990ee1d71100bb6c684c02256 +F test/multiplex.test 4a0d07a2490dc958e6b676a6825cf761be33d869 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ca9868cdae19045dc522490b34dee0f14d928ebe -R aa3d0aadabff4023861d0c947293977e +P 6b785e92f279cb65746834d5cd25594fd3333342 +R 91be173298089741f719dedf8b4f59f4 U drh -Z 0e144877cde1d96e38c3dc4f9894f6d4 +Z 69b6c3f54a5e9094db9af1965ce3b4cb diff --git a/manifest.uuid b/manifest.uuid index 194b99221e..2ce81d04f9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6b785e92f279cb65746834d5cd25594fd3333342 \ No newline at end of file +c7303d0139f7e7f4fa7060b52942e6c6c6d4b622 \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index 45a1edfbbe..427cc65ad7 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -1176,14 +1176,20 @@ int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefault){ ** THIS ROUTINE IS NOT THREADSAFE. Call this routine exactly once while ** shutting down in order to free all remaining multiplex groups. */ -int sqlite3_multiplex_shutdown(void){ +int sqlite3_multiplex_shutdown(int eForce){ + int rc = SQLITE_OK; if( gMultiplex.isInitialized==0 ) return SQLITE_MISUSE; - if( gMultiplex.pGroups ) return SQLITE_MISUSE; + if( gMultiplex.pGroups ){ + sqlite3_log(SQLITE_MISUSE, "sqlite3_multiplex_shutdown() called " + "while database connections are still open"); + if( !eForce ) return SQLITE_MISUSE; + rc = SQLITE_MISUSE; + } gMultiplex.isInitialized = 0; sqlite3_mutex_free(gMultiplex.pMutex); sqlite3_vfs_unregister(&gMultiplex.sThisVfs); memset(&gMultiplex, 0, sizeof(gMultiplex)); - return SQLITE_OK; + return rc; } /***************************** Test Code ***********************************/ @@ -1236,13 +1242,16 @@ static int test_multiplex_shutdown( UNUSED_PARAMETER(clientData); - if( objc!=1 ){ - Tcl_WrongNumArgs(interp, 1, objv, ""); + if( objc==2 && strcmp(Tcl_GetString(objv[1]),"-force")!=0 ){ + objc = 3; + } + if( (objc!=1 && objc!=2) ){ + Tcl_WrongNumArgs(interp, 1, objv, "?-force?"); return TCL_ERROR; } /* Call sqlite3_multiplex_shutdown() */ - rc = sqlite3_multiplex_shutdown(); + rc = sqlite3_multiplex_shutdown(objc==2); Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_STATIC); return TCL_OK; diff --git a/src/test_multiplex.h b/src/test_multiplex.h index 9fe2f0f2b2..d973e4af27 100644 --- a/src/test_multiplex.h +++ b/src/test_multiplex.h @@ -90,7 +90,7 @@ extern int sqlite3_multiplex_initialize(const char *zOrigVfsName, int makeDefaul ** THIS ROUTINE IS NOT THREADSAFE. Call this routine exactly once while ** shutting down in order to free all remaining multiplex groups. */ -extern int sqlite3_multiplex_shutdown(void); +extern int sqlite3_multiplex_shutdown(int eForce); #ifdef __cplusplus } /* End of the 'extern "C"' block */ diff --git a/test/multiplex.test b/test/multiplex.test index 32c87d9a52..55fab013c4 100644 --- a/test/multiplex.test +++ b/test/multiplex.test @@ -68,6 +68,12 @@ proc multiplex_delete {name} { } db close +sqlite3_shutdown +test_sqlite3_log xLog +proc xLog {error_code msg} { + lappend ::log $error_code $msg +} +unset -nocomplain log multiplex_delete test.db multiplex_delete test2.db @@ -188,12 +194,16 @@ do_test multiplex-2.3.1 { } {} +unset -nocomplain ::log do_test multiplex-2.4.1 { sqlite3_multiplex_shutdown } {SQLITE_MISUSE} do_test multiplex-2.4.2 { execsql { INSERT INTO t1 VALUES(3, randomblob(1100)) } } {} +do_test multiplex-2.4.3 { + set ::log +} {SQLITE_MISUSE {sqlite3_multiplex_shutdown() called while database connections are still open}} do_test multiplex-2.4.4 { file size [multiplex_name test.x 0] } {7168} do_test multiplex-2.4.5 { db close From c690461ea1bc3a4dab5a43f19fa6a46db5dd1973 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Jul 2014 17:21:37 +0000 Subject: [PATCH 165/710] Mark some invariants in the vdbesort.c logic when SQLITE_MAX_WORKER_THREADS==0. FossilOrigin-Name: 721cd965859c9ccc24d2a1d2851c914229e584b3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 20 +++++++++++++++++--- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 7a190a12d7..eead314fba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\strunk\schanges,\sand\sespecially\sthe\sfix\sfor\sthe\sCREATE\sUNIQUE\sINDEX\nproblem\sof\sticket\s[9a6daf340df99ba9]. -D 2014-07-30T14:44:24.940 +C Mark\ssome\sinvariants\sin\sthe\svdbesort.c\slogic\swhen\sSQLITE_MAX_WORKER_THREADS==0. +D 2014-07-30T17:21:37.320 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c cab84b480d5c616e830d9b20c8903e42de3b60b3 +F src/vdbesort.c e37e494274fb9a05955b44eb61ead2230ae3e321 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1190,7 +1190,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bd9ee0ea69181526cfc3cadac33a5ec5190112b0 6b785e92f279cb65746834d5cd25594fd3333342 -R 5abc11219614f612d7ef9f82b784bc0c +P 5b50a8380b2b678c1646ff303e3696efc1d7d92c +R 79135dabbf9021221308446429439ade U drh -Z f20a2d1678a8f4c3f64fec3d49fad416 +Z af737cf3847cc90d4e2e364aefe53a30 diff --git a/manifest.uuid b/manifest.uuid index 96ada87e5a..a98bddf92c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b50a8380b2b678c1646ff303e3696efc1d7d92c \ No newline at end of file +721cd965859c9ccc24d2a1d2851c914229e584b3 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 9110bd67b0..8ff68d54c8 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1860,6 +1860,10 @@ static void vdbeMergeEngineCompare( /* ** Allowed values for the eMode parameter to vdbeMergeEngineInit() ** and vdbePmaReaderIncrMergeInit(). +** +** Only INCRINIT_NORMAL is valid in single-threaded builds (when +** SQLITE_MAX_WORKER_THREADS==0). The other values are only used +** when there exists one or more separate worker threads. */ #define INCRINIT_NORMAL 0 #define INCRINIT_TASK 1 @@ -1898,12 +1902,15 @@ static int vdbeMergeEngineInit( int i; /* For looping over PmaReader objects */ int nTree = pMerger->nTree; + /* eMode is always INCRINIT_NORMAL in single-threaded mode */ + assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); + /* Verify that the MergeEngine is assigned to a single thread */ assert( pMerger->pTask==0 ); // || pMerger->pTask==pTask ); pMerger->pTask = pTask; for(i=0; i0 && eMode==INCRINIT_ROOT ){ /* PmaReaders should be normally initialized in order, as if they are ** reading from the same temp file this makes for more linear file IO. ** However, in the INCRINIT_ROOT case, if PmaReader aReadr[nTask-1] is @@ -1962,6 +1969,10 @@ static int vdbeMergeEngineInit( static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ int rc = SQLITE_OK; IncrMerger *pIncr = pReadr->pIncr; + + /* eMode is always INCRINIT_NORMAL in single-threaded mode */ + assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); + if( pIncr ){ SortSubtask *pTask = pIncr->pTask; sqlite3 *db = pTask->pSorter->db; @@ -2005,7 +2016,9 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ } #endif - if( rc==SQLITE_OK && eMode!=INCRINIT_TASK ){ + if( rc==SQLITE_OK + && (SQLITE_MAX_WORKER_THREADS==0 || eMode!=INCRINIT_TASK) + ){ rc = vdbePmaReaderNext(pReadr); } } @@ -2186,7 +2199,8 @@ static int vdbeSorterMergeTreeBuild( for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ SortSubtask *pTask = &pSorter->aTask[iTask]; - if( pTask->nPMA ){ + assert( pTask->nPMA>0 || SQLITE_MAX_WORKER_THREADS>0 ); + if( SQLITE_MAX_WORKER_THREADS==0 || pTask->nPMA ){ MergeEngine *pRoot = 0; /* Root node of tree for this task */ int nDepth = vdbeSorterTreeDepth(pTask->nPMA); i64 iReadOff = 0; From c0fea3cf53015606fbc1725937fd0b74fc523112 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Jul 2014 18:47:12 +0000 Subject: [PATCH 166/710] Add a new sqlite3FaultSim() call to vdbePmaReaderSeek() to facilitate tests of error handling in the sorter. FossilOrigin-Name: 655d8cfc752b3f5f63521a57f2155f8e14aaf7c4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index eead314fba..e5ae89aca5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Mark\ssome\sinvariants\sin\sthe\svdbesort.c\slogic\swhen\sSQLITE_MAX_WORKER_THREADS==0. -D 2014-07-30T17:21:37.320 +C Add\sa\snew\ssqlite3FaultSim()\scall\sto\svdbePmaReaderSeek()\sto\sfacilitate\ntests\sof\serror\shandling\sin\sthe\ssorter. +D 2014-07-30T18:47:12.395 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 F src/vdbeaux.c 3f1d2baa4a8cbdad33cb255a5f4fd1af7a414683 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 -F src/vdbesort.c e37e494274fb9a05955b44eb61ead2230ae3e321 +F src/vdbesort.c b9a830685826c057cfd41993902a5afc6fe436e1 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1190,7 +1190,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5b50a8380b2b678c1646ff303e3696efc1d7d92c -R 79135dabbf9021221308446429439ade +P 721cd965859c9ccc24d2a1d2851c914229e584b3 +R 1fd8d4a346a6db9102e067303c7fe957 U drh -Z af737cf3847cc90d4e2e364aefe53a30 +Z a72347f7edf8a1215db60e10b80e325d diff --git a/manifest.uuid b/manifest.uuid index a98bddf92c..a8cbf441d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -721cd965859c9ccc24d2a1d2851c914229e584b3 \ No newline at end of file +655d8cfc752b3f5f63521a57f2155f8e14aaf7c4 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 8ff68d54c8..6915a2a9b9 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -623,6 +623,7 @@ static int vdbePmaReaderSeek( assert( pReadr->pIncr==0 || pReadr->pIncr->bEof==0 ); + if( sqlite3FaultSim(201) ) return SQLITE_IOERR_READ; if( pReadr->aMap ){ sqlite3OsUnfetch(pReadr->pFd, 0, pReadr->aMap); pReadr->aMap = 0; From d42d0bed943d1c807eed4b69256f4211665cee24 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 30 Jul 2014 21:10:12 +0000 Subject: [PATCH 167/710] Add three new static mutexes for use by the application. This is a partial import of changes from the threads branch. FossilOrigin-Name: 3aad01960f92c5e77dba64ac1a6c6b063378fb97 --- manifest | 20 +++---- manifest.uuid | 2 +- src/mutex.c | 2 +- src/mutex_noop.c | 2 +- src/mutex_unix.c | 8 ++- src/mutex_w32.c | 153 ++++++++++++++++++++++++----------------------- src/sqlite.h.in | 9 ++- 7 files changed, 105 insertions(+), 91 deletions(-) diff --git a/manifest b/manifest index ac3ca46634..276387fbe1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"eForce"\sparameter\sto\sthe\ssqlite3_multiplex_shutdown()\sentry\spoint\nin\stest_multiplex.c.\s\sShutdown\sis\sforced\sif\strue.\s\sShutdown\sis\snot\sdone\sif\nthere\sare\spending\sdatabase\sconnections\sand\seForce\sis\sfalse,\sbut\san\serror\slog\nentry\sis\smade\sinstead. -D 2014-07-30T15:43:05.568 +C Add\sthree\snew\sstatic\smutexes\sfor\suse\sby\sthe\sapplication.\s\sThis\sis\sa\spartial\nimport\sof\schanges\sfrom\sthe\sthreads\sbranch. +D 2014-07-30T21:10:12.606 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -197,11 +197,11 @@ F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c 74670012946c4adc8a6ad84d03acc80959c3e529 F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 -F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc +F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea -F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 -F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc -F src/mutex_w32.c 45020ed78735a202ff14efcf19415d29f556f16d +F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 +F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 +F src/mutex_w32.c 92e8ead41598a289c93ac64dfd5e78820c4dabc1 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e @@ -224,7 +224,7 @@ F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 6762c62e11b504aa014edceab8886495165e3a77 F src/shell.c 191129c3f7a9cf241aea90ff6a6be3e74d3767f0 -F src/sqlite.h.in 5967719136e14183b2573ba619993438e2b9856e +F src/sqlite.h.in 9bbc5815c73b0e77e68b5275481a5e3e7814a804 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 068e42f41a09ce6b9edbe194ac8a470ab53145df @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6b785e92f279cb65746834d5cd25594fd3333342 -R 91be173298089741f719dedf8b4f59f4 +P c7303d0139f7e7f4fa7060b52942e6c6c6d4b622 +R 9a0604d781d4eb725027fc152740bd2b U drh -Z 69b6c3f54a5e9094db9af1965ce3b4cb +Z b2c6542aa553cebe25ffe186e01d871a diff --git a/manifest.uuid b/manifest.uuid index 2ce81d04f9..03e9ad26fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c7303d0139f7e7f4fa7060b52942e6c6c6d4b622 \ No newline at end of file +3aad01960f92c5e77dba64ac1a6c6b063378fb97 \ No newline at end of file diff --git a/src/mutex.c b/src/mutex.c index b567e7c27e..bad5a7c113 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -81,7 +81,7 @@ int sqlite3MutexEnd(void){ */ sqlite3_mutex *sqlite3_mutex_alloc(int id){ #ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return 0; + if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; #endif return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } diff --git a/src/mutex_noop.c b/src/mutex_noop.c index 456e82a25e..1a900c225a 100644 --- a/src/mutex_noop.c +++ b/src/mutex_noop.c @@ -107,7 +107,7 @@ static int debugMutexEnd(void){ return SQLITE_OK; } ** that means that a mutex could not be allocated. */ static sqlite3_mutex *debugMutexAlloc(int id){ - static sqlite3_debug_mutex aStatic[6]; + static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_APP3 - 1]; sqlite3_debug_mutex *pNew = 0; switch( id ){ case SQLITE_MUTEX_FAST: diff --git a/src/mutex_unix.c b/src/mutex_unix.c index eca7295831..c8663144e8 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -96,10 +96,13 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU **
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -133,6 +136,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; sqlite3_mutex *p; diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 48f3bc6969..78d1a6fba4 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -9,16 +9,11 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains the C functions that implement mutexes for Win32. +** This file contains the C functions that implement mutexes for win32 */ #include "sqliteInt.h" #if SQLITE_OS_WIN -/* -** Include code that is common to all os_*.c files -*/ -#include "os_common.h" - /* ** Include the header file for the Windows VFS. */ @@ -27,7 +22,7 @@ /* ** The code in this file is only used if we are compiling multithreaded -** on a Win32 system. +** on a win32 system. */ #ifdef SQLITE_MUTEX_W32 @@ -40,24 +35,50 @@ struct sqlite3_mutex { #ifdef SQLITE_DEBUG volatile int nRef; /* Number of enterances */ volatile DWORD owner; /* Thread holding this mutex */ - volatile int trace; /* True to trace changes */ + int trace; /* True to trace changes */ #endif }; - -/* -** These are the initializer values used when declaring a "static" mutex -** on Win32. It should be noted that all mutexes require initialization -** on the Win32 platform. -*/ #define SQLITE_W32_MUTEX_INITIALIZER { 0 } - #ifdef SQLITE_DEBUG -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, \ - 0L, (DWORD)0, 0 } +#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, 0L, (DWORD)0, 0 } #else #define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 } #endif +/* +** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, +** or WinCE. Return false (zero) for Win95, Win98, or WinME. +** +** Here is an interesting observation: Win95, Win98, and WinME lack +** the LockFileEx() API. But we can still statically link against that +** API as long as we don't call it win running Win95/98/ME. A call to +** this routine is used to determine if the host is Win95/98/ME or +** WinNT/2K/XP so that we will know whether or not we can safely call +** the LockFileEx() API. +** +** mutexIsNT() is only used for the TryEnterCriticalSection() API call, +** which is only available if your application was compiled with +** _WIN32_WINNT defined to a value >= 0x0400. Currently, the only +** call to TryEnterCriticalSection() is #ifdef'ed out, so #ifdef +** this out as well. +*/ +#if 0 +#if SQLITE_OS_WINCE || SQLITE_OS_WINRT +# define mutexIsNT() (1) +#else + static int mutexIsNT(void){ + static int osType = 0; + if( osType==0 ){ + OSVERSIONINFO sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + GetVersionEx(&sInfo); + osType = sInfo.dwPlatformId==VER_PLATFORM_WIN32_NT ? 2 : 1; + } + return osType==2; + } +#endif /* SQLITE_OS_WINCE || SQLITE_OS_WINRT */ +#endif + #ifdef SQLITE_DEBUG /* ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are @@ -66,21 +87,23 @@ struct sqlite3_mutex { static int winMutexHeld(sqlite3_mutex *p){ return p->nRef!=0 && p->owner==GetCurrentThreadId(); } - static int winMutexNotheld2(sqlite3_mutex *p, DWORD tid){ return p->nRef==0 || p->owner!=tid; } - static int winMutexNotheld(sqlite3_mutex *p){ - DWORD tid = GetCurrentThreadId(); + DWORD tid = GetCurrentThreadId(); return winMutexNotheld2(p, tid); } #endif + /* ** Initialize and deinitialize the mutex subsystem. */ -static sqlite3_mutex winMutex_staticMutexes[6] = { +static sqlite3_mutex winMutex_staticMutexes[] = { + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, + SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER, @@ -88,20 +111,17 @@ static sqlite3_mutex winMutex_staticMutexes[6] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; - static int winMutex_isInit = 0; - -/* As winMutexInit() and winMutexEnd() are called as part of the -** sqlite3_initialize() and sqlite3_shutdown() processing, the -** "interlocked" magic used in this section may not strictly -** necessary. +/* As winMutexInit() and winMutexEnd() are called as part +** of the sqlite3_initialize and sqlite3_shutdown() +** processing, the "interlocked" magic is probably not +** strictly necessary. */ static LONG winMutex_lock = 0; -int sqlite3_win32_is_nt(void); /* os_win.c */ void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ -static int winMutexInit(void){ +static int winMutexInit(void){ /* The first to increment to 1 does actual initialization */ if( InterlockedCompareExchange(&winMutex_lock, 1, 0)==0 ){ int i; @@ -114,17 +134,16 @@ static int winMutexInit(void){ } winMutex_isInit = 1; }else{ - /* Another thread is (in the process of) initializing the static - ** mutexes */ + /* Someone else is in the process of initing the static mutexes */ while( !winMutex_isInit ){ sqlite3_win32_sleep(1); } } - return SQLITE_OK; + return SQLITE_OK; } -static int winMutexEnd(void){ - /* The first to decrement to 0 does actual shutdown +static int winMutexEnd(void){ + /* The first to decrement to 0 does actual shutdown ** (which should be the last to shutdown.) */ if( InterlockedCompareExchange(&winMutex_lock, 0, 1)==1 ){ if( winMutex_isInit==1 ){ @@ -135,7 +154,7 @@ static int winMutexEnd(void){ winMutex_isInit = 0; } } - return SQLITE_OK; + return SQLITE_OK; } /* @@ -150,10 +169,13 @@ static int winMutexEnd(void){ **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU **
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 +**
  • SQLITE_MUTEX_STATIC_APP3 ** ** ** The first two constants cause sqlite3_mutex_alloc() to create @@ -176,7 +198,7 @@ static int winMutexEnd(void){ ** ** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. But for the static +** returns a different mutex on every call. But for the static ** mutex types, the same mutex is returned on every call that has ** the same type number. */ @@ -187,12 +209,9 @@ static sqlite3_mutex *winMutexAlloc(int iType){ case SQLITE_MUTEX_FAST: case SQLITE_MUTEX_RECURSIVE: { p = sqlite3MallocZero( sizeof(*p) ); - if( p ){ + if( p ){ #ifdef SQLITE_DEBUG p->id = iType; -#ifdef SQLITE_WIN32_MUTEX_TRACE_DYNAMIC - p->trace = 1; -#endif #endif #if SQLITE_OS_WINRT InitializeCriticalSectionEx(&p->mutex, 0, 0); @@ -203,15 +222,12 @@ static sqlite3_mutex *winMutexAlloc(int iType){ break; } default: { + assert( winMutex_isInit==1 ); assert( iType-2 >= 0 ); assert( iType-2 < ArraySize(winMutex_staticMutexes) ); - assert( winMutex_isInit==1 ); p = &winMutex_staticMutexes[iType-2]; #ifdef SQLITE_DEBUG p->id = iType; -#ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC - p->trace = 1; -#endif #endif break; } @@ -227,10 +243,9 @@ static sqlite3_mutex *winMutexAlloc(int iType){ */ static void winMutexFree(sqlite3_mutex *p){ assert( p ); -#ifdef SQLITE_DEBUG assert( p->nRef==0 && p->owner==0 ); assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); -#endif + assert( winMutex_isInit==1 ); DeleteCriticalSection(&p->mutex); sqlite3_free(p); } @@ -247,38 +262,31 @@ static void winMutexFree(sqlite3_mutex *p){ ** more than once, the behavior is undefined. */ static void winMutexEnter(sqlite3_mutex *p){ -#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - DWORD tid = GetCurrentThreadId(); -#endif #ifdef SQLITE_DEBUG - assert( p ); + DWORD tid = GetCurrentThreadId(); assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); -#else - assert( p ); #endif + assert( winMutex_isInit==1 ); EnterCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG assert( p->nRef>0 || p->owner==0 ); - p->owner = tid; + p->owner = tid; p->nRef++; if( p->trace ){ - OSTRACE(("ENTER-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", - tid, p, p->trace, p->nRef)); + printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); } #endif } - static int winMutexTry(sqlite3_mutex *p){ -#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - DWORD tid = GetCurrentThreadId(); +#ifndef NDEBUG + DWORD tid = GetCurrentThreadId(); #endif int rc = SQLITE_BUSY; - assert( p ); assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); /* ** The sqlite3_mutex_try() routine is very rarely used, and when it ** is used it is merely an optimization. So it is OK for it to always - ** fail. + ** fail. ** ** The TryEnterCriticalSection() interface is only available on WinNT. ** And some windows compilers complain if you try to use it without @@ -286,21 +294,19 @@ static int winMutexTry(sqlite3_mutex *p){ ** For that reason, we will omit this optimization for now. See ** ticket #2685. */ -#if defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x0400 - if( sqlite3_win32_is_nt() && TryEnterCriticalSection(&p->mutex) ){ -#ifdef SQLITE_DEBUG +#if 0 + assert( winMutex_isInit==1 ); + if( mutexIsNT() && TryEnterCriticalSection(&p->mutex) ){ p->owner = tid; p->nRef++; -#endif rc = SQLITE_OK; } #else UNUSED_PARAMETER(p); #endif #ifdef SQLITE_DEBUG - if( p->trace ){ - OSTRACE(("TRY-MUTEX tid=%lu, mutex=%p (%d), owner=%lu, nRef=%d, rc=%s\n", - tid, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); + if( rc==SQLITE_OK && p->trace ){ + printf("try mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); } #endif return rc; @@ -313,22 +319,19 @@ static int winMutexTry(sqlite3_mutex *p){ ** is not currently allocated. SQLite will never do either. */ static void winMutexLeave(sqlite3_mutex *p){ -#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) +#ifndef NDEBUG DWORD tid = GetCurrentThreadId(); -#endif - assert( p ); -#ifdef SQLITE_DEBUG assert( p->nRef>0 ); assert( p->owner==tid ); p->nRef--; if( p->nRef==0 ) p->owner = 0; assert( p->nRef==0 || p->id==SQLITE_MUTEX_RECURSIVE ); #endif + assert( winMutex_isInit==1 ); LeaveCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG if( p->trace ){ - OSTRACE(("LEAVE-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", - tid, p, p->trace, p->nRef)); + printf("leave mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); } #endif } @@ -350,7 +353,7 @@ sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ 0 #endif }; + return &sMutex; } - #endif /* SQLITE_MUTEX_W32 */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 8d51632285..8470d9e8c0 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5875,10 +5875,12 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); **
  • SQLITE_MUTEX_RECURSIVE **
  • SQLITE_MUTEX_STATIC_MASTER **
  • SQLITE_MUTEX_STATIC_MEM -**
  • SQLITE_MUTEX_STATIC_MEM2 +**
  • SQLITE_MUTEX_STATIC_OPEN **
  • SQLITE_MUTEX_STATIC_PRNG **
  • SQLITE_MUTEX_STATIC_LRU -**
  • SQLITE_MUTEX_STATIC_LRU2 +**
  • SQLITE_MUTEX_STATIC_PMEM +**
  • SQLITE_MUTEX_STATIC_APP1 +**
  • SQLITE_MUTEX_STATIC_APP2 ** )^ ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) @@ -6082,6 +6084,9 @@ int sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ #define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ #define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ +#define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ /* ** CAPI3REF: Retrieve the mutex for a database connection From 0174ffa976f7a042a5ae1adaf58b1b125d5f0eb1 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 30 Jul 2014 23:11:16 +0000 Subject: [PATCH 168/710] Re-integrate the recent changes from the 'winMutex' branch back into the Win32 mutex subsystem. FossilOrigin-Name: 5360ecb0b8891d0c27f3f02d81b6c7b548361a10 --- manifest | 14 ++--- manifest.uuid | 2 +- src/mutex_w32.c | 138 +++++++++++++++++++++++++----------------------- 3 files changed, 80 insertions(+), 74 deletions(-) diff --git a/manifest b/manifest index 276387fbe1..c1369a4aa5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthree\snew\sstatic\smutexes\sfor\suse\sby\sthe\sapplication.\s\sThis\sis\sa\spartial\nimport\sof\schanges\sfrom\sthe\sthreads\sbranch. -D 2014-07-30T21:10:12.606 +C Re-integrate\sthe\srecent\schanges\sfrom\sthe\s'winMutex'\sbranch\sback\sinto\sthe\sWin32\smutex\ssubsystem. +D 2014-07-30T23:11:16.516 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -201,7 +201,7 @@ F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 -F src/mutex_w32.c 92e8ead41598a289c93ac64dfd5e78820c4dabc1 +F src/mutex_w32.c 08890085b81ce181d2cbc6fc2636e3444ae30b27 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c7303d0139f7e7f4fa7060b52942e6c6c6d4b622 -R 9a0604d781d4eb725027fc152740bd2b -U drh -Z b2c6542aa553cebe25ffe186e01d871a +P 3aad01960f92c5e77dba64ac1a6c6b063378fb97 +R 30d9c6511ca3c35312cef2a09e79ecbc +U mistachkin +Z c0143953af6d35d374b1fbf6a22594a7 diff --git a/manifest.uuid b/manifest.uuid index 03e9ad26fa..ab5604951f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3aad01960f92c5e77dba64ac1a6c6b063378fb97 \ No newline at end of file +5360ecb0b8891d0c27f3f02d81b6c7b548361a10 \ No newline at end of file diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 78d1a6fba4..c34f28f0e4 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -9,11 +9,16 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains the C functions that implement mutexes for win32 +** This file contains the C functions that implement mutexes for Win32. */ #include "sqliteInt.h" #if SQLITE_OS_WIN +/* +** Include code that is common to all os_*.c files +*/ +#include "os_common.h" + /* ** Include the header file for the Windows VFS. */ @@ -22,7 +27,7 @@ /* ** The code in this file is only used if we are compiling multithreaded -** on a win32 system. +** on a Win32 system. */ #ifdef SQLITE_MUTEX_W32 @@ -35,48 +40,22 @@ struct sqlite3_mutex { #ifdef SQLITE_DEBUG volatile int nRef; /* Number of enterances */ volatile DWORD owner; /* Thread holding this mutex */ - int trace; /* True to trace changes */ + volatile int trace; /* True to trace changes */ #endif }; -#define SQLITE_W32_MUTEX_INITIALIZER { 0 } -#ifdef SQLITE_DEBUG -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, 0L, (DWORD)0, 0 } -#else -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 } -#endif /* -** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, -** or WinCE. Return false (zero) for Win95, Win98, or WinME. -** -** Here is an interesting observation: Win95, Win98, and WinME lack -** the LockFileEx() API. But we can still statically link against that -** API as long as we don't call it win running Win95/98/ME. A call to -** this routine is used to determine if the host is Win95/98/ME or -** WinNT/2K/XP so that we will know whether or not we can safely call -** the LockFileEx() API. -** -** mutexIsNT() is only used for the TryEnterCriticalSection() API call, -** which is only available if your application was compiled with -** _WIN32_WINNT defined to a value >= 0x0400. Currently, the only -** call to TryEnterCriticalSection() is #ifdef'ed out, so #ifdef -** this out as well. +** These are the initializer values used when declaring a "static" mutex +** on Win32. It should be noted that all mutexes require initialization +** on the Win32 platform. */ -#if 0 -#if SQLITE_OS_WINCE || SQLITE_OS_WINRT -# define mutexIsNT() (1) +#define SQLITE_W32_MUTEX_INITIALIZER { 0 } + +#ifdef SQLITE_DEBUG +#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, \ + 0L, (DWORD)0, 0 } #else - static int mutexIsNT(void){ - static int osType = 0; - if( osType==0 ){ - OSVERSIONINFO sInfo; - sInfo.dwOSVersionInfoSize = sizeof(sInfo); - GetVersionEx(&sInfo); - osType = sInfo.dwPlatformId==VER_PLATFORM_WIN32_NT ? 2 : 1; - } - return osType==2; - } -#endif /* SQLITE_OS_WINCE || SQLITE_OS_WINRT */ +#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 } #endif #ifdef SQLITE_DEBUG @@ -87,16 +66,17 @@ struct sqlite3_mutex { static int winMutexHeld(sqlite3_mutex *p){ return p->nRef!=0 && p->owner==GetCurrentThreadId(); } + static int winMutexNotheld2(sqlite3_mutex *p, DWORD tid){ return p->nRef==0 || p->owner!=tid; } + static int winMutexNotheld(sqlite3_mutex *p){ - DWORD tid = GetCurrentThreadId(); + DWORD tid = GetCurrentThreadId(); return winMutexNotheld2(p, tid); } #endif - /* ** Initialize and deinitialize the mutex subsystem. */ @@ -111,17 +91,19 @@ static sqlite3_mutex winMutex_staticMutexes[] = { SQLITE3_MUTEX_INITIALIZER, SQLITE3_MUTEX_INITIALIZER }; + static int winMutex_isInit = 0; -/* As winMutexInit() and winMutexEnd() are called as part -** of the sqlite3_initialize and sqlite3_shutdown() -** processing, the "interlocked" magic is probably not -** strictly necessary. + +/* As the winMutexInit() and winMutexEnd() functions are called as part +** of the sqlite3_initialize() and sqlite3_shutdown() processing, the +** "interlocked" magic used here is probably not strictly necessary. */ static LONG winMutex_lock = 0; +int sqlite3_win32_is_nt(void); /* os_win.c */ void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ -static int winMutexInit(void){ +static int winMutexInit(void){ /* The first to increment to 1 does actual initialization */ if( InterlockedCompareExchange(&winMutex_lock, 1, 0)==0 ){ int i; @@ -134,16 +116,17 @@ static int winMutexInit(void){ } winMutex_isInit = 1; }else{ - /* Someone else is in the process of initing the static mutexes */ + /* Another thread is (in the process of) initializing the static + ** mutexes */ while( !winMutex_isInit ){ sqlite3_win32_sleep(1); } } - return SQLITE_OK; + return SQLITE_OK; } -static int winMutexEnd(void){ - /* The first to decrement to 0 does actual shutdown +static int winMutexEnd(void){ + /* The first to decrement to 0 does actual shutdown ** (which should be the last to shutdown.) */ if( InterlockedCompareExchange(&winMutex_lock, 0, 1)==1 ){ if( winMutex_isInit==1 ){ @@ -154,7 +137,7 @@ static int winMutexEnd(void){ winMutex_isInit = 0; } } - return SQLITE_OK; + return SQLITE_OK; } /* @@ -198,7 +181,7 @@ static int winMutexEnd(void){ ** ** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. But for the static +** returns a different mutex on every call. But for the static ** mutex types, the same mutex is returned on every call that has ** the same type number. */ @@ -209,9 +192,12 @@ static sqlite3_mutex *winMutexAlloc(int iType){ case SQLITE_MUTEX_FAST: case SQLITE_MUTEX_RECURSIVE: { p = sqlite3MallocZero( sizeof(*p) ); - if( p ){ + if( p ){ #ifdef SQLITE_DEBUG p->id = iType; +#ifdef SQLITE_WIN32_MUTEX_TRACE_DYNAMIC + p->trace = 1; +#endif #endif #if SQLITE_OS_WINRT InitializeCriticalSectionEx(&p->mutex, 0, 0); @@ -222,12 +208,15 @@ static sqlite3_mutex *winMutexAlloc(int iType){ break; } default: { - assert( winMutex_isInit==1 ); assert( iType-2 >= 0 ); assert( iType-2 < ArraySize(winMutex_staticMutexes) ); + assert( winMutex_isInit==1 ); p = &winMutex_staticMutexes[iType-2]; #ifdef SQLITE_DEBUG p->id = iType; +#ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC + p->trace = 1; +#endif #endif break; } @@ -243,8 +232,10 @@ static sqlite3_mutex *winMutexAlloc(int iType){ */ static void winMutexFree(sqlite3_mutex *p){ assert( p ); +#ifdef SQLITE_DEBUG assert( p->nRef==0 && p->owner==0 ); assert( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ); +#endif assert( winMutex_isInit==1 ); DeleteCriticalSection(&p->mutex); sqlite3_free(p); @@ -262,31 +253,39 @@ static void winMutexFree(sqlite3_mutex *p){ ** more than once, the behavior is undefined. */ static void winMutexEnter(sqlite3_mutex *p){ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + DWORD tid = GetCurrentThreadId(); +#endif #ifdef SQLITE_DEBUG - DWORD tid = GetCurrentThreadId(); + assert( p ); assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); +#else + assert( p ); #endif assert( winMutex_isInit==1 ); EnterCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG assert( p->nRef>0 || p->owner==0 ); - p->owner = tid; + p->owner = tid; p->nRef++; if( p->trace ){ - printf("enter mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + OSTRACE(("ENTER-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", + tid, p, p->trace, p->nRef)); } #endif } + static int winMutexTry(sqlite3_mutex *p){ -#ifndef NDEBUG - DWORD tid = GetCurrentThreadId(); +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + DWORD tid = GetCurrentThreadId(); #endif int rc = SQLITE_BUSY; + assert( p ); assert( p->id==SQLITE_MUTEX_RECURSIVE || winMutexNotheld2(p, tid) ); /* ** The sqlite3_mutex_try() routine is very rarely used, and when it ** is used it is merely an optimization. So it is OK for it to always - ** fail. + ** fail. ** ** The TryEnterCriticalSection() interface is only available on WinNT. ** And some windows compilers complain if you try to use it without @@ -294,19 +293,22 @@ static int winMutexTry(sqlite3_mutex *p){ ** For that reason, we will omit this optimization for now. See ** ticket #2685. */ -#if 0 +#if defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x0400 assert( winMutex_isInit==1 ); - if( mutexIsNT() && TryEnterCriticalSection(&p->mutex) ){ + if( sqlite3_win32_is_nt() && TryEnterCriticalSection(&p->mutex) ){ +#ifdef SQLITE_DEBUG p->owner = tid; p->nRef++; +#endif rc = SQLITE_OK; } #else UNUSED_PARAMETER(p); #endif #ifdef SQLITE_DEBUG - if( rc==SQLITE_OK && p->trace ){ - printf("try mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + if( p->trace ){ + OSTRACE(("TRY-MUTEX tid=%lu, mutex=%p (%d), owner=%lu, nRef=%d, rc=%s\n", + tid, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); } #endif return rc; @@ -319,8 +321,11 @@ static int winMutexTry(sqlite3_mutex *p){ ** is not currently allocated. SQLite will never do either. */ static void winMutexLeave(sqlite3_mutex *p){ -#ifndef NDEBUG +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) DWORD tid = GetCurrentThreadId(); +#endif + assert( p ); +#ifdef SQLITE_DEBUG assert( p->nRef>0 ); assert( p->owner==tid ); p->nRef--; @@ -331,7 +336,8 @@ static void winMutexLeave(sqlite3_mutex *p){ LeaveCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG if( p->trace ){ - printf("leave mutex %p (%d) with nRef=%d\n", p, p->trace, p->nRef); + OSTRACE(("LEAVE-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", + tid, p, p->trace, p->nRef)); } #endif } @@ -353,7 +359,7 @@ sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ 0 #endif }; - return &sMutex; } + #endif /* SQLITE_MUTEX_W32 */ From 4387006c18f99a63fffbed12ce30a0a099999c18 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 31 Jul 2014 15:44:44 +0000 Subject: [PATCH 169/710] Deactivate the DISTINCT in a SELECT on the right-hand side of an IN operator, since it should not make any difference in the output but dues consume extra memory and CPU time. FossilOrigin-Name: f4cb53651b1e352fae7378878b830a902bcd9248 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 11 +++++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index c1369a4aa5..240778ae2c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Re-integrate\sthe\srecent\schanges\sfrom\sthe\s'winMutex'\sbranch\sback\sinto\sthe\sWin32\smutex\ssubsystem. -D 2014-07-30T23:11:16.516 +C Deactivate\sthe\sDISTINCT\sin\sa\sSELECT\son\sthe\sright-hand\sside\sof\san\sIN\soperator,\nsince\sit\sshould\snot\smake\sany\sdifference\sin\sthe\soutput\sbut\sdues\sconsume\sextra\nmemory\sand\sCPU\stime. +D 2014-07-31T15:44:44.199 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c b989d07fc7c8780fff77365a4fc59881223e340c +F src/expr.c 77ca517a25a589f8088df88ace671e1c1d7dd3de F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c a549cff9fe8b736cdae21650ea0af6de29b77619 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3aad01960f92c5e77dba64ac1a6c6b063378fb97 -R 30d9c6511ca3c35312cef2a09e79ecbc -U mistachkin -Z c0143953af6d35d374b1fbf6a22594a7 +P 5360ecb0b8891d0c27f3f02d81b6c7b548361a10 +R fa406e6fe8ac4b7f1ba9a949d0399141 +U drh +Z ae156d6c18fec68cfd3675b456914b76 diff --git a/manifest.uuid b/manifest.uuid index ab5604951f..1f35b44ebe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5360ecb0b8891d0c27f3f02d81b6c7b548361a10 \ No newline at end of file +f4cb53651b1e352fae7378878b830a902bcd9248 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 72286dfdf9..94647e5148 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1624,7 +1624,7 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ } if( eType==0 ){ - /* Could not found an existing table or index to use as the RHS b-tree. + /* Could not find an existing table or index to use as the RHS b-tree. ** We will have to generate an ephemeral table to do the job. */ u32 savedNQueryLoop = pParse->nQueryLoop; @@ -1754,6 +1754,7 @@ int sqlite3CodeSubselect( ** Generate code to write the results of the select into the temporary ** table allocated and opened above. */ + Select *pSelect = pExpr->x.pSelect; SelectDest dest; ExprList *pEList; @@ -1761,13 +1762,15 @@ int sqlite3CodeSubselect( sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); dest.affSdst = (u8)affinity; assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); - pExpr->x.pSelect->iLimit = 0; + pSelect->iLimit = 0; + testcase( pSelect->selFlags & SF_Distinct ); + pSelect->selFlags &= ~SF_Distinct; testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ - if( sqlite3Select(pParse, pExpr->x.pSelect, &dest) ){ + if( sqlite3Select(pParse, pSelect, &dest) ){ sqlite3KeyInfoUnref(pKeyInfo); return 0; } - pEList = pExpr->x.pSelect->pEList; + pEList = pSelect->pEList; assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ assert( pEList!=0 ); assert( pEList->nExpr>0 ); From e8559837e68730202854a92235c5abd7bd5e97b2 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 31 Jul 2014 17:35:40 +0000 Subject: [PATCH 170/710] Fix a leaked statement handle in pager2.test. FossilOrigin-Name: 47457b0488abcbec2137abf706c1d677563b9ea5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/tester.tcl | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 240778ae2c..c990fe1cc9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Deactivate\sthe\sDISTINCT\sin\sa\sSELECT\son\sthe\sright-hand\sside\sof\san\sIN\soperator,\nsince\sit\sshould\snot\smake\sany\sdifference\sin\sthe\soutput\sbut\sdues\sconsume\sextra\nmemory\sand\sCPU\stime. -D 2014-07-31T15:44:44.199 +C Fix\sa\sleaked\sstatement\shandle\sin\spager2.test. +D 2014-07-31T17:35:40.036 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -863,7 +863,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 8ec228b0db5d7ebc4ee9b458fc28cb9e7873f5e1 -F test/tester.tcl f31bea1483ea1d39620f982130026e76f872d744 +F test/tester.tcl b4ff83a8b069633f4aca788236d10a7086112a49 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5360ecb0b8891d0c27f3f02d81b6c7b548361a10 -R fa406e6fe8ac4b7f1ba9a949d0399141 -U drh -Z ae156d6c18fec68cfd3675b456914b76 +P f4cb53651b1e352fae7378878b830a902bcd9248 +R 6cb5ac9ad4e6d98c300e9ffde83ac7f3 +U dan +Z 2d65f06df1c6f3ff1350a732b5e2220f diff --git a/manifest.uuid b/manifest.uuid index 1f35b44ebe..0528abb823 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4cb53651b1e352fae7378878b830a902bcd9248 \ No newline at end of file +47457b0488abcbec2137abf706c1d677563b9ea5 \ No newline at end of file diff --git a/test/tester.tcl b/test/tester.tcl index 1c4e93937c..3bf92f2b4f 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -863,6 +863,7 @@ proc speed_trial_summary {name} { # proc finish_test {} { catch {db close} + catch {db1 close} catch {db2 close} catch {db3 close} if {0==[info exists ::SLAVE]} { finalize_testing } From f6296cafc54513b5072c1a9aa14f554ce7cc22be Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 31 Jul 2014 18:14:37 +0000 Subject: [PATCH 171/710] Add a missing call to "test_sqlite3_log" to multiplex.test. FossilOrigin-Name: 0708f9df23a325349f658741358c5994b5c4c873 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/multiplex.test | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index c990fe1cc9..13634c500c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sleaked\sstatement\shandle\sin\spager2.test. -D 2014-07-31T17:35:40.036 +C Add\sa\smissing\scall\sto\s"test_sqlite3_log"\sto\smultiplex.test. +D 2014-07-31T18:14:37.512 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -719,7 +719,7 @@ F test/mmap1.test 93d167b328255cbe6679fe1e1a23be1b1197d07b F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 F test/mmap3.test c92273e16eb8d23c1d55c9815b446bb72ef0512e F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 -F test/multiplex.test 4a0d07a2490dc958e6b676a6825cf761be33d869 +F test/multiplex.test efd015ca0b5b4a57dc9535b8feb1273eebeadb60 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f4cb53651b1e352fae7378878b830a902bcd9248 -R 6cb5ac9ad4e6d98c300e9ffde83ac7f3 +P 47457b0488abcbec2137abf706c1d677563b9ea5 +R 40079bb444b74388b9936aaa1a8a8059 U dan -Z 2d65f06df1c6f3ff1350a732b5e2220f +Z a0b960eb92c17485651b9cbc9a7c3b70 diff --git a/manifest.uuid b/manifest.uuid index 0528abb823..fa6a4ad3e8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -47457b0488abcbec2137abf706c1d677563b9ea5 \ No newline at end of file +0708f9df23a325349f658741358c5994b5c4c873 \ No newline at end of file diff --git a/test/multiplex.test b/test/multiplex.test index 55fab013c4..5db56f264a 100644 --- a/test/multiplex.test +++ b/test/multiplex.test @@ -593,5 +593,9 @@ do_test multiplex-6.99 { } +catch { db close } catch { sqlite3_multiplex_shutdown } +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize finish_test From 202cb641898ae45a957a1543e2e014335601bbe9 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 31 Jul 2014 18:54:01 +0000 Subject: [PATCH 172/710] Optimizations to the OS sub-type checking in the Win32 VFS. FossilOrigin-Name: 1e5489faff093d6a8e538061e45532f9050e9459 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/mutex_w32.c | 10 ++++++++-- src/os_win.c | 20 +++++++++++++------- src/test1.c | 4 ++-- 5 files changed, 33 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 13634c500c..498fa5f658 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smissing\scall\sto\s"test_sqlite3_log"\sto\smultiplex.test. -D 2014-07-31T18:14:37.512 +C Optimizations\sto\sthe\sOS\ssub-type\schecking\sin\sthe\sWin32\sVFS. +D 2014-07-31T18:54:01.916 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -201,14 +201,14 @@ F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 -F src/mutex_w32.c 08890085b81ce181d2cbc6fc2636e3444ae30b27 +F src/mutex_w32.c c50939b72368f1cfbddb58520372081a50558548 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c a7baf1b30f3c58ba20b813e01aab23b18ae44f85 -F src/os_win.c c29e3a80b47ebdbabd61fc3d4015e52d2654d8c5 +F src/os_win.c c67bec43e5dadde8029470ab7b7825ccc9abe578 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -232,7 +232,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd -F src/test1.c 3c8bc491d2f8de5adbbf306533cefc343c733927 +F src/test1.c 14409a611e9c27c6c522c610bbff5561f05c1558 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 47457b0488abcbec2137abf706c1d677563b9ea5 -R 40079bb444b74388b9936aaa1a8a8059 -U dan -Z a0b960eb92c17485651b9cbc9a7c3b70 +P 0708f9df23a325349f658741358c5994b5c4c873 +R 33905802a155aabab7e0415ce706b3b1 +U mistachkin +Z f67dc40e0d7f1ad1a9a6019eb7b3fb85 diff --git a/manifest.uuid b/manifest.uuid index fa6a4ad3e8..42ebb3917e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0708f9df23a325349f658741358c5994b5c4c873 \ No newline at end of file +1e5489faff093d6a8e538061e45532f9050e9459 \ No newline at end of file diff --git a/src/mutex_w32.c b/src/mutex_w32.c index c34f28f0e4..218342d2b9 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -93,12 +93,13 @@ static sqlite3_mutex winMutex_staticMutexes[] = { }; static int winMutex_isInit = 0; +static int winMutex_isNt = -1; /* <0 means "need to query" */ /* As the winMutexInit() and winMutexEnd() functions are called as part ** of the sqlite3_initialize() and sqlite3_shutdown() processing, the ** "interlocked" magic used here is probably not strictly necessary. */ -static LONG winMutex_lock = 0; +static LONG volatile winMutex_lock = 0; int sqlite3_win32_is_nt(void); /* os_win.c */ void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ @@ -295,7 +296,12 @@ static int winMutexTry(sqlite3_mutex *p){ */ #if defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x0400 assert( winMutex_isInit==1 ); - if( sqlite3_win32_is_nt() && TryEnterCriticalSection(&p->mutex) ){ + assert( winMutex_isNt>=-1 && winMutex_isNt<=1 ); + if( winMutex_isNt<0 ){ + winMutex_isNt = sqlite3_win32_is_nt(); + } + assert( winMutex_isNt==0 || winMutex_isNt==1 ); + if( winMutex_isNt && TryEnterCriticalSection(&p->mutex) ){ #ifdef SQLITE_DEBUG p->owner = tid; p->nRef++; diff --git a/src/os_win.c b/src/os_win.c index 842a4ad7cf..fd2997206c 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -414,10 +414,10 @@ const sqlite3_mem_methods *sqlite3MemGetWin32(void); ** can manually set this value to 1 to emulate Win98 behavior. */ #ifdef SQLITE_TEST -int sqlite3_os_type = 0; +LONG volatile sqlite3_os_type = 0; #elif !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ defined(SQLITE_WIN32_HAS_ANSI) && defined(SQLITE_WIN32_HAS_WIDE) -static int sqlite3_os_type = 0; +static LONG volatile sqlite3_os_type = 0; #endif #ifndef SYSCALL @@ -1048,6 +1048,11 @@ static struct win_syscall { #define osCreateFileMappingFromApp ((HANDLE(WINAPI*)(HANDLE, \ LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[75].pCurrent) + { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, + +#define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG volatile*, \ + LONG,LONG))aSyscall[76].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -1298,7 +1303,7 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #elif !defined(SQLITE_WIN32_HAS_WIDE) # define osIsNT() (0) #else -# define osIsNT() (sqlite3_win32_is_nt()) +# define osIsNT() ((sqlite3_os_type==2) || sqlite3_win32_is_nt()) #endif /* @@ -1306,7 +1311,7 @@ void sqlite3_win32_sleep(DWORD milliseconds){ ** based on the NT kernel. */ int sqlite3_win32_is_nt(void){ - if( sqlite3_os_type==0 ){ + if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ #if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WIN8 OSVERSIONINFOW sInfo; sInfo.dwOSVersionInfoSize = sizeof(sInfo); @@ -1316,9 +1321,10 @@ int sqlite3_win32_is_nt(void){ sInfo.dwOSVersionInfoSize = sizeof(sInfo); osGetVersionExA(&sInfo); #endif - sqlite3_os_type = (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1; + osInterlockedCompareExchange(&sqlite3_os_type, + (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); } - return (sqlite3_os_type == 2); + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; } #ifdef SQLITE_WIN32_MALLOC @@ -5475,7 +5481,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==76 ); + assert( ArraySize(aSyscall)==77 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); diff --git a/src/test1.c b/src/test1.c index 8d2249cbb8..56487f6abb 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6581,7 +6581,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ extern int sqlite3_pager_writedb_count; extern int sqlite3_pager_writej_count; #if SQLITE_OS_WIN - extern int sqlite3_os_type; + extern LONG volatile sqlite3_os_type; #endif #ifdef SQLITE_DEBUG extern int sqlite3WhereTrace; @@ -6639,7 +6639,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #endif #if SQLITE_OS_WIN Tcl_LinkVar(interp, "sqlite_os_type", - (char*)&sqlite3_os_type, TCL_LINK_INT); + (char*)&sqlite3_os_type, TCL_LINK_LONG); #endif #ifdef SQLITE_TEST { From 37e08081f3da7ceab00ff4db996510f924de931a Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 31 Jul 2014 20:16:08 +0000 Subject: [PATCH 173/710] Omit a pointless OP_Null when processing a value-list RHS of an IN operator where the LHS is a rowid. FossilOrigin-Name: 1361450a9dfe9476e8df98f370a3695752252245 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 498fa5f658..8e34956aba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimizations\sto\sthe\sOS\ssub-type\schecking\sin\sthe\sWin32\sVFS. -D 2014-07-31T18:54:01.916 +C Omit\sa\spointless\sOP_Null\swhen\sprocessing\sa\svalue-list\sRHS\sof\san\sIN\soperator\nwhere\sthe\sLHS\sis\sa\srowid. +D 2014-07-31T20:16:08.031 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 77ca517a25a589f8088df88ace671e1c1d7dd3de +F src/expr.c 0e2a6c1cd3b752bea188a92bd826a2304c79e7bf F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c a549cff9fe8b736cdae21650ea0af6de29b77619 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0708f9df23a325349f658741358c5994b5c4c873 -R 33905802a155aabab7e0415ce706b3b1 -U mistachkin -Z f67dc40e0d7f1ad1a9a6019eb7b3fb85 +P 1e5489faff093d6a8e538061e45532f9050e9459 +R cbfd634bf9c7df43faab424574b6f5df +U drh +Z a5f816e75181f7c6a373a521ba2c396b diff --git a/manifest.uuid b/manifest.uuid index 42ebb3917e..13453542e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e5489faff093d6a8e538061e45532f9050e9459 \ No newline at end of file +1361450a9dfe9476e8df98f370a3695752252245 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 94647e5148..aa55ff7af2 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1801,7 +1801,7 @@ int sqlite3CodeSubselect( /* Loop through each expression in . */ r1 = sqlite3GetTempReg(pParse); r2 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_Null, 0, r2); + if( isRowid ) sqlite3VdbeAddOp2(v, OP_Null, 0, r2); for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ Expr *pE2 = pItem->pExpr; int iValToIns; From 5f1d1d9c870f2377d6907ec05df3c6d38d75cd57 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 31 Jul 2014 22:59:04 +0000 Subject: [PATCH 174/710] Refactoring: Change "pIndex->onError!=OE_None" to use a macro: "IsUniqueIndex(pIndex)". Easier to understand that way. FossilOrigin-Name: e75b26ee357bb3d3c1a539b05d633ebf314726d7 --- manifest | 26 +++++++++++++------------- manifest.uuid | 2 +- src/analyze.c | 2 +- src/build.c | 8 ++++---- src/expr.c | 2 +- src/fkey.c | 2 +- src/insert.c | 2 +- src/pragma.c | 2 +- src/sqliteInt.h | 3 +++ src/where.c | 8 ++++---- 10 files changed, 30 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 8e34956aba..7cd7515a7e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sa\spointless\sOP_Null\swhen\sprocessing\sa\svalue-list\sRHS\sof\san\sIN\soperator\nwhere\sthe\sLHS\sis\sa\srowid. -D 2014-07-31T20:16:08.031 +C Refactoring:\s\sChange\s"pIndex->onError!=OE_None"\sto\suse\sa\smacro:\n"IsUniqueIndex(pIndex)".\s\sEasier\sto\sunderstand\sthat\sway. +D 2014-07-31T22:59:04.121 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,7 +161,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c de34a73b86db9dc3a16beef12cc5573c50223956 +F src/analyze.c f98a351908da29f7b44741cfeb9eb20dda648ba0 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 @@ -170,21 +170,21 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c b5531339cd826af46b9621e4a9323971a9380e12 F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 7ba21d8f0f5f1e8b5a0ed21aab9be2b39d1af516 +F src/build.c c67a915cd5aabda9ac170f2af8ea25434476b66f F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 0e2a6c1cd3b752bea188a92bd826a2304c79e7bf +F src/expr.c 3be0e9f90bb1c475a99a821a11eecde53ecefc1d F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c a549cff9fe8b736cdae21650ea0af6de29b77619 +F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc F src/global.c 1e4bd956dc2f608f87d2a929abc4a20db65f30e4 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c a038daeadfb818aaadafa854f40f5623b6929d84 +F src/insert.c 991e4964e9295da3993e2c0f81c7faf642371848 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b @@ -216,7 +216,7 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c 30f3b2ac09fef58320375d78e0e18b976198fc69 +F src/pragma.c f9268bd5fa072afb3a174149129859727efb4326 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c af06f66927919730f03479fed6ae9854f73419f4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -227,7 +227,7 @@ F src/shell.c 191129c3f7a9cf241aea90ff6a6be3e74d3767f0 F src/sqlite.h.in 9bbc5815c73b0e77e68b5275481a5e3e7814a804 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 068e42f41a09ce6b9edbe194ac8a470ab53145df +F src/sqliteInt.h 3b17ba74eec22781f51e7b3e4c73d2cbd458f89b F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 4dfcd80380d154be434c4b51e890e17ce9754b3e +F src/where.c a7d0434de56d6a4b60cc98ec661969d521201d39 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1e5489faff093d6a8e538061e45532f9050e9459 -R cbfd634bf9c7df43faab424574b6f5df +P 1361450a9dfe9476e8df98f370a3695752252245 +R 114928731acdabb64615a67d8eb69386 U drh -Z a5f816e75181f7c6a373a521ba2c396b +Z 0cd7191a5ffc11f0c0a8249763204863 diff --git a/manifest.uuid b/manifest.uuid index 13453542e7..0455b18c12 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1361450a9dfe9476e8df98f370a3695752252245 \ No newline at end of file +e75b26ee357bb3d3c1a539b05d633ebf314726d7 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 43aff141b2..f9c03dc848 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1129,7 +1129,7 @@ static void analyzeOneTable( */ sqlite3VdbeAddOp0(v, OP_Goto); addrNextRow = sqlite3VdbeCurrentAddr(v); - if( nColTest==1 && pIdx->nKeyCol==1 && pIdx->onError!=OE_None ){ + if( nColTest==1 && pIdx->nKeyCol==1 && IsUniqueIndex(pIdx) ){ /* For a single-column UNIQUE index, once we have found a non-NULL ** row, we know that all the rest will be distinct, so skip ** subsequent distinctness tests. */ diff --git a/src/build.c b/src/build.c index 384802c533..28205c4c61 100644 --- a/src/build.c +++ b/src/build.c @@ -2707,7 +2707,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v); assert( pKey!=0 || db->mallocFailed || pParse->nErr ); - if( pIndex->onError!=OE_None && pKey!=0 ){ + if( IsUniqueIndex(pIndex) && pKey!=0 ){ int j2 = sqlite3VdbeCurrentAddr(v) + 3; sqlite3VdbeAddOp2(v, OP_Goto, 0, j2); addr2 = sqlite3VdbeCurrentAddr(v); @@ -3104,9 +3104,9 @@ Index *sqlite3CreateIndex( Index *pIdx; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int k; - assert( pIdx->onError!=OE_None ); + assert( IsUniqueIndex(pIdx) ); assert( pIdx->idxType!=SQLITE_IDXTYPE_APPDEF ); - assert( pIndex->onError!=OE_None ); + assert( IsUniqueIndex(pIndex) ); if( pIdx->nKeyCol!=pIndex->nKeyCol ) continue; for(k=0; knKeyCol; k++){ @@ -3297,7 +3297,7 @@ void sqlite3DefaultRowEst(Index *pIdx){ } assert( 0==sqlite3LogEst(1) ); - if( pIdx->onError!=OE_None ) a[pIdx->nKeyCol] = 0; + if( IsUniqueIndex(pIdx) ) a[pIdx->nKeyCol] = 0; } /* diff --git a/src/expr.c b/src/expr.c index aa55ff7af2..3b254f3d3e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1604,7 +1604,7 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ if( (pIdx->aiColumn[0]==iCol) && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq - && (!mustBeUnique || (pIdx->nKeyCol==1 && pIdx->onError!=OE_None)) + && (!mustBeUnique || (pIdx->nKeyCol==1 && IsUniqueIndex(pIdx))) ){ int iAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); diff --git a/src/fkey.c b/src/fkey.c index c3cac276a9..50c10da822 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -225,7 +225,7 @@ int sqlite3FkLocateIndex( } for(pIdx=pParent->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->nKeyCol==nCol && pIdx->onError!=OE_None ){ + if( pIdx->nKeyCol==nCol && IsUniqueIndex(pIdx) ){ /* pIdx is a UNIQUE index (or a PRIMARY KEY) and has the right number ** of columns. If each indexed column corresponds to a foreign key ** column of pFKey, then this index is a winner. */ diff --git a/src/insert.c b/src/insert.c index 7b427d1eb6..5964b91ca4 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1886,7 +1886,7 @@ static int xferOptimization( } } for(pDestIdx=pDest->pIndex; pDestIdx; pDestIdx=pDestIdx->pNext){ - if( pDestIdx->onError!=OE_None ){ + if( IsUniqueIndex(pDestIdx) ){ destHasUniqueIdx = 1; } for(pSrcIdx=pSrc->pIndex; pSrcIdx; pSrcIdx=pSrcIdx->pNext){ diff --git a/src/pragma.c b/src/pragma.c index c4374cc71c..a4a1b2a250 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1544,7 +1544,7 @@ void sqlite3Pragma( for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ sqlite3VdbeAddOp2(v, OP_Integer, i, 1); sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0); - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->onError!=OE_None, 3); + sqlite3VdbeAddOp2(v, OP_Integer, IsUniqueIndex(pIdx), 3); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5d72295d66..1aede95c14 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1720,6 +1720,9 @@ struct Index { /* Return true if index X is a PRIMARY KEY index */ #define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY) +/* Return true if index X is a UNIQUE index */ +#define IsUniqueIndex(X) ((X)->onError!=OE_None) + /* ** Each sample stored in the sqlite_stat3 table is represented in memory ** using a structure of this type. See documentation at the top of the diff --git a/src/where.c b/src/where.c index 38d1014ac0..3cc66a34b1 100644 --- a/src/where.c +++ b/src/where.c @@ -1470,7 +1470,7 @@ static int isDistinctRedundant( ** contain a "col=X" term are subject to a NOT NULL constraint. */ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->onError==OE_None ) continue; + if( !IsUniqueIndex(pIdx) ) continue; for(i=0; inKeyCol; i++){ i16 iCol = pIdx->aiColumn[i]; if( 0==findTerm(pWC, iBase, iCol, ~(Bitmask)0, WO_EQ, pIdx) ){ @@ -4376,7 +4376,7 @@ static int whereLoopAddBtreeIndex( }else if( eOp & (WO_EQ) ){ pNew->wsFlags |= WHERE_COLUMN_EQ; if( iCol<0 || (nInMul==0 && pNew->u.btree.nEq==pProbe->nKeyCol-1) ){ - if( iCol>=0 && pProbe->onError==OE_None ){ + if( iCol>=0 && !IsUniqueIndex(pProbe) ){ pNew->wsFlags |= WHERE_UNQ_WANTED; }else{ pNew->wsFlags |= WHERE_ONEROW; @@ -5231,7 +5231,7 @@ static i8 wherePathSatisfiesOrderBy( nColumn = pIndex->nColumn; assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); assert( pIndex->aiColumn[nColumn-1]==(-1) || !HasRowid(pIndex->pTable)); - isOrderDistinct = pIndex->onError!=OE_None; + isOrderDistinct = IsUniqueIndex(pIndex); } /* Loop through all columns of the index and deal with the ones @@ -5746,7 +5746,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pLoop->aLTermSpace==pLoop->aLTerm ); assert( ArraySize(pLoop->aLTermSpace)==4 ); - if( pIdx->onError==OE_None + if( !IsUniqueIndex(pIdx) || pIdx->pPartIdxWhere!=0 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) ) continue; From cefc87fca5bbfd6bd8f918373de15efd9fe5b413 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 1 Aug 2014 01:40:33 +0000 Subject: [PATCH 175/710] Enhance the PRAGMA integrity_check command to detect UNIQUE and NOT NULL constraint violations. FossilOrigin-Name: 9abcf2698c09f4f6a44a68e74f9f6b538f3253d6 --- manifest | 16 ++++++------- manifest.uuid | 2 +- src/pragma.c | 60 +++++++++++++++++++++++++++++++++++++++++++----- src/vdbe.c | 10 ++++---- test/pragma.test | 27 +++++++++++++++++++++- 5 files changed, 94 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 7cd7515a7e..f13081bfc1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactoring:\s\sChange\s"pIndex->onError!=OE_None"\sto\suse\sa\smacro:\n"IsUniqueIndex(pIndex)".\s\sEasier\sto\sunderstand\sthat\sway. -D 2014-07-31T22:59:04.121 +C Enhance\sthe\sPRAGMA\sintegrity_check\scommand\sto\sdetect\sUNIQUE\sand\sNOT\sNULL\nconstraint\sviolations. +D 2014-08-01T01:40:33.869 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -216,7 +216,7 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c f9268bd5fa072afb3a174149129859727efb4326 +F src/pragma.c d4a33151f057e35e5a2024adf8e41d2817b5c105 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c af06f66927919730f03479fed6ae9854f73419f4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -283,7 +283,7 @@ F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c f87f77b0049cbef1fa68b331c3551d82b4d9fba4 +F src/vdbe.c 115e08834b883964d9a480f685d8601826c5792a F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 @@ -754,7 +754,7 @@ F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 F test/permutations.test bc474bafb022cc5014ef3a9c3d5ab61d6d6f587c -F test/pragma.test adb21a90875bc54a880fa939c4d7c46598905aa0 +F test/pragma.test 19d0241a007bcdd77fc2606ec60fc60357e7fc8b F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/printf2.test bed79b4c3e5da08ba88ad637c0bf62586843cfb1 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1361450a9dfe9476e8df98f370a3695752252245 -R 114928731acdabb64615a67d8eb69386 +P e75b26ee357bb3d3c1a539b05d633ebf314726d7 +R 50d518176c08d9fef67243df4085599b U drh -Z 0cd7191a5ffc11f0c0a8249763204863 +Z ae546b77e53655a859301e9ce422b440 diff --git a/manifest.uuid b/manifest.uuid index 0455b18c12..087d6ad801 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e75b26ee357bb3d3c1a539b05d633ebf314726d7 \ No newline at end of file +9abcf2698c09f4f6a44a68e74f9f6b538f3253d6 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index a4a1b2a250..b37499f7bd 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1908,28 +1908,76 @@ void sqlite3Pragma( pParse->nMem = MAX(pParse->nMem, 8+j); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); + /* Verify that all NOT NULL columns really are NOT NULL */ + for(j=0; jnCol; j++){ + char *zErr; + int jmp2, jmp3; + if( j==pTab->iPKey ) continue; + if( pTab->aCol[j].notNull==0 ) continue; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ + zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, + pTab->aCol[j].zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); + jmp3 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); + sqlite3VdbeAddOp0(v, OP_Halt); + sqlite3VdbeJumpHere(v, jmp2); + sqlite3VdbeJumpHere(v, jmp3); + } + /* Validate index entries for the current row */ for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - int jmp2, jmp3, jmp4; + int jmp2, jmp3, jmp4, jmp5; + int ckUniq = sqlite3VdbeMakeLabel(v); if( pPk==pIdx ) continue; r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, pPrior, r1); pPrior = pIdx; sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1); /* increment entry count */ - jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, 0, r1, + /* Verify that an index entry exists for the current table row */ + jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, pIdx->nColumn); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, "row ", P4_STATIC); sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, " missing from index ", - P4_STATIC); + sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + " missing from index ", P4_STATIC); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); - sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, pIdx->zName, P4_TRANSIENT); + jmp5 = sqlite3VdbeAddOp4(v, OP_String8, 0, 4, 0, + pIdx->zName, P4_TRANSIENT); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); sqlite3VdbeAddOp0(v, OP_Halt); - sqlite3VdbeJumpHere(v, jmp4); sqlite3VdbeJumpHere(v, jmp2); + /* For UNIQUE indexes, verify that only one entry exists with the + ** current key. The entry is unique if (1) any column is NULL + ** or (2) the next entry has a different key */ + if( IsUniqueIndex(pIdx) ){ + int uniqOk = sqlite3VdbeMakeLabel(v); + int jmp6; + int kk; + for(kk=0; kknKeyCol; kk++){ + int iCol = pIdx->aiColumn[kk]; + assert( iCol>=0 && iColnCol ); + if( pTab->aCol[iCol].notNull ) continue; + sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); + VdbeCoverage(v); + } + jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, uniqOk); + sqlite3VdbeJumpHere(v, jmp6); + sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, + pIdx->nKeyCol); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, + "non-unique entry in index ", P4_STATIC); + sqlite3VdbeAddOp2(v, OP_Goto, 0, jmp5); + sqlite3VdbeResolveLabel(v, uniqOk); + } + sqlite3VdbeJumpHere(v, jmp4); sqlite3ResolvePartIdxLabel(pParse, jmp3); } sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); diff --git a/src/vdbe.c b/src/vdbe.c index 52f0f3aa74..07514951a4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3747,9 +3747,9 @@ case OP_Seek: { /* in2 */ ** is a prefix of any entry in P1 then a jump is made to P2 and ** P1 is left pointing at the matching entry. ** -** This operation leaves the cursor in a state where it cannot be -** advanced in either direction. In other words, the Next and Prev -** opcodes do not work after this operation. +** This operation leaves the cursor in a state where it can be +** advanced in the forward direction. The Next instruction will work, +** but not the Prev instruction. ** ** See also: NotFound, NoConflict, NotExists. SeekGe */ @@ -3816,7 +3816,7 @@ case OP_Found: { /* jump, in3 */ pC = p->apCsr[pOp->p1]; assert( pC!=0 ); #ifdef SQLITE_DEBUG - pC->seekOp = 0; + pC->seekOp = pOp->opcode; #endif pIn3 = &aMem[pOp->p3]; assert( pC->pCursor!=0 ); @@ -4673,7 +4673,7 @@ case OP_Next: /* jump */ ** The Prev opcode is only used after SeekLT, SeekLE, and Last. */ assert( pOp->opcode!=OP_Next || pOp->opcode!=OP_NextIfOpen || pC->seekOp==OP_SeekGT || pC->seekOp==OP_SeekGE - || pC->seekOp==OP_Rewind ); + || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found); assert( pOp->opcode!=OP_Prev || pOp->opcode!=OP_PrevIfOpen || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE || pC->seekOp==OP_Last ); diff --git a/test/pragma.test b/test/pragma.test index 8f54e695d7..539d867366 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -431,7 +431,32 @@ Page 6 is never used} {row 1 missing from index i2}} db eval {PRAGMA integrity_check} } {ok} } -#exit + +# Verify that PRAGMA integrity_check catches UNIQUE and NOT NULL +# constraint violations. +# +do_execsql_test pragma-3.20 { + CREATE TABLE t1(a,b); + CREATE INDEX t1a ON t1(a); + INSERT INTO t1 VALUES(1,1),(2,2),(3,3),(2,4),(NULL,5),(NULL,6); + PRAGMA writable_schema=ON; + UPDATE sqlite_master SET sql='CREATE UNIQUE INDEX t1a ON t1(a)' + WHERE name='t1a'; + UPDATE sqlite_master SET sql='CREATE TABLE t1(a NOT NULL,b)' + WHERE name='t1'; + PRAGMA writable_schema=OFF; + ALTER TABLE t1 RENAME TO t1x; + PRAGMA integrity_check; +} {{non-unique entry in index t1a} {NULL value in t1x.a} {non-unique entry in index t1a} {NULL value in t1x.a}} +do_execsql_test pragma-3.21 { + PRAGMA integrity_check(3); +} {{non-unique entry in index t1a} {NULL value in t1x.a} {non-unique entry in index t1a}} +do_execsql_test pragma-3.22 { + PRAGMA integrity_check(2); +} {{non-unique entry in index t1a} {NULL value in t1x.a}} +do_execsql_test pragma-3.21 { + PRAGMA integrity_check(1); +} {{non-unique entry in index t1a}} # Test modifying the cache_size of an attached database. ifcapable pager_pragmas&&attach { From 3a85625d87c2b5d0f4cd504cb4fdc5d8fc5ee8c4 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 1 Aug 2014 14:46:57 +0000 Subject: [PATCH 176/710] Begin making changes to the IN operator in an attempt to make it run faster and to make the code easier to understand. FossilOrigin-Name: ee0fd6aaf94cda1dce3fe752bfe3b0f83e0043f1 --- manifest | 19 ++++++++------- manifest.uuid | 2 +- src/expr.c | 64 +++++++++++++++++++++++++++++-------------------- src/sqliteInt.h | 20 ++++++++++++---- src/where.c | 2 +- 5 files changed, 66 insertions(+), 41 deletions(-) diff --git a/manifest b/manifest index f13081bfc1..3fae6c98bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sPRAGMA\sintegrity_check\scommand\sto\sdetect\sUNIQUE\sand\sNOT\sNULL\nconstraint\sviolations. -D 2014-08-01T01:40:33.869 +C Begin\smaking\schanges\sto\sthe\sIN\soperator\sin\san\sattempt\sto\smake\sit\srun\sfaster\nand\sto\smake\sthe\scode\seasier\sto\sunderstand. +D 2014-08-01T14:46:57.155 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 3be0e9f90bb1c475a99a821a11eecde53ecefc1d +F src/expr.c 7c52ea8b322992a91a241c0092a5bf97b141d353 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -227,7 +227,7 @@ F src/shell.c 191129c3f7a9cf241aea90ff6a6be3e74d3767f0 F src/sqlite.h.in 9bbc5815c73b0e77e68b5275481a5e3e7814a804 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 3b17ba74eec22781f51e7b3e4c73d2cbd458f89b +F src/sqliteInt.h 17ece600d3c9d36cc0ee2b74a30507507f3e0937 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c a7d0434de56d6a4b60cc98ec661969d521201d39 +F src/where.c ce1b9a3a2573033cd15e0882719db7f211f21cdd F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1185,7 +1185,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e75b26ee357bb3d3c1a539b05d633ebf314726d7 -R 50d518176c08d9fef67243df4085599b +P 9abcf2698c09f4f6a44a68e74f9f6b538f3253d6 +R 1cc42777f2b661f6d0caf162eadc6d85 +T *branch * IN-operator-improvements +T *sym-IN-operator-improvements * +T -sym-trunk * U drh -Z ae546b77e53655a859301e9ce422b440 +Z 08adfdc6013d75458685b1e962a583ca diff --git a/manifest.uuid b/manifest.uuid index 087d6ad801..1b3e9a1eeb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9abcf2698c09f4f6a44a68e74f9f6b538f3253d6 \ No newline at end of file +ee0fd6aaf94cda1dce3fe752bfe3b0f83e0043f1 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 3b254f3d3e..dd7b7c3de1 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1484,7 +1484,7 @@ int sqlite3CodeOnce(Parse *pParse){ ** be used either to test for membership in the RHS set or to iterate through ** all members of the RHS set, skipping duplicates. ** -** A cursor is opened on the b-tree object that the RHS of the IN operator +** A cursor is opened on the b-tree object that is the RHS of the IN operator ** and pX->iTable is set to the index of that cursor. ** ** The returned value of this function indicates the b-tree type, as follows: @@ -1494,6 +1494,8 @@ int sqlite3CodeOnce(Parse *pParse){ ** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. ** IN_INDEX_EPH - The cursor was opened on a specially created and ** populated epheremal table. +** IN_INDEX_NOOP - No cursor was allocated. The in operator must be +** implemented as a sequence of comparisons. ** ** An existing b-tree might be used if the RHS expression pX is a simple ** subquery such as: @@ -1503,23 +1505,37 @@ int sqlite3CodeOnce(Parse *pParse){ ** If the RHS of the IN operator is a list or a more complex subquery, then ** an ephemeral table might need to be generated from the RHS and then ** pX->iTable made to point to the ephermeral table instead of an -** existing table. +** existing table. ** -** If the prNotFound parameter is 0, then the b-tree will be used to iterate -** through the set members, skipping any duplicates. In this case an -** epheremal table must be used unless the selected is guaranteed +** The inFlags parameter must contain exactly one of the bits +** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP. If inFlags contains +** IN_INDEX_MEMBERSHIP, then the generated table will be used for a +** fast membership test. When the IN_INDEX_LOOP bit is set, the +** IN index will be used to loop over all values of the RHS of the +** IN operator. +** +** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate +** through the set members) then the b-tree must not contain duplicates. +** An epheremal table must be used unless the selected is guaranteed ** to be unique - either because it is an INTEGER PRIMARY KEY or it ** has a UNIQUE constraint or UNIQUE index. ** -** If the prNotFound parameter is not 0, then the b-tree will be used -** for fast set membership tests. In this case an epheremal table must +** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used +** for fast set membership tests) then an epheremal table must ** be used unless is an INTEGER PRIMARY KEY or an index can ** be found with as its left-most column. ** +** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and +** if the RHS of the IN operator is a list (not a subquery) then this +** routine might decide that creating an ephemeral b-tree for membership +** testing is too expensive and return IN_INDEX_NOOP. IN that case, the +** calling routine should implement the IN operator using a sequence +** of Eq or Ne comparison operations. +** ** When the b-tree is being used for membership tests, the calling function -** needs to know whether or not the structure contains an SQL NULL -** value in order to correctly evaluate expressions like "X IN (Y, Z)". -** If there is any chance that the (...) might contain a NULL value at +** might need to know whether or not the RHS side of the IN operator +** contains a NULL. If prNotFound is not NULL and +** if there is any chance that the (...) might contain a NULL value at ** runtime, then a register is allocated and the register number written ** to *prNotFound. If there is no chance that the (...) contains a ** NULL value, then *prNotFound is left unchanged. @@ -1540,14 +1556,15 @@ int sqlite3CodeOnce(Parse *pParse){ ** test more often than is necessary. */ #ifndef SQLITE_OMIT_SUBQUERY -int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ +int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prNotFound){ Select *p; /* SELECT to the right of IN operator */ int eType = 0; /* Type of RHS table. IN_INDEX_* */ int iTab = pParse->nTab++; /* Cursor of the RHS table */ - int mustBeUnique = (prNotFound==0); /* True if RHS must be unique */ + int mustBeUnique; /* True if RHS must be unique */ Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ assert( pX->op==TK_IN ); + mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; /* Check to see if an existing table or index can be used to ** satisfy the query. This is preferable to generating a new @@ -1630,14 +1647,14 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ u32 savedNQueryLoop = pParse->nQueryLoop; int rMayHaveNull = 0; eType = IN_INDEX_EPH; - if( prNotFound ){ - *prNotFound = rMayHaveNull = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound); - }else{ + if( inFlags & IN_INDEX_LOOP ){ pParse->nQueryLoop = 0; if( pX->pLeft->iColumn<0 && !ExprHasProperty(pX, EP_xIsSelect) ){ eType = IN_INDEX_ROWID; } + }else if( prNotFound ){ + *prNotFound = rMayHaveNull = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, rMayHaveNull); } sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID); pParse->nQueryLoop = savedNQueryLoop; @@ -1668,15 +1685,9 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ ** ** If rMayHaveNull is non-zero, that means that the operation is an IN ** (not a SELECT or EXISTS) and that the RHS might contains NULLs. -** Furthermore, the IN is in a WHERE clause and that we really want -** to iterate over the RHS of the IN operator in order to quickly locate -** all corresponding LHS elements. All this routine does is initialize -** the register given by rMayHaveNull to NULL. Calling routines will take -** care of changing this register value to non-NULL if the RHS is NULL-free. -** -** If rMayHaveNull is zero, that means that the subquery is being used -** for membership testing only. There is no need to initialize any -** registers to indicate the presence or absence of NULLs on the RHS. +** All this routine does is initialize the register given by rMayHaveNull +** to NULL. Calling routines will take care of changing this register +** value to non-NULL if the RHS is NULL-free. ** ** For a SELECT or EXISTS operator, return the register that holds the ** result. For IN operators or if an error occurs, the return value is 0. @@ -1928,7 +1939,8 @@ static void sqlite3ExprCodeIN( v = pParse->pVdbe; assert( v!=0 ); /* OOM detected prior to this routine */ VdbeNoopComment((v, "begin IN expr")); - eType = sqlite3FindInIndex(pParse, pExpr, &rRhsHasNull); + eType = sqlite3FindInIndex(pParse, pExpr, 0, + destIfFalse==destIfNull ? 0 : &rRhsHasNull); /* Figure out the affinity to use to create a key from the results ** of the expression. affinityStr stores a static string suitable for diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 1aede95c14..784fd0fd93 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3591,11 +3591,21 @@ const char *sqlite3JournalModename(int); #define sqlite3EndBenignMalloc() #endif -#define IN_INDEX_ROWID 1 -#define IN_INDEX_EPH 2 -#define IN_INDEX_INDEX_ASC 3 -#define IN_INDEX_INDEX_DESC 4 -int sqlite3FindInIndex(Parse *, Expr *, int*); +/* +** Allowed return values from sqlite3FindInIndex() +*/ +#define IN_INDEX_ROWID 1 /* Search the rowid of the table */ +#define IN_INDEX_EPH 2 /* Search an ephemeral b-tree */ +#define IN_INDEX_INDEX_ASC 3 /* Existing index ASCENDING */ +#define IN_INDEX_INDEX_DESC 4 /* Existing index DESCENDING */ +#define IN_INDEX_NOOP 5 /* No table available. Use comparisons */ +/* +** Allowed flags for the 3rd parameter to sqlite3FindInIndex(). +*/ +#define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ +#define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ +#define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ +int sqlite3FindInIndex(Parse *, Expr *, u32, int*); #ifdef SQLITE_ENABLE_ATOMIC_WRITE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); diff --git a/src/where.c b/src/where.c index 3cc66a34b1..20823046f7 100644 --- a/src/where.c +++ b/src/where.c @@ -2522,7 +2522,7 @@ static int codeEqualityTerm( } assert( pX->op==TK_IN ); iReg = iTarget; - eType = sqlite3FindInIndex(pParse, pX, 0); + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0); if( eType==IN_INDEX_INDEX_DESC ){ testcase( bRev ); bRev = !bRev; From e80c9b9ad5e62af9ccd8a4ce8f069c965dde6070 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 1 Aug 2014 15:34:36 +0000 Subject: [PATCH 177/710] The idea of coding IN operator with a short list on the RHS as an OR expression turns out to be helpful. If the list is of length 1 or 2, the OR expression is very slightly faster, but the ephemeral table approach is clearly better for all list lengths greater than 2. Better to keep the code simple. FossilOrigin-Name: e13175d3579e1045165bab091b3b28951d691704 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/expr.c | 14 +++----------- src/sqliteInt.h | 6 ++---- 4 files changed, 13 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 3fae6c98bd..76ab0745e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\smaking\schanges\sto\sthe\sIN\soperator\sin\san\sattempt\sto\smake\sit\srun\sfaster\nand\sto\smake\sthe\scode\seasier\sto\sunderstand. -D 2014-08-01T14:46:57.155 +C The\sidea\sof\scoding\sIN\soperator\swith\sa\sshort\slist\son\sthe\sRHS\sas\san\sOR\sexpression\nturns\sout\sto\sbe\shelpful.\s\sIf\sthe\slist\sis\sof\slength\s1\sor\s2,\sthe\sOR\sexpression\nis\svery\sslightly\sfaster,\sbut\sthe\sephemeral\stable\sapproach\sis\sclearly\sbetter\sfor\nall\slist\slengths\sgreater\sthan\s2.\s\sBetter\sto\skeep\sthe\scode\ssimple. +D 2014-08-01T15:34:36.551 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 7c52ea8b322992a91a241c0092a5bf97b141d353 +F src/expr.c 1f7fa7ddbdffd89898a3ce5ff1e57d137ec87aa5 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -227,7 +227,7 @@ F src/shell.c 191129c3f7a9cf241aea90ff6a6be3e74d3767f0 F src/sqlite.h.in 9bbc5815c73b0e77e68b5275481a5e3e7814a804 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 17ece600d3c9d36cc0ee2b74a30507507f3e0937 +F src/sqliteInt.h 5cee19f34f6efe6e6f7733d55a5c59e3a35a378a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1185,10 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9abcf2698c09f4f6a44a68e74f9f6b538f3253d6 -R 1cc42777f2b661f6d0caf162eadc6d85 -T *branch * IN-operator-improvements -T *sym-IN-operator-improvements * -T -sym-trunk * +P ee0fd6aaf94cda1dce3fe752bfe3b0f83e0043f1 +R b49330a02feffafe84f051ae1dbb9076 U drh -Z 08adfdc6013d75458685b1e962a583ca +Z f695386e744e4159654587b52ac0d789 diff --git a/manifest.uuid b/manifest.uuid index 1b3e9a1eeb..9b33d2f23a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee0fd6aaf94cda1dce3fe752bfe3b0f83e0043f1 \ No newline at end of file +e13175d3579e1045165bab091b3b28951d691704 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index dd7b7c3de1..d818827107 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1494,8 +1494,6 @@ int sqlite3CodeOnce(Parse *pParse){ ** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. ** IN_INDEX_EPH - The cursor was opened on a specially created and ** populated epheremal table. -** IN_INDEX_NOOP - No cursor was allocated. The in operator must be -** implemented as a sequence of comparisons. ** ** An existing b-tree might be used if the RHS expression pX is a simple ** subquery such as: @@ -1525,13 +1523,6 @@ int sqlite3CodeOnce(Parse *pParse){ ** be used unless is an INTEGER PRIMARY KEY or an index can ** be found with as its left-most column. ** -** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and -** if the RHS of the IN operator is a list (not a subquery) then this -** routine might decide that creating an ephemeral b-tree for membership -** testing is too expensive and return IN_INDEX_NOOP. IN that case, the -** calling routine should implement the IN operator using a sequence -** of Eq or Ne comparison operations. -** ** When the b-tree is being used for membership tests, the calling function ** might need to know whether or not the RHS side of the IN operator ** contains a NULL. If prNotFound is not NULL and @@ -1939,7 +1930,7 @@ static void sqlite3ExprCodeIN( v = pParse->pVdbe; assert( v!=0 ); /* OOM detected prior to this routine */ VdbeNoopComment((v, "begin IN expr")); - eType = sqlite3FindInIndex(pParse, pExpr, 0, + eType = sqlite3FindInIndex(pParse, pExpr, IN_INDEX_MEMBERSHIP, destIfFalse==destIfNull ? 0 : &rRhsHasNull); /* Figure out the affinity to use to create a key from the results @@ -1986,7 +1977,8 @@ static void sqlite3ExprCodeIN( ** contains one or more NULL values, then the result of the ** expression is also NULL. */ - if( rRhsHasNull==0 || destIfFalse==destIfNull ){ + assert( destIfFalse!=destIfNull || rRhsHasNull==0 ); + if( rRhsHasNull==0 ){ /* This branch runs if it is known at compile time that the RHS ** cannot contain NULL values. This happens as the result ** of a "NOT NULL" constraint in the database schema. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 784fd0fd93..97fb2d46e4 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3598,13 +3598,11 @@ const char *sqlite3JournalModename(int); #define IN_INDEX_EPH 2 /* Search an ephemeral b-tree */ #define IN_INDEX_INDEX_ASC 3 /* Existing index ASCENDING */ #define IN_INDEX_INDEX_DESC 4 /* Existing index DESCENDING */ -#define IN_INDEX_NOOP 5 /* No table available. Use comparisons */ /* ** Allowed flags for the 3rd parameter to sqlite3FindInIndex(). */ -#define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ -#define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ -#define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ +#define IN_INDEX_MEMBERSHIP 0x0001 /* IN operator used for membership test */ +#define IN_INDEX_LOOP 0x0002 /* IN operator used as a loop */ int sqlite3FindInIndex(Parse *, Expr *, u32, int*); #ifdef SQLITE_ENABLE_ATOMIC_WRITE From e21a6e1dfe2196184bf17e9d9212f3eae0b346e5 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 1 Aug 2014 18:00:24 +0000 Subject: [PATCH 178/710] Remove an unnecessary OP_Null in the IN-operator logic. Attempt to clarify comments explaining the IN-operator code, though it is not clear that the comments are correct even yet - more work to be done. FossilOrigin-Name: c11e55fabbc718cb324ecd3540453c25db98f50c --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/expr.c | 42 +++++++++++++++++++++++------------------- src/vdbe.c | 4 ++-- 4 files changed, 33 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index d534cb249b..2762bc4675 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\sthe\sIN\soperator\scode\sgeneration\slogic\sto\smake\sit\seasier\sto\sreason\nabout.\s\sIn\sthe\sprocess,\simprove\scode\sgeneration\sto\somit\ssome\sunused\sOP_Null\noperations. -D 2014-08-01T15:51:36.528 +C Remove\san\sunnecessary\sOP_Null\sin\sthe\sIN-operator\slogic.\s\sAttempt\sto\sclarify\ncomments\sexplaining\sthe\sIN-operator\scode,\sthough\sit\sis\snot\sclear\sthat\sthe\ncomments\sare\scorrect\seven\syet\s-\smore\swork\sto\sbe\sdone. +D 2014-08-01T18:00:24.770 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 1f7fa7ddbdffd89898a3ce5ff1e57d137ec87aa5 +F src/expr.c 45da7f2dd6b5c52b3210387956711d578bdceb37 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -283,7 +283,7 @@ F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 115e08834b883964d9a480f685d8601826c5792a +F src/vdbe.c b9e6866e43a61ca4080410f27c4bb52823495186 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 @@ -1185,8 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9abcf2698c09f4f6a44a68e74f9f6b538f3253d6 e13175d3579e1045165bab091b3b28951d691704 -R b49330a02feffafe84f051ae1dbb9076 -T +closed e13175d3579e1045165bab091b3b28951d691704 +P 7c6fbcfe6ed5739e8e4639b7b123fbf9828cbfc0 +R 76f22de612006799aae8055deaa164fe U drh -Z e21436d3fd2e7daacda65f50d724cf15 +Z ebc1ba4c7f530e221b3bc2fcb7e1119a diff --git a/manifest.uuid b/manifest.uuid index 06cf7d6673..ef13592241 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c6fbcfe6ed5739e8e4639b7b123fbf9828cbfc0 \ No newline at end of file +c11e55fabbc718cb324ecd3540453c25db98f50c \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index d818827107..591ea9845c 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1525,29 +1525,34 @@ int sqlite3CodeOnce(Parse *pParse){ ** ** When the b-tree is being used for membership tests, the calling function ** might need to know whether or not the RHS side of the IN operator -** contains a NULL. If prNotFound is not NULL and +** contains a NULL. If prRhsHasNull is not a NULL pointer and ** if there is any chance that the (...) might contain a NULL value at ** runtime, then a register is allocated and the register number written -** to *prNotFound. If there is no chance that the (...) contains a -** NULL value, then *prNotFound is left unchanged. +** to *prRhsHasNull. If there is no chance that the (...) contains a +** NULL value, then *prRhsHasNull is left unchanged. ** -** If a register is allocated and its location stored in *prNotFound, then -** its initial value is NULL. If the (...) does not remain constant -** for the duration of the query (i.e. the SELECT within the (...) -** is a correlated subquery) then the value of the allocated register is -** reset to NULL each time the subquery is rerun. This allows the -** caller to use vdbe code equivalent to the following: +** If a register is allocated and its location stored in *prRhsHasNull, then +** the value in that register will be: ** -** if( register==NULL ){ -** has_null = -** register = 1 +** 0 if the (...) contains no NULL values +** 1 if the (...) does not contain NULL values +** NULL if we do not yet know if (...) contains NULLs +** +** If the (...) does not remain constant for the duration of the query +** (i.e. the SELECT within the (...) is a correlated subquery) then the +** value of the allocated register is reset to NULL each time the subquery +** is rerun. This allows the caller to use vdbe code equivalent to the +** following: +** +** if( r[*prRhsHasNull] IS NULL ){ +** r[*prRhsHasNull] = ** } ** ** in order to avoid running the ** test more often than is necessary. */ #ifndef SQLITE_OMIT_SUBQUERY -int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prNotFound){ +int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ Select *p; /* SELECT to the right of IN operator */ int eType = 0; /* Type of RHS table. IN_INDEX_* */ int iTab = pParse->nTab++; /* Cursor of the RHS table */ @@ -1621,9 +1626,9 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prNotFound){ assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; - if( prNotFound && !pTab->aCol[iCol].notNull ){ - *prNotFound = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound); + if( prRhsHasNull && !pTab->aCol[iCol].notNull ){ + *prRhsHasNull = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, *prRhsHasNull); } sqlite3VdbeJumpHere(v, iAddr); } @@ -1643,9 +1648,8 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prNotFound){ if( pX->pLeft->iColumn<0 && !ExprHasProperty(pX, EP_xIsSelect) ){ eType = IN_INDEX_ROWID; } - }else if( prNotFound ){ - *prNotFound = rMayHaveNull = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, rMayHaveNull); + }else if( prRhsHasNull ){ + *prRhsHasNull = rMayHaveNull = ++pParse->nMem; } sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID); pParse->nQueryLoop = savedNQueryLoop; diff --git a/src/vdbe.c b/src/vdbe.c index 07514951a4..ade82b054a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2229,13 +2229,13 @@ case OP_Once: { /* jump */ ** ** Jump to P2 if the value in register P1 is true. The value ** is considered true if it is numeric and non-zero. If the value -** in P1 is NULL then take the jump if P3 is non-zero. +** in P1 is NULL then take the jump if and only if P3 is non-zero. */ /* Opcode: IfNot P1 P2 P3 * * ** ** Jump to P2 if the value in register P1 is False. The value ** is considered false if it has a numeric value of zero. If the value -** in P1 is NULL then take the jump if P3 is zero. +** in P1 is NULL then take the jump if and only if P3 is non-zero. */ case OP_If: /* jump, in1 */ case OP_IfNot: { /* jump, in1 */ From 6be515ebe0c6976352efd82cd6cddebe4b43361f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 1 Aug 2014 21:00:53 +0000 Subject: [PATCH 179/710] Improved detection and handling of NULL values on the RHS of a IN operator. FossilOrigin-Name: 468e730036edac22cfeb9ea3515aa16e6bcd6650 --- manifest | 12 +++---- manifest.uuid | 2 +- src/expr.c | 94 ++++++++++++++++++++++----------------------------- 3 files changed, 47 insertions(+), 61 deletions(-) diff --git a/manifest b/manifest index 2762bc4675..5a22fd3d13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sOP_Null\sin\sthe\sIN-operator\slogic.\s\sAttempt\sto\sclarify\ncomments\sexplaining\sthe\sIN-operator\scode,\sthough\sit\sis\snot\sclear\sthat\sthe\ncomments\sare\scorrect\seven\syet\s-\smore\swork\sto\sbe\sdone. -D 2014-08-01T18:00:24.770 +C Improved\sdetection\sand\shandling\sof\sNULL\svalues\son\sthe\sRHS\sof\sa\sIN\soperator. +D 2014-08-01T21:00:53.855 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 45da7f2dd6b5c52b3210387956711d578bdceb37 +F src/expr.c ac35f4c83ccc090e4b31bf3c839d519762a86fd5 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7c6fbcfe6ed5739e8e4639b7b123fbf9828cbfc0 -R 76f22de612006799aae8055deaa164fe +P c11e55fabbc718cb324ecd3540453c25db98f50c +R d4c66ce9989f6295fb51f65762a6a30b U drh -Z ebc1ba4c7f530e221b3bc2fcb7e1119a +Z 526f21f790d2042539b4ae116e22530d diff --git a/manifest.uuid b/manifest.uuid index ef13592241..4742ee228e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c11e55fabbc718cb324ecd3540453c25db98f50c \ No newline at end of file +468e730036edac22cfeb9ea3515aa16e6bcd6650 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 591ea9845c..6440dea397 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1475,6 +1475,22 @@ int sqlite3CodeOnce(Parse *pParse){ return sqlite3VdbeAddOp1(v, OP_Once, pParse->nOnce++); } +/* +** Generate code that checks the single-column index table iCur to see if +** contains any NULL entries. Cause the register at regHasNull to be set +** to a non-NULL value if iCur contains no NULLs. Cause register regHasNull +** to be set to NULL if iCur contains one or more NULL values. +*/ +static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ + int j1; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regHasNull); + j1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); + sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); + VdbeComment((v, "")); + sqlite3VdbeJumpHere(v, j1); +} + /* ** This function is used by the implementation of the IN (...) operator. ** The pX parameter is the expression on the RHS of the IN operator, which @@ -1532,24 +1548,9 @@ int sqlite3CodeOnce(Parse *pParse){ ** NULL value, then *prRhsHasNull is left unchanged. ** ** If a register is allocated and its location stored in *prRhsHasNull, then -** the value in that register will be: -** -** 0 if the (...) contains no NULL values -** 1 if the (...) does not contain NULL values -** NULL if we do not yet know if (...) contains NULLs -** -** If the (...) does not remain constant for the duration of the query -** (i.e. the SELECT within the (...) is a correlated subquery) then the -** value of the allocated register is reset to NULL each time the subquery -** is rerun. This allows the caller to use vdbe code equivalent to the -** following: -** -** if( r[*prRhsHasNull] IS NULL ){ -** r[*prRhsHasNull] = -** } -** -** in order to avoid running the -** test more often than is necessary. +** the value in that register will be NULL if the b-tree contains one or more +** NULL values, and it will be some non-NULL value if the b-tree contains no +** NULL values. */ #ifndef SQLITE_OMIT_SUBQUERY int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ @@ -1628,7 +1629,7 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ if( prRhsHasNull && !pTab->aCol[iCol].notNull ){ *prRhsHasNull = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Null, 0, *prRhsHasNull); + sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); } sqlite3VdbeJumpHere(v, iAddr); } @@ -1691,10 +1692,10 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ int sqlite3CodeSubselect( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The IN, SELECT, or EXISTS operator */ - int rMayHaveNull, /* Register that records whether NULLs exist in RHS */ + int rHasNullFlag, /* Register that records whether NULLs exist in RHS */ int isRowid /* If true, LHS of IN operator is a rowid */ ){ - int testAddr = -1; /* One-time test address */ + int jmpIfDynamic = -1; /* One-time test address */ int rReg = 0; /* Register storing resulting */ Vdbe *v = sqlite3GetVdbe(pParse); if( NEVER(v==0) ) return 0; @@ -1711,13 +1712,13 @@ int sqlite3CodeSubselect( ** save the results, and reuse the same result on subsequent invocations. */ if( !ExprHasProperty(pExpr, EP_VarSelect) ){ - testAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); + jmpIfDynamic = sqlite3CodeOnce(pParse); VdbeCoverage(v); } #ifndef SQLITE_OMIT_EXPLAIN if( pParse->explain==2 ){ char *zMsg = sqlite3MPrintf( - pParse->db, "EXECUTE %s%s SUBQUERY %d", testAddr>=0?"":"CORRELATED ", + pParse->db, "EXECUTE %s%s SUBQUERY %d", jmpIfDynamic>=0?"":"CORRELATED ", pExpr->op==TK_IN?"LIST":"SCALAR", pParse->iNextSelectId ); sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); @@ -1731,10 +1732,6 @@ int sqlite3CodeSubselect( Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ KeyInfo *pKeyInfo = 0; /* Key information */ - if( rMayHaveNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, rMayHaveNull); - } - affinity = sqlite3ExprAffinity(pLeft); /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' @@ -1817,9 +1814,9 @@ int sqlite3CodeSubselect( ** this code only executes once. Because for a non-constant ** expression we need to rerun this code each time. */ - if( testAddr>=0 && !sqlite3ExprIsConstant(pE2) ){ - sqlite3VdbeChangeToNoop(v, testAddr); - testAddr = -1; + if( jmpIfDynamic>=0 && !sqlite3ExprIsConstant(pE2) ){ + sqlite3VdbeChangeToNoop(v, jmpIfDynamic); + jmpIfDynamic = -1; } /* Evaluate the expression and insert it into the temp table */ @@ -1889,8 +1886,12 @@ int sqlite3CodeSubselect( } } - if( testAddr>=0 ){ - sqlite3VdbeJumpHere(v, testAddr); + if( rHasNullFlag ){ + sqlite3SetHasNullFlag(v, pExpr->iTable, rHasNullFlag); + } + + if( jmpIfDynamic>=0 ){ + sqlite3VdbeJumpHere(v, jmpIfDynamic); } sqlite3ExprCachePop(pParse); @@ -1911,7 +1912,7 @@ int sqlite3CodeSubselect( ** if the LHS is NULL or if the LHS is not contained within the RHS and the ** RHS contains one or more NULL values. ** -** This routine generates code will jump to destIfFalse if the LHS is not +** This routine generates code that jumps to destIfFalse if the LHS is not ** contained within the RHS. If due to NULLs we cannot determine if the LHS ** is contained in the RHS then jump to destIfNull. If the LHS is contained ** within the RHS then fall through. @@ -1997,34 +1998,19 @@ static void sqlite3ExprCodeIN( ** the presence of a NULL on the RHS makes a difference in the ** outcome. */ - int j1, j2; + int j1; /* First check to see if the LHS is contained in the RHS. If so, - ** then the presence of NULLs in the RHS does not matter, so jump - ** over all of the code that follows. + ** then the answer is TRUE the presence of NULLs in the RHS does + ** not matter. If the LHS is not contained in the RHS, then the + ** answer is NULL if the RHS contains NULLs and the answer is + ** FALSE if the RHS is NULL-free. */ j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); VdbeCoverage(v); - - /* Here we begin generating code that runs if the LHS is not - ** contained within the RHS. Generate additional code that - ** tests the RHS for NULLs. If the RHS contains a NULL then - ** jump to destIfNull. If there are no NULLs in the RHS then - ** jump to destIfFalse. - */ - sqlite3VdbeAddOp2(v, OP_If, rRhsHasNull, destIfNull); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_IfNot, rRhsHasNull, destIfFalse); VdbeCoverage(v); - j2 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, rRhsHasNull, 1); + sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Integer, 0, rRhsHasNull); sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); - sqlite3VdbeJumpHere(v, j2); - sqlite3VdbeAddOp2(v, OP_Integer, 1, rRhsHasNull); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); - - /* The OP_Found at the top of this branch jumps here when true, - ** causing the overall IN expression evaluation to fall through. - */ sqlite3VdbeJumpHere(v, j1); } } From 4c259e9f40a0580c60455f9f4b7492f1d99749a7 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 1 Aug 2014 21:12:35 +0000 Subject: [PATCH 180/710] A better comment on the generated code for the NULL-in-RHS-of-IN detection logic. FossilOrigin-Name: 9bc1c730a366e75b760b58e7a343d39165b2a469 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 5a22fd3d13..6b5c8679ce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sdetection\sand\shandling\sof\sNULL\svalues\son\sthe\sRHS\sof\sa\sIN\soperator. -D 2014-08-01T21:00:53.855 +C A\sbetter\scomment\son\sthe\sgenerated\scode\sfor\sthe\sNULL-in-RHS-of-IN\sdetection\nlogic. +D 2014-08-01T21:12:35.509 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c ac35f4c83ccc090e4b31bf3c839d519762a86fd5 +F src/expr.c 564c28f4f68c30103ba4d8ef60b53f178f60eeb3 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c11e55fabbc718cb324ecd3540453c25db98f50c -R d4c66ce9989f6295fb51f65762a6a30b +P 468e730036edac22cfeb9ea3515aa16e6bcd6650 +R 894229228ea8340c8d39c1accaa43c91 U drh -Z 526f21f790d2042539b4ae116e22530d +Z 2180e0f63fb08551c10fb11eafeeb559 diff --git a/manifest.uuid b/manifest.uuid index 4742ee228e..764209da3f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -468e730036edac22cfeb9ea3515aa16e6bcd6650 \ No newline at end of file +9bc1c730a366e75b760b58e7a343d39165b2a469 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 6440dea397..c32affe34c 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1476,8 +1476,8 @@ int sqlite3CodeOnce(Parse *pParse){ } /* -** Generate code that checks the single-column index table iCur to see if -** contains any NULL entries. Cause the register at regHasNull to be set +** Generate code that checks the left-most column of index table iCur to see if +** it contains any NULL entries. Cause the register at regHasNull to be set ** to a non-NULL value if iCur contains no NULLs. Cause register regHasNull ** to be set to NULL if iCur contains one or more NULL values. */ @@ -1487,7 +1487,7 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ j1 = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Column, iCur, 0, regHasNull); sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); - VdbeComment((v, "")); + VdbeComment((v, "first_entry_in(%d)", iCur)); sqlite3VdbeJumpHere(v, j1); } From 16d511a664c58b7410ce00fefabcbe8a72e347fe Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 2 Aug 2014 20:44:13 +0000 Subject: [PATCH 181/710] Remove (newly) incorrect preprocessor check to fix build on WinRT. FossilOrigin-Name: ba7826542908eac2e14789d183d0b3e35b143fed --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 6b5c8679ce..ae65d8580f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\sbetter\scomment\son\sthe\sgenerated\scode\sfor\sthe\sNULL-in-RHS-of-IN\sdetection\nlogic. -D 2014-08-01T21:12:35.509 +C Remove\s(newly)\sincorrect\spreprocessor\scheck\sto\sfix\sbuild\son\sWinRT. +D 2014-08-02T20:44:13.563 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c a7baf1b30f3c58ba20b813e01aab23b18ae44f85 -F src/os_win.c c67bec43e5dadde8029470ab7b7825ccc9abe578 +F src/os_win.c e5f0fc446a682b70db3397d14cca9806d9a15d12 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 468e730036edac22cfeb9ea3515aa16e6bcd6650 -R 894229228ea8340c8d39c1accaa43c91 -U drh -Z 2180e0f63fb08551c10fb11eafeeb559 +P 9bc1c730a366e75b760b58e7a343d39165b2a469 +R 8c9bd54f21254a5c39cdbf67a68f6f01 +U mistachkin +Z a3832f522ef0a90c3ccd915e060c6930 diff --git a/manifest.uuid b/manifest.uuid index 764209da3f..d8a5cdba49 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9bc1c730a366e75b760b58e7a343d39165b2a469 \ No newline at end of file +ba7826542908eac2e14789d183d0b3e35b143fed \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index fd2997206c..f770f13862 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -415,8 +415,7 @@ const sqlite3_mem_methods *sqlite3MemGetWin32(void); */ #ifdef SQLITE_TEST LONG volatile sqlite3_os_type = 0; -#elif !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ - defined(SQLITE_WIN32_HAS_ANSI) && defined(SQLITE_WIN32_HAS_WIDE) +#else static LONG volatile sqlite3_os_type = 0; #endif From bb53ecb1dbd36de81519086bab2c6589640daf49 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 2 Aug 2014 21:03:33 +0000 Subject: [PATCH 182/710] Enhancements to the code generator for the IN operator that result in much faster queries in some cases, for example when the RHS of the IN operator changes for each row of a large table scan. FossilOrigin-Name: 436e884215e2b33ca3fbb555362237b12827c07a --- manifest | 23 +++-- manifest.uuid | 2 +- src/expr.c | 195 ++++++++++++++++++++++++++++----------- src/sqliteInt.h | 6 +- test/in4.test | 4 +- test/tkt-80e031a00f.test | 4 + 6 files changed, 165 insertions(+), 69 deletions(-) diff --git a/manifest b/manifest index ae65d8580f..0a8cdbcd1c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\s(newly)\sincorrect\spreprocessor\scheck\sto\sfix\sbuild\son\sWinRT. -D 2014-08-02T20:44:13.563 +C Enhancements\sto\sthe\scode\sgenerator\sfor\sthe\sIN\soperator\sthat\sresult\sin\smuch\nfaster\squeries\sin\ssome\scases,\sfor\sexample\swhen\sthe\sRHS\sof\sthe\sIN\soperator\nchanges\sfor\seach\srow\sof\sa\slarge\stable\sscan. +D 2014-08-02T21:03:33.699 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 564c28f4f68c30103ba4d8ef60b53f178f60eeb3 +F src/expr.c 89574df1ed8ca967fdd5be0e370722e508b91c2b F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -227,7 +227,7 @@ F src/shell.c 191129c3f7a9cf241aea90ff6a6be3e74d3767f0 F src/sqlite.h.in 9bbc5815c73b0e77e68b5275481a5e3e7814a804 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 5cee19f34f6efe6e6f7733d55a5c59e3a35a378a +F src/sqliteInt.h 17ece600d3c9d36cc0ee2b74a30507507f3e0937 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -607,7 +607,7 @@ F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 -F test/in4.test 41c1c031aa46b1eb4411df2687ed2ed498da23b5 +F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 @@ -901,7 +901,7 @@ F test/tkt-78e04e52ea.test 813779f8888f3ca226df656c4eef078f9635f3c9 F test/tkt-7a31705a7e6.test e75a2bba4eec801b92c8040eb22096ac6d35e844 F test/tkt-7bbfb7d442.test 7b2cd79c7a17ae6750e75ec1a7846712a69c9d18 F test/tkt-80ba201079.test 105a721e6aad0ae3c5946d7615d1e4d03f6145b8 -F test/tkt-80e031a00f.test 9a154173461a4dbe2de49cda73963e04842d52f7 +F test/tkt-80e031a00f.test f50046f474ecf67ad5c50cd9200da04ff887d7cd F test/tkt-8454a207b9.test c583a9f814a82a2b5ba95207f55001c9f0cd816c F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-8c63ff0ec.test 258b7fc8d7e4e1cb5362c7d65c143528b9c4cbed @@ -1185,7 +1185,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9bc1c730a366e75b760b58e7a343d39165b2a469 -R 8c9bd54f21254a5c39cdbf67a68f6f01 -U mistachkin -Z a3832f522ef0a90c3ccd915e060c6930 +P ba7826542908eac2e14789d183d0b3e35b143fed +R 93a5c3d553517a616682ed62728bfb2c +T *branch * IN-operator-improvements +T *sym-IN-operator-improvements * +T -sym-trunk * +U drh +Z c474e62d8861a07a170ddf9e71c3b1a9 diff --git a/manifest.uuid b/manifest.uuid index d8a5cdba49..0ddf2ad117 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba7826542908eac2e14789d183d0b3e35b143fed \ No newline at end of file +436e884215e2b33ca3fbb555362237b12827c07a \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index c32affe34c..7565d41b90 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1491,6 +1491,24 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ sqlite3VdbeJumpHere(v, j1); } + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** The argument is an IN operator with a list (not a subquery) on the +** right-hand side. Return TRUE if that list is constant. +*/ +static int sqlite3InRhsIsConstant(Expr *pIn){ + Expr *pLHS; + int res; + assert( !ExprHasProperty(pIn, EP_xIsSelect) ); + pLHS = pIn->pLeft; + pIn->pLeft = 0; + res = sqlite3ExprIsConstant(pIn); + pIn->pLeft = pLHS; + return res; +} +#endif + /* ** This function is used by the implementation of the IN (...) operator. ** The pX parameter is the expression on the RHS of the IN operator, which @@ -1510,6 +1528,8 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ ** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. ** IN_INDEX_EPH - The cursor was opened on a specially created and ** populated epheremal table. +** IN_INDEX_NOOP - No cursor was allocated. The IN operator must be +** implemented as a sequence of comparisons. ** ** An existing b-tree might be used if the RHS expression pX is a simple ** subquery such as: @@ -1539,6 +1559,13 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ ** be used unless is an INTEGER PRIMARY KEY or an index can ** be found with as its left-most column. ** +** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and +** if the RHS of the IN operator is a list (not a subquery) then this +** routine might decide that creating an ephemeral b-tree for membership +** testing is too expensive and return IN_INDEX_NOOP. In that case, the +** calling routine should implement the IN operator using a sequence +** of Eq or Ne comparison operations. +** ** When the b-tree is being used for membership tests, the calling function ** might need to know whether or not the RHS side of the IN operator ** contains a NULL. If prRhsHasNull is not a NULL pointer and @@ -1637,6 +1664,22 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ } } + /* If no preexisting index is available for the IN clause + ** and IN_INDEX_NOOP is an allowed reply + ** and the RHS of the IN operator is a list, not a subquery + ** and the RHS is not contant or has two or fewer terms, + ** then it is not worth creating an ephermeral table to evaluate + ** the IN operator so return IN_INDEX_NOOP. + */ + if( eType==0 + && (inFlags & IN_INDEX_NOOP_OK) + && !ExprHasProperty(pX, EP_xIsSelect) + && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2) + ){ + eType = IN_INDEX_NOOP; + } + + if( eType==0 ){ /* Could not find an existing table or index to use as the RHS b-tree. ** We will have to generate an ephemeral table to do the job. @@ -1935,7 +1978,8 @@ static void sqlite3ExprCodeIN( v = pParse->pVdbe; assert( v!=0 ); /* OOM detected prior to this routine */ VdbeNoopComment((v, "begin IN expr")); - eType = sqlite3FindInIndex(pParse, pExpr, IN_INDEX_MEMBERSHIP, + eType = sqlite3FindInIndex(pParse, pExpr, + IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK, destIfFalse==destIfNull ? 0 : &rRhsHasNull); /* Figure out the affinity to use to create a key from the results @@ -1950,68 +1994,111 @@ static void sqlite3ExprCodeIN( r1 = sqlite3GetTempReg(pParse); sqlite3ExprCode(pParse, pExpr->pLeft, r1); - /* If the LHS is NULL, then the result is either false or NULL depending - ** on whether the RHS is empty or not, respectively. + /* If sqlite3FindInIndex() did not find or create an index that is + ** suitable for evaluating the IN operator, then evaluate using a + ** sequence of comparisons. */ - if( destIfNull==destIfFalse ){ - /* Shortcut for the common case where the false and NULL outcomes are - ** the same. */ + if( eType==IN_INDEX_NOOP ){ + ExprList *pList = pExpr->x.pList; + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + int labelOk = sqlite3VdbeMakeLabel(v); + int r2, regToFree; + int regCkNull = 0; + int ii; + assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); + if( destIfNull!=destIfFalse ){ + regCkNull = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regCkNull); + } + for(ii=0; iinExpr; ii++){ + r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); + if( regCkNull ){ + sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); + } + if( iinExpr-1 || destIfNull!=destIfFalse ){ + sqlite3VdbeAddOp4(v, OP_Eq, r1, labelOk, r2, + (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, affinity); + }else{ + assert( destIfNull==destIfFalse ); + sqlite3VdbeAddOp4(v, OP_Ne, r1, destIfFalse, r2, + (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, affinity | SQLITE_JUMPIFNULL); + } + sqlite3ReleaseTempReg(pParse, regToFree); + } + if( regCkNull ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + } + sqlite3VdbeResolveLabel(v, labelOk); + sqlite3ReleaseTempReg(pParse, regCkNull); }else{ - int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); - sqlite3VdbeJumpHere(v, addr1); - } - - if( eType==IN_INDEX_ROWID ){ - /* In this case, the RHS is the ROWID of table b-tree + + /* If the LHS is NULL, then the result is either false or NULL depending + ** on whether the RHS is empty or not, respectively. */ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); - VdbeCoverage(v); - }else{ - /* In this case, the RHS is an index b-tree. - */ - sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); - - /* If the set membership test fails, then the result of the - ** "x IN (...)" expression must be either 0 or NULL. If the set - ** contains no NULL values, then the result is 0. If the set - ** contains one or more NULL values, then the result of the - ** expression is also NULL. - */ - assert( destIfFalse!=destIfNull || rRhsHasNull==0 ); - if( rRhsHasNull==0 ){ - /* This branch runs if it is known at compile time that the RHS - ** cannot contain NULL values. This happens as the result - ** of a "NOT NULL" constraint in the database schema. - ** - ** Also run this branch if NULL is equivalent to FALSE - ** for this particular IN operator. + if( destIfNull==destIfFalse ){ + /* Shortcut for the common case where the false and NULL outcomes are + ** the same. */ + sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); + }else{ + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + sqlite3VdbeJumpHere(v, addr1); + } + + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree */ - sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); + sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); VdbeCoverage(v); }else{ - /* In this branch, the RHS of the IN might contain a NULL and - ** the presence of a NULL on the RHS makes a difference in the - ** outcome. + /* In this case, the RHS is an index b-tree. */ - int j1; - - /* First check to see if the LHS is contained in the RHS. If so, - ** then the answer is TRUE the presence of NULLs in the RHS does - ** not matter. If the LHS is not contained in the RHS, then the - ** answer is NULL if the RHS contains NULLs and the answer is - ** FALSE if the RHS is NULL-free. + sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); + + /* If the set membership test fails, then the result of the + ** "x IN (...)" expression must be either 0 or NULL. If the set + ** contains no NULL values, then the result is 0. If the set + ** contains one or more NULL values, then the result of the + ** expression is also NULL. */ - j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); - sqlite3VdbeJumpHere(v, j1); + assert( destIfFalse!=destIfNull || rRhsHasNull==0 ); + if( rRhsHasNull==0 ){ + /* This branch runs if it is known at compile time that the RHS + ** cannot contain NULL values. This happens as the result + ** of a "NOT NULL" constraint in the database schema. + ** + ** Also run this branch if NULL is equivalent to FALSE + ** for this particular IN operator. + */ + sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); + VdbeCoverage(v); + }else{ + /* In this branch, the RHS of the IN might contain a NULL and + ** the presence of a NULL on the RHS makes a difference in the + ** outcome. + */ + int j1; + + /* First check to see if the LHS is contained in the RHS. If so, + ** then the answer is TRUE the presence of NULLs in the RHS does + ** not matter. If the LHS is not contained in the RHS, then the + ** answer is NULL if the RHS contains NULLs and the answer is + ** FALSE if the RHS is NULL-free. + */ + j1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + sqlite3VdbeJumpHere(v, j1); + } } } sqlite3ReleaseTempReg(pParse, r1); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 97fb2d46e4..784fd0fd93 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3598,11 +3598,13 @@ const char *sqlite3JournalModename(int); #define IN_INDEX_EPH 2 /* Search an ephemeral b-tree */ #define IN_INDEX_INDEX_ASC 3 /* Existing index ASCENDING */ #define IN_INDEX_INDEX_DESC 4 /* Existing index DESCENDING */ +#define IN_INDEX_NOOP 5 /* No table available. Use comparisons */ /* ** Allowed flags for the 3rd parameter to sqlite3FindInIndex(). */ -#define IN_INDEX_MEMBERSHIP 0x0001 /* IN operator used for membership test */ -#define IN_INDEX_LOOP 0x0002 /* IN operator used as a loop */ +#define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ +#define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ +#define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ int sqlite3FindInIndex(Parse *, Expr *, u32, int*); #ifdef SQLITE_ENABLE_ATOMIC_WRITE diff --git a/test/in4.test b/test/in4.test index 0a4a75008b..a89961f82b 100644 --- a/test/in4.test +++ b/test/in4.test @@ -231,11 +231,11 @@ do_execsql_test in4-3.44 { SELECT * FROM t3 WHERE x IN (10); } {~/OpenEphemeral/} do_execsql_test in4-3.45 { - SELECT * FROM t3 WHERE x NOT IN (10,11); + SELECT * FROM t3 WHERE x NOT IN (10,11,99999); } {1 1 1} do_execsql_test in4-3.46 { EXPLAIN - SELECT * FROM t3 WHERE x NOT IN (10,11); + SELECT * FROM t3 WHERE x NOT IN (10,11,99999); } {/OpenEphemeral/} do_execsql_test in4-3.47 { SELECT * FROM t3 WHERE x NOT IN (10); diff --git a/test/tkt-80e031a00f.test b/test/tkt-80e031a00f.test index 95372abf04..af1d636eed 100644 --- a/test/tkt-80e031a00f.test +++ b/test/tkt-80e031a00f.test @@ -160,6 +160,10 @@ do_execsql_test tkt-80e031a00f.322 {SELECT 'b' IN t8} 1 do_execsql_test tkt-80e031a00f.323 {SELECT 'c' NOT IN t8} 0 do_execsql_test tkt-80e031a00f.324 {SELECT 'c' IN t8n} 1 do_execsql_test tkt-80e031a00f.325 {SELECT 'd' NOT IN t8n} 0 +do_execsql_test tkt-80e031a00f.326 {SELECT 'a' IN (NULL,'a')} 1 +do_execsql_test tkt-80e031a00f.327 {SELECT 'a' IN (NULL,'b')} {{}} +do_execsql_test tkt-80e031a00f.328 {SELECT 'a' NOT IN (NULL,'a')} 0 +do_execsql_test tkt-80e031a00f.329 {SELECT 'a' NOT IN (NULL,'b')} {{}} # # Row 4: do_execsql_test tkt-80e031a00f.400 {SELECT 1 IN (2,3,4,null)} {{}} From a976979b6e1c01dbdfaeaddee5b3c80fc924ef94 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 4 Aug 2014 16:39:39 +0000 Subject: [PATCH 183/710] Refinements to the enhanced IN-operator logic. FossilOrigin-Name: 92ba2821468ecbfac2469161d81c873de67b2243 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/expr.c | 7 +++---- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 0a8cdbcd1c..d358a967d8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhancements\sto\sthe\scode\sgenerator\sfor\sthe\sIN\soperator\sthat\sresult\sin\smuch\nfaster\squeries\sin\ssome\scases,\sfor\sexample\swhen\sthe\sRHS\sof\sthe\sIN\soperator\nchanges\sfor\seach\srow\sof\sa\slarge\stable\sscan. -D 2014-08-02T21:03:33.699 +C Refinements\sto\sthe\senhanced\sIN-operator\slogic. +D 2014-08-04T16:39:39.657 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 89574df1ed8ca967fdd5be0e370722e508b91c2b +F src/expr.c 8f5e763623d79e71d60998a4e714324ca0e9380d F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -1185,10 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ba7826542908eac2e14789d183d0b3e35b143fed -R 93a5c3d553517a616682ed62728bfb2c -T *branch * IN-operator-improvements -T *sym-IN-operator-improvements * -T -sym-trunk * +P 436e884215e2b33ca3fbb555362237b12827c07a +R 0000d96605770a7d4078c73a9916f937 U drh -Z c474e62d8861a07a170ddf9e71c3b1a9 +Z 246e982e6330087cd1ec561e7d991220 diff --git a/manifest.uuid b/manifest.uuid index 0ddf2ad117..01804c86aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -436e884215e2b33ca3fbb555362237b12827c07a \ No newline at end of file +92ba2821468ecbfac2469161d81c873de67b2243 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 7565d41b90..86787a9c34 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2006,14 +2006,13 @@ static void sqlite3ExprCodeIN( int regCkNull = 0; int ii; assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); if( destIfNull!=destIfFalse ){ regCkNull = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regCkNull); + sqlite3VdbeAddOp3(v, OP_BitAnd, r1, r1, regCkNull); } for(ii=0; iinExpr; ii++){ r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); - if( regCkNull ){ + if( regCkNull && sqlite3ExprCanBeNull(pList->a[ii].pExpr) ){ sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); } if( iinExpr-1 || destIfNull!=destIfFalse ){ @@ -2722,7 +2721,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ addr = sqlite3VdbeAddOp1(v, op, r1); VdbeCoverageIf(v, op==TK_ISNULL); VdbeCoverageIf(v, op==TK_NOTNULL); - sqlite3VdbeAddOp2(v, OP_AddImm, target, -1); + sqlite3VdbeAddOp2(v, OP_Integer, 0, target); sqlite3VdbeJumpHere(v, addr); break; } From 7248a8b2b95a99d7fb7af7e5a3481f9cfcb95e3e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 4 Aug 2014 18:50:54 +0000 Subject: [PATCH 184/710] Further enhancements to IN-operator processing. FossilOrigin-Name: 7fdf26da1d2f40b80f9e44ff6f5af22ace8f95f3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 25 +++++++++++++++---------- src/update.c | 5 +++-- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index d358a967d8..a6de05b0b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refinements\sto\sthe\senhanced\sIN-operator\slogic. -D 2014-08-04T16:39:39.657 +C Further\senhancements\sto\sIN-operator\sprocessing. +D 2014-08-04T18:50:54.734 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 8f5e763623d79e71d60998a4e714324ca0e9380d +F src/expr.c 8ac2d0e8a0c1bc84eba94ab39867ba4d07d84f75 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -279,7 +279,7 @@ F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb -F src/update.c 01564b3c430f6c7b0a35afaf7aba7987206fa3a5 +F src/update.c 510c59a21bd8ba315db173b38d79688cc15261df F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 436e884215e2b33ca3fbb555362237b12827c07a -R 0000d96605770a7d4078c73a9916f937 +P 92ba2821468ecbfac2469161d81c873de67b2243 +R d7c5ed9f11c47855b5c64fea1267f119 U drh -Z 246e982e6330087cd1ec561e7d991220 +Z 6f0f219998d1d32472de07ef9dc89d8e diff --git a/manifest.uuid b/manifest.uuid index 01804c86aa..bf4d5ed437 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92ba2821468ecbfac2469161d81c873de67b2243 \ No newline at end of file +7fdf26da1d2f40b80f9e44ff6f5af22ace8f95f3 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 86787a9c34..6816d560d3 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1368,6 +1368,9 @@ int sqlite3ExprCanBeNull(const Expr *p){ case TK_FLOAT: case TK_BLOB: return 0; + case TK_COLUMN: + assert( p->pTab!=0 ); + return p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0; default: return 1; } @@ -2038,16 +2041,18 @@ static void sqlite3ExprCodeIN( /* If the LHS is NULL, then the result is either false or NULL depending ** on whether the RHS is empty or not, respectively. */ - if( destIfNull==destIfFalse ){ - /* Shortcut for the common case where the false and NULL outcomes are - ** the same. */ - sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); - }else{ - int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); - sqlite3VdbeJumpHere(v, addr1); + if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ + if( destIfNull==destIfFalse ){ + /* Shortcut for the common case where the false and NULL outcomes are + ** the same. */ + sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); + }else{ + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfNull); + sqlite3VdbeJumpHere(v, addr1); + } } if( eType==IN_INDEX_ROWID ){ diff --git a/src/update.c b/src/update.c index 3e04e00316..fe717a2ac1 100644 --- a/src/update.c +++ b/src/update.c @@ -437,8 +437,9 @@ void sqlite3Update( VdbeCoverageNeverTaken(v); } labelContinue = labelBreak; - sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak); - VdbeCoverage(v); + if( pPk==0 ){ + sqlite3VdbeAddOp2(v, OP_IsNull, regOldRowid, labelBreak); VdbeCoverage(v); + } }else if( pPk ){ labelContinue = sqlite3VdbeMakeLabel(v); sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v); From 2c3ea0694feef861c13fb4e3ff818558d4655be8 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 4 Aug 2014 21:26:58 +0000 Subject: [PATCH 185/710] Part of the change in the previous check-in was incorrect and can result in an incorrect UPDATE for WITHOUT ROWID tables. This check-in fixes the problem. FossilOrigin-Name: ee5f6eae57a656d09a4b3f7fbef664b2c696ddb4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/update.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a6de05b0b9..6f9201a593 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\senhancements\sto\sIN-operator\sprocessing. -D 2014-08-04T18:50:54.734 +C Part\sof\sthe\schange\sin\sthe\sprevious\scheck-in\swas\sincorrect\sand\scan\sresult\nin\san\sincorrect\sUPDATE\sfor\sWITHOUT\sROWID\stables.\s\sThis\scheck-in\sfixes\sthe\nproblem. +D 2014-08-04T21:26:58.927 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -279,7 +279,7 @@ F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb -F src/update.c 510c59a21bd8ba315db173b38d79688cc15261df +F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 92ba2821468ecbfac2469161d81c873de67b2243 -R d7c5ed9f11c47855b5c64fea1267f119 +P 7fdf26da1d2f40b80f9e44ff6f5af22ace8f95f3 +R bbb8ed3fbc770a15faae84bee68a96b7 U drh -Z 6f0f219998d1d32472de07ef9dc89d8e +Z dee92a9842f8b9e58ec243da3f9d9eb9 diff --git a/manifest.uuid b/manifest.uuid index bf4d5ed437..9bd8699697 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fdf26da1d2f40b80f9e44ff6f5af22ace8f95f3 \ No newline at end of file +ee5f6eae57a656d09a4b3f7fbef664b2c696ddb4 \ No newline at end of file diff --git a/src/update.c b/src/update.c index fe717a2ac1..e152e9057c 100644 --- a/src/update.c +++ b/src/update.c @@ -437,9 +437,9 @@ void sqlite3Update( VdbeCoverageNeverTaken(v); } labelContinue = labelBreak; - if( pPk==0 ){ - sqlite3VdbeAddOp2(v, OP_IsNull, regOldRowid, labelBreak); VdbeCoverage(v); - } + sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak); + VdbeCoverageIf(v, pPk==0); + VdbeCoverageIf(v, pPk!=0); }else if( pPk ){ labelContinue = sqlite3VdbeMakeLabel(v); sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v); From 4336b0e64a42bbf3f7a97511805a300797c681fd Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 5 Aug 2014 00:53:51 +0000 Subject: [PATCH 186/710] Improved VdbeCoverage() macros. A few minor simplifications to generated VDBE code. FossilOrigin-Name: 01f60027ad1841051fa493a646141445f8971357 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 4 +++- src/pragma.c | 11 +++++------ src/select.c | 3 +-- src/vdbe.c | 20 +++++++++++--------- 6 files changed, 30 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 6f9201a593..e1ddd56052 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Part\sof\sthe\schange\sin\sthe\sprevious\scheck-in\swas\sincorrect\sand\scan\sresult\nin\san\sincorrect\sUPDATE\sfor\sWITHOUT\sROWID\stables.\s\sThis\scheck-in\sfixes\sthe\nproblem. -D 2014-08-04T21:26:58.927 +C Improved\sVdbeCoverage()\smacros.\s\sA\sfew\sminor\ssimplifications\sto\sgenerated\nVDBE\scode. +D 2014-08-05T00:53:51.727 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 8ac2d0e8a0c1bc84eba94ab39867ba4d07d84f75 +F src/expr.c ef474fc0e73a2fc14835a2dc5282d3c28f8e1eaa F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -216,13 +216,13 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c d4a33151f057e35e5a2024adf8e41d2817b5c105 +F src/pragma.c d10ef67c4de79f78188b965b4b7988aff1d66f2e F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c af06f66927919730f03479fed6ae9854f73419f4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c 6762c62e11b504aa014edceab8886495165e3a77 +F src/select.c 1529c49075464c5a95fde77314073612b1b8d595 F src/shell.c 191129c3f7a9cf241aea90ff6a6be3e74d3767f0 F src/sqlite.h.in 9bbc5815c73b0e77e68b5275481a5e3e7814a804 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -283,7 +283,7 @@ F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c b9e6866e43a61ca4080410f27c4bb52823495186 +F src/vdbe.c aa93cf7a215a37d1c7ae565202a44869c78dbf8d F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7fdf26da1d2f40b80f9e44ff6f5af22ace8f95f3 -R bbb8ed3fbc770a15faae84bee68a96b7 +P ee5f6eae57a656d09a4b3f7fbef664b2c696ddb4 +R 05961687f6977550c21111dd4304208a U drh -Z dee92a9842f8b9e58ec243da3f9d9eb9 +Z b7d896cc5839c70221fa827c578afa41 diff --git a/manifest.uuid b/manifest.uuid index 9bd8699697..c10e21e526 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ee5f6eae57a656d09a4b3f7fbef664b2c696ddb4 \ No newline at end of file +01f60027ad1841051fa493a646141445f8971357 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 6816d560d3..e6ac84db94 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2020,7 +2020,9 @@ static void sqlite3ExprCodeIN( } if( iinExpr-1 || destIfNull!=destIfFalse ){ sqlite3VdbeAddOp4(v, OP_Eq, r1, labelOk, r2, - (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); + (void*)pColl, P4_COLLSEQ); + VdbeCoverageIf(v, iinExpr-1); + VdbeCoverageIf(v, ii==pList->nExpr-1); sqlite3VdbeChangeP5(v, affinity); }else{ assert( destIfNull==destIfFalse ); diff --git a/src/pragma.c b/src/pragma.c index b37499f7bd..9ed5e13eb0 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1794,9 +1794,8 @@ void sqlite3Pragma( */ static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { - { OP_AddImm, 1, 0, 0}, /* 0 */ - { OP_IfNeg, 1, 0, 0}, /* 1 */ - { OP_String8, 0, 3, 0}, /* 2 */ + { OP_IfNeg, 1, 0, 0}, /* 0 */ + { OP_String8, 0, 3, 0}, /* 1 */ { OP_ResultRow, 3, 1, 0}, }; @@ -2002,9 +2001,9 @@ void sqlite3Pragma( } } addr = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); - sqlite3VdbeChangeP2(v, addr, -mxErr); - sqlite3VdbeJumpHere(v, addr+1); - sqlite3VdbeChangeP4(v, addr+2, "ok", P4_STATIC); + sqlite3VdbeChangeP3(v, addr, -mxErr); + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeChangeP4(v, addr+1, "ok", P4_STATIC); } break; #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ diff --git a/src/select.c b/src/select.c index 6ceb3fe946..f5456c893d 100644 --- a/src/select.c +++ b/src/select.c @@ -541,8 +541,7 @@ static void codeOffset( ){ if( iOffset>0 ){ int addr; - sqlite3VdbeAddOp2(v, OP_AddImm, iOffset, -1); - addr = sqlite3VdbeAddOp1(v, OP_IfNeg, iOffset); VdbeCoverage(v); + addr = sqlite3VdbeAddOp3(v, OP_IfNeg, iOffset, 0, -1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); VdbeComment((v, "skip OFFSET records")); sqlite3VdbeJumpHere(v, addr); diff --git a/src/vdbe.c b/src/vdbe.c index ade82b054a..16d9eee969 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -116,6 +116,12 @@ int sqlite3_found_count = 0; ** branch can go. It is usually 2. "I" is the direction the branch ** goes. 0 means falls through. 1 means branch is taken. 2 means the ** second alternative branch is taken. +** +** iSrcLine is the source code line (from the __LINE__ macro) that +** generated the VDBE instruction. This instrumentation assumes that all +** source code is in a single file (the amalgamation). Special values 1 +** and 2 for the iSrcLine parameter mean that this particular branch is +** always taken or never taken, respectively. */ #if !defined(SQLITE_VDBE_COVERAGE) # define VdbeBranchTaken(I,M) @@ -5600,17 +5606,16 @@ case OP_IfPos: { /* jump, in1 */ break; } -/* Opcode: IfNeg P1 P2 * * * -** Synopsis: if r[P1]<0 goto P2 +/* Opcode: IfNeg P1 P2 P3 * * +** Synopsis: r[P1]+=P3, if r[P1]<0 goto P2 ** -** If the value of register P1 is less than zero, jump to P2. -** -** It is illegal to use this instruction on a register that does -** not contain an integer. An assertion fault will result if you try. +** Register P1 must contain an intger. Add literal P3 to the value in +** register P1 then if the value of register P1 is less than zero, jump to P2. */ case OP_IfNeg: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); + pIn1->u.i += pOp->p3; VdbeBranchTaken(pIn1->u.i<0, 2); if( pIn1->u.i<0 ){ pc = pOp->p2 - 1; @@ -5623,9 +5628,6 @@ case OP_IfNeg: { /* jump, in1 */ ** ** The register P1 must contain an integer. Add literal P3 to the ** value in register P1. If the result is exactly 0, jump to P2. -** -** It is illegal to use this instruction on a register that does -** not contain an integer. An assertion fault will result if you try. */ case OP_IfZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; From 2c5e35ffd159686bdba701a561292beee4351781 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 5 Aug 2014 11:04:21 +0000 Subject: [PATCH 187/710] Rename the internal Schema.flags field to Schema.schemaFlags. FossilOrigin-Name: 5ae80b3c8f032528359c8c762505ce24da8db96f --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/btree.c | 2 +- src/build.c | 2 +- src/callback.c | 4 ++-- src/sqliteInt.h | 10 +++++----- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index ae65d8580f..2e9232ccf3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\s(newly)\sincorrect\spreprocessor\scheck\sto\sfix\sbuild\son\sWinRT. -D 2014-08-02T20:44:13.563 +C Rename\sthe\sinternal\sSchema.flags\sfield\sto\sSchema.schemaFlags. +D 2014-08-05T11:04:21.126 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,11 +167,11 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c b5531339cd826af46b9621e4a9323971a9380e12 +F src/btree.c 99d162e57af6e72ffd7db5bf79568a134cd87d5b F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c c67a915cd5aabda9ac170f2af8ea25434476b66f -F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 +F src/build.c 5abf794fe8a605f2005b422e98a3cedad9b9ef5b +F src/callback.c fcff28cf0df2403dd2f313bb8d1b8f31f6f3cd64 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 @@ -227,7 +227,7 @@ F src/shell.c 191129c3f7a9cf241aea90ff6a6be3e74d3767f0 F src/sqlite.h.in 9bbc5815c73b0e77e68b5275481a5e3e7814a804 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 5cee19f34f6efe6e6f7733d55a5c59e3a35a378a +F src/sqliteInt.h 85c03beef501eb79db37f7d963597810aaa581a5 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9bc1c730a366e75b760b58e7a343d39165b2a469 -R 8c9bd54f21254a5c39cdbf67a68f6f01 -U mistachkin -Z a3832f522ef0a90c3ccd915e060c6930 +P ba7826542908eac2e14789d183d0b3e35b143fed +R 5db7ff3ac4350fd50475a1b030dd8291 +U drh +Z 42ccecdc76986fb0ef00d15b10cbc956 diff --git a/manifest.uuid b/manifest.uuid index d8a5cdba49..01721976cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba7826542908eac2e14789d183d0b3e35b143fed \ No newline at end of file +5ae80b3c8f032528359c8c762505ce24da8db96f \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b72a2ab01d..17719caf91 100644 --- a/src/btree.c +++ b/src/btree.c @@ -162,7 +162,7 @@ static int hasSharedCacheTableLock( ** the correct locks are held. So do not bother - just return true. ** This case does not come up very often anyhow. */ - if( isIndex && (!pSchema || (pSchema->flags&DB_SchemaLoaded)==0) ){ + if( isIndex && (!pSchema || (pSchema->schemaFlags&DB_SchemaLoaded)==0) ){ return 1; } diff --git a/src/build.c b/src/build.c index 28205c4c61..a9a8f21793 100644 --- a/src/build.c +++ b/src/build.c @@ -2130,7 +2130,7 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ pSelTab->aCol = 0; sqlite3DeleteTable(db, pSelTab); assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); - pTable->pSchema->flags |= DB_UnresetViews; + pTable->pSchema->schemaFlags |= DB_UnresetViews; }else{ pTable->nCol = 0; nErr++; diff --git a/src/callback.c b/src/callback.c index 260fe806bb..46fbe2c21a 100644 --- a/src/callback.c +++ b/src/callback.c @@ -447,9 +447,9 @@ void sqlite3SchemaClear(void *p){ sqlite3HashClear(&temp1); sqlite3HashClear(&pSchema->fkeyHash); pSchema->pSeqTab = 0; - if( pSchema->flags & DB_SchemaLoaded ){ + if( pSchema->schemaFlags & DB_SchemaLoaded ){ pSchema->iGeneration++; - pSchema->flags &= ~DB_SchemaLoaded; + pSchema->schemaFlags &= ~DB_SchemaLoaded; } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 97fb2d46e4..ed9f44584d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -876,7 +876,7 @@ struct Schema { Table *pSeqTab; /* The sqlite_sequence table used by AUTOINCREMENT */ u8 file_format; /* Schema format version for this file */ u8 enc; /* Text encoding used by this database */ - u16 flags; /* Flags associated with this schema */ + u16 schemaFlags; /* Flags associated with this schema */ int cache_size; /* Number of pages to use in the cache */ }; @@ -884,10 +884,10 @@ struct Schema { ** These macros can be used to test, set, or clear bits in the ** Db.pSchema->flags field. */ -#define DbHasProperty(D,I,P) (((D)->aDb[I].pSchema->flags&(P))==(P)) -#define DbHasAnyProperty(D,I,P) (((D)->aDb[I].pSchema->flags&(P))!=0) -#define DbSetProperty(D,I,P) (D)->aDb[I].pSchema->flags|=(P) -#define DbClearProperty(D,I,P) (D)->aDb[I].pSchema->flags&=~(P) +#define DbHasProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))==(P)) +#define DbHasAnyProperty(D,I,P) (((D)->aDb[I].pSchema->schemaFlags&(P))!=0) +#define DbSetProperty(D,I,P) (D)->aDb[I].pSchema->schemaFlags|=(P) +#define DbClearProperty(D,I,P) (D)->aDb[I].pSchema->schemaFlags&=~(P) /* ** Allowed values for the DB.pSchema->flags field. From 1cfc9aa993ee0210d71b6c58c81f2a20c5590b7e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 5 Aug 2014 21:31:08 +0000 Subject: [PATCH 188/710] Ensure that aggregate functions are not used when evaluating a default value for a table column. Candidate fix for ticket [3a88d85f36704eebe134f7]. FossilOrigin-Name: 29ba812825bf06ef230f2480bba0579653f0a52d --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/expr.c | 5 +++++ test/table.test | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f744c3e087..9183753c67 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sability\sto\sevaluate\sIN\soperators\sas\sa\ssequence\sof\scomparisons\sas\nan\salternative\sto\sthe\slong-standing\salgorithm\sof\sbuilding\sa\slookup\stable.\nUse\sthe\snew\simplementation\sin\scircumstances\swhere\sit\sis\slikely\sto\sbe\sfaster,\nsuch\sas\swhen\sthe\sRHS\sof\sthe\sIN\schanges\sbetween\ssuccessive\sevaluations. -D 2014-08-05T19:16:22.008 +C Ensure\sthat\saggregate\sfunctions\sare\snot\sused\swhen\sevaluating\sa\sdefault\nvalue\sfor\sa\stable\scolumn.\nCandidate\sfix\sfor\sticket\s[3a88d85f36704eebe134f7]. +D 2014-08-05T21:31:08.768 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c ef474fc0e73a2fc14835a2dc5282d3c28f8e1eaa +F src/expr.c 94d26c8e47bb25957e3963cc3d116b09ed8e12cd F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -856,7 +856,7 @@ F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/table.test 580d23530187026d4502fae74a490f0408cf2cc7 +F test/table.test 5b985827973a7b7b24ce155c8bda5fe3544e8442 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -1185,8 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5ae80b3c8f032528359c8c762505ce24da8db96f 01f60027ad1841051fa493a646141445f8971357 -R c68854702374dc69a486e042d017018d -T +closed 01f60027ad1841051fa493a646141445f8971357 +P 952868216854e8355edf57af62bd1a6bcb70ce61 +R cc49bc1b5436eaa2e754c5d5e9ba9fd5 U drh -Z d50dbb21b6962a7518b2d56fd33bd762 +Z 79f28300cbbdd20d7d6bedff33de2779 diff --git a/manifest.uuid b/manifest.uuid index 130e8a5420..b2aee10dd3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -952868216854e8355edf57af62bd1a6bcb70ce61 \ No newline at end of file +29ba812825bf06ef230f2480bba0579653f0a52d \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index e6ac84db94..9eb893d265 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2768,6 +2768,11 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); break; } + if( pDef->xFunc==0 ){ + sqlite3ErrorMsg(pParse, "misuse of aggregate function: %.*s()", + nId, zId); + break; + } /* Attempt a direct implementation of the built-in COALESCE() and ** IFNULL() functions. This avoids unnecessary evalation of diff --git a/test/table.test b/test/table.test index ed9efc02cd..777cc1723d 100644 --- a/test/table.test +++ b/test/table.test @@ -726,4 +726,50 @@ do_test table-15.2 { execsql {COMMIT} } {} +# Ticket 3a88d85f36704eebe134f7f48aebf00cd6438c1a (2014-08-05) +# The following SQL script segfaults while running the INSERT statement: +# +# CREATE TABLE t1(x DEFAULT(max(1))); +# INSERT INTO t1(rowid) VALUES(1); +# +# The problem appears to be the use of an aggregate function as part of +# the default value for a column. This problem has been in the code since +# at least 2006-01-01 and probably before that. This problem was detected +# and reported on the sqlite-users@sqlite.org mailing list by Zsbán Ambrus. +# +do_execsql_test table-16.1 { + CREATE TABLE t16(x DEFAULT(max(1))); + INSERT INTO t16(x) VALUES(123); + SELECT rowid, x FROM t16; +} {1 123} +do_catchsql_test table-16.2 { + INSERT INTO t16(rowid) VALUES(4); +} {1 {misuse of aggregate function: max()}} +do_execsql_test table-16.3 { + DROP TABLE t16; + CREATE TABLE t16(x DEFAULT(abs(1))); + INSERT INTO t16(rowid) VALUES(4); + SELECT rowid, x FROM t16; +} {4 1} +do_catchsql_test table-16.4 { + DROP TABLE t16; + CREATE TABLE t16(x DEFAULT(avg(1))); + INSERT INTO t16(rowid) VALUES(123); + SELECT rowid, x FROM t16; +} {1 {misuse of aggregate function: avg()}} +do_catchsql_test table-16.5 { + DROP TABLE t16; + CREATE TABLE t16(x DEFAULT(count())); + INSERT INTO t16(rowid) VALUES(123); + SELECT rowid, x FROM t16; +} {1 {misuse of aggregate function: count()}} +do_catchsql_test table-16.6 { + DROP TABLE t16; + CREATE TABLE t16(x DEFAULT(group_concat('x',','))); + INSERT INTO t16(rowid) VALUES(123); + SELECT rowid, x FROM t16; +} {1 {misuse of aggregate function: group_concat()}} + + + finish_test From 0c4de2d96de879f19e4ff8f952a82713c9a9c18b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 6 Aug 2014 00:29:06 +0000 Subject: [PATCH 189/710] A simpler fix for ticket [3a88d85f36704eebe1] - one that uses less code. The error message is not quite as good, but as this error has apparently not previously occurred in over 8 years of heavy use, that is not seen as a serious problem. FossilOrigin-Name: 0ad1ed8ef0b5fb5d8db44479373b2b93d8fcfd66 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 7 +------ test/table.test | 13 +++++++------ 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 9183753c67..829712eb8e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\saggregate\sfunctions\sare\snot\sused\swhen\sevaluating\sa\sdefault\nvalue\sfor\sa\stable\scolumn.\nCandidate\sfix\sfor\sticket\s[3a88d85f36704eebe134f7]. -D 2014-08-05T21:31:08.768 +C A\ssimpler\sfix\sfor\sticket\s[3a88d85f36704eebe1]\s-\sone\sthat\suses\sless\scode.\nThe\serror\smessage\sis\snot\squite\sas\sgood,\sbut\sas\sthis\serror\shas\sapparently\nnot\spreviously\soccurred\sin\sover\s8\syears\sof\sheavy\suse,\sthat\sis\snot\sseen\sas\na\sserious\sproblem. +D 2014-08-06T00:29:06.807 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf -F src/expr.c 94d26c8e47bb25957e3963cc3d116b09ed8e12cd +F src/expr.c f749009cf4a8534efb5e0d5cd7c9fb1fb0f2836c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc @@ -856,7 +856,7 @@ F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/table.test 5b985827973a7b7b24ce155c8bda5fe3544e8442 +F test/table.test 2a1d2fa52c531de5915f28023747d9a8c27b6f31 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 952868216854e8355edf57af62bd1a6bcb70ce61 -R cc49bc1b5436eaa2e754c5d5e9ba9fd5 +P 29ba812825bf06ef230f2480bba0579653f0a52d +R f634869bc839ef6a8575d1173c369416 U drh -Z 79f28300cbbdd20d7d6bedff33de2779 +Z 8e016893764edede1871bf058b88fac4 diff --git a/manifest.uuid b/manifest.uuid index b2aee10dd3..741aa44ffb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29ba812825bf06ef230f2480bba0579653f0a52d \ No newline at end of file +0ad1ed8ef0b5fb5d8db44479373b2b93d8fcfd66 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 9eb893d265..0d2292e943 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2764,15 +2764,10 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ zId = pExpr->u.zToken; nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(db, zId, nId, nFarg, enc, 0); - if( pDef==0 ){ + if( pDef==0 || pDef->xFunc==0 ){ sqlite3ErrorMsg(pParse, "unknown function: %.*s()", nId, zId); break; } - if( pDef->xFunc==0 ){ - sqlite3ErrorMsg(pParse, "misuse of aggregate function: %.*s()", - nId, zId); - break; - } /* Attempt a direct implementation of the built-in COALESCE() and ** IFNULL() functions. This avoids unnecessary evalation of diff --git a/test/table.test b/test/table.test index 777cc1723d..656884ca73 100644 --- a/test/table.test +++ b/test/table.test @@ -744,7 +744,7 @@ do_execsql_test table-16.1 { } {1 123} do_catchsql_test table-16.2 { INSERT INTO t16(rowid) VALUES(4); -} {1 {misuse of aggregate function: max()}} +} {1 {unknown function: max()}} do_execsql_test table-16.3 { DROP TABLE t16; CREATE TABLE t16(x DEFAULT(abs(1))); @@ -756,20 +756,21 @@ do_catchsql_test table-16.4 { CREATE TABLE t16(x DEFAULT(avg(1))); INSERT INTO t16(rowid) VALUES(123); SELECT rowid, x FROM t16; -} {1 {misuse of aggregate function: avg()}} +} {1 {unknown function: avg()}} do_catchsql_test table-16.5 { DROP TABLE t16; CREATE TABLE t16(x DEFAULT(count())); INSERT INTO t16(rowid) VALUES(123); SELECT rowid, x FROM t16; -} {1 {misuse of aggregate function: count()}} +} {1 {unknown function: count()}} do_catchsql_test table-16.6 { DROP TABLE t16; CREATE TABLE t16(x DEFAULT(group_concat('x',','))); INSERT INTO t16(rowid) VALUES(123); SELECT rowid, x FROM t16; -} {1 {misuse of aggregate function: group_concat()}} - - +} {1 {unknown function: group_concat()}} +do_catchsql_test table-16.7 { + INSERT INTO t16 DEFAULT VALUES; +} {1 {unknown function: group_concat()}} finish_test From bc5cf3813ea1bf56ed19a35028061e779827fffc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 6 Aug 2014 01:08:07 +0000 Subject: [PATCH 190/710] Fix typos in the opcode documentation. Comment changes only. No changes to code. FossilOrigin-Name: 717245d48714c08156c9b7636aaa6c3a402bad66 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 829712eb8e..a945d8dafc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\ssimpler\sfix\sfor\sticket\s[3a88d85f36704eebe1]\s-\sone\sthat\suses\sless\scode.\nThe\serror\smessage\sis\snot\squite\sas\sgood,\sbut\sas\sthis\serror\shas\sapparently\nnot\spreviously\soccurred\sin\sover\s8\syears\sof\sheavy\suse,\sthat\sis\snot\sseen\sas\na\sserious\sproblem. -D 2014-08-06T00:29:06.807 +C Fix\stypos\sin\sthe\sopcode\sdocumentation.\s\sComment\schanges\sonly.\s\sNo\schanges\nto\scode. +D 2014-08-06T01:08:07.269 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c aa93cf7a215a37d1c7ae565202a44869c78dbf8d +F src/vdbe.c 2db817cf7c51fa775ecc68160352aefc3d9870f7 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 29ba812825bf06ef230f2480bba0579653f0a52d -R f634869bc839ef6a8575d1173c369416 +P 0ad1ed8ef0b5fb5d8db44479373b2b93d8fcfd66 +R 1045cc0c95f68e0f0e2e47642b0ec158 U drh -Z 8e016893764edede1871bf058b88fac4 +Z fecab6dfa8d111d2725130892a38124f diff --git a/manifest.uuid b/manifest.uuid index 741aa44ffb..c81900087c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0ad1ed8ef0b5fb5d8db44479373b2b93d8fcfd66 \ No newline at end of file +717245d48714c08156c9b7636aaa6c3a402bad66 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 16d9eee969..6c46d76569 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -796,7 +796,7 @@ case OP_InitCoroutine: { /* jump */ /* Opcode: EndCoroutine P1 * * * * ** -** The instruction at the address in register P1 is an Yield. +** The instruction at the address in register P1 is a Yield. ** Jump to the P2 parameter of that Yield. ** After the jump, register P1 becomes undefined. ** @@ -989,7 +989,7 @@ case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ ** Synopsis: r[P2]='P4' ** ** P4 points to a nul terminated UTF-8 string. This opcode is transformed -** into an OP_String before it is executed for the first time. During +** into a String before it is executed for the first time. During ** this transformation, the length of string P4 is computed and stored ** as the P1 parameter. */ @@ -3513,7 +3513,7 @@ case OP_Close: { ** greater than or equal to the key and P2 is not zero, then jump to P2. ** ** This opcode leaves the cursor configured to move in forward order, -** from the begining toward the end. In other words, the cursor is +** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. ** ** See also: Found, NotFound, SeekLt, SeekGt, SeekLe @@ -4217,7 +4217,7 @@ case OP_InsertInt: { ** The cursor will be left pointing at either the next or the previous ** record in the table. If it is left pointing at the next record, then ** the next Next instruction will be a no-op. Hence it is OK to delete -** a record from within an Next loop. +** a record from within a Next loop. ** ** If the OPFLAG_NCHANGE flag of P2 is set, then the row change count is ** incremented (otherwise not). @@ -4280,7 +4280,7 @@ case OP_ResetCount: { ** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2 ** ** P1 is a sorter cursor. This instruction compares a prefix of the -** the record blob in register P3 against a prefix of the entry that +** record blob in register P3 against a prefix of the entry that ** the sorter cursor currently points to. Only the first P4 fields ** of r[P3] and the sorter record are compared. ** @@ -5609,7 +5609,7 @@ case OP_IfPos: { /* jump, in1 */ /* Opcode: IfNeg P1 P2 P3 * * ** Synopsis: r[P1]+=P3, if r[P1]<0 goto P2 ** -** Register P1 must contain an intger. Add literal P3 to the value in +** Register P1 must contain an integer. Add literal P3 to the value in ** register P1 then if the value of register P1 is less than zero, jump to P2. */ case OP_IfNeg: { /* jump, in1 */ From 17835a5443f826613451412fc847fcfed8534757 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 6 Aug 2014 03:06:01 +0000 Subject: [PATCH 191/710] In the Win32 VFS, work around InterlockedCompareExchange() being a macro on some platforms (e.g. x64). FossilOrigin-Name: 7be244ce129d6502f3a3e3f3e8a1dd61ef71d878 --- manifest | 14 ++-- manifest.uuid | 2 +- src/os_win.c | 179 ++++++++++++++++++++++++++------------------------ 3 files changed, 102 insertions(+), 93 deletions(-) diff --git a/manifest b/manifest index a945d8dafc..33c5c487e2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypos\sin\sthe\sopcode\sdocumentation.\s\sComment\schanges\sonly.\s\sNo\schanges\nto\scode. -D 2014-08-06T01:08:07.269 +C In\sthe\sWin32\sVFS,\swork\saround\sInterlockedCompareExchange()\sbeing\sa\smacro\son\ssome\splatforms\s(e.g.\sx64). +D 2014-08-06T03:06:01.637 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c a7baf1b30f3c58ba20b813e01aab23b18ae44f85 -F src/os_win.c e5f0fc446a682b70db3397d14cca9806d9a15d12 +F src/os_win.c dddffe56dd21fd3687eaec36e01499988918a4fb F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0ad1ed8ef0b5fb5d8db44479373b2b93d8fcfd66 -R 1045cc0c95f68e0f0e2e47642b0ec158 -U drh -Z fecab6dfa8d111d2725130892a38124f +P 717245d48714c08156c9b7636aaa6c3a402bad66 +R c63f8cfa87b32fea35b14f1ae3f3b367 +U mistachkin +Z 0f9f130c5300af8c7a887c050bb16bac diff --git a/manifest.uuid b/manifest.uuid index c81900087c..87d1cc7902 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -717245d48714c08156c9b7636aaa6c3a402bad66 \ No newline at end of file +7be244ce129d6502f3a3e3f3e8a1dd61ef71d878 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index f770f13862..0df311cbee 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -204,7 +204,7 @@ WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); ** Some Microsoft compilers lack this definition. */ #ifndef INVALID_FILE_ATTRIBUTES -# define INVALID_FILE_ATTRIBUTES ((DWORD)-1) +# define INVALID_FILE_ATTRIBUTES ((DWORD)-1) #endif #ifndef FILE_FLAG_MASK @@ -254,7 +254,7 @@ struct winFile { int szChunk; /* Chunk size configured by FCNTL_CHUNK_SIZE */ #if SQLITE_OS_WINCE LPWSTR zDeleteOnClose; /* Name of file to delete when closing */ - HANDLE hMutex; /* Mutex used to control access to shared lock */ + HANDLE hMutex; /* Mutex used to control access to shared lock */ HANDLE hShared; /* Shared memory segment used for locking */ winceLock local; /* Locks obtained by this instance of winFile */ winceLock *shared; /* Global shared lock memory for the file */ @@ -1047,10 +1047,19 @@ static struct win_syscall { #define osCreateFileMappingFromApp ((HANDLE(WINAPI*)(HANDLE, \ LPSECURITY_ATTRIBUTES,ULONG,ULONG64,LPCWSTR))aSyscall[75].pCurrent) +/* +** NOTE: On some sub-platforms, the InterlockedCompareExchange "function" +** is really just a macro that uses a compiler intrinsic (e.g. x64). +*/ + +#if defined(InterlockedCompareExchange) +#define osInterlockedCompareExchange InterlockedCompareExchange +#else { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, #define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG volatile*, \ LONG,LONG))aSyscall[76].pCurrent) +#endif /* defined(InterlockedCompareExchange) */ }; /* End of the overrideable system calls */ @@ -1532,7 +1541,7 @@ void sqlite3MemSetDefault(void){ #endif /* SQLITE_WIN32_MALLOC */ /* -** Convert a UTF-8 string to Microsoft Unicode (UTF-16?). +** Convert a UTF-8 string to Microsoft Unicode (UTF-16?). ** ** Space to hold the returned string is obtained from malloc. */ @@ -1585,7 +1594,7 @@ static char *winUnicodeToUtf8(LPCWSTR zWideFilename){ /* ** Convert an ANSI string to Microsoft Unicode, based on the ** current codepage settings for file apis. -** +** ** Space to hold the returned string is obtained ** from sqlite3_malloc. */ @@ -1659,7 +1668,7 @@ char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ } /* -** Convert UTF-8 to multibyte character string. Space to hold the +** Convert UTF-8 to multibyte character string. Space to hold the ** returned string is obtained from sqlite3_malloc(). */ char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ @@ -1799,11 +1808,11 @@ static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ ** ** This routine is invoked after an error occurs in an OS function. ** It logs a message using sqlite3_log() containing the current value of -** error code and, if possible, the human-readable equivalent from +** error code and, if possible, the human-readable equivalent from ** FormatMessage. ** ** The first argument passed to the macro should be the error code that -** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). ** The two subsequent arguments should be the name of the OS function that ** failed and the associated file-system path, if any. */ @@ -1834,7 +1843,7 @@ static int winLogErrorAtLine( /* ** The number of times that a ReadFile(), WriteFile(), and DeleteFile() -** will be retried following a locking error - probably caused by +** will be retried following a locking error - probably caused by ** antivirus software. Also the initial delay before the first retry. ** The delay increases linearly with each retry. */ @@ -1909,7 +1918,7 @@ static int winRetryIoerr(int *pnRetry, DWORD *pError){ */ static void winLogIoerr(int nRetry){ if( nRetry ){ - sqlite3_log(SQLITE_IOERR, + sqlite3_log(SQLITE_IOERR, "delayed %dms for lock/sharing conflict", winIoerrRetryDelay*nRetry*(nRetry+1)/2 ); @@ -2003,17 +2012,17 @@ static int winceCreateLock(const char *zFilename, winFile *pFile){ /* Acquire the mutex before continuing */ winceMutexAcquire(pFile->hMutex); - - /* Since the names of named mutexes, semaphores, file mappings etc are + + /* Since the names of named mutexes, semaphores, file mappings etc are ** case-sensitive, take advantage of that by uppercasing the mutex name ** and using that as the shared filemapping name. */ osCharUpperW(zName); pFile->hShared = osCreateFileMappingW(INVALID_HANDLE_VALUE, NULL, PAGE_READWRITE, 0, sizeof(winceLock), - zName); + zName); - /* Set a flag that indicates we're the first to create the memory so it + /* Set a flag that indicates we're the first to create the memory so it ** must be zero-initialized */ lastErrno = osGetLastError(); if (lastErrno == ERROR_ALREADY_EXISTS){ @@ -2024,7 +2033,7 @@ static int winceCreateLock(const char *zFilename, winFile *pFile){ /* If we succeeded in making the shared memory handle, map it. */ if( pFile->hShared ){ - pFile->shared = (winceLock*)osMapViewOfFile(pFile->hShared, + pFile->shared = (winceLock*)osMapViewOfFile(pFile->hShared, FILE_MAP_READ|FILE_MAP_WRITE, 0, 0, sizeof(winceLock)); /* If mapping failed, close the shared memory handle and erase it */ if( !pFile->shared ){ @@ -2050,7 +2059,7 @@ static int winceCreateLock(const char *zFilename, winFile *pFile){ pFile->hMutex = NULL; return SQLITE_IOERR; } - + /* Initialize the shared memory if we're supposed to */ if( bInit ){ memset(pFile->shared, 0, sizeof(winceLock)); @@ -2088,13 +2097,13 @@ static void winceDestroyLock(winFile *pFile){ osCloseHandle(pFile->hShared); /* Done with the mutex */ - winceMutexRelease(pFile->hMutex); + winceMutexRelease(pFile->hMutex); osCloseHandle(pFile->hMutex); pFile->hMutex = NULL; } } -/* +/* ** An implementation of the LockFile() API of Windows for CE */ static BOOL winceLockFile( @@ -2305,8 +2314,8 @@ static BOOL winUnlockFile( #endif /* -** Move the current position of the file handle passed as the first -** argument to offset iOffset within the file. If successful, return 0. +** Move the current position of the file handle passed as the first +** argument to offset iOffset within the file. If successful, return 0. ** Otherwise, set pFile->lastErrno and return non-zero. */ static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ @@ -2321,11 +2330,11 @@ static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ upperBits = (LONG)((iOffset>>32) & 0x7fffffff); lowerBits = (LONG)(iOffset & 0xffffffff); - /* API oddity: If successful, SetFilePointer() returns a dword + /* API oddity: If successful, SetFilePointer() returns a dword ** containing the lower 32-bits of the new file-offset. Or, if it fails, - ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, - ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine - ** whether an error has actually occurred, it is also necessary to call + ** it returns INVALID_SET_FILE_POINTER. However according to MSDN, + ** INVALID_SET_FILE_POINTER may also be a valid new offset. So to determine + ** whether an error has actually occurred, it is also necessary to call ** GetLastError(). */ dwRet = osSetFilePointer(pFile->h, lowerBits, &upperBits, FILE_BEGIN); @@ -2408,7 +2417,7 @@ static int winClose(sqlite3_file *id){ int cnt = 0; while( osDeleteFileW(pFile->zDeleteOnClose)==0 - && osGetFileAttributesW(pFile->zDeleteOnClose)!=0xffffffff + && osGetFileAttributesW(pFile->zDeleteOnClose)!=0xffffffff && cnt++ < WINCE_DELETION_ATTEMPTS ){ sqlite3_win32_sleep(100); /* Wait a little before trying again */ @@ -3256,7 +3265,7 @@ static int winDeviceCharacteristics(sqlite3_file *id){ ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); } -/* +/* ** Windows will only let you create file view mappings ** on allocation size granularity boundaries. ** During sqlite3_os_init() we do a GetSystemInfo() @@ -3268,11 +3277,11 @@ static SYSTEM_INFO winSysInfo; /* ** Helper functions to obtain and relinquish the global mutex. The -** global mutex is used to protect the winLockInfo objects used by +** global mutex is used to protect the winLockInfo objects used by ** this file, all of which may be shared by multiple threads. ** -** Function winShmMutexHeld() is used to assert() that the global mutex -** is held when required. This function is only used as part of assert() +** Function winShmMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() ** statements. e.g. ** ** winShmEnterMutex() @@ -3302,10 +3311,10 @@ static int winShmMutexHeld(void) { ** this object or while reading or writing the following fields: ** ** nRef -** pNext +** pNext ** ** The following fields are read-only after the object is created: -** +** ** fid ** zFilename ** @@ -3401,7 +3410,7 @@ static int winShmSystemLock( if( lockType == _SHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); } - + if( rc!= 0 ){ rc = SQLITE_OK; }else{ @@ -3497,7 +3506,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ } pNew->zFilename = (char*)&pNew[1]; sqlite3_snprintf(nName+15, pNew->zFilename, "%s-shm", pDbFd->zPath); - sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); + sqlite3FileSuffix3(pDbFd->zPath, pNew->zFilename); /* Look to see if there is an existing winShmNode that can be used. ** If no matching winShmNode currently exists, create a new one. @@ -3534,7 +3543,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ } /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. + ** If not, truncate the file to zero length. */ if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); @@ -3563,7 +3572,7 @@ static int winOpenSharedMemory(winFile *pDbFd){ ** the cover of the winShmEnterMutex() mutex and the pointer from the ** new (struct winShm) object to the pShmNode has been set. All that is ** left to do is to link the new object into the linked list starting - ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex + ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex ** mutex. */ sqlite3_mutex_enter(pShmNode->mutex); @@ -3583,7 +3592,7 @@ shm_open_err: } /* -** Close a connection to shared-memory. Delete the underlying +** Close a connection to shared-memory. Delete the underlying ** storage if deleteFlag is true. */ static int winShmUnmap( @@ -3672,7 +3681,7 @@ static int winShmLock( if( rc==SQLITE_OK ){ p->exclMask &= ~mask; p->sharedMask &= ~mask; - } + } }else if( flags & SQLITE_SHM_SHARED ){ u16 allShared = 0; /* Union of locks held by connections other than "p" */ @@ -3711,7 +3720,7 @@ static int winShmLock( break; } } - + /* Get the exclusive locks at the system level. Then if successful ** also mark the local connection as being locked. */ @@ -3731,7 +3740,7 @@ static int winShmLock( } /* -** Implement a memory barrier or memory fence on shared memory. +** Implement a memory barrier or memory fence on shared memory. ** ** All loads and stores begun before the barrier must complete before ** any load or store begun after the barrier. @@ -3746,22 +3755,22 @@ static void winShmBarrier( } /* -** This function is called to obtain a pointer to region iRegion of the -** shared-memory associated with the database file fd. Shared-memory regions -** are numbered starting from zero. Each shared-memory region is szRegion +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion ** bytes in size. ** ** If an error occurs, an error code is returned and *pp is set to NULL. ** ** Otherwise, if the isWrite parameter is 0 and the requested shared-memory ** region has not been allocated (by any client, including one running in a -** separate process), then *pp is set to NULL and SQLITE_OK returned. If -** isWrite is non-zero and the requested shared-memory region has not yet +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** isWrite is non-zero and the requested shared-memory region has not yet ** been allocated, it is allocated by this function. ** ** If the shared-memory region has already been allocated or is allocated by -** this call as described above, then it is mapped into this processes -** address space (if it is not already), *pp is set to point to the mapped +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped ** memory and SQLITE_OK returned. */ static int winShmMap( @@ -3833,17 +3842,17 @@ static int winShmMap( while( pShmNode->nRegion<=iRegion ){ HANDLE hMap = NULL; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ - + #if SQLITE_OS_WINRT hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, NULL, PAGE_READWRITE, nByte, NULL ); #elif defined(SQLITE_WIN32_HAS_WIDE) - hMap = osCreateFileMappingW(pShmNode->hFile.h, + hMap = osCreateFileMappingW(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); #elif defined(SQLITE_WIN32_HAS_ANSI) - hMap = osCreateFileMappingA(pShmNode->hFile.h, + hMap = osCreateFileMappingA(pShmNode->hFile.h, NULL, PAGE_READWRITE, 0, nByte, NULL ); #endif @@ -3940,14 +3949,14 @@ static int winUnmapfile(winFile *pFile){ /* ** Memory map or remap the file opened by file-descriptor pFd (if the file -** is already mapped, the existing mapping is replaced by the new). Or, if -** there already exists a mapping for this file, and there are still +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still ** outstanding xFetch() references to it, this function is a no-op. ** -** If parameter nByte is non-negative, then it is the requested size of -** the mapping to create. Otherwise, if nByte is less than zero, then the +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the ** requested size is the size of the file on disk. The actual size of the -** created mapping is either the requested size or the value configured +** created mapping is either the requested size or the value configured ** using SQLITE_FCNTL_MMAP_SIZE, whichever is smaller. ** ** SQLITE_OK is returned if no error occurs (even if the mapping is not @@ -3976,7 +3985,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ nMap = pFd->mmapSizeMax; } nMap &= ~(sqlite3_int64)(winSysInfo.dwPageSize - 1); - + if( nMap==0 && pFd->mmapSize>0 ){ winUnmapfile(pFd); } @@ -4048,7 +4057,7 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ ** Finally, if an error does occur, return an SQLite error code. The final ** value of *pp is undefined in this case. ** -** If this function does return a pointer, the caller must eventually +** If this function does return a pointer, the caller must eventually ** release the reference by calling winUnfetch(). */ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ @@ -4083,20 +4092,20 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ } /* -** If the third argument is non-NULL, then this function releases a +** If the third argument is non-NULL, then this function releases a ** reference obtained by an earlier call to winFetch(). The second ** argument passed to this function must be the same as the corresponding -** argument that was passed to the winFetch() invocation. +** argument that was passed to the winFetch() invocation. ** -** Or, if the third argument is NULL, then this function is being called -** to inform the VFS layer that, according to POSIX, any existing mapping +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping ** may now be invalid and should be unmapped. */ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ #if SQLITE_MAX_MMAP_SIZE>0 winFile *pFd = (winFile*)fd; /* The underlying database file */ - /* If p==0 (unmap the entire file) then there must be no outstanding + /* If p==0 (unmap the entire file) then there must be no outstanding ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), ** then there must be at least one outstanding. */ assert( (p==0)==(pFd->nFetchOut==0) ); @@ -4242,7 +4251,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this - ** function failing. + ** function failing. */ SimulateIOError( return SQLITE_IOERR ); @@ -4424,7 +4433,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ } /* - ** Check that the output buffer is large enough for the temporary file + ** Check that the output buffer is large enough for the temporary file ** name in the following format: ** ** "/etilqs_XXXXXXXXXXXXXXX\0\0" @@ -4527,8 +4536,8 @@ static int winOpen( #ifndef NDEBUG int isOpenJournal = (isCreate && ( - eType==SQLITE_OPEN_MASTER_JOURNAL - || eType==SQLITE_OPEN_MAIN_JOURNAL + eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_WAL )); #endif @@ -4536,9 +4545,9 @@ static int winOpen( OSTRACE(("OPEN name=%s, pFile=%p, flags=%x, pOutFlags=%p\n", zUtf8Name, id, flags, pOutFlags)); - /* Check the following statements are true: + /* Check the following statements are true: ** - ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and ** (b) if CREATE is set, then READWRITE must also be set, and ** (c) if EXCLUSIVE is set, then CREATE must also be set. ** (d) if DELETEONCLOSE is set, then CREATE must also be set. @@ -4548,7 +4557,7 @@ static int winOpen( assert(isExclusive==0 || isCreate); assert(isDelete==0 || isCreate); - /* The main DB, main journal, WAL file and master journal are never + /* The main DB, main journal, WAL file and master journal are never ** automatically deleted. Nor are they ever temporary files. */ assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); @@ -4556,9 +4565,9 @@ static int winOpen( assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); /* Assert that the upper layer has set one of the "file-type" flags. */ - assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB - || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL - || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); @@ -4573,8 +4582,8 @@ static int winOpen( } #endif - /* If the second argument to this function is NULL, generate a - ** temporary file name to use + /* If the second argument to this function is NULL, generate a + ** temporary file name to use */ if( !zUtf8Name ){ assert( isDelete && !isOpenJournal ); @@ -4614,8 +4623,8 @@ static int winOpen( dwDesiredAccess = GENERIC_READ; } - /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is - ** created. SQLite doesn't use it to indicate "exclusive access" + /* SQLITE_OPEN_EXCLUSIVE is used to make sure that a new file is + ** created. SQLite doesn't use it to indicate "exclusive access" ** as it is usually understood. */ if( isExclusive ){ @@ -4704,7 +4713,7 @@ static int winOpen( sqlite3_free(zConverted); sqlite3_free(zTmpname); if( isReadWrite && !isExclusive ){ - return winOpen(pVfs, zName, id, + return winOpen(pVfs, zName, id, ((flags|SQLITE_OPEN_READONLY) & ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); @@ -4913,14 +4922,14 @@ static int winAccess( WIN32_FILE_ATTRIBUTE_DATA sAttrData; memset(&sAttrData, 0, sizeof(sAttrData)); while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, - GetFileExInfoStandard, + GetFileExInfoStandard, &sAttrData)) && winRetryIoerr(&cnt, &lastErrno) ){} if( rc ){ /* For an SQLITE_ACCESS_EXISTS query, treat a zero-length file ** as if it does not exist. */ if( flags==SQLITE_ACCESS_EXISTS - && sAttrData.nFileSizeHigh==0 + && sAttrData.nFileSizeHigh==0 && sAttrData.nFileSizeLow==0 ){ attr = INVALID_FILE_ATTRIBUTES; }else{ @@ -5019,7 +5028,7 @@ static int winFullPathname( int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ - + #if defined(__CYGWIN__) SimulateIOError( return SQLITE_ERROR ); UNUSED_PARAMETER(nFull); @@ -5332,12 +5341,12 @@ int sqlite3_current_time = 0; /* Fake system time in seconds since 1970. */ ** epoch of noon in Greenwich on November 24, 4714 B.C according to the ** proleptic Gregorian calendar. ** -** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date +** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date ** cannot be found. */ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ - /* FILETIME structure is a 64-bit value representing the number of - 100-nanosecond intervals since January 1, 1601 (= JD 2305813.5). + /* FILETIME structure is a 64-bit value representing the number of + 100-nanosecond intervals since January 1, 1601 (= JD 2305813.5). */ FILETIME ft; static const sqlite3_int64 winFiletimeEpoch = 23058135*(sqlite3_int64)8640000; @@ -5345,7 +5354,7 @@ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; #endif /* 2^32 - to avoid use of LL and warnings in gcc */ - static const sqlite3_int64 max32BitValue = + static const sqlite3_int64 max32BitValue = (sqlite3_int64)2000000000 + (sqlite3_int64)2000000000 + (sqlite3_int64)294967296; @@ -5361,7 +5370,7 @@ static int winCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *piNow){ #endif *piNow = winFiletimeEpoch + - ((((sqlite3_int64)ft.dwHighDateTime)*max32BitValue) + + ((((sqlite3_int64)ft.dwHighDateTime)*max32BitValue) + (sqlite3_int64)ft.dwLowDateTime)/(sqlite3_int64)10000; #ifdef SQLITE_TEST @@ -5498,10 +5507,10 @@ int sqlite3_os_init(void){ sqlite3_vfs_register(&winLongPathVfs, 0); #endif - return SQLITE_OK; + return SQLITE_OK; } -int sqlite3_os_end(void){ +int sqlite3_os_end(void){ #if SQLITE_OS_WINRT if( sleepObj!=NULL ){ osCloseHandle(sleepObj); From 2abe6a281c3937ce4c2a12475dc8c13ac769b656 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 6 Aug 2014 11:57:54 +0000 Subject: [PATCH 192/710] On the windows VFS, do not try to make InterlockedCompareExchange an overloadable function, since sometimes it is a macro. FossilOrigin-Name: ab1a751e1304749bef5bc5c833f9abed8950f7d0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 11 ++--------- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 33c5c487e2..7fa415777a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sWin32\sVFS,\swork\saround\sInterlockedCompareExchange()\sbeing\sa\smacro\son\ssome\splatforms\s(e.g.\sx64). -D 2014-08-06T03:06:01.637 +C On\sthe\swindows\sVFS,\sdo\snot\stry\sto\smake\sInterlockedCompareExchange\san\noverloadable\sfunction,\ssince\ssometimes\sit\sis\sa\smacro. +D 2014-08-06T11:57:54.546 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c a7baf1b30f3c58ba20b813e01aab23b18ae44f85 -F src/os_win.c dddffe56dd21fd3687eaec36e01499988918a4fb +F src/os_win.c 3fca1bfdf78338705bf536059a407d0fb04016d5 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 717245d48714c08156c9b7636aaa6c3a402bad66 -R c63f8cfa87b32fea35b14f1ae3f3b367 -U mistachkin -Z 0f9f130c5300af8c7a887c050bb16bac +P 7be244ce129d6502f3a3e3f3e8a1dd61ef71d878 +R 40d3ff3c08b7d4b2cddf84583f93f135 +U drh +Z 1b7c539afa0e75fe0851e437eb151215 diff --git a/manifest.uuid b/manifest.uuid index 87d1cc7902..7113e4822f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7be244ce129d6502f3a3e3f3e8a1dd61ef71d878 \ No newline at end of file +ab1a751e1304749bef5bc5c833f9abed8950f7d0 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 0df311cbee..5f1d84a3fb 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1050,16 +1050,9 @@ static struct win_syscall { /* ** NOTE: On some sub-platforms, the InterlockedCompareExchange "function" ** is really just a macro that uses a compiler intrinsic (e.g. x64). +** So do not try to make this is into a redefinable interface. */ - -#if defined(InterlockedCompareExchange) #define osInterlockedCompareExchange InterlockedCompareExchange -#else - { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, - -#define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG volatile*, \ - LONG,LONG))aSyscall[76].pCurrent) -#endif /* defined(InterlockedCompareExchange) */ }; /* End of the overrideable system calls */ @@ -5489,7 +5482,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==77 ); + assert( ArraySize(aSyscall)==76 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); From ede7ae31f166a8901817afc2ac9b8924be7020e1 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 6 Aug 2014 11:58:40 +0000 Subject: [PATCH 193/710] Fix an obsolete comment in the func.c source file. No changes to code. FossilOrigin-Name: 5c6bb57d90bad32785d6d9cdf110a825bbc5ec73 --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/func.c | 9 +++------ 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 7fa415777a..f2683e69ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C On\sthe\swindows\sVFS,\sdo\snot\stry\sto\smake\sInterlockedCompareExchange\san\noverloadable\sfunction,\ssince\ssometimes\sit\sis\sa\smacro. -D 2014-08-06T11:57:54.546 +C Fix\san\sobsolete\scomment\sin\sthe\sfunc.c\ssource\sfile.\s\sNo\schanges\sto\scode. +D 2014-08-06T11:58:40.249 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,7 +179,7 @@ F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf F src/expr.c f749009cf4a8534efb5e0d5cd7c9fb1fb0f2836c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 -F src/func.c 3bc223ea36cd29a91c481485343d0ee4257ab8dc +F src/func.c bbb724b74ed96ca42675a7274646a71dd52bcda7 F src/global.c 1e4bd956dc2f608f87d2a929abc4a20db65f30e4 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 @@ -1185,7 +1185,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7be244ce129d6502f3a3e3f3e8a1dd61ef71d878 -R 40d3ff3c08b7d4b2cddf84583f93f135 +P ab1a751e1304749bef5bc5c833f9abed8950f7d0 +Q +7a145c9409e70b92946534eb44c2e205f7309897 +R c5d9492d210d0878e8d4fad26b1bb9f1 U drh -Z 1b7c539afa0e75fe0851e437eb151215 +Z c0ef6bc5c4f2711fca6f4933f6dbf08d diff --git a/manifest.uuid b/manifest.uuid index 7113e4822f..89b8c4d9e9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ab1a751e1304749bef5bc5c833f9abed8950f7d0 \ No newline at end of file +5c6bb57d90bad32785d6d9cdf110a825bbc5ec73 \ No newline at end of file diff --git a/src/func.c b/src/func.c index efbb3efbb7..dbc8375541 100644 --- a/src/func.c +++ b/src/func.c @@ -9,12 +9,9 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains the C functions that implement various SQL -** functions of SQLite. -** -** There is only one exported symbol in this file - the function -** sqliteRegisterBuildinFunctions() found at the bottom of the file. -** All other code has file scope. +** This file contains the C-language implementions for many of the SQL +** functions of SQLite. (Some function, and in particular the date and +** time functions, are implemented separately.) */ #include "sqliteInt.h" #include From 5402710b0b4a7903f62a3a0ceec928ff045e67cd Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 6 Aug 2014 14:36:53 +0000 Subject: [PATCH 194/710] Fix harmless compiler warnings. In the command-line shell, report if the ".system" command returns a non-zero result. FossilOrigin-Name: 1202e9771f6d699dbf5268368108a3527d06d799 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/main.c | 2 +- src/shell.c | 5 +++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index f2683e69ef..bdbb4d649d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sobsolete\scomment\sin\sthe\sfunc.c\ssource\sfile.\s\sNo\schanges\sto\scode. -D 2014-08-06T11:58:40.249 +C Fix\sharmless\scompiler\swarnings.\s\sIn\sthe\scommand-line\sshell,\sreport\sif\sthe\n".system"\scommand\sreturns\sa\snon-zero\sresult. +D 2014-08-06T14:36:53.910 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -189,7 +189,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c bbf7f993843fc3fc496089747f6849272d77e824 +F src/main.c 6bc7cd1b896a47f2803f69a56737191abf76918a F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -223,7 +223,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 1529c49075464c5a95fde77314073612b1b8d595 -F src/shell.c 191129c3f7a9cf241aea90ff6a6be3e74d3767f0 +F src/shell.c 4af71e7f25af6da6debda87dcbd6be37da710916 F src/sqlite.h.in 9bbc5815c73b0e77e68b5275481a5e3e7814a804 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1185,8 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ab1a751e1304749bef5bc5c833f9abed8950f7d0 -Q +7a145c9409e70b92946534eb44c2e205f7309897 -R c5d9492d210d0878e8d4fad26b1bb9f1 +P 5c6bb57d90bad32785d6d9cdf110a825bbc5ec73 +R 70e92229f0d1d2ca5063839c778d94a2 U drh -Z c0ef6bc5c4f2711fca6f4933f6dbf08d +Z 832222a1addbce429c4a8001874a3f39 diff --git a/manifest.uuid b/manifest.uuid index 89b8c4d9e9..3b487cbb12 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c6bb57d90bad32785d6d9cdf110a825bbc5ec73 \ No newline at end of file +1202e9771f6d699dbf5268368108a3527d06d799 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 990e62e6e4..a95200641f 100644 --- a/src/main.c +++ b/src/main.c @@ -1056,7 +1056,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** Return a static string containing the name corresponding to the error code ** specified in the argument. */ -#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) +#if (defined(SQLITE_DEBUG) && SQLITE_OS_WIN) || defined(SQLITE_TEST) const char *sqlite3ErrName(int rc){ const char *zName = 0; int i, origRc = rc; diff --git a/src/shell.c b/src/shell.c index 371efa024e..e9ecf194b6 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3126,7 +3126,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ && (strncmp(azArg[0], "shell", n)==0 || strncmp(azArg[0],"system",n)==0) ){ char *zCmd; - int i; + int i, x; if( nArg<2 ){ fprintf(stderr, "Usage: .system COMMAND\n"); rc = 1; @@ -3137,8 +3137,9 @@ static int do_meta_command(char *zLine, struct callback_data *p){ zCmd = sqlite3_mprintf(strchr(azArg[i],' ')==0?"%z %s":"%z \"%s\"", zCmd, azArg[i]); } - (void)system(zCmd); + x = system(zCmd); sqlite3_free(zCmd); + if( x ) fprintf(stderr, "System command returns %d\n", x); }else if( c=='s' && strncmp(azArg[0], "show", n)==0 ){ From e8f2c9dc71c9c3d7b08552a8db5fbd4ab5b5fb86 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 6 Aug 2014 17:49:13 +0000 Subject: [PATCH 195/710] Fix two more harmless compiler warnings. Make sure the fts3_unicode2.c file is in sync with mkunicode.tcl. FossilOrigin-Name: a2a60307ea68a3230952a56cb65369ba0a208967 --- ext/fts3/fts3_unicode2.c | 2 +- ext/fts3/unicode/mkunicode.tcl | 6 +++--- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- 5 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c index 4837dbd272..a2ca622c6e 100644 --- a/ext/fts3/fts3_unicode2.c +++ b/ext/fts3/fts3_unicode2.c @@ -39,7 +39,7 @@ int sqlite3FtsUnicodeIsalnum(int c){ ** C. It is not possible to represent a range larger than 1023 codepoints ** using this format. */ - const static unsigned int aEntry[] = { + static const unsigned int aEntry[] = { 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index 2da17c51a5..66698bb055 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -298,7 +298,7 @@ proc an_print_range_array {lRange} { ** using this format. */ }] - puts -nonewline " const static unsigned int aEntry\[\] = \{" + puts -nonewline " static const unsigned int aEntry\[\] = \{" set i 0 foreach range $lRange { foreach {iFirst nRange} $range {} @@ -732,7 +732,7 @@ proc print_fileheader {} { */ }] puts "" - puts "#if defined(SQLITE_ENABLE_FTS4_UNICODE61)" + puts "#ifndef SQLITE_DISABLE_FTS3_UNICODE" puts "#if defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4)" puts "" puts "#include " @@ -808,4 +808,4 @@ if {$::generate_test_code} { } puts "#endif /* defined(SQLITE_ENABLE_FTS3) || defined(SQLITE_ENABLE_FTS4) */" -puts "#endif /* !defined(SQLITE_ENABLE_FTS4_UNICODE61) */" +puts "#endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */" diff --git a/manifest b/manifest index bdbb4d649d..4f407ebf1a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings.\s\sIn\sthe\scommand-line\sshell,\sreport\sif\sthe\n".system"\scommand\sreturns\sa\snon-zero\sresult. -D 2014-08-06T14:36:53.910 +C Fix\stwo\smore\sharmless\scompiler\swarnings.\s\sMake\ssure\sthe\sfts3_unicode2.c\sfile\nis\sin\ssync\swith\smkunicode.tcl. +D 2014-08-06T17:49:13.360 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -95,14 +95,14 @@ F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c e80eef8a11f2020dc9c1eb95c5b405b9012f2fbe -F ext/fts3/fts3_unicode2.c c8adda75aad0c6c252ef3dd555f811f437485044 +F ext/fts3/fts3_unicode2.c b81e5f0f7991e5f706544f7f7a9931c491f8ac07 F ext/fts3/fts3_write.c 8260388626516a7005d06a9dce94f9e55c6c2a41 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 3986531f2fc0ceca0c89c31ec7d0589b6adb19d6 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl dc6f268eb526710e2c6e496c372471d773d0c368 +F ext/fts3/unicode/mkunicode.tcl ddeb6629fbcd5eb9b55f8999dbad5e4c63e5dd5a F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 @@ -283,7 +283,7 @@ F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 2db817cf7c51fa775ecc68160352aefc3d9870f7 +F src/vdbe.c cd8d7e3ecd3e0e0684f6bf48469966335c666883 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5c6bb57d90bad32785d6d9cdf110a825bbc5ec73 -R 70e92229f0d1d2ca5063839c778d94a2 +P 1202e9771f6d699dbf5268368108a3527d06d799 +R d4b78aa6bcc5216879daa10572b102f7 U drh -Z 832222a1addbce429c4a8001874a3f39 +Z 6c4725b3b555727ccd6e011265bbf365 diff --git a/manifest.uuid b/manifest.uuid index 3b487cbb12..7c82aa8be9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1202e9771f6d699dbf5268368108a3527d06d799 \ No newline at end of file +a2a60307ea68a3230952a56cb65369ba0a208967 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 6c46d76569..10b81a7992 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3266,7 +3266,7 @@ case OP_ReopenIdx: { assert( pOp->p5==0 ); assert( pOp->p4type==P4_KEYINFO ); pCur = p->apCsr[pOp->p1]; - if( pCur && pCur->pgnoRoot==pOp->p2 ){ + if( pCur && pCur->pgnoRoot==(u32)pOp->p2 ){ assert( pCur->iDb==pOp->p3 ); /* Guaranteed by the code generator */ break; } From 858b638d1f76b0bb1a7f47f3eab72ca69a3a587e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 6 Aug 2014 18:50:51 +0000 Subject: [PATCH 196/710] A couple more harmless compiler warnings eliminated. FossilOrigin-Name: bcf6d775f90f4d1ba018a1b965f2f710df130f01 --- ext/fts3/fts3_unicode2.c | 4 ++-- ext/fts3/unicode/mkunicode.tcl | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ext/fts3/fts3_unicode2.c b/ext/fts3/fts3_unicode2.c index a2ca622c6e..20b7a25dbf 100644 --- a/ext/fts3/fts3_unicode2.c +++ b/ext/fts3/fts3_unicode2.c @@ -131,7 +131,7 @@ int sqlite3FtsUnicodeIsalnum(int c){ return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); }else if( c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; - int iRes; + int iRes = 0; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; int iLo = 0; while( iHi>=iLo ){ @@ -202,7 +202,7 @@ static int remove_diacritic(int c){ } assert( key>=aDia[iRes] ); return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); -}; +} /* diff --git a/ext/fts3/unicode/mkunicode.tcl b/ext/fts3/unicode/mkunicode.tcl index 66698bb055..c3083ee368 100644 --- a/ext/fts3/unicode/mkunicode.tcl +++ b/ext/fts3/unicode/mkunicode.tcl @@ -160,7 +160,7 @@ proc print_rd {map} { } assert( key>=aDia[iRes] ); return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]);} - puts "\};" + puts "\}" } proc print_isdiacritic {zFunc map} { @@ -349,7 +349,7 @@ proc print_isalnum {zFunc lRange} { return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); }else if( c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; - int iRes; + int iRes = 0; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; int iLo = 0; while( iHi>=iLo ){ diff --git a/manifest b/manifest index 4f407ebf1a..e0d03aef40 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stwo\smore\sharmless\scompiler\swarnings.\s\sMake\ssure\sthe\sfts3_unicode2.c\sfile\nis\sin\ssync\swith\smkunicode.tcl. -D 2014-08-06T17:49:13.360 +C A\scouple\smore\sharmless\scompiler\swarnings\seliminated. +D 2014-08-06T18:50:51.299 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -95,14 +95,14 @@ F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c e80eef8a11f2020dc9c1eb95c5b405b9012f2fbe -F ext/fts3/fts3_unicode2.c b81e5f0f7991e5f706544f7f7a9931c491f8ac07 +F ext/fts3/fts3_unicode2.c c3d01968d497bd7001e7dc774ba75b372738c057 F ext/fts3/fts3_write.c 8260388626516a7005d06a9dce94f9e55c6c2a41 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 3986531f2fc0ceca0c89c31ec7d0589b6adb19d6 F ext/fts3/unicode/CaseFolding.txt 8c678ca52ecc95e16bc7afc2dbf6fc9ffa05db8c F ext/fts3/unicode/UnicodeData.txt cd07314edb62d49fde34debdaf92fa2aa69011e7 -F ext/fts3/unicode/mkunicode.tcl ddeb6629fbcd5eb9b55f8999dbad5e4c63e5dd5a +F ext/fts3/unicode/mkunicode.tcl a2567f9d6ad6779879a2e394c120ad8718557e65 F ext/icu/README.txt d9fbbad0c2f647c3fdf715fc9fd64af53aedfc43 F ext/icu/icu.c d415ccf984defeb9df2c0e1afcfaa2f6dc05eacb F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1202e9771f6d699dbf5268368108a3527d06d799 -R d4b78aa6bcc5216879daa10572b102f7 +P a2a60307ea68a3230952a56cb65369ba0a208967 +R 1442001d079992194e2cf55642b869fa U drh -Z 6c4725b3b555727ccd6e011265bbf365 +Z bb5786216c21ae2a69b537cdb50bf939 diff --git a/manifest.uuid b/manifest.uuid index 7c82aa8be9..ff8a700478 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a2a60307ea68a3230952a56cb65369ba0a208967 \ No newline at end of file +bcf6d775f90f4d1ba018a1b965f2f710df130f01 \ No newline at end of file From ddef5dc044de3ec73c9223359d45f1a6a55a792d Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 7 Aug 2014 16:50:00 +0000 Subject: [PATCH 197/710] When the estimated cost to do a sort overwhelms the estimated cost to do individual table lookups, make sure that the table lookup costs are still taken into consideration when selecting the lookup algorithm. FossilOrigin-Name: ec5d84ba69c100d9565425ed74040a49e410ea03 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 37 ++++++++++++++++++++++++++++++------- test/whereJ.test | 46 +++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 83 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index e0d03aef40..1e4aea7641 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\scouple\smore\sharmless\scompiler\swarnings\seliminated. -D 2014-08-06T18:50:51.299 +C When\sthe\sestimated\scost\sto\sdo\sa\ssort\soverwhelms\sthe\sestimated\scost\sto\sdo\nindividual\stable\slookups,\smake\ssure\sthat\sthe\stable\slookup\scosts\sare\sstill\ntaken\sinto\sconsideration\swhen\sselecting\sthe\slookup\salgorithm. +D 2014-08-07T16:50:00.581 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c ce1b9a3a2573033cd15e0882719db7f211f21cdd +F src/where.c 5a099c6d2a1bfbf46bc64979e89330dee3a1f3b9 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1117,7 +1117,7 @@ F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test 69f5ec4b15760a8c860f80e2d55525669390aab3 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test 1d89199697919d4930be05a71e7fe620f114e622 -F test/whereJ.test 7dde28284d20f358b559ca592e294db03e1d7103 +F test/whereJ.test 35a40a50d0e13aa6b0de7cc5d4b204e5f9f9669f F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a2a60307ea68a3230952a56cb65369ba0a208967 -R 1442001d079992194e2cf55642b869fa +P bcf6d775f90f4d1ba018a1b965f2f710df130f01 +R da285ec7c1bbef437389fa691998cee0 U drh -Z bb5786216c21ae2a69b537cdb50bf939 +Z 083887f7ebac8cb5fec0c6d1cdc7c751 diff --git a/manifest.uuid b/manifest.uuid index ff8a700478..04f61bd019 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bcf6d775f90f4d1ba018a1b965f2f710df130f01 \ No newline at end of file +ec5d84ba69c100d9565425ed74040a49e410ea03 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 20823046f7..ff65cc3b36 100644 --- a/src/where.c +++ b/src/where.c @@ -5424,6 +5424,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ LogEst rCost; /* Cost of a path */ LogEst nOut; /* Number of outputs */ LogEst mxCost = 0; /* Maximum cost of a set of paths */ + LogEst mxOut = 0; /* nOut value for maximum cost path */ int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ WherePath *aFrom; /* All nFrom paths at the previous level */ WherePath *aTo; /* The nTo best paths at the current level */ @@ -5441,7 +5442,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** For joins of 3 or more tables, track the 10 best paths */ mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); assert( nLoop<=pWInfo->pTabList->nSrc ); - WHERETRACE(0x002, ("---- begin solver\n")); + WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst)); /* Allocate and initialize space for aTo and aFrom */ ii = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; @@ -5529,7 +5530,13 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ }else{ revMask = pFrom->revLoop; } - /* Check to see if pWLoop should be added to the mxChoice best so far */ + /* Check to see if pWLoop should be added to the set of + ** mxChoice best-so-far paths. + ** + ** First look for an existing path among best-so-far paths + ** that covers the same set of loops and has the same isOrdered + ** setting as the current path candidate. + */ for(jj=0, pTo=aTo; jjmaskLoop==maskNew && ((pTo->isOrdered^isOrdered)&80)==0 @@ -5539,7 +5546,14 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } } if( jj>=nTo ){ - if( nTo>=mxChoice && rCost>=mxCost ){ + /* None of the existing best-so-far paths match the candidate. */ +if( nTo>=mxChoice && rCost==mxCost ) printf("nOut=%d mxOut=%d\n", nOut, mxOut); + if( nTo>=mxChoice + && (rCost>mxCost || (rCost==mxCost && nOut>=mxOut)) + ){ + /* The current candidate is no better than any of the mxChoice + ** paths currently in the best-so-far buffer. So discard + ** this candidate as not viable. */ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n", @@ -5549,7 +5563,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ #endif continue; } - /* Add a new Path to the aTo[] set */ + /* If we reach this points it means that the new candidate path + ** needs to be added to the set of best-so-far paths. */ if( nTorCost<=rCost ){ + /* Control reaches here if best-so-far path pTo=aTo[jj] covers the + ** same set of loops and has the sam isOrdered setting as the + ** candidate path. Check to see if the candidate should replace + ** pTo or if the candidate should be skipped */ + if( pTo->rCostrCost==rCost && pTo->nRow<=nOut) ){ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( @@ -5578,11 +5597,13 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); } #endif + /* Discard the candidate path from further consideration */ testcase( pTo->rCost==rCost ); continue; } testcase( pTo->rCost==rCost+1 ); - /* A new and better score for a previously created equivalent path */ + /* Control reaches here if the candidate path is better than the + ** pTo path. Replace pTo with the candidate. */ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( @@ -5606,9 +5627,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( nTo>=mxChoice ){ mxI = 0; mxCost = aTo[0].rCost; + mxOut = aTo[0].nRow; for(jj=1, pTo=&aTo[1]; jjrCost>mxCost ){ + if( pTo->rCost>mxCost || (pTo->rCost==mxCost && pTo->nRow>mxOut) ){ mxCost = pTo->rCost; + mxOut = pTo->nRow; mxI = jj; } } diff --git a/test/whereJ.test b/test/whereJ.test index 3cdeae890b..5209f16193 100644 --- a/test/whereJ.test +++ b/test/whereJ.test @@ -9,7 +9,7 @@ # #*********************************************************************** # -# This file implements a single regression test for a complex +# This file implements regression tests for a complex # query planning case. # @@ -328,4 +328,48 @@ do_execsql_test whereJ-1.4 { GROUP BY aid; } {/B-TREE/} +############################################################################ +# Ensure that the sorting cost does not swamp the loop costs and cause +# distinctions between individual loop costs to get lost, and hence for +# sub-optimal loops to be chosen. +# +do_execsql_test whereJ-2.1 { + CREATE TABLE tab( + id INTEGER PRIMARY KEY, + minChild INTEGER REFERENCES t1, + maxChild INTEGER REFERENCES t1, + x INTEGER + ); + EXPLAIN QUERY PLAN + SELECT t4.x + FROM tab AS t0, tab AS t1, tab AS t2, tab AS t3, tab AS t4 + WHERE t0.id=0 + AND t1.id BETWEEN t0.minChild AND t0.maxChild + AND t2.id BETWEEN t1.minChild AND t1.maxChild + AND t3.id BETWEEN t2.minChild AND t2.maxChild + AND t4.id BETWEEN t3.minChild AND t3.maxChild + ORDER BY t4.x; +} {~/SCAN/} +do_execsql_test whereJ-2.2 { + EXPLAIN QUERY PLAN + SELECT t4.x + FROM tab AS t0a, tab AS t0b, + tab AS t1a, tab AS t1b, + tab AS t2a, tab AS t2b, + tab AS t3a, tab AS t3b, + tab AS t4 + WHERE 1 + AND t0a.id=1 + AND t1a.id BETWEEN t0a.minChild AND t0a.maxChild + AND t2a.id BETWEEN t1a.minChild AND t1a.maxChild + AND t3a.id BETWEEN t2a.minChild AND t2a.maxChild + AND t0b.id=2 + AND t1b.id BETWEEN t0b.minChild AND t0b.maxChild + AND t2b.id BETWEEN t1b.minChild AND t1b.maxChild + AND t3b.id BETWEEN t2b.minChild AND t2b.maxChild + AND t4.id BETWEEN t3a.minChild AND t3b.maxChild + ORDER BY t4.x; +} {~/SCAN/} + + finish_test From b28ce65f45ef4fabef86e0fc1dc55a211ca8d191 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 7 Aug 2014 20:25:37 +0000 Subject: [PATCH 198/710] Remove the extraneous debugging printf() from the previous check-in. FossilOrigin-Name: 8f04d2c0084afa6381e78847c9aa296498d448cb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1e4aea7641..833fa0e284 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sthe\sestimated\scost\sto\sdo\sa\ssort\soverwhelms\sthe\sestimated\scost\sto\sdo\nindividual\stable\slookups,\smake\ssure\sthat\sthe\stable\slookup\scosts\sare\sstill\ntaken\sinto\sconsideration\swhen\sselecting\sthe\slookup\salgorithm. -D 2014-08-07T16:50:00.581 +C Remove\sthe\sextraneous\sdebugging\sprintf()\sfrom\sthe\sprevious\scheck-in. +D 2014-08-07T20:25:37.284 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 5a099c6d2a1bfbf46bc64979e89330dee3a1f3b9 +F src/where.c 5e4fe929798e75f9d5c6b5b9c545fbc006966c33 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bcf6d775f90f4d1ba018a1b965f2f710df130f01 -R da285ec7c1bbef437389fa691998cee0 +P ec5d84ba69c100d9565425ed74040a49e410ea03 +R 85fb26fb0f7344228780016c4074b483 U drh -Z 083887f7ebac8cb5fec0c6d1cdc7c751 +Z 218dd74820a0f9854df8a36e666b7aca diff --git a/manifest.uuid b/manifest.uuid index 04f61bd019..4e34981ca6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec5d84ba69c100d9565425ed74040a49e410ea03 \ No newline at end of file +8f04d2c0084afa6381e78847c9aa296498d448cb \ No newline at end of file diff --git a/src/where.c b/src/where.c index ff65cc3b36..ff50f52b45 100644 --- a/src/where.c +++ b/src/where.c @@ -5547,7 +5547,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } if( jj>=nTo ){ /* None of the existing best-so-far paths match the candidate. */ -if( nTo>=mxChoice && rCost==mxCost ) printf("nOut=%d mxOut=%d\n", nOut, mxOut); if( nTo>=mxChoice && (rCost>mxCost || (rCost==mxCost && nOut>=mxOut)) ){ From f2a90306cacfab863d3561024287bcdb88245bcc Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 7 Aug 2014 20:37:01 +0000 Subject: [PATCH 199/710] Clarify the computation of compatible isOrdered by in the plan solver of the query planner. FossilOrigin-Name: b5e8fd575a80334160de0aac8084ed5cd28816a5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 833fa0e284..a3788a8b47 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sextraneous\sdebugging\sprintf()\sfrom\sthe\sprevious\scheck-in. -D 2014-08-07T20:25:37.284 +C Clarify\sthe\scomputation\sof\scompatible\sisOrdered\sby\sin\sthe\splan\ssolver\sof\nthe\squery\splanner. +D 2014-08-07T20:37:01.008 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 5e4fe929798e75f9d5c6b5b9c545fbc006966c33 +F src/where.c 4cd4b3622caa81b71ebce8fea8f7811ae052d664 F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ec5d84ba69c100d9565425ed74040a49e410ea03 -R 85fb26fb0f7344228780016c4074b483 +P 8f04d2c0084afa6381e78847c9aa296498d448cb +R 444666601d1d38c7f4902aa29891f8d2 U drh -Z 218dd74820a0f9854df8a36e666b7aca +Z fa58944021488910925d31a2d03bf4a8 diff --git a/manifest.uuid b/manifest.uuid index 4e34981ca6..42afde34f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f04d2c0084afa6381e78847c9aa296498d448cb \ No newline at end of file +b5e8fd575a80334160de0aac8084ed5cd28816a5 \ No newline at end of file diff --git a/src/where.c b/src/where.c index ff50f52b45..4dd6579e2f 100644 --- a/src/where.c +++ b/src/where.c @@ -5536,10 +5536,14 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** First look for an existing path among best-so-far paths ** that covers the same set of loops and has the same isOrdered ** setting as the current path candidate. + ** + ** The term "((pTo->isOrdered^isOrdered)&0x80)==0" is equivalent + ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range + ** of legal values for isOrdered, -1..64. */ for(jj=0, pTo=aTo; jjmaskLoop==maskNew - && ((pTo->isOrdered^isOrdered)&80)==0 + && ((pTo->isOrdered^isOrdered)&0x80)==0 ){ testcase( jj==nTo-1 ); break; From 1d8ba02445b68bb2f66a7cffffe52aed3f7508ae Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Aug 2014 12:51:42 +0000 Subject: [PATCH 200/710] Reworking the documentation on integer result codes. This is a comment and documentation change only. There are no changes to code. FossilOrigin-Name: 54f1df7b63166f14b2f0d5955c546820e5cbdec9 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 36 ++++++++++++++---------------------- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index be7f215708..71ec5214cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sthe\sestimated\ssorting\scost\soverwhelms\sthe\sestimated\slookup\scost,\sensure\nthat\slookup\scosts\sare\sstill\staken\sinto\saccount\swhen\sselecting\sa\slookup\nalgorithm. -D 2014-08-07T20:42:33.655 +C Reworking\sthe\sdocumentation\son\sinteger\sresult\scodes.\s\sThis\sis\sa\scomment\nand\sdocumentation\schange\sonly.\s\sThere\sare\sno\schanges\sto\scode. +D 2014-08-08T12:51:42.280 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -224,7 +224,7 @@ F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 1529c49075464c5a95fde77314073612b1b8d595 F src/shell.c 4af71e7f25af6da6debda87dcbd6be37da710916 -F src/sqlite.h.in 9bbc5815c73b0e77e68b5275481a5e3e7814a804 +F src/sqlite.h.in c9c7328b51a94633c229b435eda8092fcd4e1ea3 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 641f8fbb65ca2084c8df95b525f6f82c7a1e91ae @@ -1185,8 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bcf6d775f90f4d1ba018a1b965f2f710df130f01 b5e8fd575a80334160de0aac8084ed5cd28816a5 -R 444666601d1d38c7f4902aa29891f8d2 -T +closed b5e8fd575a80334160de0aac8084ed5cd28816a5 +P 2af630c5720a4d71f22a952af29346a09bd8dfd0 +R b6800f247d253d726e2d1b70eaab6208 U drh -Z f100e3a960f67faf0129a27666ad7927 +Z 92cc627e0fe24841ecb93d0e5157435c diff --git a/manifest.uuid b/manifest.uuid index 421b35294d..3d8e83ed97 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2af630c5720a4d71f22a952af29346a09bd8dfd0 \ No newline at end of file +54f1df7b63166f14b2f0d5955c546820e5cbdec9 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 8470d9e8c0..e3287071b2 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -264,7 +264,7 @@ typedef sqlite_uint64 sqlite3_uint64; ** ** ^The sqlite3_close() and sqlite3_close_v2() routines are destructors ** for the [sqlite3] object. -** ^Calls to sqlite3_close() and sqlite3_close_v2() return SQLITE_OK if +** ^Calls to sqlite3_close() and sqlite3_close_v2() return [SQLITE_OK] if ** the [sqlite3] object is successfully destroyed and all associated ** resources are deallocated. ** @@ -285,7 +285,7 @@ typedef sqlite_uint64 sqlite3_uint64; ** with the [sqlite3] object prior to attempting to close the object. ^If ** sqlite3_close_v2() is called on a [database connection] that still has ** outstanding [prepared statements], [BLOB handles], and/or -** [sqlite3_backup] objects then it returns SQLITE_OK but the deallocation +** [sqlite3_backup] objects then it returns [SQLITE_OK] but the deallocation ** of resources is deferred until all [prepared statements], [BLOB handles], ** and [sqlite3_backup] objects are also destroyed. ** @@ -381,16 +381,14 @@ int sqlite3_exec( /* ** CAPI3REF: Result Codes -** KEYWORDS: SQLITE_OK {error code} {error codes} -** KEYWORDS: {result code} {result codes} +** KEYWORDS: {result code definitions} ** ** Many SQLite functions return an integer result code from the set shown ** here in order to indicate success or failure. ** ** New error codes may be added in future versions of SQLite. ** -** See also: [SQLITE_IOERR_READ | extended result codes], -** [sqlite3_vtab_on_conflict()] [SQLITE_ROLLBACK | result codes]. +** See also: [extended result code definitions] */ #define SQLITE_OK 0 /* Successful result */ /* beginning-of-error-codes */ @@ -428,26 +426,19 @@ int sqlite3_exec( /* ** CAPI3REF: Extended Result Codes -** KEYWORDS: {extended error code} {extended error codes} -** KEYWORDS: {extended result code} {extended result codes} +** KEYWORDS: {extended result code definitions} ** -** In its default configuration, SQLite API routines return one of 26 integer -** [SQLITE_OK | result codes]. However, experience has shown that many of +** In its default configuration, SQLite API routines return one of 30 integer +** [result codes]. However, experience has shown that many of ** these result codes are too coarse-grained. They do not provide as ** much information about problems as programmers might like. In an effort to ** address this, newer versions of SQLite (version 3.3.8 and later) include ** support for additional result codes that provide more detailed information -** about errors. The extended result codes are enabled or disabled +** about errors. These [extended result codes] are enabled or disabled ** on a per database connection basis using the -** [sqlite3_extended_result_codes()] API. -** -** Some of the available extended result codes are listed here. -** One may expect the number of extended result codes will increase -** over time. Software that uses extended result codes should expect -** to see new result codes in future releases of SQLite. -** -** The SQLITE_OK result code will never be extended. It will always -** be exactly zero. +** [sqlite3_extended_result_codes()] API. Or, the extended code for +** the most recent error can be obtained using +** [sqlite3_extended_errcode()]. */ #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) @@ -2523,8 +2514,8 @@ int sqlite3_set_authorizer( ** [sqlite3_set_authorizer | authorizer documentation] for additional ** information. ** -** Note that SQLITE_IGNORE is also used as a [SQLITE_ROLLBACK | return code] -** from the [sqlite3_vtab_on_conflict()] interface. +** Note that SQLITE_IGNORE is also used as a [conflict resolution mode] +** returned from the [sqlite3_vtab_on_conflict()] interface. */ #define SQLITE_DENY 1 /* Abort the SQL statement with an error */ #define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ @@ -7364,6 +7355,7 @@ int sqlite3_vtab_on_conflict(sqlite3 *); /* ** CAPI3REF: Conflict resolution modes +** KEYWORDS: {conflict resolution mode} ** ** These constants are returned by [sqlite3_vtab_on_conflict()] to ** inform a [virtual table] implementation what the [ON CONFLICT] mode From 3c19bbea0abf6fa092f5a6a4eeb3a45655b0c5e5 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Aug 2014 15:38:11 +0000 Subject: [PATCH 201/710] The SQLITE_IOERR_BLOCKED extended error code is not longer used, so remove assert() statements and documentation for that error code. Also make other documentation improvements. FossilOrigin-Name: 36b7c5cefcad6bad044806092593c84876fee8bc --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/main.c | 1 - src/os_unix.c | 10 ---------- src/sqlite.h.in | 26 ++++++-------------------- src/vdbeaux.c | 1 - 6 files changed, 16 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index 71ec5214cd..e81ad39217 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reworking\sthe\sdocumentation\son\sinteger\sresult\scodes.\s\sThis\sis\sa\scomment\nand\sdocumentation\schange\sonly.\s\sThere\sare\sno\schanges\sto\scode. -D 2014-08-08T12:51:42.280 +C The\sSQLITE_IOERR_BLOCKED\sextended\serror\scode\sis\snot\slonger\sused,\sso\sremove\nassert()\sstatements\sand\sdocumentation\sfor\sthat\serror\scode.\s\sAlso\smake\sother\ndocumentation\simprovements. +D 2014-08-08T15:38:11.174 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -189,7 +189,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 6bc7cd1b896a47f2803f69a56737191abf76918a +F src/main.c 5cd4d5037e07af703d12035fdfdaf802abf2b7f8 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -207,7 +207,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c a7baf1b30f3c58ba20b813e01aab23b18ae44f85 +F src/os_unix.c f0a58d439d58b207c06f4a25463113dcba7bd9da F src/os_win.c 3fca1bfdf78338705bf536059a407d0fb04016d5 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 @@ -224,7 +224,7 @@ F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 1529c49075464c5a95fde77314073612b1b8d595 F src/shell.c 4af71e7f25af6da6debda87dcbd6be37da710916 -F src/sqlite.h.in c9c7328b51a94633c229b435eda8092fcd4e1ea3 +F src/sqlite.h.in 66a923e0d8a3a771a7050f292cc8028b8c711652 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 641f8fbb65ca2084c8df95b525f6f82c7a1e91ae @@ -287,7 +287,7 @@ F src/vdbe.c cd8d7e3ecd3e0e0684f6bf48469966335c666883 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 -F src/vdbeaux.c ac063f36c929f88bf6cecdbcc413000e272265bb +F src/vdbeaux.c bbe934b0d472c98f57433829db91fc052e90fa17 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2af630c5720a4d71f22a952af29346a09bd8dfd0 -R b6800f247d253d726e2d1b70eaab6208 +P 54f1df7b63166f14b2f0d5955c546820e5cbdec9 +R 7baa54a57d0f22faa98f7b7348dc4b71 U drh -Z 92cc627e0fe24841ecb93d0e5157435c +Z 8357d07b76c5300f8dd47c47cbfcc4a1 diff --git a/manifest.uuid b/manifest.uuid index 3d8e83ed97..dcae5f47f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54f1df7b63166f14b2f0d5955c546820e5cbdec9 \ No newline at end of file +36b7c5cefcad6bad044806092593c84876fee8bc \ No newline at end of file diff --git a/src/main.c b/src/main.c index a95200641f..bf5c38a47e 100644 --- a/src/main.c +++ b/src/main.c @@ -1091,7 +1091,6 @@ const char *sqlite3ErrName(int rc){ case SQLITE_IOERR_UNLOCK: zName = "SQLITE_IOERR_UNLOCK"; break; case SQLITE_IOERR_RDLOCK: zName = "SQLITE_IOERR_RDLOCK"; break; case SQLITE_IOERR_DELETE: zName = "SQLITE_IOERR_DELETE"; break; - case SQLITE_IOERR_BLOCKED: zName = "SQLITE_IOERR_BLOCKED"; break; case SQLITE_IOERR_NOMEM: zName = "SQLITE_IOERR_NOMEM"; break; case SQLITE_IOERR_ACCESS: zName = "SQLITE_IOERR_ACCESS"; break; case SQLITE_IOERR_CHECKRESERVEDLOCK: diff --git a/src/os_unix.c b/src/os_unix.c index 347e82220c..28feb1182c 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -761,16 +761,6 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { case EPERM: return SQLITE_PERM; - /* EDEADLK is only possible if a call to fcntl(F_SETLKW) is made. And - ** this module never makes such a call. And the code in SQLite itself - ** asserts that SQLITE_IOERR_BLOCKED is never returned. For these reasons - ** this case is also commented out. If the system does set errno to EDEADLK, - ** the default SQLITE_IOERR_XXX code will be returned. */ -#if 0 - case EDEADLK: - return SQLITE_IOERR_BLOCKED; -#endif - #if EOPNOTSUPP!=ENOTSUP case EOPNOTSUPP: /* something went terribly awry, unless during file system support diff --git a/src/sqlite.h.in b/src/sqlite.h.in index e3287071b2..c89cbc1f17 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -671,7 +671,7 @@ struct sqlite3_file { ** locking strategy (for example to use dot-file locks), to inquire ** about the status of a lock, or to break stale locks. The SQLite ** core reserves all opcodes less than 100 for its own use. -** A [SQLITE_FCNTL_LOCKSTATE | list of opcodes] less than 100 is available. +** A [file control opcodes | list of opcodes] less than 100 is available. ** Applications that define a custom xFileControl method should use opcodes ** greater than 100 to avoid conflicts. VFS implementations should ** return [SQLITE_NOTFOUND] for file control opcodes that they do not @@ -744,6 +744,7 @@ struct sqlite3_io_methods { /* ** CAPI3REF: Standard File Control Opcodes +** KEYWORDS: {file control opcodes} {file control opcode} ** ** These integer constants are opcodes for the xFileControl method ** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] @@ -2031,7 +2032,7 @@ int sqlite3_complete16(const void *sql); ** The sqlite3_busy_handler() interface is used to implement ** [sqlite3_busy_timeout()] and [PRAGMA busy_timeout]. ** -** ^If the busy callback is NULL, then [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] +** ^If the busy callback is NULL, then [SQLITE_BUSY] ** is returned immediately upon encountering the lock. ^If the busy callback ** is not NULL, then the callback might be invoked with two arguments. ** @@ -2040,7 +2041,7 @@ int sqlite3_complete16(const void *sql); ** the busy handler callback is the number of times that the busy handler has ** been invoked for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to -** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned +** access the database and [SQLITE_BUSY] is returned ** to the application. ** ^If the callback returns non-zero, then another attempt ** is made to access the database and the cycle repeats. @@ -2048,7 +2049,7 @@ int sqlite3_complete16(const void *sql); ** The presence of a busy handler does not guarantee that it will be invoked ** when there is lock contention. ^If SQLite determines that invoking the busy ** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] -** or [SQLITE_IOERR_BLOCKED] to the application instead of invoking the +** to the application instead of invoking the ** busy handler. ** Consider a scenario where one process is holding a read lock that ** it is trying to promote to a reserved lock and @@ -2063,21 +2064,6 @@ int sqlite3_complete16(const void *sql); ** ** ^The default busy callback is NULL. ** -** ^The [SQLITE_BUSY] error is converted to [SQLITE_IOERR_BLOCKED] -** when SQLite is in the middle of a large transaction where all the -** changes will not fit into the in-memory cache. SQLite will -** already hold a RESERVED lock on the database file, but it needs -** to promote this lock to EXCLUSIVE so that it can spill cache -** pages into the database file without harm to concurrent -** readers. ^If it is unable to promote the lock, then the in-memory -** cache will be left in an inconsistent state and so the error -** code is promoted from the relatively benign [SQLITE_BUSY] to -** the more severe [SQLITE_IOERR_BLOCKED]. ^This error code promotion -** forces an automatic rollback of the changes. See the -** -** CorruptionFollowingBusyError wiki page for a discussion of why -** this is important. -** ** ^(There can only be a single busy handler defined for each ** [database connection]. Setting a new busy handler clears any ** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] @@ -2102,7 +2088,7 @@ int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); ** will sleep multiple times until at least "ms" milliseconds of sleeping ** have accumulated. ^After at least "ms" milliseconds of sleeping, ** the handler returns 0 which causes [sqlite3_step()] to return -** [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED]. +** [SQLITE_BUSY]. ** ** ^Calling this routine with an argument less than or equal to zero ** turns off all busy handlers. diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 240085bfbf..dba11eed67 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2301,7 +2301,6 @@ int sqlite3VdbeHalt(Vdbe *p){ /* Check for one of the special errors */ mrc = p->rc & 0xff; - assert( p->rc!=SQLITE_IOERR_BLOCKED ); /* This error no longer exists */ isSpecialError = mrc==SQLITE_NOMEM || mrc==SQLITE_IOERR || mrc==SQLITE_INTERRUPT || mrc==SQLITE_FULL; if( isSpecialError ){ From 50ae31e6f678b476a8473df2b707c341029ce6cc Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 8 Aug 2014 16:52:28 +0000 Subject: [PATCH 202/710] Because SQLite internally calculates query plan costs using a logarithmic scale, very large estimated sorting costs can cause all other estimated costs to be rounded down to zero. In these cases break ties between plans with the same total cost by comparing the costs with sorting excluded. This is an alternative fix for the same problem as addressed by [2af630c572]. FossilOrigin-Name: 299b9570279ded7158d22349ef93384286a5c755 --- manifest | 19 +++--- manifest.uuid | 2 +- src/where.c | 158 ++++++++++++++++++++++++++++++++----------------- src/whereInt.h | 1 + 4 files changed, 118 insertions(+), 62 deletions(-) diff --git a/manifest b/manifest index e81ad39217..fe79529289 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sSQLITE_IOERR_BLOCKED\sextended\serror\scode\sis\snot\slonger\sused,\sso\sremove\nassert()\sstatements\sand\sdocumentation\sfor\sthat\serror\scode.\s\sAlso\smake\sother\ndocumentation\simprovements. -D 2014-08-08T15:38:11.174 +C Because\sSQLite\sinternally\scalculates\squery\splan\scosts\susing\sa\slogarithmic\sscale,\svery\slarge\sestimated\ssorting\scosts\scan\scause\sall\sother\sestimated\scosts\sto\sbe\srounded\sdown\sto\szero.\sIn\sthese\scases\sbreak\sties\sbetween\splans\swith\sthe\ssame\stotal\scost\sby\scomparing\sthe\scosts\swith\ssorting\sexcluded.\sThis\sis\san\salternative\sfix\sfor\sthe\ssame\sproblem\sas\saddressed\sby\s[2af630c572]. +D 2014-08-08T16:52:28.259 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,8 +296,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 4cd4b3622caa81b71ebce8fea8f7811ae052d664 -F src/whereInt.h 929c1349b5355fd44f22cee5c14d72b3329c58a6 +F src/where.c 012ef65af2ae3e1061aa42bbe4eb549b409ee7e7 +F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1185,7 +1185,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 54f1df7b63166f14b2f0d5955c546820e5cbdec9 -R 7baa54a57d0f22faa98f7b7348dc4b71 -U drh -Z 8357d07b76c5300f8dd47c47cbfcc4a1 +P 36b7c5cefcad6bad044806092593c84876fee8bc +R 97e5eb0484c42f8168a99aaa07e49072 +T *branch * query-planner-fix +T *sym-query-planner-fix * +T -sym-trunk * +U dan +Z 10210eb3192fa621adfca2044e28b578 diff --git a/manifest.uuid b/manifest.uuid index dcae5f47f8..2b491ab1ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -36b7c5cefcad6bad044806092593c84876fee8bc \ No newline at end of file +299b9570279ded7158d22349ef93384286a5c755 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4dd6579e2f..ece31542b7 100644 --- a/src/where.c +++ b/src/where.c @@ -5400,6 +5400,45 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ } #endif +/* +** Return the cost of sorting nRow rows, assuming that the keys have +** nOrderby columns and that the first nSorted columns are already in +** order. +*/ +static LogEst whereSortingCost( + WhereInfo *pWInfo, + LogEst nRow, + int nOrderBy, + int nSorted +){ + /* TUNING: Estimated cost of a full external sort, where N is + ** the number of rows to sort is: + ** + ** cost = (3.0 * N * log(N)). + ** + ** Or, if the order-by clause has X terms but only the last Y + ** terms are out of order, then block-sorting will reduce the + ** sorting cost to: + ** + ** cost = (3.0 * N * log(N)) * (Y/X) + ** + ** The (Y/X) term is implemented using stack variable rScale + ** below. */ + LogEst rScale, rSortCost; + assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); + rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; + rSortCost = nRow + estLog(nRow) + rScale + 16; + + /* TUNING: The cost of implementing DISTINCT using a B-TREE is + ** similar but with a larger constant of proportionality. + ** Multiply by an additional factor of 3.0. */ + if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ + rSortCost += 16; + } + + return rSortCost; +} + /* ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine ** attempts to find the lowest cost path that visits each WhereLoop @@ -5421,10 +5460,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int ii, jj; /* Loop counters */ int mxI = 0; /* Index of next entry to replace */ int nOrderBy; /* Number of ORDER BY clause terms */ - LogEst rCost; /* Cost of a path */ - LogEst nOut; /* Number of outputs */ LogEst mxCost = 0; /* Maximum cost of a set of paths */ - LogEst mxOut = 0; /* nOut value for maximum cost path */ + LogEst mxUnsorted = 0; /* Maximum unsorted cost of a set of path */ int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ WherePath *aFrom; /* All nFrom paths at the previous level */ WherePath *aTo; /* The nTo best paths at the current level */ @@ -5432,6 +5469,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ WherePath *pTo; /* An element of aTo[] that we are working on */ WhereLoop *pWLoop; /* One of the WhereLoop objects */ WhereLoop **pX; /* Used to divy up the pSpace memory */ + LogEst *aSortCost = 0; /* Sorting and partial sorting costs */ char *pSpace; /* Temporary memory used by this routine */ pParse = pWInfo->pParse; @@ -5444,8 +5482,20 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ assert( nLoop<=pWInfo->pTabList->nSrc ); WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst)); - /* Allocate and initialize space for aTo and aFrom */ + /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this + ** case the purpose of this call is to estimate the number of rows returned + ** by the overall query. Once this estimate has been obtained, the caller + ** will invoke this function a second time, passing the estimate as the + ** nRowEst parameter. */ + if( pWInfo->pOrderBy==0 || nRowEst==0 ){ + nOrderBy = 0; + }else{ + nOrderBy = pWInfo->pOrderBy->nExpr; + } + + /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ ii = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; + ii += sizeof(LogEst) * nOrderBy; pSpace = sqlite3DbMallocRaw(db, ii); if( pSpace==0 ) return SQLITE_NOMEM; aTo = (WherePath*)pSpace; @@ -5455,6 +5505,16 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ for(ii=mxChoice*2, pFrom=aTo; ii>0; ii--, pFrom++, pX += nLoop){ pFrom->aLoop = pX; } + if( nOrderBy ){ + /* If there is an ORDER BY clause and it is not being ignored, set up + ** space for the aSortCost[] array. Each element of the aSortCost array + ** is either zero - meaning it has not yet been initialized - or the + ** cost of sorting nRowEst rows of data where the first X terms of + ** the ORDER BY clause are already in order, where X is the array + ** index. */ + aSortCost = (LogEst*)pX; + memset(aSortCost, 0, sizeof(LogEst) * (nOrderBy+1)); + } /* Seed the search with a single WherePath containing zero WhereLoops. ** @@ -5463,15 +5523,15 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** rows, then do not use the automatic index. */ aFrom[0].nRow = MIN(pParse->nQueryLoop, 46); assert( 46==sqlite3LogEst(25) ); nFrom = 1; - - /* Precompute the cost of sorting the final result set, if the caller - ** to sqlite3WhereBegin() was concerned about sorting */ - if( pWInfo->pOrderBy==0 || nRowEst==0 ){ - aFrom[0].isOrdered = 0; - nOrderBy = 0; - }else{ - aFrom[0].isOrdered = nLoop>0 ? -1 : 1; - nOrderBy = pWInfo->pOrderBy->nExpr; + assert( aFrom[0].isOrdered==0 ); + if( nOrderBy ){ + /* If nLoop is zero, then there are no FROM terms in the query. Since + ** in this case the query may return a maximum of one row, the results + ** are already in the requested order. Set isOrdered to nOrderBy to + ** indicate this. Or, if nLoop is greater than zero, set isOrdered to + ** -1, indicating that the result set may or may not be ordered, + ** depending on the loops added to the current plan. */ + aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy; } /* Compute successively longer WherePaths using the previous generation @@ -5481,55 +5541,44 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ nTo = 0; for(ii=0, pFrom=aFrom; iipLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ - Bitmask maskNew; - Bitmask revMask = 0; - i8 isOrdered = pFrom->isOrdered; + LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ + LogEst rCost; /* Cost of path (pFrom+pWLoop) */ + LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ + i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */ + Bitmask maskNew; /* Mask of src visited by (..) */ + Bitmask revMask = 0; /* Mask of rev-order loops for (..) */ + if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; /* At this point, pWLoop is a candidate to be the next loop. ** Compute its cost */ - rCost = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); - rCost = sqlite3LogEstAdd(rCost, pFrom->rCost); + rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); + rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; if( isOrdered<0 ){ isOrdered = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); - if( isOrdered>=0 && isOrdered0 && 66==sqlite3LogEst(100) ); - rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; - rSortCost = nRowEst + estLog(nRowEst) + rScale + 16; - - /* TUNING: The cost of implementing DISTINCT using a B-TREE is - ** similar but with a larger constant of proportionality. - ** Multiply by an additional factor of 3.0. */ - if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ - rSortCost += 16; - } - WHERETRACE(0x002, - ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", - rSortCost, (nOrderBy-isOrdered), nOrderBy, rCost, - sqlite3LogEstAdd(rCost,rSortCost))); - rCost = sqlite3LogEstAdd(rCost, rSortCost); - } }else{ revMask = pFrom->revLoop; } + if( isOrdered>=0 && isOrdered=nTo ){ /* None of the existing best-so-far paths match the candidate. */ if( nTo>=mxChoice - && (rCost>mxCost || (rCost==mxCost && nOut>=mxOut)) + && (rCost>mxCost || (rCost==mxCost && rUnsorted>=mxUnsorted)) ){ /* The current candidate is no better than any of the mxChoice ** paths currently in the best-so-far buffer. So discard @@ -5624,17 +5673,20 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pTo->revLoop = revMask; pTo->nRow = nOut; pTo->rCost = rCost; + pTo->rUnsorted = rUnsorted; pTo->isOrdered = isOrdered; memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); pTo->aLoop[iLoop] = pWLoop; if( nTo>=mxChoice ){ mxI = 0; mxCost = aTo[0].rCost; - mxOut = aTo[0].nRow; + mxUnsorted = aTo[0].nRow; for(jj=1, pTo=&aTo[1]; jjrCost>mxCost || (pTo->rCost==mxCost && pTo->nRow>mxOut) ){ + if( pTo->rCost>mxCost + || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) + ){ mxCost = pTo->rCost; - mxOut = pTo->nRow; + mxUnsorted = pTo->rUnsorted; mxI = jj; } } diff --git a/src/whereInt.h b/src/whereInt.h index 72e7530db9..81f4a03667 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -183,6 +183,7 @@ struct WherePath { Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ LogEst nRow; /* Estimated number of rows generated by this path */ LogEst rCost; /* Total cost of this path */ + LogEst rUnsorted; /* Total cost of this path ignoring sorting costs */ i8 isOrdered; /* No. of ORDER BY terms satisfied. -1 for unknown */ WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ }; From e2c278513d56f27725b0b9118c19aef2fc77e695 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 8 Aug 2014 17:25:33 +0000 Subject: [PATCH 203/710] Fix a buffer overrun in the previous commit. FossilOrigin-Name: 43c59c85436dc8001c81f4aac7f5231b13d741cb --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/where.c | 11 +++++++---- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index fe79529289..3634881a4d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Because\sSQLite\sinternally\scalculates\squery\splan\scosts\susing\sa\slogarithmic\sscale,\svery\slarge\sestimated\ssorting\scosts\scan\scause\sall\sother\sestimated\scosts\sto\sbe\srounded\sdown\sto\szero.\sIn\sthese\scases\sbreak\sties\sbetween\splans\swith\sthe\ssame\stotal\scost\sby\scomparing\sthe\scosts\swith\ssorting\sexcluded.\sThis\sis\san\salternative\sfix\sfor\sthe\ssame\sproblem\sas\saddressed\sby\s[2af630c572]. -D 2014-08-08T16:52:28.259 +C Fix\sa\sbuffer\soverrun\sin\sthe\sprevious\scommit. +D 2014-08-08T17:25:33.967 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 012ef65af2ae3e1061aa42bbe4eb549b409ee7e7 +F src/where.c ab20f9c24a422ee8900831b343c3d1e5e7aca87b F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1185,10 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 36b7c5cefcad6bad044806092593c84876fee8bc -R 97e5eb0484c42f8168a99aaa07e49072 -T *branch * query-planner-fix -T *sym-query-planner-fix * -T -sym-trunk * +P 299b9570279ded7158d22349ef93384286a5c755 +R 1f8265817308d29fd5a420f3d8d1525f U dan -Z 10210eb3192fa621adfca2044e28b578 +Z 2535cee87377895ac3ecbfed51081b0b diff --git a/manifest.uuid b/manifest.uuid index 2b491ab1ab..b3ea543e1e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -299b9570279ded7158d22349ef93384286a5c755 \ No newline at end of file +43c59c85436dc8001c81f4aac7f5231b13d741cb \ No newline at end of file diff --git a/src/where.c b/src/where.c index ece31542b7..9c30136e87 100644 --- a/src/where.c +++ b/src/where.c @@ -5471,6 +5471,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ WhereLoop **pX; /* Used to divy up the pSpace memory */ LogEst *aSortCost = 0; /* Sorting and partial sorting costs */ char *pSpace; /* Temporary memory used by this routine */ + int nSpace; /* Bytes of space allocated at pSpace */ pParse = pWInfo->pParse; db = pParse->db; @@ -5494,9 +5495,9 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ - ii = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; - ii += sizeof(LogEst) * nOrderBy; - pSpace = sqlite3DbMallocRaw(db, ii); + nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; + nSpace += sizeof(LogEst) * nOrderBy; + pSpace = sqlite3DbMallocRaw(db, nSpace); if( pSpace==0 ) return SQLITE_NOMEM; aTo = (WherePath*)pSpace; aFrom = aTo+mxChoice; @@ -5513,8 +5514,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** the ORDER BY clause are already in order, where X is the array ** index. */ aSortCost = (LogEst*)pX; - memset(aSortCost, 0, sizeof(LogEst) * (nOrderBy+1)); + memset(aSortCost, 0, sizeof(LogEst) * nOrderBy); } + assert( aSortCost==0 || &pSpace[nSpace]==(char*)&aSortCost[nOrderBy] ); + assert( aSortCost!=0 || &pSpace[nSpace]==(char*)pX ); /* Seed the search with a single WherePath containing zero WhereLoops. ** From e8a537eea7a57ad1b42bb75cd7d7a4ee29d7653c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 8 Aug 2014 18:26:20 +0000 Subject: [PATCH 204/710] Update requirements marks. No changes to code. FossilOrigin-Name: 7556bd9aa5db1b9cd92d0e48effcda9fe96f3128 --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- src/resolve.c | 2 +- test/e_createtable.test | 6 +++--- test/e_expr.test | 2 +- 5 files changed, 14 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index e2bfea2510..61a8d036f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\sway\sthe\squery\splanner\shandles\ssorting\scosts,\sso\sthat\nvery\slarge\ssorting\scosts\sdo\snot\soverwhelm\sthe\sloop\scosts. -D 2014-08-08T17:49:55.898 +C Update\srequirements\smarks.\s\sNo\schanges\sto\scode. +D 2014-08-08T18:26:20.575 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -220,7 +220,7 @@ F src/pragma.c d10ef67c4de79f78188b965b4b7988aff1d66f2e F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c af06f66927919730f03479fed6ae9854f73419f4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c 5fc110baeacf120a73fe34e103f052632ff11a02 +F src/resolve.c 44e2e434deaef2b41d4410ede695d867deb89886 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 1529c49075464c5a95fde77314073612b1b8d595 F src/shell.c 4af71e7f25af6da6debda87dcbd6be37da710916 @@ -442,11 +442,11 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 -F test/e_createtable.test ed82efcedc4b3656b27a5fcd12335cdb7e20eeee +F test/e_createtable.test 181653f6f45e3adde73f8686600ce5ad7515466b F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 F test/e_dropview.test 0c9f7f60989164a70a67a9d9c26d1083bc808306 -F test/e_expr.test 5c71d183fbf519a4769fd2e2124afdc70b5b1f42 +F test/e_expr.test 5f4528f3dea1c3c401d19ab1d543ec7959e67464 F test/e_fkey.test a1783fe1f759e1990e6a11adfcf0702dac4d0707 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 F test/e_insert.test 7b2fa9cd1456f83474d6c5d27db3abaeb8be2023 @@ -1185,8 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 36b7c5cefcad6bad044806092593c84876fee8bc 43c59c85436dc8001c81f4aac7f5231b13d741cb -R 1f8265817308d29fd5a420f3d8d1525f -T +closed 43c59c85436dc8001c81f4aac7f5231b13d741cb +P bdaa6947371a60a31b6a13267b0ba6e46df2a8ce +R 72bdb7fa5001d300e89054929fd995d2 U drh -Z 45c189f6c18b0be836f726dcb23b5c12 +Z 671dba8d2ee0bfa11f311df8108f5873 diff --git a/manifest.uuid b/manifest.uuid index c94e49bd0f..4ea12bd8df 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bdaa6947371a60a31b6a13267b0ba6e46df2a8ce \ No newline at end of file +7556bd9aa5db1b9cd92d0e48effcda9fe96f3128 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index ce26f00c66..0226d06462 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -354,7 +354,7 @@ static int lookupName( } } if( iCol>=pTab->nCol && sqlite3IsRowid(zCol) && HasRowid(pTab) ){ - /* IMP: R-24309-18625 */ + /* IMP: R-51414-32910 */ /* IMP: R-44911-55124 */ iCol = -1; } diff --git a/test/e_createtable.test b/test/e_createtable.test index 42fc017e2a..08f606f65b 100644 --- a/test/e_createtable.test +++ b/test/e_createtable.test @@ -1175,9 +1175,9 @@ do_createtable_tests 4.2 -repair { # EVIDENCE-OF: R-59124-61339 Each row in a table with a primary key must # have a unique combination of values in its primary key columns. # -# EVIDENCE-OF: R-39102-06737 If an INSERT or UPDATE statement attempts -# to modify the table content so that two or more rows feature identical -# primary key values, it is a constraint violation. +# EVIDENCE-OF: R-06471-16287 If an INSERT or UPDATE statement attempts +# to modify the table content so that two or more rows have identical +# primary key values, that is a constraint violation. # drop_all_tables do_execsql_test 4.3.0 { diff --git a/test/e_expr.test b/test/e_expr.test index a743c0c390..60e345441a 100644 --- a/test/e_expr.test +++ b/test/e_expr.test @@ -450,7 +450,7 @@ do_execsql_test e_expr-10.3.4 { SELECT typeof('isn''t') } {text} # containing hexadecimal data and preceded by a single "x" or "X" # character. # -# EVIDENCE-OF: R-39344-59787 For example: X'53514C697465' +# EVIDENCE-OF: R-19836-11244 Example: X'53514C697465' # do_execsql_test e_expr-10.4.1 { SELECT typeof(X'0123456789ABCDEF') } blob do_execsql_test e_expr-10.4.2 { SELECT typeof(x'0123456789ABCDEF') } blob From 91be7dc32078819a51ef024ba548024d55484179 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 11 Aug 2014 13:53:30 +0000 Subject: [PATCH 205/710] Changes that will perhaps enable SQLite to work better on VxWorks. FossilOrigin-Name: de27c742c0dcda20b51339598bf6094a8dcf5fb9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 19 ++++++++++++------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 61a8d036f4..8b2db457d5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\srequirements\smarks.\s\sNo\schanges\sto\scode. -D 2014-08-08T18:26:20.575 +C Changes\sthat\swill\sperhaps\senable\sSQLite\sto\swork\sbetter\son\sVxWorks. +D 2014-08-11T13:53:30.969 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -207,7 +207,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c f0a58d439d58b207c06f4a25463113dcba7bd9da +F src/os_unix.c 17e7ab0f9160a78c964d615b15b1658ab2090d42 F src/os_win.c 3fca1bfdf78338705bf536059a407d0fb04016d5 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bdaa6947371a60a31b6a13267b0ba6e46df2a8ce -R 72bdb7fa5001d300e89054929fd995d2 +P 7556bd9aa5db1b9cd92d0e48effcda9fe96f3128 +R 07fe7ef4c70cea44dc683244e34b47a2 U drh -Z 671dba8d2ee0bfa11f311df8108f5873 +Z c5b6316215c9354c3470fdb843673338 diff --git a/manifest.uuid b/manifest.uuid index 4ea12bd8df..b1e2fcea2d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7556bd9aa5db1b9cd92d0e48effcda9fe96f3128 \ No newline at end of file +de27c742c0dcda20b51339598bf6094a8dcf5fb9 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 28feb1182c..feb9314167 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -94,11 +94,10 @@ #include #include #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 -#include +# include #endif - -#if SQLITE_ENABLE_LOCKING_STYLE +#if SQLITE_ENABLE_LOCKING_STYLE || OS_VXWORKS # include # if OS_VXWORKS # include @@ -318,7 +317,11 @@ static int posixOpen(const char *zFile, int flags, int mode){ ** we are not running as root. */ static int posixFchown(int fd, uid_t uid, gid_t gid){ +#if OS_VXWORKS + return 0; +#else return geteuid() ? 0 : fchown(fd,uid,gid); +#endif } /* Forward reference */ @@ -374,7 +377,7 @@ static struct unix_syscall { { "read", (sqlite3_syscall_ptr)read, 0 }, #define osRead ((ssize_t(*)(int,void*,size_t))aSyscall[8].pCurrent) -#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE +#if defined(USE_PREAD) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) { "pread", (sqlite3_syscall_ptr)pread, 0 }, #else { "pread", (sqlite3_syscall_ptr)0, 0 }, @@ -391,7 +394,7 @@ static struct unix_syscall { { "write", (sqlite3_syscall_ptr)write, 0 }, #define osWrite ((ssize_t(*)(int,const void*,size_t))aSyscall[11].pCurrent) -#if defined(USE_PREAD) || SQLITE_ENABLE_LOCKING_STYLE +#if defined(USE_PREAD) || (SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS) { "pwrite", (sqlite3_syscall_ptr)pwrite, 0 }, #else { "pwrite", (sqlite3_syscall_ptr)0, 0 }, @@ -1295,7 +1298,11 @@ static int findInodeInfo( static int fileHasMoved(unixFile *pFile){ struct stat buf; return pFile->pInode!=0 && +#if OS_VXWORKS + pFile->pId!=pFile->pInode->fileId.Pid; +#else (osStat(pFile->zPath, &buf)!=0 || buf.st_ino!=pFile->pInode->fileId.ino); +#endif } @@ -2438,7 +2445,6 @@ static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { /* Otherwise see if some other process holds it. */ if( !reserved ){ sem_t *pSem = pFile->pInode->pSem; - struct stat statBuf; if( sem_trywait(pSem)==-1 ){ int tErrno = errno; @@ -2491,7 +2497,6 @@ static int semCheckReservedLock(sqlite3_file *id, int *pResOut) { */ static int semLock(sqlite3_file *id, int eFileLock) { unixFile *pFile = (unixFile*)id; - int fd; sem_t *pSem = pFile->pInode->pSem; int rc = SQLITE_OK; From 490fe86f1a74ca24703b7b63a656915d49180b72 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 11 Aug 2014 14:21:32 +0000 Subject: [PATCH 206/710] Fix harmless compiler warnings. FossilOrigin-Name: 52b03f045edf6fc29f9ba9a5cac53a59f0feb0eb --- ext/fts3/fts3_unicode.c | 2 +- ext/misc/fileio.c | 5 +---- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/shell.c | 5 +---- 5 files changed, 12 insertions(+), 18 deletions(-) diff --git a/ext/fts3/fts3_unicode.c b/ext/fts3/fts3_unicode.c index 57e7a4d3c9..94fc27b5b4 100644 --- a/ext/fts3/fts3_unicode.c +++ b/ext/fts3/fts3_unicode.c @@ -318,7 +318,7 @@ static int unicodeNext( ){ unicode_cursor *pCsr = (unicode_cursor *)pC; unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); - int iCode; + int iCode = 0; char *zOut; const unsigned char *z = &pCsr->aInput[pCsr->iOff]; const unsigned char *zStart = z; diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c index f80949bff2..fbe2d030c0 100644 --- a/ext/misc/fileio.c +++ b/ext/misc/fileio.c @@ -61,7 +61,6 @@ static void writefileFunc( ){ FILE *out; const char *z; - int n; sqlite3_int64 rc; const char *zFile; @@ -71,11 +70,9 @@ static void writefileFunc( if( out==0 ) return; z = (const char*)sqlite3_value_blob(argv[1]); if( z==0 ){ - n = 0; rc = 0; }else{ - n = sqlite3_value_bytes(argv[1]); - rc = fwrite(z, 1, n, out); + rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out); } fclose(out); sqlite3_result_int64(context, rc); diff --git a/manifest b/manifest index 8b2db457d5..0039113b6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sthat\swill\sperhaps\senable\sSQLite\sto\swork\sbetter\son\sVxWorks. -D 2014-08-11T13:53:30.969 +C Fix\sharmless\scompiler\swarnings. +D 2014-08-11T14:21:32.900 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -94,7 +94,7 @@ F ext/fts3/fts3_tokenize_vtab.c 011170fe9eba5ff062f1a31d3188e00267716706 F ext/fts3/fts3_tokenizer.c bbdc731bc91338050675c6d1da9ab82147391e16 F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 -F ext/fts3/fts3_unicode.c e80eef8a11f2020dc9c1eb95c5b405b9012f2fbe +F ext/fts3/fts3_unicode.c a93f5edc0aff44ef8b06d7cb55b52026541ca145 F ext/fts3/fts3_unicode2.c c3d01968d497bd7001e7dc774ba75b372738c057 F ext/fts3/fts3_write.c 8260388626516a7005d06a9dce94f9e55c6c2a41 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 @@ -109,7 +109,7 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 678056a4bfcd83c4e82dea81d37543cd1d6dbee1 F ext/misc/closure.c 636024302cde41b2bf0c542f81c40c624cfb7012 F ext/misc/compress.c 76e45655f4046e756064ab10c62e18f2eb846b9f -F ext/misc/fileio.c beea82bb5055b6590cffe2d2e6d922905894f691 +F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 136533c53cfce0957f0b48fa11dba27e21c5c01d F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 @@ -223,7 +223,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 44e2e434deaef2b41d4410ede695d867deb89886 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 1529c49075464c5a95fde77314073612b1b8d595 -F src/shell.c 4af71e7f25af6da6debda87dcbd6be37da710916 +F src/shell.c 75bb7bd2c80bb44861598f322a417c4bafe98fd7 F src/sqlite.h.in 66a923e0d8a3a771a7050f292cc8028b8c711652 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7556bd9aa5db1b9cd92d0e48effcda9fe96f3128 -R 07fe7ef4c70cea44dc683244e34b47a2 +P de27c742c0dcda20b51339598bf6094a8dcf5fb9 +R b22e48d145e96418fb870a5b38049ca4 U drh -Z c5b6316215c9354c3470fdb843673338 +Z 67a30f4afb77c4b424be11cabe826966 diff --git a/manifest.uuid b/manifest.uuid index b1e2fcea2d..832743ad81 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de27c742c0dcda20b51339598bf6094a8dcf5fb9 \ No newline at end of file +52b03f045edf6fc29f9ba9a5cac53a59f0feb0eb \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index e9ecf194b6..a74dcdb85d 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1694,7 +1694,6 @@ static void writefileFunc( ){ FILE *out; const char *z; - int n; sqlite3_int64 rc; const char *zFile; @@ -1704,11 +1703,9 @@ static void writefileFunc( if( out==0 ) return; z = (const char*)sqlite3_value_blob(argv[1]); if( z==0 ){ - n = 0; rc = 0; }else{ - n = sqlite3_value_bytes(argv[1]); - rc = fwrite(z, 1, n, out); + rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out); } fclose(out); sqlite3_result_int64(context, rc); From ddb17cae386ce5a4ccb920419e9065e7efd65324 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 11 Aug 2014 15:54:11 +0000 Subject: [PATCH 207/710] Updates to evidence marks and requirements. No changes to code. FossilOrigin-Name: 62d38308b519a5362f559b296a0cf1acccf8f673 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/legacy.c | 3 +++ src/main.c | 2 ++ src/resolve.c | 6 +++++- src/sqlite.h.in | 4 ++-- test/func3.test | 24 +++++++++++++++++------- 7 files changed, 40 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 0039113b6b..0781e2b0bb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2014-08-11T14:21:32.900 +C Updates\sto\sevidence\smarks\sand\srequirements.\s\sNo\schanges\sto\scode. +D 2014-08-11T15:54:11.189 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -186,10 +186,10 @@ F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 991e4964e9295da3993e2c0f81c7faf642371848 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d -F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 +F src/legacy.c febc2a9e7ad6c1a6191c7b5b9170b325d263f343 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 5cd4d5037e07af703d12035fdfdaf802abf2b7f8 +F src/main.c f86a887bf46a3df0058eae1af991a12f59e5d1ef F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -220,11 +220,11 @@ F src/pragma.c d10ef67c4de79f78188b965b4b7988aff1d66f2e F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c af06f66927919730f03479fed6ae9854f73419f4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c 44e2e434deaef2b41d4410ede695d867deb89886 +F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 1529c49075464c5a95fde77314073612b1b8d595 F src/shell.c 75bb7bd2c80bb44861598f322a417c4bafe98fd7 -F src/sqlite.h.in 66a923e0d8a3a771a7050f292cc8028b8c711652 +F src/sqlite.h.in ed9d35990c61f0388ca6405706455c4095310553 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 641f8fbb65ca2084c8df95b525f6f82c7a1e91ae @@ -589,7 +589,7 @@ F test/fts4unicode.test 01ec3fe2a7c3cfff3b4c0581b83caa11b33efa36 F test/full.test 6b3c8fb43c6beab6b95438c1675374b95fab245d F test/func.test ae97561957aba6ca9e3a7b8a13aac41830d701ef F test/func2.test 772d66227e4e6684b86053302e2d74a2500e1e0f -F test/func3.test e82d16b13739f1a0c5efb20048583c79b767b625 +F test/func3.test d202a7606d23f90988a664e88e268aed1087c11c F test/func4.test 6beacdfcb0e18c358e6c2dcacf1b65d1fa80955f F test/func5.test cdd224400bc3e48d891827cc913a57051a426fa4 F test/fuzz-oss1.test 4912e528ec9cf2f42134456933659d371c9e0d74 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P de27c742c0dcda20b51339598bf6094a8dcf5fb9 -R b22e48d145e96418fb870a5b38049ca4 +P 52b03f045edf6fc29f9ba9a5cac53a59f0feb0eb +R 8606a5b0f380536d0eae822aaf63b02e U drh -Z 67a30f4afb77c4b424be11cabe826966 +Z 15a1264271dfbd20eedca74e84319e8d diff --git a/manifest.uuid b/manifest.uuid index 832743ad81..a43bc55fab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -52b03f045edf6fc29f9ba9a5cac53a59f0feb0eb \ No newline at end of file +62d38308b519a5362f559b296a0cf1acccf8f673 \ No newline at end of file diff --git a/src/legacy.c b/src/legacy.c index 94649ae705..1913f0b5af 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -96,6 +96,9 @@ int sqlite3_exec( } } if( xCallback(pArg, nCol, azVals, azCols) ){ + /* EVIDENCE-OF: R-38229-40159 If the callback function to + ** sqlite3_exec() returns non-zero, then sqlite3_exec() will + ** return SQLITE_ABORT. */ rc = SQLITE_ABORT; sqlite3VdbeFinalize((Vdbe *)pStmt); pStmt = 0; diff --git a/src/main.c b/src/main.c index bf5c38a47e..d945106d73 100644 --- a/src/main.c +++ b/src/main.c @@ -827,6 +827,8 @@ static int connectionIsBusy(sqlite3 *db){ */ static int sqlite3Close(sqlite3 *db, int forceZombie){ if( !db ){ + /* EVIDENCE-OF: R-63257-11740 Calling sqlite3_close() or + ** sqlite3_close_v2() with a NULL pointer argument is a harmless no-op. */ return SQLITE_OK; } if( !sqlite3SafetyCheckSickOrOk(db) ){ diff --git a/src/resolve.c b/src/resolve.c index 0226d06462..935d311346 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -710,7 +710,11 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is equivalent to ** likelihood(X, 0.0625). ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is short-hand for - ** likelihood(X,0.0625). */ + ** likelihood(X,0.0625). + ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand for + ** likelihood(X,0.9375). + ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent to + ** likelihood(X,0.9375). */ /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ pExpr->iTable = pDef->zName[0]=='u' ? 62 : 938; } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c89cbc1f17..230f8d4028 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -272,7 +272,7 @@ typedef sqlite_uint64 sqlite3_uint64; ** statements or unfinished sqlite3_backup objects then sqlite3_close() ** will leave the database connection open and return [SQLITE_BUSY]. ** ^If sqlite3_close_v2() is called with unfinalized prepared statements -** and unfinished sqlite3_backups, then the database connection becomes +** and/or unfinished sqlite3_backups, then the database connection becomes ** an unusable "zombie" which will automatically be deallocated when the ** last prepared statement is finalized or the last sqlite3_backup is ** finished. The sqlite3_close_v2() interface is intended for use with @@ -285,7 +285,7 @@ typedef sqlite_uint64 sqlite3_uint64; ** with the [sqlite3] object prior to attempting to close the object. ^If ** sqlite3_close_v2() is called on a [database connection] that still has ** outstanding [prepared statements], [BLOB handles], and/or -** [sqlite3_backup] objects then it returns [SQLITE_OK] but the deallocation +** [sqlite3_backup] objects then it returns [SQLITE_OK] and the deallocation ** of resources is deferred until all [prepared statements], [BLOB handles], ** and [sqlite3_backup] objects are also destroyed. ** diff --git a/test/func3.test b/test/func3.test index b5a082a4ca..3b1613b56c 100644 --- a/test/func3.test +++ b/test/func3.test @@ -153,28 +153,38 @@ do_test func3-5.39 { db eval {EXPLAIN SELECT unlikely(min(1.0+'2.0',4*11))} } [db eval {EXPLAIN SELECT min(1.0+'2.0',4*11)}] -do_execsql_test func3-5.40 { + +# EVIDENCE-OF: R-23735-03107 The likely(X) function returns the argument +# X unchanged. +# +do_execsql_test func3-5.50 { SELECT likely(9223372036854775807); } {9223372036854775807} -do_execsql_test func3-5.41 { +do_execsql_test func3-5.51 { SELECT likely(-9223372036854775808); } {-9223372036854775808} -do_execsql_test func3-5.42 { +do_execsql_test func3-5.52 { SELECT likely(14.125); } {14.125} -do_execsql_test func3-5.43 { +do_execsql_test func3-5.53 { SELECT likely(NULL); } {{}} -do_execsql_test func3-5.44 { +do_execsql_test func3-5.54 { SELECT likely('test-string'); } {test-string} -do_execsql_test func3-5.45 { +do_execsql_test func3-5.55 { SELECT quote(likely(x'010203000405')); } {X'010203000405'} -do_test func3-5.49 { + +# EVIDENCE-OF: R-43464-09689 The likely(X) function is a no-op that the +# code generator optimizes away so that it consumes no CPU cycles at +# run-time (that is, during calls to sqlite3_step()). +# +do_test func3-5.59 { db eval {EXPLAIN SELECT likely(min(1.0+'2.0',4*11))} } [db eval {EXPLAIN SELECT min(1.0+'2.0',4*11)}] + finish_test From 9f959b07de5fcc58f5e9e470d3b741d3c9c9f2c2 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 11 Aug 2014 17:37:27 +0000 Subject: [PATCH 208/710] Add a few more requirements tests. FossilOrigin-Name: b5652439d5d770f0edeb80c8f55fa7cc515482e3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 2 +- test/e_expr.test | 7 +++++++ 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 0781e2b0bb..f7cef7e70f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sevidence\smarks\sand\srequirements.\s\sNo\schanges\sto\scode. -D 2014-08-11T15:54:11.189 +C Add\sa\sfew\smore\srequirements\stests. +D 2014-08-11T17:37:27.183 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -189,7 +189,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c febc2a9e7ad6c1a6191c7b5b9170b325d263f343 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c f86a887bf46a3df0058eae1af991a12f59e5d1ef +F src/main.c 1cf92c5c6468f2b6ed99b638706781ccc9c60b42 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -446,7 +446,7 @@ F test/e_createtable.test 181653f6f45e3adde73f8686600ce5ad7515466b F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 F test/e_dropview.test 0c9f7f60989164a70a67a9d9c26d1083bc808306 -F test/e_expr.test 5f4528f3dea1c3c401d19ab1d543ec7959e67464 +F test/e_expr.test 8f5fdd7261e2d746813b0c6a1c0e34824ad3c5ad F test/e_fkey.test a1783fe1f759e1990e6a11adfcf0702dac4d0707 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 F test/e_insert.test 7b2fa9cd1456f83474d6c5d27db3abaeb8be2023 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 52b03f045edf6fc29f9ba9a5cac53a59f0feb0eb -R 8606a5b0f380536d0eae822aaf63b02e +P 62d38308b519a5362f559b296a0cf1acccf8f673 +R 02be8e57f1690f8a6be367e01ea8228e U drh -Z 15a1264271dfbd20eedca74e84319e8d +Z 0a8f19538bd793702427af9b3a2d2400 diff --git a/manifest.uuid b/manifest.uuid index a43bc55fab..f63115401a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62d38308b519a5362f559b296a0cf1acccf8f673 \ No newline at end of file +b5652439d5d770f0edeb80c8f55fa7cc515482e3 \ No newline at end of file diff --git a/src/main.c b/src/main.c index d945106d73..cea72829df 100644 --- a/src/main.c +++ b/src/main.c @@ -2077,7 +2077,7 @@ static const int aHardLimit[] = { SQLITE_MAX_FUNCTION_ARG, SQLITE_MAX_ATTACHED, SQLITE_MAX_LIKE_PATTERN_LENGTH, - SQLITE_MAX_VARIABLE_NUMBER, + SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ SQLITE_MAX_TRIGGER_DEPTH, }; diff --git a/test/e_expr.test b/test/e_expr.test index 60e345441a..271635f944 100644 --- a/test/e_expr.test +++ b/test/e_expr.test @@ -1596,6 +1596,13 @@ do_expr_test e_expr-30.4.1 { CAST('' AS INTEGER) } integer 0 do_expr_test e_expr-30.4.2 { CAST('not a number' AS INTEGER) } integer 0 do_expr_test e_expr-30.4.3 { CAST('XXI' AS INTEGER) } integer 0 +# EVIDENCE-OF: R-08980-53124 The CAST operator understands decimal +# integers only — conversion of hexadecimal integers stops at +# the "x" in the "0x" prefix of the hexadecimal integer string and thus +# result of the CAST is always zero. +do_expr_test e_expr-30.5.1 { CAST('0x1234' AS INTEGER) } integer 0 +do_expr_test e_expr-30.5.2 { CAST('0X1234' AS INTEGER) } integer 0 + # EVIDENCE-OF: R-02752-50091 A cast of a REAL value into an INTEGER # results in the integer between the REAL value and zero that is closest # to the REAL value. From 91d1249748846897684d285466c809a602ca57dc Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 11 Aug 2014 17:38:38 +0000 Subject: [PATCH 209/710] Fix for #ifdef issue with GetVersionEx in the Win32 VFS. FossilOrigin-Name: 1a0d466dd48163c86de4774035fe4a30c1c55311 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 22 ++++++++++++++++++---- 3 files changed, 26 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index f7cef7e70f..8aa852cdf5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sfew\smore\srequirements\stests. -D 2014-08-11T17:37:27.183 +C Fix\sfor\s#ifdef\sissue\swith\sGetVersionEx\sin\sthe\sWin32\sVFS. +D 2014-08-11T17:38:38.381 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 17e7ab0f9160a78c964d615b15b1658ab2090d42 -F src/os_win.c 3fca1bfdf78338705bf536059a407d0fb04016d5 +F src/os_win.c d37c3e70e85f9c9bfb2e0dad1b1d8714077fbf8d F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 62d38308b519a5362f559b296a0cf1acccf8f673 -R 02be8e57f1690f8a6be367e01ea8228e -U drh -Z 0a8f19538bd793702427af9b3a2d2400 +P b5652439d5d770f0edeb80c8f55fa7cc515482e3 +R da55da95544cc788a74ff6acfb8e9051 +U mistachkin +Z 276e7597dfac53b7c7af380d74f891b3 diff --git a/manifest.uuid b/manifest.uuid index f63115401a..2da18ac2ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5652439d5d770f0edeb80c8f55fa7cc515482e3 \ No newline at end of file +1a0d466dd48163c86de4774035fe4a30c1c55311 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 5f1d84a3fb..0f391f9521 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -155,7 +155,7 @@ ** [sometimes] not used by the code (e.g. via conditional compilation). */ #ifndef UNUSED_VARIABLE_VALUE -# define UNUSED_VARIABLE_VALUE(x) (void)(x) +# define UNUSED_VARIABLE_VALUE(x) (void)(x) #endif /* @@ -1052,7 +1052,16 @@ static struct win_syscall { ** is really just a macro that uses a compiler intrinsic (e.g. x64). ** So do not try to make this is into a redefinable interface. */ +#if defined(InterlockedCompareExchange) + { "InterlockedCompareExchange", (SYSCALL)0, 0 }, + #define osInterlockedCompareExchange InterlockedCompareExchange +#else + { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, + +#define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG volatile*, \ + LONG,LONG))aSyscall[76].pCurrent) +#endif /* defined(InterlockedCompareExchange) */ }; /* End of the overrideable system calls */ @@ -1312,12 +1321,14 @@ void sqlite3_win32_sleep(DWORD milliseconds){ ** based on the NT kernel. */ int sqlite3_win32_is_nt(void){ +#if defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ -#if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WIN8 +#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ + defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WIN8 OSVERSIONINFOW sInfo; sInfo.dwOSVersionInfoSize = sizeof(sInfo); osGetVersionExW(&sInfo); -#else +#elif defined(SQLITE_WIN32_HAS_ANSI) OSVERSIONINFOA sInfo; sInfo.dwOSVersionInfoSize = sizeof(sInfo); osGetVersionExA(&sInfo); @@ -1326,6 +1337,9 @@ int sqlite3_win32_is_nt(void){ (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); } return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; +#else + return 1; +#endif } #ifdef SQLITE_WIN32_MALLOC @@ -5482,7 +5496,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==76 ); + assert( ArraySize(aSyscall)==77 ); /* get memory map allocation granularity */ memset(&winSysInfo, 0, sizeof(SYSTEM_INFO)); From c96c7e3c15195d6bd532a31215a7075db3601b0f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 11 Aug 2014 17:40:30 +0000 Subject: [PATCH 210/710] Clarify the comment explaining the meaning of the SQLITE_WIN32_GETVERSIONEX macro in the Windows VFS. FossilOrigin-Name: 69714287dbbdbf4ef2a2c5e99ed740ae7d27e051 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 12 ++++-------- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 8aa852cdf5..ab513ec919 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sfor\s#ifdef\sissue\swith\sGetVersionEx\sin\sthe\sWin32\sVFS. -D 2014-08-11T17:38:38.381 +C Clarify\sthe\scomment\sexplaining\sthe\smeaning\sof\sthe\sSQLITE_WIN32_GETVERSIONEX\nmacro\sin\sthe\sWindows\sVFS. +D 2014-08-11T17:40:30.234 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 17e7ab0f9160a78c964d615b15b1658ab2090d42 -F src/os_win.c d37c3e70e85f9c9bfb2e0dad1b1d8714077fbf8d +F src/os_win.c f8affe82a8b28990879a564c84d78c51aab4fa90 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b5652439d5d770f0edeb80c8f55fa7cc515482e3 -R da55da95544cc788a74ff6acfb8e9051 -U mistachkin -Z 276e7597dfac53b7c7af380d74f891b3 +P 1a0d466dd48163c86de4774035fe4a30c1c55311 +R 1a63d8cf8de05d642fa8f087d8e134e9 +U drh +Z 62728ef38d819e244d119f9b4cea00bc diff --git a/manifest.uuid b/manifest.uuid index 2da18ac2ad..c5ea92e575 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a0d466dd48163c86de4774035fe4a30c1c55311 \ No newline at end of file +69714287dbbdbf4ef2a2c5e99ed740ae7d27e051 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 0f391f9521..0ad87006bf 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -72,18 +72,14 @@ #endif /* -** Check if the GetVersionEx[AW] functions should be considered deprecated -** and avoid using them in that case. It should be noted here that if the -** value of the SQLITE_WIN32_GETVERSIONEX pre-processor macro is zero -** (whether via this block or via being manually specified), that implies -** the underlying operating system will always be based on the Windows NT -** Kernel. +** Check to see if the GetVersionEx[AW] functions are deprecated on the +** target. GetVersionEx[AW] were first deprecated in Win8.1. The */ #ifndef SQLITE_WIN32_GETVERSIONEX # if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WINBLUE -# define SQLITE_WIN32_GETVERSIONEX 0 +# define SQLITE_WIN32_GETVERSIONEX 0 /* GetVersionEx() is deprecated */ # else -# define SQLITE_WIN32_GETVERSIONEX 1 +# define SQLITE_WIN32_GETVERSIONEX 1 /* GetVersionEx() is current */ # endif #endif From e5e20d354d4b07cfcdf3fcc6e7c9eb19dc09a160 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 11 Aug 2014 17:41:53 +0000 Subject: [PATCH 211/710] Further clarification and typo fixes for the previous comment change. FossilOrigin-Name: 87ef9e2f293afd89901fed6be45b0f0052df6846 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ab513ec919..b027033737 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clarify\sthe\scomment\sexplaining\sthe\smeaning\sof\sthe\sSQLITE_WIN32_GETVERSIONEX\nmacro\sin\sthe\sWindows\sVFS. -D 2014-08-11T17:40:30.234 +C Further\sclarification\sand\stypo\sfixes\sfor\sthe\sprevious\scomment\schange. +D 2014-08-11T17:41:53.298 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 17e7ab0f9160a78c964d615b15b1658ab2090d42 -F src/os_win.c f8affe82a8b28990879a564c84d78c51aab4fa90 +F src/os_win.c 80e4b93cd46b6c4c6375c68faf61ef5cbb0a251a F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1a0d466dd48163c86de4774035fe4a30c1c55311 -R 1a63d8cf8de05d642fa8f087d8e134e9 +P 69714287dbbdbf4ef2a2c5e99ed740ae7d27e051 +R b95f2777181460436236a8c0d3ecfe02 U drh -Z 62728ef38d819e244d119f9b4cea00bc +Z 9211ee242443aa3aa6c7a0b0186317b6 diff --git a/manifest.uuid b/manifest.uuid index c5ea92e575..7c1d97ec7a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69714287dbbdbf4ef2a2c5e99ed740ae7d27e051 \ No newline at end of file +87ef9e2f293afd89901fed6be45b0f0052df6846 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 0ad87006bf..72ffa746d2 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -73,7 +73,7 @@ /* ** Check to see if the GetVersionEx[AW] functions are deprecated on the -** target. GetVersionEx[AW] were first deprecated in Win8.1. The +** target system. GetVersionEx was first deprecated in Win8.1. */ #ifndef SQLITE_WIN32_GETVERSIONEX # if defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WINBLUE From f0740a947d44f585ddfdf5e1fc9e0b91eed7e0df Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 11 Aug 2014 17:51:23 +0000 Subject: [PATCH 212/710] Modify GetVersionEx Win32 VFS fix for use with the test suite. FossilOrigin-Name: fd2221768b45d9006bbb2919a4977cf2791826d6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b027033737..49af3799cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sclarification\sand\stypo\sfixes\sfor\sthe\sprevious\scomment\schange. -D 2014-08-11T17:41:53.298 +C Modify\sGetVersionEx\sWin32\sVFS\sfix\sfor\suse\swith\sthe\stest\ssuite. +D 2014-08-11T17:51:23.396 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 17e7ab0f9160a78c964d615b15b1658ab2090d42 -F src/os_win.c 80e4b93cd46b6c4c6375c68faf61ef5cbb0a251a +F src/os_win.c e0260a7ba735b4a30b5eccf1a3ace9d6b9718204 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 69714287dbbdbf4ef2a2c5e99ed740ae7d27e051 -R b95f2777181460436236a8c0d3ecfe02 -U drh -Z 9211ee242443aa3aa6c7a0b0186317b6 +P 87ef9e2f293afd89901fed6be45b0f0052df6846 +R e0e522e37a27751d028bda9decd5982c +U mistachkin +Z eaad92a848e90bf0dd1d823726bec5b2 diff --git a/manifest.uuid b/manifest.uuid index 7c1d97ec7a..903be479d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -87ef9e2f293afd89901fed6be45b0f0052df6846 \ No newline at end of file +fd2221768b45d9006bbb2919a4977cf2791826d6 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 72ffa746d2..7d98490ab3 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1333,6 +1333,8 @@ int sqlite3_win32_is_nt(void){ (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); } return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; +#elif SQLITE_TEST + return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; #else return 1; #endif From fcf2a7757e9c95b62783dd231c814723b1e9f73c Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Aug 2014 01:23:07 +0000 Subject: [PATCH 213/710] Fix an assert() statement in the SELECT code generator that was incorrect following an OOM error. FossilOrigin-Name: a179e41e40dba4c19a488985f77777acd27b689d --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 49af3799cd..7a1fd203a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sGetVersionEx\sWin32\sVFS\sfix\sfor\suse\swith\sthe\stest\ssuite. -D 2014-08-11T17:51:23.396 +C Fix\san\sassert()\sstatement\sin\sthe\sSELECT\scode\sgenerator\sthat\swas\nincorrect\sfollowing\san\sOOM\serror. +D 2014-08-12T01:23:07.241 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/printf.c af06f66927919730f03479fed6ae9854f73419f4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c 1529c49075464c5a95fde77314073612b1b8d595 +F src/select.c ea48e891406ccdf748f3eb02893e056d134a0fea F src/shell.c 75bb7bd2c80bb44861598f322a417c4bafe98fd7 F src/sqlite.h.in ed9d35990c61f0388ca6405706455c4095310553 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1185,7 +1185,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 87ef9e2f293afd89901fed6be45b0f0052df6846 -R e0e522e37a27751d028bda9decd5982c -U mistachkin -Z eaad92a848e90bf0dd1d823726bec5b2 +P fd2221768b45d9006bbb2919a4977cf2791826d6 +R b6cc4377208f7ed66435f59cee05c5c1 +U drh +Z b6d2068c4cafb891d688b19239264b19 diff --git a/manifest.uuid b/manifest.uuid index 903be479d6..c78b961e4d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fd2221768b45d9006bbb2919a4977cf2791826d6 \ No newline at end of file +a179e41e40dba4c19a488985f77777acd27b689d \ No newline at end of file diff --git a/src/select.c b/src/select.c index f5456c893d..932874d8fe 100644 --- a/src/select.c +++ b/src/select.c @@ -706,7 +706,7 @@ static void selectInnerLoop( sqlite3VdbeChangeP4(v, -1, (const char *)pColl, P4_COLLSEQ); sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); } - assert( sqlite3VdbeCurrentAddr(v)==iJump ); + assert( sqlite3VdbeCurrentAddr(v)==iJump || pParse->db->mallocFailed ); sqlite3VdbeAddOp3(v, OP_Copy, regResult, regPrev, nResultCol-1); break; } From 793bd861fe53ab1e4d6ec105b268ec8bc2d9cae1 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 12 Aug 2014 09:36:08 +0000 Subject: [PATCH 214/710] Add a test to ensure that the problem fixed by [a179e41e40] does not recur. FossilOrigin-Name: 31356f2cae26278660e6bd360ad35e57261d977c --- manifest | 13 +++++++------ manifest.uuid | 2 +- test/mallocL.test | 43 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 51 insertions(+), 7 deletions(-) create mode 100644 test/mallocL.test diff --git a/manifest b/manifest index 7a1fd203a6..ecafc92b44 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert()\sstatement\sin\sthe\sSELECT\scode\sgenerator\sthat\swas\nincorrect\sfollowing\san\sOOM\serror. -D 2014-08-12T01:23:07.241 +C Add\sa\stest\sto\sensure\sthat\sthe\sproblem\sfixed\sby\s[a179e41e40]\sdoes\snot\srecur. +D 2014-08-12T09:36:08.939 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -696,6 +696,7 @@ F test/mallocH.test 79b65aed612c9b3ed2dcdaa727c85895fd1bfbdb F test/mallocI.test a88c2b9627c8506bf4703d8397420043a786cdb6 F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e F test/mallocK.test 3cff7c0f64735f6883bacdd294e45a6ed5714817 +F test/mallocL.test 252ddc7eb4fbf75364eab17b938816085ff1fc17 F test/malloc_common.tcl 58e54229c4132ef882a11fab6419ec4cd3073589 F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f @@ -1185,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fd2221768b45d9006bbb2919a4977cf2791826d6 -R b6cc4377208f7ed66435f59cee05c5c1 -U drh -Z b6d2068c4cafb891d688b19239264b19 +P a179e41e40dba4c19a488985f77777acd27b689d +R fc85cc8ab2d6ab6a18c32ae79fed8287 +U dan +Z 3ae13b57bd03139bdf57f8a227f614c2 diff --git a/manifest.uuid b/manifest.uuid index c78b961e4d..7def0f461f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a179e41e40dba4c19a488985f77777acd27b689d \ No newline at end of file +31356f2cae26278660e6bd360ad35e57261d977c \ No newline at end of file diff --git a/test/mallocL.test b/test/mallocL.test new file mode 100644 index 0000000000..6532ca5e14 --- /dev/null +++ b/test/mallocL.test @@ -0,0 +1,43 @@ +# 2014 August 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test script is designed to show that the assert() fix at +# [f1cb48f412] really is required. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix mallocL + +do_test 1.0 { + for {set i 0} {$i < 40} {incr i} { + lappend cols "c$i" + lappend vals $i + } + + execsql "CREATE TABLE t1([join $cols ,])" + execsql "CREATE INDEX i1 ON t1([join $cols ,])" + execsql "INSERT INTO t1 VALUES([join $vals ,])" +} {} + +for {set j 1} {$j < 40} {incr j} { + set ::sql "SELECT DISTINCT [join [lrange $cols 0 $j] ,] FROM t1" + do_faultsim_test 1.$j -faults oom* -body { + execsql $::sql + } -test { + faultsim_test_result [list 0 [lrange $::vals 0 $::j]] + } +} + + +finish_test + From 61ffea549087e3489eb59a80bc152ed6c9c65804 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Aug 2014 12:19:25 +0000 Subject: [PATCH 215/710] Fix typos in the VxWorks code of os_unix.c. FossilOrigin-Name: 19682e8fdc4a3b7884dba3e4387763e435ec16e6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index ecafc92b44..8868264630 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\sto\sensure\sthat\sthe\sproblem\sfixed\sby\s[a179e41e40]\sdoes\snot\srecur. -D 2014-08-12T09:36:08.939 +C Fix\stypos\sin\sthe\sVxWorks\scode\sof\sos_unix.c. +D 2014-08-12T12:19:25.484 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -207,7 +207,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 17e7ab0f9160a78c964d615b15b1658ab2090d42 +F src/os_unix.c 119f4f1fb94e74373d53e4946d33e4bc31e91b26 F src/os_win.c e0260a7ba735b4a30b5eccf1a3ace9d6b9718204 F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a179e41e40dba4c19a488985f77777acd27b689d -R fc85cc8ab2d6ab6a18c32ae79fed8287 -U dan -Z 3ae13b57bd03139bdf57f8a227f614c2 +P 31356f2cae26278660e6bd360ad35e57261d977c +R 3b3addf1401dd2b7481a8053af2cf7fe +U drh +Z 2e7aab7a2b3236c3b8e957ada63bd33e diff --git a/manifest.uuid b/manifest.uuid index 7def0f461f..fda807ffed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -31356f2cae26278660e6bd360ad35e57261d977c \ No newline at end of file +19682e8fdc4a3b7884dba3e4387763e435ec16e6 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index feb9314167..d8b5db7680 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -1296,12 +1296,12 @@ static int findInodeInfo( ** Return TRUE if pFile has been renamed or unlinked since it was first opened. */ static int fileHasMoved(unixFile *pFile){ +#if OS_VXWORKS + return pFile->pInode!=0 && pFile->pId!=pFile->pInode->fileId.pId; +#else struct stat buf; return pFile->pInode!=0 && -#if OS_VXWORKS - pFile->pId!=pFile->pInode->fileId.Pid; -#else - (osStat(pFile->zPath, &buf)!=0 || buf.st_ino!=pFile->pInode->fileId.ino); + (osStat(pFile->zPath, &buf)!=0 || buf.st_ino!=pFile->pInode->fileId.ino); #endif } From 76ccd89d13590fad2c34922753c75efe120c3491 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 12 Aug 2014 13:38:52 +0000 Subject: [PATCH 216/710] If SQLITE_TEST_REALLOC_STRESS is defined, extend the op-code array used by virtual-machine programs by one element at a time, instead of doubling its size with each realloc(). FossilOrigin-Name: 4c291827224b84487a38e7ccba2edabc0f15b5ba --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/vdbeaux.c | 26 +++++++++++++++++++------- test/incrblob_err.test | 1 + test/malloc.test | 1 + test/malloc_common.tcl | 1 + 6 files changed, 33 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 8868264630..60d54fa586 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypos\sin\sthe\sVxWorks\scode\sof\sos_unix.c. -D 2014-08-12T12:19:25.484 +C If\sSQLITE_TEST_REALLOC_STRESS\sis\sdefined,\sextend\sthe\sop-code\sarray\sused\sby\svirtual-machine\sprograms\sby\sone\selement\sat\sa\stime,\sinstead\sof\sdoubling\sits\ssize\swith\seach\srealloc(). +D 2014-08-12T13:38:52.837 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -287,7 +287,7 @@ F src/vdbe.c cd8d7e3ecd3e0e0684f6bf48469966335c666883 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 -F src/vdbeaux.c bbe934b0d472c98f57433829db91fc052e90fa17 +F src/vdbeaux.c d83ca171e5df5fdea95e61b6e935d78c02b9d982 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 @@ -613,7 +613,7 @@ F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 F test/incrblob4.test f26502a5697893e5acea268c910f16478c2f0fab -F test/incrblob_err.test d2562d2771ebffd4b3af89ef64c140dd44371597 +F test/incrblob_err.test af1f12ba60d220c9752073ff2bda2ad59e88960d F test/incrblobfault.test 280474078f6da9e732cd2a215d3d854969014b6e F test/incrvacuum.test d2a6ddf5e429720b5fe502766af747915ccf6c32 F test/incrvacuum2.test 379eeb8740b0ef60c372c439ad4cbea20b34bb9b @@ -676,7 +676,7 @@ F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 F test/lookaside.test 93f07bac140c5bb1d49f3892d2684decafdc7af2 F test/main.test 39c4bb8a157f57298ed1659d6df89d9f35aaf2c8 F test/make-where7.tcl 05c16b5d4f5d6512881dfec560cb793915932ef9 -F test/malloc.test fd368e31fe98d4779ed80442f311ed9f03bcd1f7 +F test/malloc.test 4eb83876dfe4915766c179b687b8640437f14abf F test/malloc3.test e3b32c724b5a124b57cb0ed177f675249ad0c66a F test/malloc4.test 957337613002b7058a85116493a262f679f3a261 F test/malloc5.test fafce0aa9157060445cd1a56ad50fc79d82f28c3 @@ -697,7 +697,7 @@ F test/mallocI.test a88c2b9627c8506bf4703d8397420043a786cdb6 F test/mallocJ.test b5d1839da331d96223e5f458856f8ffe1366f62e F test/mallocK.test 3cff7c0f64735f6883bacdd294e45a6ed5714817 F test/mallocL.test 252ddc7eb4fbf75364eab17b938816085ff1fc17 -F test/malloc_common.tcl 58e54229c4132ef882a11fab6419ec4cd3073589 +F test/malloc_common.tcl 3663f9001ce3e29bbaa9677ffe15cd468e3ec7e3 F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test fcb5297b321b562084fc79d64d5a12a1cd2b639b @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 31356f2cae26278660e6bd360ad35e57261d977c -R 3b3addf1401dd2b7481a8053af2cf7fe -U drh -Z 2e7aab7a2b3236c3b8e957ada63bd33e +P 19682e8fdc4a3b7884dba3e4387763e435ec16e6 +R c6c0e18a93e2092b23f216934f9dfe20 +U dan +Z 311715db7c795982c324ba776d8b3111 diff --git a/manifest.uuid b/manifest.uuid index fda807ffed..4788e494de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19682e8fdc4a3b7884dba3e4387763e435ec16e6 \ No newline at end of file +4c291827224b84487a38e7ccba2edabc0f15b5ba \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index dba11eed67..62bde7da84 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -84,18 +84,30 @@ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ } /* -** Resize the Vdbe.aOp array so that it is at least one op larger than -** it was. +** Resize the Vdbe.aOp array so that it is at least nOp elements larger +** its current size. nOp is guaranteed to be less than or equal to 1024. ** ** If an out-of-memory error occurs while resizing the array, return -** SQLITE_NOMEM. In this case Vdbe.aOp and Vdbe.nOpAlloc remain +** SQLITE_NOMEM. In this case Vdbe.aOp and Parse.nOpAlloc remain ** unchanged (this is so that any opcodes already allocated can be ** correctly deallocated along with the rest of the Vdbe). */ -static int growOpArray(Vdbe *v){ +static int growOpArray(Vdbe *v, int nOp){ VdbeOp *pNew; Parse *p = v->pParse; + + /* If SQLITE_TEST_REALLOC_STRESS is defined and the current op array is + ** less than 512 entries in size, grow the op array by the minimum amount + ** required. Otherwise, allocate either double the current size of the + ** array or 1KB of space, whichever is smaller. */ +#ifdef SQLITE_TEST_REALLOC_STRESS + int nNew = (p->nOpAlloc>=512 ? p->nOpAlloc*2 : p->nOpAlloc+nOp); +#else int nNew = (p->nOpAlloc ? p->nOpAlloc*2 : (int)(1024/sizeof(Op))); + UNUSED_PARAMETER(nOp); +#endif + + assert( nNew>=(p->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); if( pNew ){ p->nOpAlloc = sqlite3DbMallocSize(p->db, pNew)/sizeof(Op); @@ -139,7 +151,7 @@ int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ assert( p->magic==VDBE_MAGIC_INIT ); assert( op>0 && op<0xff ); if( p->pParse->nOpAlloc<=i ){ - if( growOpArray(p) ){ + if( growOpArray(p, 1) ){ return 1; } } @@ -541,7 +553,7 @@ VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg){ int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ int addr; assert( p->magic==VDBE_MAGIC_INIT ); - if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p) ){ + if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){ return 0; } addr = p->nOp; @@ -726,7 +738,7 @@ void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ ** Change the opcode at addr into OP_Noop */ void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ - if( p->aOp ){ + if( addrnOp ){ VdbeOp *pOp = &p->aOp[addr]; sqlite3 *db = p->db; freeP4(db, pOp->p4type, pOp->p4.p); diff --git a/test/incrblob_err.test b/test/incrblob_err.test index 35d37fee92..a08bea3e4e 100644 --- a/test/incrblob_err.test +++ b/test/incrblob_err.test @@ -14,6 +14,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set ::testprefix incrblob_err ifcapable {!incrblob || !memdebug || !tclvar} { finish_test diff --git a/test/malloc.test b/test/malloc.test index 5d03aa8fe8..4276b58bba 100644 --- a/test/malloc.test +++ b/test/malloc.test @@ -20,6 +20,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set ::testprefix malloc # Only run these tests if memory debugging is turned on. diff --git a/test/malloc_common.tcl b/test/malloc_common.tcl index 160897057a..b586c88d1f 100644 --- a/test/malloc_common.tcl +++ b/test/malloc_common.tcl @@ -409,6 +409,7 @@ proc do_malloc_test {tn args} { if {[string is integer $tn]} { set tn malloc-$tn + catch { set tn $::testprefix-$tn } } if {[info exists ::mallocopts(-start)]} { set start $::mallocopts(-start) From d79d27aedade556f73c98daa1b91b098fa2e1899 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 12 Aug 2014 14:06:13 +0000 Subject: [PATCH 217/710] Run a test with TEST_REALLOC_STRESS and OMIT_LOOKASIDE defined as part of releasetest.tcl on Linux/x86-64. FossilOrigin-Name: a1baf3a7b177728cdfcd6d9345a0d6bf0a8887c0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/releasetest.tcl | 7 +++++++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 60d54fa586..90eaca8c90 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sSQLITE_TEST_REALLOC_STRESS\sis\sdefined,\sextend\sthe\sop-code\sarray\sused\sby\svirtual-machine\sprograms\sby\sone\selement\sat\sa\stime,\sinstead\sof\sdoubling\sits\ssize\swith\seach\srealloc(). -D 2014-08-12T13:38:52.837 +C Run\sa\stest\swith\sTEST_REALLOC_STRESS\sand\sOMIT_LOOKASIDE\sdefined\sas\spart\sof\sreleasetest.tcl\son\sLinux/x86-64. +D 2014-08-12T14:06:13.039 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -773,7 +773,7 @@ F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a -F test/releasetest.tcl 06d289d8255794073a58d2850742f627924545ce +F test/releasetest.tcl a0df0dfc5e3ee83ade87b9cc96db41b52d590b9e F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 19682e8fdc4a3b7884dba3e4387763e435ec16e6 -R c6c0e18a93e2092b23f216934f9dfe20 +P 4c291827224b84487a38e7ccba2edabc0f15b5ba +R 1284079e980f8822b336e39d17ba6126 U dan -Z 311715db7c795982c324ba776d8b3111 +Z b89cd6a3a53bd57f0d4f91bd2d9a8d6b diff --git a/manifest.uuid b/manifest.uuid index 4788e494de..e3d0ee73b5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c291827224b84487a38e7ccba2edabc0f15b5ba \ No newline at end of file +a1baf3a7b177728cdfcd6d9345a0d6bf0a8887c0 \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index b635061c55..eb2e440013 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -176,6 +176,12 @@ array set ::Configs { -DSQLITE_DISABLE_FTS4_DEFERRED -DSQLITE_ENABLE_RTREE } + + "No-lookaside" { + -DSQLITE_TEST_REALLOC_STRESS=1 + -DSQLITE_OMIT_LOOKASIDE=1 + -DHAVE_USLEEP=1 + } } array set ::Platforms { @@ -188,6 +194,7 @@ array set ::Platforms { "Extra-Robustness" test "Device-Two" test "Ftrapv" test + "No-lookaside" test "Default" "threadtest test" "Device-One" fulltest } From 81e069eee5bd94f57c8756cb4195deebd74763b2 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 12 Aug 2014 14:29:20 +0000 Subject: [PATCH 218/710] Improve the comments associated with SQLITE_TEST_REALLOC_STRESS and add an extra assert() to prove an assumption. FossilOrigin-Name: 35c454616321d480ecbc4efdf6869bbcdf0d3aa2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 15 ++++++++++----- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 90eaca8c90..04ef69ea7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Run\sa\stest\swith\sTEST_REALLOC_STRESS\sand\sOMIT_LOOKASIDE\sdefined\sas\spart\sof\sreleasetest.tcl\son\sLinux/x86-64. -D 2014-08-12T14:06:13.039 +C Improve\sthe\scomments\sassociated\swith\sSQLITE_TEST_REALLOC_STRESS\sand\sadd\nan\sextra\sassert()\sto\sprove\san\sassumption. +D 2014-08-12T14:29:20.012 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -287,7 +287,7 @@ F src/vdbe.c cd8d7e3ecd3e0e0684f6bf48469966335c666883 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 -F src/vdbeaux.c d83ca171e5df5fdea95e61b6e935d78c02b9d982 +F src/vdbeaux.c 25d62ef82cf1be2a1255eacac636fa0d943d8b3d F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4c291827224b84487a38e7ccba2edabc0f15b5ba -R 1284079e980f8822b336e39d17ba6126 -U dan -Z b89cd6a3a53bd57f0d4f91bd2d9a8d6b +P a1baf3a7b177728cdfcd6d9345a0d6bf0a8887c0 +R f180a595651acb4f056167092db3a8a0 +U drh +Z 0f6249f5f46f843c4c724ed2717337d8 diff --git a/manifest.uuid b/manifest.uuid index e3d0ee73b5..7672239134 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a1baf3a7b177728cdfcd6d9345a0d6bf0a8887c0 \ No newline at end of file +35c454616321d480ecbc4efdf6869bbcdf0d3aa2 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 62bde7da84..fb3f7c3a8c 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -85,7 +85,8 @@ void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ /* ** Resize the Vdbe.aOp array so that it is at least nOp elements larger -** its current size. nOp is guaranteed to be less than or equal to 1024. +** than its current size. nOp is guaranteed to be less than or equal +** to 1024/sizeof(Op). ** ** If an out-of-memory error occurs while resizing the array, return ** SQLITE_NOMEM. In this case Vdbe.aOp and Parse.nOpAlloc remain @@ -96,10 +97,13 @@ static int growOpArray(Vdbe *v, int nOp){ VdbeOp *pNew; Parse *p = v->pParse; - /* If SQLITE_TEST_REALLOC_STRESS is defined and the current op array is - ** less than 512 entries in size, grow the op array by the minimum amount - ** required. Otherwise, allocate either double the current size of the - ** array or 1KB of space, whichever is smaller. */ + /* The SQLITE_TEST_REALLOC_STRESS compile-time option is designed to force + ** more frequent reallocs and hence provide more opportunities for + ** simulated OOM faults. SQLITE_TEST_REALLOC_STRESS is generally used + ** during testing only. With SQLITE_TEST_REALLOC_STRESS grow the op array + ** by the minimum* amount required until the size reaches 512. Normal + ** operation (without SQLITE_TEST_REALLOC_STRESS) is to double the current + ** size of the op array or add 1KB of space, whichever is smaller. */ #ifdef SQLITE_TEST_REALLOC_STRESS int nNew = (p->nOpAlloc>=512 ? p->nOpAlloc*2 : p->nOpAlloc+nOp); #else @@ -107,6 +111,7 @@ static int growOpArray(Vdbe *v, int nOp){ UNUSED_PARAMETER(nOp); #endif + assert( nOp<=(1024/sizeof(Op)) ); assert( nNew>=(p->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); if( pNew ){ From 4eb4fefe2dd2c451b7f6ec8e73cf06ba73a83407 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 12 Aug 2014 16:13:37 +0000 Subject: [PATCH 219/710] Fix compilation issue in the Win32 VFS when manually defining SQLITE_WIN32_NO_ANSI. FossilOrigin-Name: 6715991296886c2a02b9a285a1e61189ad1f79c0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 4 +++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 04ef69ea7d..353ccbf28d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\scomments\sassociated\swith\sSQLITE_TEST_REALLOC_STRESS\sand\sadd\nan\sextra\sassert()\sto\sprove\san\sassumption. -D 2014-08-12T14:29:20.012 +C Fix\scompilation\sissue\sin\sthe\sWin32\sVFS\swhen\smanually\sdefining\sSQLITE_WIN32_NO_ANSI. +D 2014-08-12T16:13:37.205 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 119f4f1fb94e74373d53e4946d33e4bc31e91b26 -F src/os_win.c e0260a7ba735b4a30b5eccf1a3ace9d6b9718204 +F src/os_win.c 1c936c7b0659d0eb12b868e2cd710a570e78873e F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a1baf3a7b177728cdfcd6d9345a0d6bf0a8887c0 -R f180a595651acb4f056167092db3a8a0 -U drh -Z 0f6249f5f46f843c4c724ed2717337d8 +P 35c454616321d480ecbc4efdf6869bbcdf0d3aa2 +R b496e5fc455ad86aa746ec5d2b99190f +U mistachkin +Z fae056f6598b8171729329dfe1ec3668 diff --git a/manifest.uuid b/manifest.uuid index 7672239134..47dd9e4073 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35c454616321d480ecbc4efdf6869bbcdf0d3aa2 \ No newline at end of file +6715991296886c2a02b9a285a1e61189ad1f79c0 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 7d98490ab3..f479de3a1b 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1324,13 +1324,15 @@ int sqlite3_win32_is_nt(void){ OSVERSIONINFOW sInfo; sInfo.dwOSVersionInfoSize = sizeof(sInfo); osGetVersionExW(&sInfo); + osInterlockedCompareExchange(&sqlite3_os_type, + (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); #elif defined(SQLITE_WIN32_HAS_ANSI) OSVERSIONINFOA sInfo; sInfo.dwOSVersionInfoSize = sizeof(sInfo); osGetVersionExA(&sInfo); -#endif osInterlockedCompareExchange(&sqlite3_os_type, (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); +#endif } return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; #elif SQLITE_TEST From 299b24688aa73b1921ae09e03d88ad54f6bf86f2 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 12 Aug 2014 20:13:22 +0000 Subject: [PATCH 220/710] Fix where9.test so that it works with the "no_optimization" permutation. FossilOrigin-Name: d46adf9d8f741838b9c30f915231d6a3986c3591 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/where9.test | 5 +++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 353ccbf28d..8f0418df43 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompilation\sissue\sin\sthe\sWin32\sVFS\swhen\smanually\sdefining\sSQLITE_WIN32_NO_ANSI. -D 2014-08-12T16:13:37.205 +C Fix\swhere9.test\sso\sthat\sit\sworks\swith\sthe\s"no_optimization"\spermutation. +D 2014-08-12T20:13:22.252 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1108,7 +1108,7 @@ F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5a4b0abc207d71da4deecd734ad8579e8dd40aa8 F test/where8.test 806f1dcec4088be2b826b33f757fe6e17c3236a1 F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 -F test/where9.test 4f3eab951353a3ae164befc521c777dfa903e46c +F test/where9.test 729c3ba9b47e8f9f1aab96bae7dad2a524f1d1a2 F test/whereA.test 4d253178d135ec46d1671e440cd8f2b916aa6e6b F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 35c454616321d480ecbc4efdf6869bbcdf0d3aa2 -R b496e5fc455ad86aa746ec5d2b99190f -U mistachkin -Z fae056f6598b8171729329dfe1ec3668 +P 6715991296886c2a02b9a285a1e61189ad1f79c0 +R 710d8f15d7faf7079d5cbb8c78df8572 +U dan +Z 37c0d23955a964bca892f819d09ecb97 diff --git a/manifest.uuid b/manifest.uuid index 47dd9e4073..e6ce5cc412 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6715991296886c2a02b9a285a1e61189ad1f79c0 \ No newline at end of file +d46adf9d8f741838b9c30f915231d6a3986c3591 \ No newline at end of file diff --git a/test/where9.test b/test/where9.test index 1c32ff82f7..d073074d43 100644 --- a/test/where9.test +++ b/test/where9.test @@ -781,7 +781,12 @@ do_test where9-6.8.2 { OR (b NOT NULL AND c NOT NULL AND d IS NULL) } } {1 {no query solution}} + +set solution_possible 0 ifcapable stat4||stat3 { + if {[permutation] != "no_optimization"} { set solution_possible 1 } +} +if $solution_possible { # When STAT3 is enabled, the "b NOT NULL" terms get translated # into b>NULL, which can be satified by the index t1b. It is a very # expensive way to do the query, but it works, and so a solution is possible. From bd9455457a5d752399c288f203439f8ce3105c13 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 13 Aug 2014 11:39:42 +0000 Subject: [PATCH 221/710] Minor change to unixDelete for VxWorks with a DOS filesystem. FossilOrigin-Name: f01d42cc8b00d2b7c4f14defcb05fdc493cf1bfd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 6 +++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8f0418df43..2718e0bc1a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\swhere9.test\sso\sthat\sit\sworks\swith\sthe\s"no_optimization"\spermutation. -D 2014-08-12T20:13:22.252 +C Minor\schange\sto\sunixDelete\sfor\sVxWorks\swith\sa\sDOS\sfilesystem. +D 2014-08-13T11:39:42.875 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -207,7 +207,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 119f4f1fb94e74373d53e4946d33e4bc31e91b26 +F src/os_unix.c bd7df3094a60915c148517504c76df4fca24e542 F src/os_win.c 1c936c7b0659d0eb12b868e2cd710a570e78873e F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6715991296886c2a02b9a285a1e61189ad1f79c0 -R 710d8f15d7faf7079d5cbb8c78df8572 -U dan -Z 37c0d23955a964bca892f819d09ecb97 +P d46adf9d8f741838b9c30f915231d6a3986c3591 +R 0bcddbf637a1ae28dd56c0df43bdd1a6 +U drh +Z a0023172094bb1f589643de5ae500c6e diff --git a/manifest.uuid b/manifest.uuid index e6ce5cc412..12fc1aa48f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d46adf9d8f741838b9c30f915231d6a3986c3591 \ No newline at end of file +f01d42cc8b00d2b7c4f14defcb05fdc493cf1bfd \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index d8b5db7680..b1a0bedcff 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5883,7 +5883,11 @@ static int unixDelete( UNUSED_PARAMETER(NotUsed); SimulateIOError(return SQLITE_IOERR_DELETE); if( osUnlink(zPath)==(-1) ){ - if( errno==ENOENT ){ + if( errno==ENOENT +#if OS_VXWORKS + || errno==0x380003 +#endif + ){ rc = SQLITE_IOERR_DELETE_NOENT; }else{ rc = unixLogError(SQLITE_IOERR_DELETE, "unlink", zPath); From cf4312c9adc64c68b5927c09df5a32a030333da4 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 14 Aug 2014 02:59:51 +0000 Subject: [PATCH 222/710] Add icon to the Windows shell executable. This is a resource change only, no changes to code. FossilOrigin-Name: f5cce9db109a2bbe9a05b4c01b076a51d5fad8ba --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite3.rc | 8 ++++++++ 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2718e0bc1a..d4223a7aff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schange\sto\sunixDelete\sfor\sVxWorks\swith\sa\sDOS\sfilesystem. -D 2014-08-13T11:39:42.875 +C Add\sicon\sto\sthe\sWindows\sshell\sexecutable.\s\sThis\sis\sa\sresource\schange\sonly,\sno\schanges\sto\scode. +D 2014-08-14T02:59:51.389 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -225,7 +225,7 @@ F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c ea48e891406ccdf748f3eb02893e056d134a0fea F src/shell.c 75bb7bd2c80bb44861598f322a417c4bafe98fd7 F src/sqlite.h.in ed9d35990c61f0388ca6405706455c4095310553 -F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e +F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 641f8fbb65ca2084c8df95b525f6f82c7a1e91ae F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d46adf9d8f741838b9c30f915231d6a3986c3591 -R 0bcddbf637a1ae28dd56c0df43bdd1a6 -U drh -Z a0023172094bb1f589643de5ae500c6e +P f01d42cc8b00d2b7c4f14defcb05fdc493cf1bfd +R 3949cef84e476358af94bafdce6dc1cb +U mistachkin +Z 4ed18721cd3f6e17fb0699ef60e0d747 diff --git a/manifest.uuid b/manifest.uuid index 12fc1aa48f..ce1b5bf024 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f01d42cc8b00d2b7c4f14defcb05fdc493cf1bfd \ No newline at end of file +f5cce9db109a2bbe9a05b4c01b076a51d5fad8ba \ No newline at end of file diff --git a/src/sqlite3.rc b/src/sqlite3.rc index aedbb63ebd..04dd086488 100644 --- a/src/sqlite3.rc +++ b/src/sqlite3.rc @@ -35,6 +35,14 @@ LANGUAGE LANG_ENGLISH, SUBLANG_ENGLISH_US #pragma code_page(1252) #endif /* defined(_WIN32) */ +/* + * Icon + */ + +#define IDI_SQLITE 101 + +IDI_SQLITE ICON "..\\art\\sqlite370.ico" + /* * Version */ From 4ed2fb9d84bdc76c3085c0e8baf80a53d8d57d10 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 14 Aug 2014 13:06:25 +0000 Subject: [PATCH 223/710] Fix typos in comments used to help generate documentation. No changes to code. FossilOrigin-Name: 13a2d90a2869c53b79754de39045bbbdbc7688e3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d4223a7aff..a9e401b96b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sicon\sto\sthe\sWindows\sshell\sexecutable.\s\sThis\sis\sa\sresource\schange\sonly,\sno\schanges\sto\scode. -D 2014-08-14T02:59:51.389 +C Fix\stypos\sin\scomments\sused\sto\shelp\sgenerate\sdocumentation.\s\sNo\schanges\sto\scode. +D 2014-08-14T13:06:25.245 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c cd8d7e3ecd3e0e0684f6bf48469966335c666883 +F src/vdbe.c f7f4066e4d6e3858878d76ce9288ea603e12ddf6 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f01d42cc8b00d2b7c4f14defcb05fdc493cf1bfd -R 3949cef84e476358af94bafdce6dc1cb -U mistachkin -Z 4ed18721cd3f6e17fb0699ef60e0d747 +P f5cce9db109a2bbe9a05b4c01b076a51d5fad8ba +R b4842944a4c3679f5b3af787e9b322df +U drh +Z 52db703ea373cd617fd2c598672c0ab8 diff --git a/manifest.uuid b/manifest.uuid index ce1b5bf024..5d1e011162 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f5cce9db109a2bbe9a05b4c01b076a51d5fad8ba \ No newline at end of file +13a2d90a2869c53b79754de39045bbbdbc7688e3 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 10b81a7992..61adb9cccd 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3531,7 +3531,7 @@ case OP_Close: { ** the key and P2 is not zero, then jump to P2. ** ** This opcode leaves the cursor configured to move in forward order, -** from the begining toward the end. In other words, the cursor is +** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. ** ** See also: Found, NotFound, SeekLt, SeekGe, SeekLe @@ -4548,7 +4548,7 @@ case OP_Sort: { /* jump */ ** to the following instruction. ** ** This opcode leaves the cursor configured to move in forward order, -** from the begining toward the end. In other words, the cursor is +** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. */ case OP_Rewind: { /* jump */ From ce64d6105016e56ea60c7f02534eb44cdf505a77 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 14 Aug 2014 18:31:56 +0000 Subject: [PATCH 224/710] Fix compiler warnings on WinCE. FossilOrigin-Name: cc910b8e0c45e7387024f3a729003e2fef08b198 --- manifest | 21 ++++++++++++--------- manifest.uuid | 2 +- src/mutex_w32.c | 2 +- src/os_win.c | 8 ++++---- src/os_win.h | 10 ++++++++++ 5 files changed, 28 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index a9e401b96b..44902dc52f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypos\sin\scomments\sused\sto\shelp\sgenerate\sdocumentation.\s\sNo\schanges\sto\scode. -D 2014-08-14T13:06:25.245 +C Fix\scompiler\swarnings\son\sWinCE. +D 2014-08-14T18:31:56.425 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -201,15 +201,15 @@ F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 -F src/mutex_w32.c c50939b72368f1cfbddb58520372081a50558548 +F src/mutex_w32.c 06bfff9a3a83b53389a51a967643db3967032e1e F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bd7df3094a60915c148517504c76df4fca24e542 -F src/os_win.c 1c936c7b0659d0eb12b868e2cd710a570e78873e -F src/os_win.h 057344a6720b4c8405d9bd98f58cb37a6ee46c25 +F src/os_win.c d067fce558a5032e6e6afe62899e5397bf63cf3e +F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 @@ -1186,7 +1186,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f5cce9db109a2bbe9a05b4c01b076a51d5fad8ba -R b4842944a4c3679f5b3af787e9b322df -U drh -Z 52db703ea373cd617fd2c598672c0ab8 +P 13a2d90a2869c53b79754de39045bbbdbc7688e3 +R 9127229654219d70056c71e11a62fce3 +T *branch * winCeWarn +T *sym-winCeWarn * +T -sym-trunk * +U mistachkin +Z 13b8e5c3b082e69ddfa9c76704e89cbc diff --git a/manifest.uuid b/manifest.uuid index 5d1e011162..8ce24171cb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13a2d90a2869c53b79754de39045bbbdbc7688e3 \ No newline at end of file +cc910b8e0c45e7387024f3a729003e2fef08b198 \ No newline at end of file diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 218342d2b9..da7d73f7c5 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -99,7 +99,7 @@ static int winMutex_isNt = -1; /* <0 means "need to query" */ ** of the sqlite3_initialize() and sqlite3_shutdown() processing, the ** "interlocked" magic used here is probably not strictly necessary. */ -static LONG volatile winMutex_lock = 0; +static LONG SQLITE_WIN32_VOLATILE winMutex_lock = 0; int sqlite3_win32_is_nt(void); /* os_win.c */ void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ diff --git a/src/os_win.c b/src/os_win.c index f479de3a1b..b9f13becd6 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -410,9 +410,9 @@ const sqlite3_mem_methods *sqlite3MemGetWin32(void); ** can manually set this value to 1 to emulate Win98 behavior. */ #ifdef SQLITE_TEST -LONG volatile sqlite3_os_type = 0; +LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; #else -static LONG volatile sqlite3_os_type = 0; +static LONG SQLITE_WIN32_VOLATILE sqlite3_os_type = 0; #endif #ifndef SYSCALL @@ -1055,8 +1055,8 @@ static struct win_syscall { #else { "InterlockedCompareExchange", (SYSCALL)InterlockedCompareExchange, 0 }, -#define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG volatile*, \ - LONG,LONG))aSyscall[76].pCurrent) +#define osInterlockedCompareExchange ((LONG(WINAPI*)(LONG \ + SQLITE_WIN32_VOLATILE*, LONG,LONG))aSyscall[76].pCurrent) #endif /* defined(InterlockedCompareExchange) */ }; /* End of the overrideable system calls */ diff --git a/src/os_win.h b/src/os_win.h index d662cd467d..5174ac7781 100644 --- a/src/os_win.h +++ b/src/os_win.h @@ -64,4 +64,14 @@ # define SQLITE_OS_WINRT 0 #endif +/* +** For WinCE, some API function parameters do not appear to be declared as +** volatile. +*/ +#if SQLITE_OS_WINCE +# define SQLITE_WIN32_VOLATILE +#else +# define SQLITE_WIN32_VOLATILE volatile +#endif + #endif /* _OS_WIN_H_ */ From f216e324f9dc009b20c05e1cb5e15a4316334718 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 14 Aug 2014 19:53:37 +0000 Subject: [PATCH 225/710] Fix an assert that can fail if the database file is corrupted. FossilOrigin-Name: 3f45b8192dad7fb1f027cbaa694046e3c1b3e278 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 3 ++- test/corruptI.test | 26 ++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a9e401b96b..a8fda48d5f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypos\sin\scomments\sused\sto\shelp\sgenerate\sdocumentation.\s\sNo\schanges\sto\scode. -D 2014-08-14T13:06:25.245 +C Fix\san\sassert\sthat\scan\sfail\sif\sthe\sdatabase\sfile\sis\scorrupted. +D 2014-08-14T19:53:37.748 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 99d162e57af6e72ffd7db5bf79568a134cd87d5b +F src/btree.c fa057e30794bfd867963b44a3a42710a45c335a1 F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 5abf794fe8a605f2005b422e98a3cedad9b9ef5b @@ -413,7 +413,7 @@ F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb -F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 +F test/corruptI.test 0afbba50bfae006094cc548b4605f521c1179502 F test/cost.test 19d314526616ce4473eb4e4e450fcb94499ce318 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f5cce9db109a2bbe9a05b4c01b076a51d5fad8ba -R b4842944a4c3679f5b3af787e9b322df -U drh -Z 52db703ea373cd617fd2c598672c0ab8 +P 13a2d90a2869c53b79754de39045bbbdbc7688e3 +R be8bb3bca247e348b90f7388fee24972 +U dan +Z 1fbc222a1bef7c1b8874e0319a4338ee diff --git a/manifest.uuid b/manifest.uuid index 5d1e011162..8166d1d7f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13a2d90a2869c53b79754de39045bbbdbc7688e3 \ No newline at end of file +3f45b8192dad7fb1f027cbaa694046e3c1b3e278 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 17719caf91..60bc7de41e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5741,7 +5741,8 @@ static void insertCell( if( *pRC ) return; assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); - assert( pPage->nCell<=MX_CELL(pPage->pBt) && MX_CELL(pPage->pBt)<=10921 ); + assert( MX_CELL(pPage->pBt)<=10921 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); diff --git a/test/corruptI.test b/test/corruptI.test index ed34c0f8c3..41200c5409 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -75,5 +75,31 @@ do_test 2.2 { catchsql { SELECT * FROM r WHERE x >= 10 } } {1 {database disk image is malformed}} +reset_db + +do_execsql_test 3.1 { + PRAGMA page_size = 512; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + WITH s(a, b) AS ( + SELECT 2, 'abcdefghij' + UNION ALL + SELECT a+2, b FROM s WHERe a < 40 + ) + INSERT INTO t1 SELECT * FROM s; +} {} + +do_test 3.2 { + hexio_write test.db [expr 512+3] 0054 + db close + sqlite3 db test.db + execsql { INSERT INTO t1 VALUES(5, 'klmnopqrst') } + execsql { INSERT INTO t1 VALUES(7, 'klmnopqrst') } +} {} + +db close +sqlite3 db test.db +do_catchsql_test 3.2 { + INSERT INTO t1 VALUES(9, 'klmnopqrst'); +} {1 {database disk image is malformed}} finish_test From 932de71e39d3192ec6138f0676fa7de41807ae2c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 15 Aug 2014 11:46:33 +0000 Subject: [PATCH 226/710] Version 3.8.6 FossilOrigin-Name: 9491ba7d738528f168657adb43a198238abde19e --- manifest | 13 ++++++++----- manifest.uuid | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/manifest b/manifest index a8fda48d5f..28b547a053 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sassert\sthat\scan\sfail\sif\sthe\sdatabase\sfile\sis\scorrupted. -D 2014-08-14T19:53:37.748 +C Version\s3.8.6 +D 2014-08-15T11:46:33.931 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1186,7 +1186,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 13a2d90a2869c53b79754de39045bbbdbc7688e3 +P 3f45b8192dad7fb1f027cbaa694046e3c1b3e278 R be8bb3bca247e348b90f7388fee24972 -U dan -Z 1fbc222a1bef7c1b8874e0319a4338ee +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.8.6 * +U drh +Z 74d8dd4d4aac0ae87245b4bcefd8b172 diff --git a/manifest.uuid b/manifest.uuid index 8166d1d7f4..01b81a4804 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3f45b8192dad7fb1f027cbaa694046e3c1b3e278 \ No newline at end of file +9491ba7d738528f168657adb43a198238abde19e \ No newline at end of file From dcd87a9dd3b18148cf6444a53797840a3243d01b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 18 Aug 2014 13:45:42 +0000 Subject: [PATCH 227/710] Refactor the names of state objects in the command-line shell implementation.. FossilOrigin-Name: 11a70e1ae7f05d06e4e09c9d20db0444b8881584 --- manifest | 13 +++--- manifest.uuid | 2 +- src/shell.c | 115 ++++++++++++++++++++++++++------------------------ 3 files changed, 66 insertions(+), 64 deletions(-) diff --git a/manifest b/manifest index c1d7bb58d6..6b5562f42e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings\son\sWinCE. -D 2014-08-15T16:13:07.749 +C Refactor\sthe\snames\sof\sstate\sobjects\sin\sthe\scommand-line\sshell\simplementation.. +D 2014-08-18T13:45:42.156 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c ea48e891406ccdf748f3eb02893e056d134a0fea -F src/shell.c 75bb7bd2c80bb44861598f322a417c4bafe98fd7 +F src/shell.c 41df1103617331e023301d943977f8c06bd1486f F src/sqlite.h.in ed9d35990c61f0388ca6405706455c4095310553 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1186,8 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9491ba7d738528f168657adb43a198238abde19e cc910b8e0c45e7387024f3a729003e2fef08b198 -R ad15e862b2dc4bc2854071509d952976 -T +closed cc910b8e0c45e7387024f3a729003e2fef08b198 +P 28a379fcd43389ad3249eff51a621d0bb7c81640 +R ef518fe403795d3b06bbe754ecf5c125 U drh -Z 7367bc305ed9789e95517eb698c37af7 +Z e00f86954fa90c49bf5a476ef91fbb0c diff --git a/manifest.uuid b/manifest.uuid index d364d2f83d..e975bd8d97 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28a379fcd43389ad3249eff51a621d0bb7c81640 \ No newline at end of file +11a70e1ae7f05d06e4e09c9d20db0444b8881584 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index a74dcdb85d..87b335fbb1 100644 --- a/src/shell.c +++ b/src/shell.c @@ -432,19 +432,24 @@ static char *one_input_line(FILE *in, char *zPrior, int isContinuation){ return zResult; } -struct previous_mode_data { - int valid; /* Is there legit data in here? */ - int mode; - int showHeader; - int colWidth[100]; +/* +** Shell output mode information from before ".explain on", +** saved so that it can be restored by ".explain off" +*/ +typedef struct SavedModeInfo SavedModeInfo; +struct SavedModeInfo { + int valid; /* Is there legit data in here? */ + int mode; /* Mode prior to ".explain on" */ + int showHeader; /* The ".header" setting prior to ".explain on" */ + int colWidth[100]; /* Column widths prior to ".explain on" */ }; /* -** An pointer to an instance of this structure is passed from -** the main program to the callback. This is used to communicate -** state and mode information. +** State information about the database connection is contained in an +** instance of the following structure. */ -struct callback_data { +typedef struct ShellState ShellState; +struct ShellState { sqlite3 *db; /* The database */ int echoOn; /* True to echo input commands */ int autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ @@ -464,9 +469,7 @@ struct callback_data { int actualWidth[100]; /* Actual width of each column */ char nullvalue[20]; /* The text to print when a NULL comes back from ** the database */ - struct previous_mode_data explainPrev; - /* Holds the mode information just before - ** .explain ON */ + SavedModeInfo normalMode;/* Holds the mode just before .explain ON */ char outfile[FILENAME_MAX]; /* Filename for *out */ const char *zDbFilename; /* name of the database file */ char *zFreeOnClose; /* Filename to free when closing */ @@ -522,7 +525,7 @@ static int strlen30(const char *z){ ** A callback for the sqlite3_log() interface. */ static void shellLog(void *pArg, int iErrCode, const char *zMsg){ - struct callback_data *p = (struct callback_data*)pArg; + ShellState *p = (ShellState*)pArg; if( p->pLog==0 ) return; fprintf(p->pLog, "(%d) %s\n", iErrCode, zMsg); fflush(p->pLog); @@ -664,7 +667,7 @@ static const char needCsvQuote[] = { ** the null value. Strings are quoted if necessary. The separator ** is only issued if bSep is true. */ -static void output_csv(struct callback_data *p, const char *z, int bSep){ +static void output_csv(ShellState *p, const char *z, int bSep){ FILE *out = p->out; if( z==0 ){ fprintf(out,"%s",p->nullvalue); @@ -713,7 +716,7 @@ static void interrupt_handler(int NotUsed){ */ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int *aiType){ int i; - struct callback_data *p = (struct callback_data*)pArg; + ShellState *p = (ShellState*)pArg; switch( p->mode ){ case MODE_Line: { @@ -923,11 +926,11 @@ static int callback(void *pArg, int nArg, char **azArg, char **azCol){ } /* -** Set the destination table field of the callback_data structure to +** Set the destination table field of the ShellState structure to ** the name of the table given. Escape any quote characters in the ** table name. */ -static void set_table_name(struct callback_data *p, const char *zName){ +static void set_table_name(ShellState *p, const char *zName){ int i, n; int needQuote; char *z; @@ -1017,7 +1020,7 @@ static char *appendText(char *zIn, char const *zAppend, char quote){ ** won't consume the semicolon terminator. */ static int run_table_dump_query( - struct callback_data *p, /* Query context */ + ShellState *p, /* Query context */ const char *zSelect, /* SELECT statement to extract content */ const char *zFirstRow /* Print before first row, if not NULL */ ){ @@ -1080,7 +1083,7 @@ static char *save_err_msg( */ static int display_stats( sqlite3 *db, /* Database to query */ - struct callback_data *pArg, /* Pointer to struct callback_data */ + ShellState *pArg, /* Pointer to ShellState */ int bReset /* True to reset the stats */ ){ int iCur; @@ -1187,7 +1190,7 @@ static int str_in_array(const char *zStr, const char **azArray){ /* ** If compiled statement pSql appears to be an EXPLAIN statement, allocate -** and populate the callback_data.aiIndent[] array with the number of +** and populate the ShellState.aiIndent[] array with the number of ** spaces each opcode should be indented before it is output. ** ** The indenting rules are: @@ -1203,7 +1206,7 @@ static int str_in_array(const char *zStr, const char **azArray){ ** then indent all opcodes between the earlier instruction ** and "Goto" by 2 spaces. */ -static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ +static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ const char *zSql; /* The text of the SQL statement */ const char *z; /* Used to check if this is an EXPLAIN */ int *abYield = 0; /* True if op is an OP_Yield */ @@ -1263,7 +1266,7 @@ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ /* ** Free the array allocated by explain_data_prepare(). */ -static void explain_data_delete(struct callback_data *p){ +static void explain_data_delete(ShellState *p){ sqlite3_free(p->aiIndent); p->aiIndent = 0; p->nIndent = 0; @@ -1280,12 +1283,12 @@ static void explain_data_delete(struct callback_data *p){ ** and callback data argument. */ static int shell_exec( - sqlite3 *db, /* An open database */ - const char *zSql, /* SQL to be evaluated */ + sqlite3 *db, /* An open database */ + const char *zSql, /* SQL to be evaluated */ int (*xCallback)(void*,int,char**,char**,int*), /* Callback function */ - /* (not the same as sqlite3_exec) */ - struct callback_data *pArg, /* Pointer to struct callback_data */ - char **pzErrMsg /* Error msg written here */ + /* (not the same as sqlite3_exec) */ + ShellState *pArg, /* Pointer to ShellState */ + char **pzErrMsg /* Error msg written here */ ){ sqlite3_stmt *pStmt = NULL; /* Statement to execute. */ int rc = SQLITE_OK; /* Return Code */ @@ -1453,7 +1456,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ const char *zType; const char *zSql; const char *zPrepStmt = 0; - struct callback_data *p = (struct callback_data *)pArg; + ShellState *p = (ShellState *)pArg; UNUSED_PARAMETER(azCol); if( nArg!=3 ) return 1; @@ -1549,7 +1552,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ ** "ORDER BY rowid DESC" to the end. */ static int run_schema_dump_query( - struct callback_data *p, + ShellState *p, const char *zQuery ){ int rc; @@ -1649,7 +1652,7 @@ static char zHelp[] = ; /* Forward reference */ -static int process_input(struct callback_data *p, FILE *in); +static int process_input(ShellState *p, FILE *in); /* ** Implementation of the "readfile(X)" SQL function. The entire content ** of the file named X is read and returned as a BLOB. NULL is returned @@ -1715,7 +1718,7 @@ static void writefileFunc( ** Make sure the database is open. If it is not, then open it. If ** the database fails to open, print an error message and exit. */ -static void open_db(struct callback_data *p, int keepAlive){ +static void open_db(ShellState *p, int keepAlive){ if( p->db==0 ){ sqlite3_initialize(); sqlite3_open(p->zDbFilename, &p->db); @@ -2014,7 +2017,7 @@ static char *csv_read_one_field(CSVReader *p){ ** work for WITHOUT ROWID tables. */ static void tryToCloneData( - struct callback_data *p, + ShellState *p, sqlite3 *newDb, const char *zTable ){ @@ -2127,10 +2130,10 @@ end_data_xfer: ** sqlite_master table, try again moving backwards. */ static void tryToCloneSchema( - struct callback_data *p, + ShellState *p, sqlite3 *newDb, const char *zWhere, - void (*xForEach)(struct callback_data*,sqlite3*,const char*) + void (*xForEach)(ShellState*,sqlite3*,const char*) ){ sqlite3_stmt *pQuery = 0; char *zQuery = 0; @@ -2201,7 +2204,7 @@ end_schema_xfer: ** as possible out of the main database (which might be corrupt) and write it ** into zNewDb. */ -static void tryToClone(struct callback_data *p, const char *zNewDb){ +static void tryToClone(ShellState *p, const char *zNewDb){ int rc; sqlite3 *newDb = 0; if( access(zNewDb,0)==0 ){ @@ -2226,7 +2229,7 @@ static void tryToClone(struct callback_data *p, const char *zNewDb){ /* ** Change the output file back to stdout */ -static void output_reset(struct callback_data *p){ +static void output_reset(ShellState *p){ if( p->outfile[0]=='|' ){ pclose(p->out); }else{ @@ -2242,7 +2245,7 @@ static void output_reset(struct callback_data *p){ ** ** Return 1 on error, 2 to exit, and 0 otherwise. */ -static int do_meta_command(char *zLine, struct callback_data *p){ +static int do_meta_command(char *zLine, ShellState *p){ int i = 1; int nArg = 0; int n, c; @@ -2360,7 +2363,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c=='d' && n>1 && strncmp(azArg[0], "databases", n)==0 ){ - struct callback_data data; + ShellState data; char *zErrMsg = 0; open_db(p, 0); memcpy(&data, p, sizeof(data)); @@ -2458,11 +2461,11 @@ static int do_meta_command(char *zLine, struct callback_data *p){ if( c=='e' && strncmp(azArg[0], "explain", n)==0 ){ int val = nArg>=2 ? booleanValue(azArg[1]) : 1; if(val == 1) { - if(!p->explainPrev.valid) { - p->explainPrev.valid = 1; - p->explainPrev.mode = p->mode; - p->explainPrev.showHeader = p->showHeader; - memcpy(p->explainPrev.colWidth,p->colWidth,sizeof(p->colWidth)); + if(!p->normalMode.valid) { + p->normalMode.valid = 1; + p->normalMode.mode = p->mode; + p->normalMode.showHeader = p->showHeader; + memcpy(p->normalMode.colWidth,p->colWidth,sizeof(p->colWidth)); } /* We could put this code under the !p->explainValid ** condition so that it does not execute if we are already in @@ -2482,16 +2485,16 @@ static int do_meta_command(char *zLine, struct callback_data *p){ p->colWidth[5] = 13; /* P4 */ p->colWidth[6] = 2; /* P5 */ p->colWidth[7] = 13; /* Comment */ - }else if (p->explainPrev.valid) { - p->explainPrev.valid = 0; - p->mode = p->explainPrev.mode; - p->showHeader = p->explainPrev.showHeader; - memcpy(p->colWidth,p->explainPrev.colWidth,sizeof(p->colWidth)); + }else if (p->normalMode.valid) { + p->normalMode.valid = 0; + p->mode = p->normalMode.mode; + p->showHeader = p->normalMode.showHeader; + memcpy(p->colWidth,p->normalMode.colWidth,sizeof(p->colWidth)); } }else if( c=='f' && strncmp(azArg[0], "fullschema", n)==0 ){ - struct callback_data data; + ShellState data; char *zErrMsg = 0; int doStats = 0; if( nArg!=1 ){ @@ -2712,7 +2715,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c=='i' && strncmp(azArg[0], "indices", n)==0 ){ - struct callback_data data; + ShellState data; char *zErrMsg = 0; open_db(p, 0); memcpy(&data, p, sizeof(data)); @@ -3006,7 +3009,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ }else if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){ - struct callback_data data; + ShellState data; char *zErrMsg = 0; open_db(p, 0); memcpy(&data, p, sizeof(data)); @@ -3148,7 +3151,7 @@ static int do_meta_command(char *zLine, struct callback_data *p){ } fprintf(p->out,"%9.9s: %s\n","echo", p->echoOn ? "on" : "off"); fprintf(p->out,"%9.9s: %s\n","eqp", p->autoEQP ? "on" : "off"); - fprintf(p->out,"%9.9s: %s\n","explain", p->explainPrev.valid ? "on" :"off"); + fprintf(p->out,"%9.9s: %s\n","explain", p->normalMode.valid ? "on" :"off"); fprintf(p->out,"%9.9s: %s\n","headers", p->showHeader ? "on" : "off"); fprintf(p->out,"%9.9s: %s\n","mode", modeDescr[p->mode]); fprintf(p->out,"%9.9s: ", "nullvalue"); @@ -3540,7 +3543,7 @@ static int line_is_complete(char *zSql, int nSql){ ** ** Return the number of errors. */ -static int process_input(struct callback_data *p, FILE *in){ +static int process_input(ShellState *p, FILE *in){ char *zLine = 0; /* A single input line */ char *zSql = 0; /* Accumulated SQL text */ int nLine; /* Length of current line */ @@ -3719,7 +3722,7 @@ static char *find_home_dir(void){ ** Returns the number of errors. */ static int process_sqliterc( - struct callback_data *p, /* Configuration data */ + ShellState *p, /* Configuration data */ const char *sqliterc_override /* Name of config file. NULL to use default */ ){ char *home_dir = NULL; @@ -3802,7 +3805,7 @@ static void usage(int showDetail){ /* ** Initialize the state information in data */ -static void main_init(struct callback_data *data) { +static void main_init(ShellState *data) { memset(data, 0, sizeof(*data)); data->mode = MODE_List; memcpy(data->separator,"|", 2); @@ -3850,7 +3853,7 @@ static char *cmdline_option_value(int argc, char **argv, int i){ int main(int argc, char **argv){ char *zErrMsg = 0; - struct callback_data data; + ShellState data; const char *zInitFile = 0; char *zFirstCmd = 0; int i; From edf5b1657dc1fc1301de36de3b353295c383a064 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 19 Aug 2014 09:15:41 +0000 Subject: [PATCH 228/710] Minor changes to do with the Tcl tea extension package autoconf system. FossilOrigin-Name: f10a6111262ce6ee6984c64fa0e0023642eca27d --- autoconf/tea/Makefile.in | 1 + autoconf/tea/configure.in | 6 ++++-- autoconf/tea/tclconfig/tcl.m4 | 21 ++++++++++++--------- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/tclsqlite.c | 2 +- tool/mkautoconfamal.sh | 2 +- 7 files changed, 31 insertions(+), 25 deletions(-) diff --git a/autoconf/tea/Makefile.in b/autoconf/tea/Makefile.in index 08b1a44182..a8708974b0 100644 --- a/autoconf/tea/Makefile.in +++ b/autoconf/tea/Makefile.in @@ -73,6 +73,7 @@ exec_prefix = @exec_prefix@ bindir = @bindir@ libdir = @libdir@ +datarootdir = @datarootdir@ datadir = @datadir@ mandir = @mandir@ includedir = @includedir@ diff --git a/autoconf/tea/configure.in b/autoconf/tea/configure.in index ec9c565c6f..8df0af6195 100644 --- a/autoconf/tea/configure.in +++ b/autoconf/tea/configure.in @@ -166,8 +166,10 @@ AC_DEFINE(USE_TCL_STUBS, 1, [Use Tcl stubs]) #-------------------------------------------------------------------- # Redefine fdatasync as fsync on systems that lack fdatasync #-------------------------------------------------------------------- - -AC_CHECK_FUNC(fdatasync, , AC_DEFINE(fdatasync, fsync)) +# +#AC_CHECK_FUNC(fdatasync, , AC_DEFINE(fdatasync, fsync)) +# Check for library functions that SQLite can optionally use. +AC_CHECK_FUNCS([fdatasync usleep fullfsync localtime_r gmtime_r]) AC_FUNC_STRERROR_R diff --git a/autoconf/tea/tclconfig/tcl.m4 b/autoconf/tea/tclconfig/tcl.m4 index 66214e78a2..4b4bd1e888 100644 --- a/autoconf/tea/tclconfig/tcl.m4 +++ b/autoconf/tea/tclconfig/tcl.m4 @@ -1641,6 +1641,7 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ SHLIB_CFLAGS="-fPIC" SHLIB_LD="${CC} -shared" TCL_SHLIB_LD_EXTRAS="-Wl,-soname=\$[@]" + TK_SHLIB_LD_EXTRAS="-Wl,-soname,\$[@]" SHLIB_SUFFIX=".so" LDFLAGS="" AS_IF([test $doRpath = yes], [ @@ -1651,11 +1652,15 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ LIBS=`echo $LIBS | sed s/-pthread//` CFLAGS="$CFLAGS $PTHREAD_CFLAGS" LDFLAGS="$LDFLAGS $PTHREAD_LIBS"]) - # Version numbers are dot-stripped by system policy. - TCL_TRIM_DOTS=`echo ${PACKAGE_VERSION} | tr -d .` - UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' - SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}\$\{DBGX\}.so.1' - TCL_LIB_VERSIONS_OK=nodots + case $system in + FreeBSD-3.*) + # Version numbers are dot-stripped by system policy. + TCL_TRIM_DOTS=`echo ${VERSION} | tr -d .` + UNSHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.a' + SHARED_LIB_SUFFIX='${TCL_TRIM_DOTS}.so' + TCL_LIB_VERSIONS_OK=nodots + ;; + esac ;; Darwin-*) CFLAGS_OPTIMIZE="-Os" @@ -1826,8 +1831,8 @@ AC_DEFUN([TEA_CONFIG_CFLAGS], [ SHLIB_CFLAGS="-fPIC -melf" LDFLAGS="$LDFLAGS -melf -Wl,-Bexport" ], [ - SHLIB_CFLAGS="-Kpic -belf" - LDFLAGS="$LDFLAGS -belf -Wl,-Bexport" + SHLIB_CFLAGS="-Kpic -belf" + LDFLAGS="$LDFLAGS -belf -Wl,-Bexport" ]) SHLIB_LD="ld -G" SHLIB_LD_LIBS="" @@ -4158,8 +4163,6 @@ AC_DEFUN([TEA_PATH_CELIB], [ fi fi ]) - - # Local Variables: # mode: autoconf # End: diff --git a/manifest b/manifest index 6b5562f42e..963d794593 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\snames\sof\sstate\sobjects\sin\sthe\scommand-line\sshell\simplementation.. -D 2014-08-18T13:45:42.156 +C Minor\schanges\sto\sdo\swith\sthe\sTcl\stea\sextension\spackage\sautoconf\ssystem. +D 2014-08-19T09:15:41.141 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -23,15 +23,15 @@ F autoconf/depcomp 0b26f101e3bc9fd1ff0be1da9fb4a82371142f92 x F autoconf/install-sh 06ee6336e63bb845c8439d777c32eb2eccc4fbf1 x F autoconf/ltmain.sh 7a658a24028f02331c1d2446562758083c5eadd1 F autoconf/missing d7c9981a81af13370d4ed152b24c0a82b7028585 x -F autoconf/tea/Makefile.in 5c3b0bdfb66c20d55ebff59d1718864461570ca9 +F autoconf/tea/Makefile.in d55bcc63832caf0309c2ff80358756116618cfca F autoconf/tea/README 3e9a3c060f29a44344ab50aec506f4db903fb873 F autoconf/tea/aclocal.m4 52c47aac44ce0ddb1f918b6993e8beb8eee88f43 -F autoconf/tea/configure.in e0466b881b53f31f5a4a69e7a91ad130902fb359 +F autoconf/tea/configure.in 93d43c79e936fb16556e22498177d7e8571efa04 F autoconf/tea/doc/sqlite3.n e1fe45d4f5286ee3d0ccc877aca2a0def488e9bb F autoconf/tea/license.terms 13bd403c9610fd2b76ece0ab50c4c5eda933d523 F autoconf/tea/pkgIndex.tcl.in 3ef61715cf1c7bdcff56947ffadb26bc991ca39d F autoconf/tea/tclconfig/install-sh bdd5e293591621ae60d9824d86a4b1c5f22c3d00 -F autoconf/tea/tclconfig/tcl.m4 f035b86539a5ab30689e997a11ae9e7fd2e65570 +F autoconf/tea/tclconfig/tcl.m4 66ddf0a5d5e4b1d29bff472c0985fd7fa89d0fb5 F autoconf/tea/win/makefile.vc f89d0184d0eee5f7e356ea407964dcd139939928 F autoconf/tea/win/nmakehlp.c 2070e086f39866b353a482d3a14dedaf26196506 F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 @@ -231,7 +231,7 @@ F src/sqliteInt.h 641f8fbb65ca2084c8df95b525f6f82c7a1e91ae F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c e87c99e28a145943666b51b212dacae35fcea0bd +F src/tclsqlite.c 7d100e2e7aad614bb3d7026a41a0e3827dbaaebc F src/test1.c 14409a611e9c27c6c522c610bbff5561f05c1558 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c @@ -1148,7 +1148,7 @@ F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 3ff0fec22f92dfb54e62eeb48772eddffdbeb0d6 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 -F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 +F tool/mkautoconfamal.sh 5dc5010e2e748a9e1bba67baca5956a2c2deda7b F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460 @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 28a379fcd43389ad3249eff51a621d0bb7c81640 -R ef518fe403795d3b06bbe754ecf5c125 -U drh -Z e00f86954fa90c49bf5a476ef91fbb0c +P 11a70e1ae7f05d06e4e09c9d20db0444b8881584 +R aa6c5df3344dab5b287c6351a237958f +U dan +Z b3667514ef51eabff2a360098e935ae4 diff --git a/manifest.uuid b/manifest.uuid index e975bd8d97..a39201275a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11a70e1ae7f05d06e4e09c9d20db0444b8881584 \ No newline at end of file +f10a6111262ce6ee6984c64fa0e0023642eca27d \ No newline at end of file diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 9b977e54ae..2b98b6aab4 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -2381,7 +2381,7 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ if( rc==TCL_OK ){ rc = createIncrblobChannel( - interp, pDb, zDb, zTable, zColumn, iRow, isReadonly + interp, pDb, zDb, zTable, zColumn, (sqlite3_int64)iRow, isReadonly ); } #endif diff --git a/tool/mkautoconfamal.sh b/tool/mkautoconfamal.sh index c13f7c999a..4829277234 100644 --- a/tool/mkautoconfamal.sh +++ b/tool/mkautoconfamal.sh @@ -62,7 +62,7 @@ mkdir -p tea/generic echo "#ifdef USE_SYSTEM_SQLITE" > tea/generic/tclsqlite3.c echo "# include " >> tea/generic/tclsqlite3.c echo "#else" >> tea/generic/tclsqlite3.c -echo "#include \"../../sqlite3.c\"" >> tea/generic/tclsqlite3.c +echo "#include \"sqlite3.c\"" >> tea/generic/tclsqlite3.c echo "#endif" >> tea/generic/tclsqlite3.c cat $TOP/src/tclsqlite.c >> tea/generic/tclsqlite3.c From 4b2590e44b422eb0445220624c2438d74d17ed06 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 19 Aug 2014 19:28:00 +0000 Subject: [PATCH 229/710] Improvements to output formatting with the ".trace" command in the command-line shell. FossilOrigin-Name: d09d63c07748839e9b778a769b183bdd614c6c13 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 10 +++++++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 963d794593..1735f35740 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schanges\sto\sdo\swith\sthe\sTcl\stea\sextension\spackage\sautoconf\ssystem. -D 2014-08-19T09:15:41.141 +C Improvements\sto\soutput\sformatting\swith\sthe\s".trace"\scommand\sin\sthe\ncommand-line\sshell. +D 2014-08-19T19:28:00.623 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c ea48e891406ccdf748f3eb02893e056d134a0fea -F src/shell.c 41df1103617331e023301d943977f8c06bd1486f +F src/shell.c 728d2226594d356bf4fbdbdfd08538fd78fd06f3 F src/sqlite.h.in ed9d35990c61f0388ca6405706455c4095310553 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 11a70e1ae7f05d06e4e09c9d20db0444b8881584 -R aa6c5df3344dab5b287c6351a237958f -U dan -Z b3667514ef51eabff2a360098e935ae4 +P f10a6111262ce6ee6984c64fa0e0023642eca27d +R 257d241ab9ebe5fc3f4654d7f2e169ae +U drh +Z 211a00cac8127ca939f7e6f32071c25c diff --git a/manifest.uuid b/manifest.uuid index a39201275a..1cf4ead3de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f10a6111262ce6ee6984c64fa0e0023642eca27d \ No newline at end of file +d09d63c07748839e9b778a769b183bdd614c6c13 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 87b335fbb1..68478a19ba 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1899,7 +1899,11 @@ static FILE *output_file_open(const char *zFile){ */ static void sql_trace_callback(void *pArg, const char *z){ FILE *f = (FILE*)pArg; - if( f ) fprintf(f, "%s\n", z); + if( f ){ + int i = (int)strlen(z); + while( i>0 && z[i-1]==';' ){ i--; } + fprintf(f, "%.*s;\n", i, z); + } } /* @@ -2511,7 +2515,7 @@ static int do_meta_command(char *zLine, ShellState *p){ " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_master UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " - "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%'" + "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' " "ORDER BY rowid", callback, &data, &zErrMsg ); @@ -3065,7 +3069,7 @@ static int do_meta_command(char *zLine, ShellState *p){ " (SELECT sql sql, type type, tbl_name tbl_name, name name, rowid x" " FROM sqlite_master UNION ALL" " SELECT sql, type, tbl_name, name, rowid FROM sqlite_temp_master) " - "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%'" + "WHERE type!='meta' AND sql NOTNULL AND name NOT LIKE 'sqlite_%' " "ORDER BY rowid", callback, &data, &zErrMsg ); From 2c7946a485b6e19072506e40203af4cc10b0848d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 19 Aug 2014 20:27:40 +0000 Subject: [PATCH 230/710] Make sure the sqlite3_trace() callback is invoked, even if the prepared statement was marked "expired" before it ever entered sqlite3_step(). Ticket [11d5aa455e0d98f3c1e6a08]. FossilOrigin-Name: 0d4d3df4bc5e75ce1543b5539a1e9e279d2a062f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeapi.c | 8 +++++--- test/trace.test | 16 ++++++++++++++++ 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 1735f35740..c307f54e8f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\soutput\sformatting\swith\sthe\s".trace"\scommand\sin\sthe\ncommand-line\sshell. -D 2014-08-19T19:28:00.623 +C Make\ssure\sthe\ssqlite3_trace()\scallback\sis\sinvoked,\seven\sif\sthe\sprepared\nstatement\swas\smarked\s"expired"\sbefore\sit\sever\sentered\ssqlite3_step().\nTicket\s[11d5aa455e0d98f3c1e6a08]. +D 2014-08-19T20:27:40.455 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c f7f4066e4d6e3858878d76ce9288ea603e12ddf6 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df -F src/vdbeapi.c 24e40422382beb774daab11fe9fe9d37e8a04949 +F src/vdbeapi.c ce75e452dfd9c6ba4f8c6c76be6399c88bce3142 F src/vdbeaux.c 25d62ef82cf1be2a1255eacac636fa0d943d8b3d F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 @@ -1017,7 +1017,7 @@ F test/tkt3997.test a335fa41ca3985660a139df7b734a26ef53284bd F test/tkt4018.test 7c2c9ba4df489c676a0a7a0e809a1fb9b2185bd1 F test/tokenize.test ce430a7aed48fc98301611429595883fdfcab5d7 F test/tpch01.test 04adbf8d8300fa60a222f28d901abd76e7be6dd4 -F test/trace.test 4b36a41a3e9c7842151af6da5998f5080cdad9e5 +F test/trace.test 73a5508100f7fccfbc3f8018d5f6963ed478eea0 F test/trace2.test 93b47ca6996c66b47f57224cfb146f34e07df382 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f10a6111262ce6ee6984c64fa0e0023642eca27d -R 257d241ab9ebe5fc3f4654d7f2e169ae +P d09d63c07748839e9b778a769b183bdd614c6c13 +R dc4a84c1e3dbc4c500aead2f870904cc U drh -Z 211a00cac8127ca939f7e6f32071c25c +Z 8ee9013dcfb943fe42c807d90f81accd diff --git a/manifest.uuid b/manifest.uuid index 1cf4ead3de..b804f56674 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d09d63c07748839e9b778a769b183bdd614c6c13 \ No newline at end of file +0d4d3df4bc5e75ce1543b5539a1e9e279d2a062f \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 5e5bb81366..568f67296c 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -513,10 +513,12 @@ int sqlite3_step(sqlite3_stmt *pStmt){ sqlite3_mutex_enter(db->mutex); v->doingRerun = 0; while( (rc = sqlite3Step(v))==SQLITE_SCHEMA - && cnt++ < SQLITE_MAX_SCHEMA_RETRY - && (rc2 = rc = sqlite3Reprepare(v))==SQLITE_OK ){ + && cnt++ < SQLITE_MAX_SCHEMA_RETRY ){ + int savedPc = v->pc; + rc2 = rc = sqlite3Reprepare(v); + if( rc!=SQLITE_OK) break; sqlite3_reset(pStmt); - v->doingRerun = 1; + v->doingRerun = savedPc>=0; assert( v->expired==0 ); } if( rc2!=SQLITE_OK ){ diff --git a/test/trace.test b/test/trace.test index ce5a2d712a..a64cc333fa 100644 --- a/test/trace.test +++ b/test/trace.test @@ -48,6 +48,22 @@ do_test trace-1.5 { db trace {} db trace } {} +do_test trace-1.6 { + db eval { + CREATE TABLE t1b(x TEXT PRIMARY KEY, y); + INSERT INTO t1b VALUES('abc','def'),('ghi','jkl'),('mno','pqr'); + } + set ::stmtlist {} + set xyzzy a* + db trace trace_proc + db eval { + SELECT y FROM t1b WHERE x GLOB $xyzzy + } +} {def} +do_test trace-1.7 { + set ::stmtlist +} {{SELECT y FROM t1b WHERE x GLOB 'a*'}} +db trace {} # If we prepare a statement and execute it multiple times, the trace # happens on each execution. From 5f58ae75c91e1ec56f41e0d465224a6625152f24 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 19 Aug 2014 20:41:36 +0000 Subject: [PATCH 231/710] A better fix for the sqlite3_trace() problem. Ticket [11d5aa455e0d98f3c1e6a] FossilOrigin-Name: 44d5bd4cc3f76e9a151ba0abae1092bd184af264 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c307f54e8f..d226fa4f96 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\ssqlite3_trace()\scallback\sis\sinvoked,\seven\sif\sthe\sprepared\nstatement\swas\smarked\s"expired"\sbefore\sit\sever\sentered\ssqlite3_step().\nTicket\s[11d5aa455e0d98f3c1e6a08]. -D 2014-08-19T20:27:40.455 +C A\sbetter\sfix\sfor\sthe\ssqlite3_trace()\sproblem.\nTicket\s[11d5aa455e0d98f3c1e6a] +D 2014-08-19T20:41:36.374 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -286,7 +286,7 @@ F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c f7f4066e4d6e3858878d76ce9288ea603e12ddf6 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df -F src/vdbeapi.c ce75e452dfd9c6ba4f8c6c76be6399c88bce3142 +F src/vdbeapi.c 7858d7e7cd23267d3fbca18e3a28cce8e0d162a8 F src/vdbeaux.c 25d62ef82cf1be2a1255eacac636fa0d943d8b3d F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d09d63c07748839e9b778a769b183bdd614c6c13 -R dc4a84c1e3dbc4c500aead2f870904cc +P 0d4d3df4bc5e75ce1543b5539a1e9e279d2a062f +R 3e9fe87a97e756aac5d53e0b321cfb1c U drh -Z 8ee9013dcfb943fe42c807d90f81accd +Z dae8e8935ea979fe699fbe5ad58485b9 diff --git a/manifest.uuid b/manifest.uuid index b804f56674..270766200f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d4d3df4bc5e75ce1543b5539a1e9e279d2a062f \ No newline at end of file +44d5bd4cc3f76e9a151ba0abae1092bd184af264 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 568f67296c..f6cc2d8c05 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -518,7 +518,7 @@ int sqlite3_step(sqlite3_stmt *pStmt){ rc2 = rc = sqlite3Reprepare(v); if( rc!=SQLITE_OK) break; sqlite3_reset(pStmt); - v->doingRerun = savedPc>=0; + if( savedPc>=0 ) v->doingRerun = 1; assert( v->expired==0 ); } if( rc2!=SQLITE_OK ){ From dfd1547b482f8f3cc75382934575cea4c624f005 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 19 Aug 2014 23:04:49 +0000 Subject: [PATCH 232/710] Modify the memsubsys1-3.1.4 test so that it does not fail arbitrarily due to variations in the behavior of system malloc(). FossilOrigin-Name: d280157da0b5275f3e3c875d2dcfb9998d374ac0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/memsubsys1.test | 8 +++++++- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d226fa4f96..c6aee7c716 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\sbetter\sfix\sfor\sthe\ssqlite3_trace()\sproblem.\nTicket\s[11d5aa455e0d98f3c1e6a] -D 2014-08-19T20:41:36.374 +C Modify\sthe\smemsubsys1-3.1.4\stest\sso\sthat\sit\sdoes\snot\sfail\sarbitrarily\sdue\nto\svariations\sin\sthe\sbehavior\sof\ssystem\smalloc(). +D 2014-08-19T23:04:49.204 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -702,7 +702,7 @@ F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test fcb5297b321b562084fc79d64d5a12a1cd2b639b F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 -F test/memsubsys1.test f97cfd0b30e85c2f1ed16d642e7ac58006be84b2 +F test/memsubsys1.test bf270964ab83bc2da5927960f78304a866fb9a9d F test/memsubsys2.test 3a1c1a9de48e5726faa85108b02459fae8cb9ee9 F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0d4d3df4bc5e75ce1543b5539a1e9e279d2a062f -R 3e9fe87a97e756aac5d53e0b321cfb1c +P 44d5bd4cc3f76e9a151ba0abae1092bd184af264 +R da9b4e10fd35ea0494fda63ca4df66ec U drh -Z dae8e8935ea979fe699fbe5ad58485b9 +Z 375791abd440beaccbf3e162957d1a64 diff --git a/manifest.uuid b/manifest.uuid index 270766200f..a915b839b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -44d5bd4cc3f76e9a151ba0abae1092bd184af264 \ No newline at end of file +d280157da0b5275f3e3c875d2dcfb9998d374ac0 \ No newline at end of file diff --git a/test/memsubsys1.test b/test/memsubsys1.test index 76b79d00f6..8cc7c2afc1 100644 --- a/test/memsubsys1.test +++ b/test/memsubsys1.test @@ -124,7 +124,13 @@ do_test memsubsys1-3.1.3 { } 0 do_test memsubsys1-3.1.4 { set overflow [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_OVERFLOW 0] 2] -} $max_pagecache + # Note: The measured PAGECACHE_OVERFLOW is amount malloc() returns, not what + # was requested. System malloc() implementations might (arbitrarily) return + # slightly different oversize buffers, which can result in slightly different + # PAGECACHE_OVERFLOW sizes between consecutive runs. So we cannot do an + # exact comparison. Simply verify that the amount is within 5%. + expr {$overflow>=$max_pagecache*0.95 && $overflow<=$max_pagecache*1.05} +} 1 do_test memsubsys1-3.1.5 { set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] } 0 From 75b31dc9adcd78576b8d89064c1d03f2164336ba Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Aug 2014 00:54:46 +0000 Subject: [PATCH 233/710] Minor performance improvement and size reduction for the btree-page space allocator. FossilOrigin-Name: 73637d12e31f5489efe37d8cf4ab50a1911d4c75 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 17 +++++++++++------ 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index c6aee7c716..91db86833d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\smemsubsys1-3.1.4\stest\sso\sthat\sit\sdoes\snot\sfail\sarbitrarily\sdue\nto\svariations\sin\sthe\sbehavior\sof\ssystem\smalloc(). -D 2014-08-19T23:04:49.204 +C Minor\sperformance\simprovement\sand\ssize\sreduction\sfor\sthe\sbtree-page\sspace\nallocator. +D 2014-08-20T00:54:46.809 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c fa057e30794bfd867963b44a3a42710a45c335a1 +F src/btree.c c580f3fb3b3d1bf968e5c7e6a0ad48b7b0bd4366 F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 5abf794fe8a605f2005b422e98a3cedad9b9ef5b @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 44d5bd4cc3f76e9a151ba0abae1092bd184af264 -R da9b4e10fd35ea0494fda63ca4df66ec +P d280157da0b5275f3e3c875d2dcfb9998d374ac0 +R 020501de111b3b2380806f2152969bc3 U drh -Z 375791abd440beaccbf3e162957d1a64 +Z 17df18c00e8e514c309dda1b0357fb11 diff --git a/manifest.uuid b/manifest.uuid index a915b839b0..0e6072d88f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d280157da0b5275f3e3c875d2dcfb9998d374ac0 \ No newline at end of file +73637d12e31f5489efe37d8cf4ab50a1911d4c75 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 60bc7de41e..2ed304d04b 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1197,7 +1197,6 @@ static int defragmentPage(MemPage *pPage){ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ u8 * const data = pPage->aData; /* Local cache of pPage->aData */ - int nFrag; /* Number of fragmented bytes on pPage */ int top; /* First byte of cell content area */ int gap; /* First byte of gap between cell pointers and cell content */ int rc; /* Integer return code */ @@ -1212,16 +1211,22 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ usableSize = pPage->pBt->usableSize; assert( nByte < usableSize-8 ); - nFrag = data[hdr+7]; assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; - top = get2byteNotZero(&data[hdr+5]); - if( gap>top ) return SQLITE_CORRUPT_BKPT; + assert( gap<=65536 ); + top = get2byte(&data[hdr+5]); + if( gap>top ){ + if( top==0 ){ + top = 65536; + }else{ + return SQLITE_CORRUPT_BKPT; + } + } testcase( gap+2==top ); testcase( gap+1==top ); testcase( gap==top ); - if( nFrag>=60 ){ + if( data[hdr+7]>=60 ){ /* Always defragment highly fragmented pages */ rc = defragmentPage(pPage); if( rc ) return rc; @@ -1246,7 +1251,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ /* Remove the slot from the free-list. Update the number of ** fragmented bytes within the page. */ memcpy(&data[addr], &data[pc], 2); - data[hdr+7] = (u8)(nFrag + x); + data[hdr+7] += (u8)x; }else if( size+pc > usableSize ){ return SQLITE_CORRUPT_BKPT; }else{ From 53d89cdfe98e7c5b07527ff65cffa91798fa3ed9 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 20 Aug 2014 10:42:16 +0000 Subject: [PATCH 234/710] Fix a typo in the showdb usage message. FossilOrigin-Name: 6c66beae97ba1799c908d3a33371dedbc7f3f58c --- manifest | 14 +++++++------- manifest.uuid | 2 +- tool/showdb.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 91db86833d..e9f109a9a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sperformance\simprovement\sand\ssize\sreduction\sfor\sthe\sbtree-page\sspace\nallocator. -D 2014-08-20T00:54:46.809 +C Fix\sa\stypo\sin\sthe\sshowdb\susage\smessage. +D 2014-08-20T10:42:16.102 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1164,7 +1164,7 @@ F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c F tool/pagesig.c ff0ca355fd3c2398e933da5e22439bbff89b803b F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 -F tool/showdb.c b9ee6b6c81a094bf33badbf7e9da34cdbc0cce25 +F tool/showdb.c bd073a78bce714a0e42d92ea474b3eb8cb53be5d F tool/showjournal.c 053eb1cc774710c6890b7dd6293300cc297b16a5 F tool/showstat4.c c39279d6bd37cb999b634f0064f6f86ad7af008f F tool/showwal.c 3209120269cdf9380f091459e47b776b4f81dfd3 @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d280157da0b5275f3e3c875d2dcfb9998d374ac0 -R 020501de111b3b2380806f2152969bc3 -U drh -Z 17df18c00e8e514c309dda1b0357fb11 +P 73637d12e31f5489efe37d8cf4ab50a1911d4c75 +R d268dbd0231a0e3f9784dd51ed74231a +U dan +Z b16b223d7974024b463d42100aaa47bd diff --git a/manifest.uuid b/manifest.uuid index 0e6072d88f..45aa98f21f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -73637d12e31f5489efe37d8cf4ab50a1911d4c75 \ No newline at end of file +6c66beae97ba1799c908d3a33371dedbc7f3f58c \ No newline at end of file diff --git a/tool/showdb.c b/tool/showdb.c index 1a51e9d1a2..82b8c9f14f 100644 --- a/tool/showdb.c +++ b/tool/showdb.c @@ -957,7 +957,7 @@ static void usage(const char *argv0){ " NNNbdCCC Decode cell CCC on btree page NNN\n" " NNNt Decode freelist trunk page NNN\n" " NNNtd Show leaf freelist pages on the decode\n" - " NNNtr Recurisvely decode freelist starting at NNN\n" + " NNNtr Recursively decode freelist starting at NNN\n" ); } From 4c04f3c8bd1edb345186ae80d8539a684d676d74 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Aug 2014 11:56:14 +0000 Subject: [PATCH 235/710] Further size reduction and performance improvement in btree.c:allocateSpace(). FossilOrigin-Name: 1cb1cd64930a6bc371143f7d2e77eb1c51498cb0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 24 +++++++++++------------- 3 files changed, 19 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index e9f109a9a7..6883385671 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\sshowdb\susage\smessage. -D 2014-08-20T10:42:16.102 +C Further\ssize\sreduction\sand\sperformance\simprovement\sin\sbtree.c:allocateSpace(). +D 2014-08-20T11:56:14.338 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c c580f3fb3b3d1bf968e5c7e6a0ad48b7b0bd4366 +F src/btree.c 398ecbdb4a19230940955c10b4f35de958e9a05c F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 5abf794fe8a605f2005b422e98a3cedad9b9ef5b @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 73637d12e31f5489efe37d8cf4ab50a1911d4c75 -R d268dbd0231a0e3f9784dd51ed74231a -U dan -Z b16b223d7974024b463d42100aaa47bd +P 6c66beae97ba1799c908d3a33371dedbc7f3f58c +R 28706ecaca72a1e3574ba7b8d51854c6 +U drh +Z ca9532aeb7742cb91c75177cb9ee4fbc diff --git a/manifest.uuid b/manifest.uuid index 45aa98f21f..d1bf93d7a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c66beae97ba1799c908d3a33371dedbc7f3f58c \ No newline at end of file +1cb1cd64930a6bc371143f7d2e77eb1c51498cb0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2ed304d04b..61848dcb83 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1222,20 +1222,15 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ return SQLITE_CORRUPT_BKPT; } } + + /* If there is enough space between gap and top for one more cell pointer + ** array entry offset, and if the freelist is not empty, then search the + ** freelist looking for a free slot big enough to satisfy the request. + */ testcase( gap+2==top ); testcase( gap+1==top ); testcase( gap==top ); - - if( data[hdr+7]>=60 ){ - /* Always defragment highly fragmented pages */ - rc = defragmentPage(pPage); - if( rc ) return rc; - top = get2byteNotZero(&data[hdr+5]); - }else if( gap+2<=top ){ - /* Search the freelist looking for a free slot big enough to satisfy - ** the request. The allocation is made from the first free slot in - ** the list that is large enough to accommodate it. - */ + if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ int pc, addr; for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){ int size; /* Size of the free slot */ @@ -1248,6 +1243,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( x==4 ); testcase( x==3 ); if( x<4 ){ + if( data[hdr+7]>=60 ) goto defragment_page; /* Remove the slot from the free-list. Update the number of ** fragmented bytes within the page. */ memcpy(&data[addr], &data[pc], 2); @@ -1265,11 +1261,13 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ } } - /* Check to make sure there is enough space in the gap to satisfy - ** the allocation. If not, defragment. + /* The request could not be fulfilled using a freelist slot. Check + ** to see if defragmentation is necessary. */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ +defragment_page: + assert( pPage->nCell>0 ); rc = defragmentPage(pPage); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); From 905552628c4812e47ab9172911650ef5e4778e64 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Aug 2014 13:17:43 +0000 Subject: [PATCH 236/710] Change an assert() added by the previous commit into a testcase(). Fix a separate assert() in btree.c:freeSpace(). FossilOrigin-Name: fe51d3aa0ce7021213293a0647e31164073e78f7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6883385671..7a156440dc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\ssize\sreduction\sand\sperformance\simprovement\sin\sbtree.c:allocateSpace(). -D 2014-08-20T11:56:14.338 +C Change\san\sassert()\sadded\sby\sthe\sprevious\scommit\sinto\sa\stestcase().\nFix\sa\sseparate\sassert()\sin\sbtree.c:freeSpace(). +D 2014-08-20T13:17:43.884 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 398ecbdb4a19230940955c10b4f35de958e9a05c +F src/btree.c 53eb576f72093d5138f5b7468994121c6eb6fe98 F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 5abf794fe8a605f2005b422e98a3cedad9b9ef5b @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6c66beae97ba1799c908d3a33371dedbc7f3f58c -R 28706ecaca72a1e3574ba7b8d51854c6 +P 1cb1cd64930a6bc371143f7d2e77eb1c51498cb0 +R 2da2148faf73b1d43a764ae813ef9e60 U drh -Z ca9532aeb7742cb91c75177cb9ee4fbc +Z 36b8d62cf946d2fc64ee098831079eb8 diff --git a/manifest.uuid b/manifest.uuid index d1bf93d7a5..4a04c24fb1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1cb1cd64930a6bc371143f7d2e77eb1c51498cb0 \ No newline at end of file +fe51d3aa0ce7021213293a0647e31164073e78f7 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 61848dcb83..a09b82fe5e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1267,7 +1267,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ defragment_page: - assert( pPage->nCell>0 ); + testcase( pPage->nCell==0 ); rc = defragmentPage(pPage); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); @@ -1306,7 +1306,7 @@ static int freeSpace(MemPage *pPage, int start, int size){ assert( start>=pPage->hdrOffset+6+pPage->childPtrSize ); assert( (start + size) <= (int)pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( size>=0 ); /* Minimum cell size is 4 */ + assert( size>=4 ); /* Minimum cell size is 4 */ if( pPage->pBt->btsFlags & BTS_SECURE_DELETE ){ /* Overwrite deleted information with zeros when the secure_delete From fb90841adf94e9edbfa02d4506a26d2a05da11e9 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Aug 2014 13:25:06 +0000 Subject: [PATCH 237/710] Add SQLITE_API macros in front of interface routines in the test_intarray.c extension. FossilOrigin-Name: eea0661798e10018615854c871f24be0f8bb2ff9 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/test_intarray.c | 4 ++-- src/test_intarray.h | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index e9f109a9a7..bee5fb1b55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\sshowdb\susage\smessage. -D 2014-08-20T10:42:16.102 +C Add\sSQLITE_API\smacros\sin\sfront\sof\sinterface\sroutines\sin\sthe\stest_intarray.c\nextension. +D 2014-08-20T13:25:06.147 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -252,8 +252,8 @@ F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f F src/test_func.c d3013ce36f19ac72a99c73864930fd1fa41832f8 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 -F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61 -F src/test_intarray.h 2ece66438cfd177b78d1bfda7a4180cd3a10844d +F src/test_intarray.c db4614c2262a06abc4409dc048d59c580c38320f +F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 F src/test_malloc.c 1ff5b1243d96124c9a180f3b89424820a1f337f3 @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 73637d12e31f5489efe37d8cf4ab50a1911d4c75 -R d268dbd0231a0e3f9784dd51ed74231a -U dan -Z b16b223d7974024b463d42100aaa47bd +P 6c66beae97ba1799c908d3a33371dedbc7f3f58c +R 0e361fd9748a232cee30a1097c2f8f6b +U drh +Z c85b6dec89d4d1dc359d8501fa41031d diff --git a/manifest.uuid b/manifest.uuid index 45aa98f21f..bf670ea860 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c66beae97ba1799c908d3a33371dedbc7f3f58c \ No newline at end of file +eea0661798e10018615854c871f24be0f8bb2ff9 \ No newline at end of file diff --git a/src/test_intarray.c b/src/test_intarray.c index f5c3d9e405..efcd21d404 100644 --- a/src/test_intarray.c +++ b/src/test_intarray.c @@ -216,7 +216,7 @@ static sqlite3_module intarrayModule = { ** explicitly by the application, the virtual table will be dropped implicitly ** by the system when the database connection is closed. */ -int sqlite3_intarray_create( +SQLITE_API int sqlite3_intarray_create( sqlite3 *db, const char *zName, sqlite3_intarray **ppReturn @@ -250,7 +250,7 @@ int sqlite3_intarray_create( ** any query against the corresponding virtual table. If the integer ** array does change or is deallocated undefined behavior will result. */ -int sqlite3_intarray_bind( +SQLITE_API int sqlite3_intarray_bind( sqlite3_intarray *pIntArray, /* The intarray object to bind to */ int nElements, /* Number of elements in the intarray */ sqlite3_int64 *aElements, /* Content of the intarray */ diff --git a/src/test_intarray.h b/src/test_intarray.h index 6d26235a87..84b1f3fe66 100644 --- a/src/test_intarray.h +++ b/src/test_intarray.h @@ -102,7 +102,7 @@ typedef struct sqlite3_intarray sqlite3_intarray; ** explicitly by the application, the virtual table will be dropped implicitly ** by the system when the database connection is closed. */ -int sqlite3_intarray_create( +SQLITE_API int sqlite3_intarray_create( sqlite3 *db, const char *zName, sqlite3_intarray **ppReturn @@ -115,7 +115,7 @@ int sqlite3_intarray_create( ** any query against the corresponding virtual table. If the integer ** array does change or is deallocated undefined behavior will result. */ -int sqlite3_intarray_bind( +SQLITE_API int sqlite3_intarray_bind( sqlite3_intarray *pIntArray, /* The intarray object to bind to */ int nElements, /* Number of elements in the intarray */ sqlite3_int64 *aElements, /* Content of the intarray */ From 7fb91646b56fb33bad80abd34a8d34938fb461c5 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Aug 2014 14:37:09 +0000 Subject: [PATCH 238/710] Refactor local variable names in the freeSpace() routine of btree.c for improved understandability. FossilOrigin-Name: 7e63089a191f29aefde05e89bb612f3036cfa034 --- manifest | 13 ++++---- manifest.uuid | 2 +- src/btree.c | 85 +++++++++++++++++++++++++++------------------------ 3 files changed, 52 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index c03b96d1a8..c1cbe5415f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Size\sreduction\sand\sperformance\simprovements\sin\sbtree.c\sand\sthe\nallocateSpace()\sroutine.\s\sAlso\sfix\san\sassert()\sin\sfreeSpace(). -D 2014-08-20T13:35:45.034 +C Refactor\slocal\svariable\snames\sin\sthe\sfreeSpace()\sroutine\sof\sbtree.c\sfor\nimproved\sunderstandability. +D 2014-08-20T14:37:09.167 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 53eb576f72093d5138f5b7468994121c6eb6fe98 +F src/btree.c c1235eacb8d4de12850daccb1636763218da3381 F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 5abf794fe8a605f2005b422e98a3cedad9b9ef5b @@ -1186,8 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eea0661798e10018615854c871f24be0f8bb2ff9 fe51d3aa0ce7021213293a0647e31164073e78f7 -R 3c51b363f531f6f6895a1e7e29ea4492 -T +closed fe51d3aa0ce7021213293a0647e31164073e78f7 +P 121308fa869ad490a6924798d276c0ff32759acc +R 332118792b6ef82f7eceb02b0d545622 U drh -Z 25da1aeccb4d6e0fa0390c2ee1a22b6e +Z accc32db7c4714707b5e7cc22312c49c diff --git a/manifest.uuid b/manifest.uuid index aa49654c6a..1b60255c36 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -121308fa869ad490a6924798d276c0ff32759acc \ No newline at end of file +7e63089a191f29aefde05e89bb612f3036cfa034 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index a09b82fe5e..ac13b0f3da 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1290,28 +1290,30 @@ defragment_page: /* ** Return a section of the pPage->aData to the freelist. -** The first byte of the new free block is pPage->aDisk[start] -** and the size of the block is "size" bytes. +** The first byte of the new free block is pPage->aData[iStart] +** and the size of the block is iSize bytes. ** ** Most of the effort here is involved in coalesing adjacent ** free blocks into a single big free block. */ -static int freeSpace(MemPage *pPage, int start, int size){ - int addr, pbegin, hdr; - int iLast; /* Largest possible freeblock offset */ - unsigned char *data = pPage->aData; +static int freeSpace(MemPage *pPage, int iStart, int iSize){ + int iPtr; /* Address of pointer to next freeblock */ + int iFreeBlk; /* Address of the next freeblock */ + int hdr; /* Page header size. 0 or 100 */ + int iLast; /* Largest possible freeblock offset */ + unsigned char *data = pPage->aData; /* Page content */ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - assert( start>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( (start + size) <= (int)pPage->pBt->usableSize ); + assert( iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); + assert( (iStart + iSize) <= (int)pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( size>=4 ); /* Minimum cell size is 4 */ + assert( iSize>=4 ); /* Minimum cell size is 4 */ if( pPage->pBt->btsFlags & BTS_SECURE_DELETE ){ /* Overwrite deleted information with zeros when the secure_delete ** option is enabled */ - memset(&data[start], 0, size); + memset(&data[iStart], 0, iSize); } /* Add the space back into the linked list of freeblocks. Note that @@ -1324,53 +1326,56 @@ static int freeSpace(MemPage *pPage, int start, int size){ ** the freelist. */ hdr = pPage->hdrOffset; - addr = hdr + 1; + iPtr = hdr + 1; iLast = pPage->pBt->usableSize - 4; - assert( start<=iLast ); - while( (pbegin = get2byte(&data[addr]))0 ){ - if( pbegin0 ){ + if( iFreeBlkiLast ){ + if( iFreeBlk>iLast ){ return SQLITE_CORRUPT_BKPT; } - assert( pbegin>addr || pbegin==0 ); - put2byte(&data[addr], start); - put2byte(&data[start], pbegin); - put2byte(&data[start+2], size); - pPage->nFree = pPage->nFree + (u16)size; + assert( iFreeBlk>iPtr || iFreeBlk==0 ); + put2byte(&data[iPtr], iStart); + put2byte(&data[iStart], iFreeBlk); + put2byte(&data[iStart+2], iSize); + pPage->nFree = pPage->nFree + (u16)iSize; /* Coalesce adjacent free blocks */ - addr = hdr + 1; - while( (pbegin = get2byte(&data[addr]))>0 ){ - int pnext, psize, x; - assert( pbegin>addr ); - assert( pbegin <= (int)pPage->pBt->usableSize-4 ); - pnext = get2byte(&data[pbegin]); - psize = get2byte(&data[pbegin+2]); - if( pbegin + psize + 3 >= pnext && pnext>0 ){ - int frag = pnext - (pbegin+psize); - if( (frag<0) || (frag>(int)data[hdr+7]) ){ + iPtr = hdr + 1; + while( (iFreeBlk = get2byte(&data[iPtr]))>0 ){ + int iNextBlk; /* Next freeblock after iFreeBlk */ + int szFreeBlk; /* Size of iFreeBlk */ + assert( iFreeBlk>iPtr ); + assert( iFreeBlk <= (int)pPage->pBt->usableSize-4 ); + iNextBlk = get2byte(&data[iFreeBlk]); + szFreeBlk = get2byte(&data[iFreeBlk+2]); + if( iFreeBlk + szFreeBlk + 3 >= iNextBlk && iNextBlk>0 ){ + int nFrag; /* Fragment bytes in between iFreeBlk and iNextBlk */ + int x; /* Temp value */ + nFrag = iNextBlk - (iFreeBlk+szFreeBlk); + if( (nFrag<0) || (nFrag>(int)data[hdr+7]) ){ return SQLITE_CORRUPT_BKPT; } - data[hdr+7] -= (u8)frag; - x = get2byte(&data[pnext]); - put2byte(&data[pbegin], x); - x = pnext + get2byte(&data[pnext+2]) - pbegin; - put2byte(&data[pbegin+2], x); + data[hdr+7] -= (u8)nFrag; + x = get2byte(&data[iNextBlk]); + put2byte(&data[iFreeBlk], x); + x = iNextBlk + get2byte(&data[iNextBlk+2]) - iFreeBlk; + put2byte(&data[iFreeBlk+2], x); }else{ - addr = pbegin; + iPtr = iFreeBlk; } } /* If the cell content area begins with a freeblock, remove it. */ if( data[hdr+1]==data[hdr+5] && data[hdr+2]==data[hdr+6] ){ int top; - pbegin = get2byte(&data[hdr+1]); - memcpy(&data[hdr+1], &data[pbegin], 2); - top = get2byte(&data[hdr+5]) + get2byte(&data[pbegin+2]); + iFreeBlk = get2byte(&data[hdr+1]); + memcpy(&data[hdr+1], &data[iFreeBlk], 2); + top = get2byte(&data[hdr+5]) + get2byte(&data[iFreeBlk+2]); put2byte(&data[hdr+5], top); } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); From 5f5c753efbbe23df1e52a0560c2b7abbc0f3c334 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Aug 2014 17:56:27 +0000 Subject: [PATCH 239/710] Reimplement the freeSpace() routine in btree.c so that it runs faster. FossilOrigin-Name: fe4fd014b42b7b158ca968f1535b5636c67769f6 --- manifest | 12 ++--- manifest.uuid | 2 +- src/btree.c | 125 +++++++++++++++++++++++++------------------------- 3 files changed, 70 insertions(+), 69 deletions(-) diff --git a/manifest b/manifest index c1cbe5415f..4fe8e5ec20 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\slocal\svariable\snames\sin\sthe\sfreeSpace()\sroutine\sof\sbtree.c\sfor\nimproved\sunderstandability. -D 2014-08-20T14:37:09.167 +C Reimplement\sthe\sfreeSpace()\sroutine\sin\sbtree.c\sso\sthat\sit\sruns\sfaster. +D 2014-08-20T17:56:27.585 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c c1235eacb8d4de12850daccb1636763218da3381 +F src/btree.c fe2d11c8c0d99e324614141ad012b37d0601cf59 F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 5abf794fe8a605f2005b422e98a3cedad9b9ef5b @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 121308fa869ad490a6924798d276c0ff32759acc -R 332118792b6ef82f7eceb02b0d545622 +P 7e63089a191f29aefde05e89bb612f3036cfa034 +R cbff1ae81d05ca3b78b420deaaa6083f U drh -Z accc32db7c4714707b5e7cc22312c49c +Z d684bbbc1bfe7cdee532398c2232ddf8 diff --git a/manifest.uuid b/manifest.uuid index 1b60255c36..238682a66d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e63089a191f29aefde05e89bb612f3036cfa034 \ No newline at end of file +fe4fd014b42b7b158ca968f1535b5636c67769f6 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index ac13b0f3da..c10e2f1c32 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1293,92 +1293,93 @@ defragment_page: ** The first byte of the new free block is pPage->aData[iStart] ** and the size of the block is iSize bytes. ** -** Most of the effort here is involved in coalesing adjacent -** free blocks into a single big free block. +** Adjacent freeblocks are coalesced. +** +** Note that even though the freeblock list was checked by btreeInitPage(), +** that routine will not detect overlap between cells or freeblocks. Nor +** does it detect cells or freeblocks that encrouch into the reserved bytes +** at the end of the page. So do additional corruption checks inside this +** routine and return SQLITE_CORRUPT if any problems are found. */ -static int freeSpace(MemPage *pPage, int iStart, int iSize){ - int iPtr; /* Address of pointer to next freeblock */ - int iFreeBlk; /* Address of the next freeblock */ - int hdr; /* Page header size. 0 or 100 */ - int iLast; /* Largest possible freeblock offset */ +static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ + u16 iPtr; /* Address of pointer to next freeblock */ + u16 iFreeBlk; /* Address of the next freeblock */ + u8 hdr; /* Page header size. 0 or 100 */ + u8 nFrag = 0; /* Reduction in fragmentation */ + u16 iOrigSize = iSize; /* Original value of iSize */ + u32 iLast = pPage->pBt->usableSize-4; /* Largest possible freeblock offset */ + u32 iEnd = iStart + iSize; /* First byte past the iStart buffer */ unsigned char *data = pPage->aData; /* Page content */ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( (iStart + iSize) <= (int)pPage->pBt->usableSize ); + assert( iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ + assert( iStart<=iLast ); + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ if( pPage->pBt->btsFlags & BTS_SECURE_DELETE ){ - /* Overwrite deleted information with zeros when the secure_delete - ** option is enabled */ memset(&data[iStart], 0, iSize); } - /* Add the space back into the linked list of freeblocks. Note that - ** even though the freeblock list was checked by btreeInitPage(), - ** btreeInitPage() did not detect overlapping cells or - ** freeblocks that overlapped cells. Nor does it detect when the - ** cell content area exceeds the value in the page header. If these - ** situations arise, then subsequent insert operations might corrupt - ** the freelist. So we do need to check for corruption while scanning - ** the freelist. + /* The list of freeblocks must be in ascending order. Find the + ** spot on the list where iStart should be inserted. */ hdr = pPage->hdrOffset; iPtr = hdr + 1; - iLast = pPage->pBt->usableSize - 4; - assert( iStart<=iLast ); - while( (iFreeBlk = get2byte(&data[iPtr]))0 ){ - if( iFreeBlk0 && iFreeBlkiLast ){ - return SQLITE_CORRUPT_BKPT; - } + if( iFreeBlk>iLast ) return SQLITE_CORRUPT_BKPT; assert( iFreeBlk>iPtr || iFreeBlk==0 ); - put2byte(&data[iPtr], iStart); - put2byte(&data[iStart], iFreeBlk); - put2byte(&data[iStart+2], iSize); - pPage->nFree = pPage->nFree + (u16)iSize; - /* Coalesce adjacent free blocks */ - iPtr = hdr + 1; - while( (iFreeBlk = get2byte(&data[iPtr]))>0 ){ - int iNextBlk; /* Next freeblock after iFreeBlk */ - int szFreeBlk; /* Size of iFreeBlk */ - assert( iFreeBlk>iPtr ); - assert( iFreeBlk <= (int)pPage->pBt->usableSize-4 ); - iNextBlk = get2byte(&data[iFreeBlk]); - szFreeBlk = get2byte(&data[iFreeBlk+2]); - if( iFreeBlk + szFreeBlk + 3 >= iNextBlk && iNextBlk>0 ){ - int nFrag; /* Fragment bytes in between iFreeBlk and iNextBlk */ - int x; /* Temp value */ - nFrag = iNextBlk - (iFreeBlk+szFreeBlk); - if( (nFrag<0) || (nFrag>(int)data[hdr+7]) ){ - return SQLITE_CORRUPT_BKPT; - } - data[hdr+7] -= (u8)nFrag; - x = get2byte(&data[iNextBlk]); - put2byte(&data[iFreeBlk], x); - x = iNextBlk + get2byte(&data[iNextBlk+2]) - iFreeBlk; - put2byte(&data[iFreeBlk+2], x); - }else{ - iPtr = iFreeBlk; + /* At this point: + ** iFreeBlk: First freeblock after iStart, or zero if none + ** iPtr: The address of a pointer iFreeBlk + ** + ** Check to see if iFreeBlk should be coalesced onto the end of iStart. + */ + if( iFreeBlk && iEnd+3>=iFreeBlk ){ + nFrag = iFreeBlk - iEnd; + if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_BKPT; + iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); + iSize = iEnd - iStart; + iFreeBlk = get2byte(&data[iFreeBlk]); + } + + /* If iPtr is another freeblock (that is, if iPtr is not the freelist pointer + ** in the page header) then check to see if iStart should be coalesced + ** onto the end of iPtr. + */ + if( iPtr>hdr+1 ){ + int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); + if( iPtrEnd+3>=iStart ){ + if( iPtrEnd>iStart ) return SQLITE_CORRUPT_BKPT; + nFrag += iStart - iPtrEnd; + iSize = iEnd - iPtr; + iStart = iPtr; } } + if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_BKPT; - /* If the cell content area begins with a freeblock, remove it. */ - if( data[hdr+1]==data[hdr+5] && data[hdr+2]==data[hdr+6] ){ - int top; - iFreeBlk = get2byte(&data[hdr+1]); - memcpy(&data[hdr+1], &data[iFreeBlk], 2); - top = get2byte(&data[hdr+5]) + get2byte(&data[iFreeBlk+2]); - put2byte(&data[hdr+5], top); + data[hdr+7] -= nFrag; + if( iPtr==hdr+1 && iStart==get2byte(&data[hdr+5]) ){ + /* The new freeblock is at the beginning of the cell content area, + ** so just extend the cell content area rather than create another + ** freelist entry */ + put2byte(&data[hdr+1], iFreeBlk); + put2byte(&data[hdr+5], iEnd); + }else{ + /* Insert the new freeblock into the freelist */ + put2byte(&data[iPtr], iStart); + put2byte(&data[iStart], iFreeBlk); + put2byte(&data[iStart+2], iSize); } - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + pPage->nFree += iOrigSize; return SQLITE_OK; } From 7bc4c454dab1568702db8ef230c624edc35ed815 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Aug 2014 18:43:44 +0000 Subject: [PATCH 240/710] A small performance improvement in freeSpace() by special-casing the relatively common case of an empty freelist. FossilOrigin-Name: 49f44d355ff70744e4951baca2481c7c2b6c02b3 --- manifest | 12 ++++---- manifest.uuid | 2 +- src/btree.c | 76 +++++++++++++++++++++++++++------------------------ 3 files changed, 47 insertions(+), 43 deletions(-) diff --git a/manifest b/manifest index 4fe8e5ec20..0e55755198 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reimplement\sthe\sfreeSpace()\sroutine\sin\sbtree.c\sso\sthat\sit\sruns\sfaster. -D 2014-08-20T17:56:27.585 +C A\ssmall\sperformance\simprovement\sin\sfreeSpace()\sby\sspecial-casing\sthe\nrelatively\scommon\scase\sof\san\sempty\sfreelist. +D 2014-08-20T18:43:44.510 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c fe2d11c8c0d99e324614141ad012b37d0601cf59 +F src/btree.c 4195fed5741b4dbcc9831b623aec487258f3e62d F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 5abf794fe8a605f2005b422e98a3cedad9b9ef5b @@ -1186,7 +1186,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7e63089a191f29aefde05e89bb612f3036cfa034 -R cbff1ae81d05ca3b78b420deaaa6083f +P fe4fd014b42b7b158ca968f1535b5636c67769f6 +R 865eceb09aea170bb70d44b771b85e6e U drh -Z d684bbbc1bfe7cdee532398c2232ddf8 +Z 0631afdcc5729e41fc50a75336710200 diff --git a/manifest.uuid b/manifest.uuid index 238682a66d..2bb2527d30 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fe4fd014b42b7b158ca968f1535b5636c67769f6 \ No newline at end of file +49f44d355ff70744e4951baca2481c7c2b6c02b3 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c10e2f1c32..56718b69d6 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1330,47 +1330,51 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ */ hdr = pPage->hdrOffset; iPtr = hdr + 1; - while( (iFreeBlk = get2byte(&data[iPtr]))>0 && iFreeBlkiLast ) return SQLITE_CORRUPT_BKPT; - assert( iFreeBlk>iPtr || iFreeBlk==0 ); - - /* At this point: - ** iFreeBlk: First freeblock after iStart, or zero if none - ** iPtr: The address of a pointer iFreeBlk - ** - ** Check to see if iFreeBlk should be coalesced onto the end of iStart. - */ - if( iFreeBlk && iEnd+3>=iFreeBlk ){ - nFrag = iFreeBlk - iEnd; - if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_BKPT; - iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); - iSize = iEnd - iStart; - iFreeBlk = get2byte(&data[iFreeBlk]); - } - - /* If iPtr is another freeblock (that is, if iPtr is not the freelist pointer - ** in the page header) then check to see if iStart should be coalesced - ** onto the end of iPtr. - */ - if( iPtr>hdr+1 ){ - int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); - if( iPtrEnd+3>=iStart ){ - if( iPtrEnd>iStart ) return SQLITE_CORRUPT_BKPT; - nFrag += iStart - iPtrEnd; - iSize = iEnd - iPtr; - iStart = iPtr; + if( data[iPtr+1]==0 && data[iPtr]==0 ){ + iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */ + }else{ + while( (iFreeBlk = get2byte(&data[iPtr]))>0 && iFreeBlkiLast ) return SQLITE_CORRUPT_BKPT; + assert( iFreeBlk>iPtr || iFreeBlk==0 ); + + /* At this point: + ** iFreeBlk: First freeblock after iStart, or zero if none + ** iPtr: The address of a pointer iFreeBlk + ** + ** Check to see if iFreeBlk should be coalesced onto the end of iStart. + */ + if( iFreeBlk && iEnd+3>=iFreeBlk ){ + nFrag = iFreeBlk - iEnd; + if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_BKPT; + iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); + iSize = iEnd - iStart; + iFreeBlk = get2byte(&data[iFreeBlk]); + } + + /* If iPtr is another freeblock (that is, if iPtr is not the freelist pointer + ** in the page header) then check to see if iStart should be coalesced + ** onto the end of iPtr. + */ + if( iPtr>hdr+1 ){ + int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); + if( iPtrEnd+3>=iStart ){ + if( iPtrEnd>iStart ) return SQLITE_CORRUPT_BKPT; + nFrag += iStart - iPtrEnd; + iSize = iEnd - iPtr; + iStart = iPtr; + } + } + if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_BKPT; + data[hdr+7] -= nFrag; } - if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_BKPT; - - data[hdr+7] -= nFrag; - if( iPtr==hdr+1 && iStart==get2byte(&data[hdr+5]) ){ + if( iStart==get2byte(&data[hdr+5]) ){ /* The new freeblock is at the beginning of the cell content area, ** so just extend the cell content area rather than create another ** freelist entry */ + if( iPtr!=hdr+1 ) return SQLITE_CORRUPT_BKPT; put2byte(&data[hdr+1], iFreeBlk); put2byte(&data[hdr+5], iEnd); }else{ From f3f69ac942f6cbe0f143659399281c162ad1e743 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Aug 2014 23:38:07 +0000 Subject: [PATCH 241/710] Enhancements to skip-scan such that it is operable when a middle column of an index is skipped while the left-most column is constrained in the WHERE clause. FossilOrigin-Name: bc985caa7816f1f873ad8e4467c5278399f315ce --- manifest | 13 ++++---- manifest.uuid | 2 +- src/where.c | 21 +++++++++---- test/skipscan3.test | 73 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 12 deletions(-) create mode 100644 test/skipscan3.test diff --git a/manifest b/manifest index 0e55755198..928130ab8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C A\ssmall\sperformance\simprovement\sin\sfreeSpace()\sby\sspecial-casing\sthe\nrelatively\scommon\scase\sof\san\sempty\sfreelist. -D 2014-08-20T18:43:44.510 +C Enhancements\sto\sskip-scan\ssuch\sthat\sit\sis\soperable\swhen\sa\smiddle\scolumn\sof\nan\sindex\sis\sskipped\swhile\sthe\sleft-most\scolumn\sis\sconstrained\sin\sthe\sWHERE\nclause. +D 2014-08-20T23:38:07.310 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,7 +296,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c ab20f9c24a422ee8900831b343c3d1e5e7aca87b +F src/where.c 4c499d185827a492643cf017ae5e3aa0523f9f18 F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -832,6 +832,7 @@ F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 F test/skipscan1.test 28c7faa41a0d7265040ecb0a0abd90c0904270b2 F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a +F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test d8b9692b702745a0e41c23f9da6beac81df01196 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 @@ -1186,7 +1187,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fe4fd014b42b7b158ca968f1535b5636c67769f6 -R 865eceb09aea170bb70d44b771b85e6e +P 49f44d355ff70744e4951baca2481c7c2b6c02b3 +R 96923099996337b045b25027695cfca6 U drh -Z 0631afdcc5729e41fc50a75336710200 +Z 1b9938878e462b798940573e6153541a diff --git a/manifest.uuid b/manifest.uuid index 2bb2527d30..c25a80761d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49f44d355ff70744e4951baca2481c7c2b6c02b3 \ No newline at end of file +bc985caa7816f1f873ad8e4467c5278399f315ce \ No newline at end of file diff --git a/src/where.c b/src/where.c index 9c30136e87..6a4299cc9a 100644 --- a/src/where.c +++ b/src/where.c @@ -3781,8 +3781,8 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3DebugPrintf(" %12s", pItem->zAlias ? pItem->zAlias : pTab->zName); if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ - const char *zName; - if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ + const char *zName; + if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ if( strncmp(zName, "sqlite_autoindex_", 17)==0 ){ int i = sqlite3Strlen30(zName) - 1; while( zName[i]!='_' ) i--; @@ -3803,7 +3803,11 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3DebugPrintf(" %-19s", z); sqlite3_free(z); } - sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); + if( p->wsFlags & WHERE_SKIPSCAN ){ + sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->u.btree.nSkip); + }else{ + sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); + } sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); #ifdef SQLITE_ENABLE_TREE_EXPLAIN /* If the 0x100 bit of wheretracing is set, then show all of the constraint @@ -4316,8 +4320,7 @@ static int whereLoopAddBtreeIndex( ** On the other hand, the extra seeks could end up being significantly ** more expensive. */ assert( 42==sqlite3LogEst(18) ); - if( pTerm==0 - && saved_nEq==saved_nSkip + if( saved_nEq==saved_nSkip && saved_nEq+1nKeyCol && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK @@ -4328,9 +4331,17 @@ static int whereLoopAddBtreeIndex( pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + if( pTerm ){ + /* TUNING: When estimating skip-scan for a term that is also indexable, + ** increase the cost of the skip-scan by 2x, to make it a little less + ** desirable than the regular index lookup. */ + nIter += 10; assert( 10==sqlite3LogEst(2) ); + } pNew->nOut -= nIter; whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); pNew->nOut = saved_nOut; + pNew->u.btree.nEq = saved_nEq; + pNew->u.btree.nSkip = saved_nSkip; } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ diff --git a/test/skipscan3.test b/test/skipscan3.test new file mode 100644 index 0000000000..260d11cac9 --- /dev/null +++ b/test/skipscan3.test @@ -0,0 +1,73 @@ +# 2014-08-20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests of the "skip-scan" query strategy. +# In particular, this file looks at skipping intermediate terms +# in an index. For example, if (a,b,c) are indexed, and we have +# "WHERE a=?1 AND c=?2" - verify that skip-scan can still be used. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test skipscan3-1.1 { + CREATE TABLE t1(a,b,c,d,PRIMARY KEY(a,b,c)); + WITH RECURSIVE + c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) + INSERT INTO t1(a,b,c,d) + SELECT 1, 1, x, printf('x%04d',x) FROM c; + ANALYZE; +} {} + +# This version has long used skip-scan because of the "+a" +# +do_execsql_test skipscan3-1.2eqp { + EXPLAIN QUERY PLAN SELECT d FROM t1 WHERE +a=1 AND c=32; +} {/*ANY(a) AND ANY(b)*/} +do_execsql_test skipscan3-1.2 { + SELECT d FROM t1 WHERE +a=1 AND c=32; +} {x0032} + +# This version (with "a" instead of "+a") should use skip-scan but +# did not prior to changes implemented on 2014-08-20 +# +do_execsql_test skipscan3-1.3eqp { + EXPLAIN QUERY PLAN SELECT d FROM t1 WHERE a=1 AND c=32; +} {/*ANY(a) AND ANY(b)*/} +do_execsql_test skipscan3-1.3 { + SELECT d FROM t1 WHERE a=1 AND c=32; +} {x0032} + +# Repeat the test on a WITHOUT ROWID table +# +do_execsql_test skipscan3-2.1 { + CREATE TABLE t2(a,b,c,d,PRIMARY KEY(a,b,c)) WITHOUT ROWID; + WITH RECURSIVE + c(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM c WHERE x<1000) + INSERT INTO t2(a,b,c,d) + SELECT 1, 1, x, printf('x%04d',x) FROM c; + ANALYZE; +} {} +do_execsql_test skipscan3-2.2eqp { + EXPLAIN QUERY PLAN SELECT d FROM t2 WHERE +a=1 AND c=32; +} {/*ANY(a) AND ANY(b)*/} +do_execsql_test skipscan3-2.2 { + SELECT d FROM t2 WHERE +a=1 AND c=32; +} {x0032} +do_execsql_test skipscan3-2.3eqp { + EXPLAIN QUERY PLAN SELECT d FROM t2 WHERE a=1 AND c=32; +} {/*ANY(a) AND ANY(b)*/} +do_execsql_test skipscan3-2.3 { + SELECT d FROM t2 WHERE a=1 AND c=32; +} {x0032} + + +finish_test From b614eab3d2da06fea1c23eb3dbc08816ed178b5e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 20 Aug 2014 23:42:50 +0000 Subject: [PATCH 242/710] Increase the version number to 3.8.7 FossilOrigin-Name: 91594aae0725388765070c80039dfe1bf126392d --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 2e14a9557d..4351a7e3a3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.6 +3.8.7 diff --git a/configure b/configure index 8748a98406..9b8266d812 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.6. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.7. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.6' -PACKAGE_STRING='sqlite 3.8.6' +PACKAGE_VERSION='3.8.7' +PACKAGE_STRING='sqlite 3.8.7' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.6 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.7 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.6:";; + short | recursive ) echo "Configuration of sqlite 3.8.7:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.6 +sqlite configure 3.8.7 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.6, which was +It was created by sqlite $as_me 3.8.7, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.6, which was +This file was extended by sqlite $as_me 3.8.7, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.6 +sqlite config.status 3.8.7 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index 928130ab8a..292e662207 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Enhancements\sto\sskip-scan\ssuch\sthat\sit\sis\soperable\swhen\sa\smiddle\scolumn\sof\nan\sindex\sis\sskipped\swhile\sthe\sleft-most\scolumn\sis\sconstrained\sin\sthe\sWHERE\nclause. -D 2014-08-20T23:38:07.310 +C Increase\sthe\sversion\snumber\sto\s3.8.7 +D 2014-08-20T23:42:50.860 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc 5b04e657cf08a9ac7fc47d876c5c8be962c47d6b F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION 1c877615a9db323e3cd301e3d57d853f9d5c4a07 +F VERSION 53a0b870e7f16d3b06623c31d233a304c163a6af F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 513f1e2464c3673bcdb5471b13d98e7eeb6f5ca2 x +F configure ad59a5f48b3c59a92b5506040a22fbe3f733a9d8 x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1187,7 +1187,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 49f44d355ff70744e4951baca2481c7c2b6c02b3 -R 96923099996337b045b25027695cfca6 +P bc985caa7816f1f873ad8e4467c5278399f315ce +R 69e5d59c0ae1acccebd6955e02833316 U drh -Z 1b9938878e462b798940573e6153541a +Z 5338496858d2cbdf1a890a2630150d12 diff --git a/manifest.uuid b/manifest.uuid index c25a80761d..d619efb0e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bc985caa7816f1f873ad8e4467c5278399f315ce \ No newline at end of file +91594aae0725388765070c80039dfe1bf126392d \ No newline at end of file From b6b4b79f344096791542265763467fc90f139c2e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 21 Aug 2014 14:10:23 +0000 Subject: [PATCH 243/710] Fix a faulty assert() statement. Add comments to clarify the behavior of the sqlite3OpenTableAndIndices() routine in insert.c. Add test cases to verify that the assert() statement is not firing inappropriately. Ticket [369d57fb8e5ccdff06f1]. FossilOrigin-Name: 7029b3404d3f5f698a496934f3a3f2972051b257 --- ext/rtree/rtreeF.test | 81 +++++++++++++++++++++++++++++++++++++++++++ manifest | 15 ++++---- manifest.uuid | 2 +- src/delete.c | 8 +++-- src/insert.c | 9 +++-- 5 files changed, 101 insertions(+), 14 deletions(-) create mode 100644 ext/rtree/rtreeF.test diff --git a/ext/rtree/rtreeF.test b/ext/rtree/rtreeF.test new file mode 100644 index 0000000000..c9620d34f7 --- /dev/null +++ b/ext/rtree/rtreeF.test @@ -0,0 +1,81 @@ +# 2014-08-21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file contains tests for the r-tree module. +# +# This file contains test cases for the ticket +# [369d57fb8e5ccdff06f197a37147a88f9de95cda] (2014-08-21) +# +# The following SQL causes an assertion fault while running +# sqlite3_prepare() on the DELETE statement: +# +# CREATE TABLE t1(x); +# CREATE TABLE t2(y); +# CREATE VIRTUAL TABLE t3 USING rtree(a,b,c); +# CREATE TRIGGER t2del AFTER DELETE ON t2 WHEN (SELECT 1 from t1) BEGIN +# DELETE FROM t3 WHERE a=old.y; +# END; +# DELETE FROM t2 WHERE y=1; +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl +ifcapable !rtree { finish_test ; return } + +do_execsql_test rtreeF-1.1 { + CREATE TABLE t1(x); + CREATE TABLE t2(y); + CREATE VIRTUAL TABLE t3 USING rtree(a,b,c); + CREATE TRIGGER t2dwl AFTER DELETE ON t2 WHEN (SELECT 1 from t1) BEGIN + DELETE FROM t3 WHERE a=old.y; + END; + + INSERT INTO t1(x) VALUES(999); + INSERT INTO t2(y) VALUES(1),(2),(3),(4),(5); + INSERT INTO t3(a,b,c) VALUES(1,2,3),(2,3,4),(3,4,5),(4,5,6),(5,6,7); + + SELECT a FROM t3 ORDER BY a; + SELECT '|'; + SELECT y FROM t2 ORDER BY y; +} {1 2 3 4 5 | 1 2 3 4 5} +do_execsql_test rtreeF-1.2 { + DELETE FROM t2 WHERE y=3; + + SELECT a FROM t3 ORDER BY a; + SELECT '|'; + SELECT y FROM t2 ORDER BY y; +} {1 2 4 5 | 1 2 4 5} +do_execsql_test rtreeF-1.3 { + DELETE FROM t1; + DELETE FROM t2 WHERE y=5; + + SELECT a FROM t3 ORDER BY a; + SELECT '|'; + SELECT y FROM t2 ORDER BY y; +} {1 2 4 5 | 1 2 4} +do_execsql_test rtreeF-1.4 { + INSERT INTO t1 DEFAULT VALUES; + DELETE FROM t2 WHERE y=5; + + SELECT a FROM t3 ORDER BY a; + SELECT '|'; + SELECT y FROM t2 ORDER BY y; +} {1 2 4 5 | 1 2 4} +do_execsql_test rtreeF-1.5 { + DELETE FROM t2 WHERE y=2; + + SELECT a FROM t3 ORDER BY a; + SELECT '|'; + SELECT y FROM t2 ORDER BY y; +} {1 4 5 | 1 4} + +finish_test diff --git a/manifest b/manifest index 292e662207..1e5132fbcf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.8.7 -D 2014-08-20T23:42:50.860 +C Fix\sa\sfaulty\sassert()\sstatement.\s\sAdd\scomments\sto\sclarify\sthe\sbehavior\sof\nthe\ssqlite3OpenTableAndIndices()\sroutine\sin\sinsert.c.\s\sAdd\stest\scases\sto\nverify\sthat\sthe\sassert()\sstatement\sis\snot\sfiring\sinappropriately.\nTicket\s[369d57fb8e5ccdff06f1]. +D 2014-08-21T14:10:23.770 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -138,6 +138,7 @@ F ext/rtree/rtreeB.test c85f9ce78766c4e68b8b89fbf2979ee9cfa82b4e F ext/rtree/rtreeC.test df158dcc81f1a43ce7eef361af03c48ec91f1e06 F ext/rtree/rtreeD.test 636630357638f5983701550b37f0f5867130d2ca F ext/rtree/rtreeE.test 388c1c8602c3ce55c15f03b509e9cf545fb7c41f +F ext/rtree/rtreeF.test 66deb9fd1611c7ca2e374adba63debdc2dbb12b4 F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h 83349d519fe5f518b3ea025d18dd1fe51b1684bd @@ -175,7 +176,7 @@ F src/callback.c fcff28cf0df2403dd2f313bb8d1b8f31f6f3cd64 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 -F src/delete.c bcf8f72126cea80fc3d5bc5494cf19b3f8935aaf +F src/delete.c 5adcd322c6b08fc25d215d780ca62cebce66304d F src/expr.c f749009cf4a8534efb5e0d5cd7c9fb1fb0f2836c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 @@ -184,7 +185,7 @@ F src/global.c 1e4bd956dc2f608f87d2a929abc4a20db65f30e4 F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 991e4964e9295da3993e2c0f81c7faf642371848 +F src/insert.c d1a104e67b33314d4cc5c1356147446086ab9fc8 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c febc2a9e7ad6c1a6191c7b5b9170b325d263f343 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b @@ -1187,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bc985caa7816f1f873ad8e4467c5278399f315ce -R 69e5d59c0ae1acccebd6955e02833316 +P 91594aae0725388765070c80039dfe1bf126392d +R 25e887863ed8cfd2c8a241fac46385f1 U drh -Z 5338496858d2cbdf1a890a2630150d12 +Z ed55b6e966354470a11e6c95fe10b90c diff --git a/manifest.uuid b/manifest.uuid index d619efb0e7..944f894bbc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91594aae0725388765070c80039dfe1bf126392d \ No newline at end of file +7029b3404d3f5f698a496934f3a3f2972051b257 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index c74d8eab7d..af83903c42 100644 --- a/src/delete.c +++ b/src/delete.c @@ -466,10 +466,11 @@ void sqlite3DeleteFrom( ** triggers. */ if( !isView ){ + testcase( IsVirtual(pTab) ); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, iTabCur, aToOpen, &iDataCur, &iIdxCur); - assert( pPk || iDataCur==iTabCur ); - assert( pPk || iIdxCur==iDataCur+1 ); + assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); + assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); } /* Set up a loop over the rowids/primary-keys that were found in the @@ -477,7 +478,8 @@ void sqlite3DeleteFrom( */ if( okOnePass ){ /* Just one row. Hence the top-of-loop is a no-op */ - assert( nKey==nPk ); /* OP_Found will use an unpacked key */ + assert( nKey==nPk ); /* OP_Found will use an unpacked key */ + assert( !IsVirtual(pTab) ); if( aToOpen[iDataCur-iTabCur] ){ assert( pPk!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); diff --git a/src/insert.c b/src/insert.c index 5964b91ca4..3e6982d836 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1612,6 +1612,9 @@ void sqlite3CompleteInsertion( ** For a WITHOUT ROWID table, *piDataCur will be somewhere in the range ** of *piIdxCurs, depending on where the PRIMARY KEY index appears on the ** pTab->pIndex list. +** +** If pTab is a virtual table, then this routine is a no-op and the +** *piDataCur and *piIdxCur values are left uninitialized. */ int sqlite3OpenTableAndIndices( Parse *pParse, /* Parsing context */ @@ -1630,9 +1633,9 @@ int sqlite3OpenTableAndIndices( assert( op==OP_OpenRead || op==OP_OpenWrite ); if( IsVirtual(pTab) ){ - assert( aToOpen==0 ); - *piDataCur = 0; - *piIdxCur = 1; + /* This routine is a no-op for virtual tables. Leave the output + ** variables *piDataCur and *piIdxCur uninitialized so that valgrind + ** can detect if they are used by mistake in the caller. */ return 0; } iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); From cf4200a17076f77300759f2f2a640ccaddd5004f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 21 Aug 2014 19:11:17 +0000 Subject: [PATCH 244/710] For sqlite3_win32_is_nt(), assume WinRT is NT-based. FossilOrigin-Name: 2f59e71fbf31d2bd788d11197943b2ff3f4046e1 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/os_win.c | 15 ++++++++++++--- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 1e5132fbcf..8ca8db3ef3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sstatement.\s\sAdd\scomments\sto\sclarify\sthe\sbehavior\sof\nthe\ssqlite3OpenTableAndIndices()\sroutine\sin\sinsert.c.\s\sAdd\stest\scases\sto\nverify\sthat\sthe\sassert()\sstatement\sis\snot\sfiring\sinappropriately.\nTicket\s[369d57fb8e5ccdff06f1]. -D 2014-08-21T14:10:23.770 +C For\ssqlite3_win32_is_nt(),\sassume\sWinRT\sis\sNT-based. +D 2014-08-21T19:11:17.859 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -209,7 +209,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bd7df3094a60915c148517504c76df4fca24e542 -F src/os_win.c d067fce558a5032e6e6afe62899e5397bf63cf3e +F src/os_win.c 817005660016dcaee908b44e4437c000f3666b4c F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1188,7 +1188,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 91594aae0725388765070c80039dfe1bf126392d -R 25e887863ed8cfd2c8a241fac46385f1 -U drh -Z ed55b6e966354470a11e6c95fe10b90c +P 7029b3404d3f5f698a496934f3a3f2972051b257 +R 2bf9f3345de9d3c6f464f51024249e55 +T *branch * winrt +T *sym-winrt * +T -sym-trunk * +U mistachkin +Z 677c2dec084bbb63edeb1740996a4b29 diff --git a/manifest.uuid b/manifest.uuid index 944f894bbc..a2db9504d9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7029b3404d3f5f698a496934f3a3f2972051b257 \ No newline at end of file +2f59e71fbf31d2bd788d11197943b2ff3f4046e1 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index b9f13becd6..17613d253f 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1317,10 +1317,15 @@ void sqlite3_win32_sleep(DWORD milliseconds){ ** based on the NT kernel. */ int sqlite3_win32_is_nt(void){ -#if defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX +#if SQLITE_OS_WINRT + /* + ** NOTE: The WinRT sub-platform is always assumed to be based on the NT + ** kernel. + */ + return 1; +#elif defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ -#if !SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - defined(NTDDI_VERSION) && NTDDI_VERSION >= NTDDI_WIN8 +#if defined(SQLITE_WIN32_HAS_WIDE) OSVERSIONINFOW sInfo; sInfo.dwOSVersionInfoSize = sizeof(sInfo); osGetVersionExW(&sInfo); @@ -1338,6 +1343,10 @@ int sqlite3_win32_is_nt(void){ #elif SQLITE_TEST return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; #else + /* + ** NOTE: All sub-platforms where the GetVersionEx[AW] functions are + ** deprecated are always assumed to be based on the NT kernel. + */ return 1; #endif } From acbcb7e013b0c2d54f8b0977a464badc260d7048 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 21 Aug 2014 20:26:37 +0000 Subject: [PATCH 245/710] Simplify the interface to the symbol table, saving 600 bytes of code space. FossilOrigin-Name: 14b0f561fe15622b61c6676c9c455dca6b9ba5f0 --- manifest | 26 ++++++++++---------- manifest.uuid | 2 +- src/build.c | 24 ++++++------------ src/callback.c | 8 +++--- src/fkey.c | 5 ++-- src/hash.c | 66 ++++++++++++++++++++------------------------------ src/hash.h | 6 ++--- src/main.c | 3 +-- src/trigger.c | 17 +++++-------- src/vtab.c | 11 ++++----- 10 files changed, 69 insertions(+), 99 deletions(-) diff --git a/manifest b/manifest index 1e5132fbcf..bcba60c6c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sstatement.\s\sAdd\scomments\sto\sclarify\sthe\sbehavior\sof\nthe\ssqlite3OpenTableAndIndices()\sroutine\sin\sinsert.c.\s\sAdd\stest\scases\sto\nverify\sthat\sthe\sassert()\sstatement\sis\snot\sfiring\sinappropriately.\nTicket\s[369d57fb8e5ccdff06f1]. -D 2014-08-21T14:10:23.770 +C Simplify\sthe\sinterface\sto\sthe\ssymbol\stable,\ssaving\s600\sbytes\sof\scode\sspace. +D 2014-08-21T20:26:37.728 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -171,26 +171,26 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 4195fed5741b4dbcc9831b623aec487258f3e62d F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c 5abf794fe8a605f2005b422e98a3cedad9b9ef5b -F src/callback.c fcff28cf0df2403dd2f313bb8d1b8f31f6f3cd64 +F src/build.c 058e3aadb1376521ff291735237edf4c10f438fb +F src/callback.c b97d0695ffcf6a8710ee445ffe56ee387d4d8a6f F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 5adcd322c6b08fc25d215d780ca62cebce66304d F src/expr.c f749009cf4a8534efb5e0d5cd7c9fb1fb0f2836c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c 8545f3b36da47473e10800ea4fb0810fd4062514 +F src/fkey.c 8d81a780ad78d16ec9082585758a8f1d6bf02ca3 F src/func.c bbb724b74ed96ca42675a7274646a71dd52bcda7 F src/global.c 1e4bd956dc2f608f87d2a929abc4a20db65f30e4 -F src/hash.c d139319967164f139c8d1bb8a11b14db9c4ba3cd -F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 +F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 +F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c d1a104e67b33314d4cc5c1356147446086ab9fc8 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c febc2a9e7ad6c1a6191c7b5b9170b325d263f343 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 1cf92c5c6468f2b6ed99b638706781ccc9c60b42 +F src/main.c 1e5d34fb6dee85019b4bcc44e8576457b5075174 F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -279,7 +279,7 @@ F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec -F src/trigger.c 66f3470b03b52b395e839155786966e3e037fddb +F src/trigger.c 4bddd12803275aa98f1c7ce0118fceb02b2167f6 F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e @@ -293,7 +293,7 @@ F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 -F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd +F src/vtab.c 180bfc5e69c92f2014c094bc49a66e8c37c188ac F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 91594aae0725388765070c80039dfe1bf126392d -R 25e887863ed8cfd2c8a241fac46385f1 +P 7029b3404d3f5f698a496934f3a3f2972051b257 +R 818f77c3be876451b2dad41de8a59565 U drh -Z ed55b6e966354470a11e6c95fe10b90c +Z cb34ea9e2178e12926edef43b644d982 diff --git a/manifest.uuid b/manifest.uuid index 944f894bbc..98b1fd2ac2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7029b3404d3f5f698a496934f3a3f2972051b257 \ No newline at end of file +14b0f561fe15622b61c6676c9c455dca6b9ba5f0 \ No newline at end of file diff --git a/src/build.c b/src/build.c index a9a8f21793..af776cb3d8 100644 --- a/src/build.c +++ b/src/build.c @@ -286,16 +286,14 @@ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ Table *p = 0; int i; - int nName; assert( zName!=0 ); - nName = sqlite3Strlen30(zName); /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName, nName); + p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName); if( p ) break; } return p; @@ -378,7 +376,6 @@ Table *sqlite3LocateTableItem( Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ Index *p = 0; int i; - int nName = sqlite3Strlen30(zName); /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ @@ -387,7 +384,7 @@ Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){ assert( pSchema ); if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&pSchema->idxHash, zName, nName); + p = sqlite3HashFind(&pSchema->idxHash, zName); if( p ) break; } return p; @@ -415,13 +412,11 @@ static void freeIndex(sqlite3 *db, Index *p){ */ void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char *zIdxName){ Index *pIndex; - int len; Hash *pHash; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pHash = &db->aDb[iDb].pSchema->idxHash; - len = sqlite3Strlen30(zIdxName); - pIndex = sqlite3HashInsert(pHash, zIdxName, len, 0); + pIndex = sqlite3HashInsert(pHash, zIdxName, 0); if( ALWAYS(pIndex) ){ if( pIndex->pTable->pIndex==pIndex ){ pIndex->pTable->pIndex = pIndex->pNext; @@ -581,7 +576,7 @@ void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ if( !db || db->pnBytesFreed==0 ){ char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( - &pIndex->pSchema->idxHash, zName, sqlite3Strlen30(zName), 0 + &pIndex->pSchema->idxHash, zName, 0 ); assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); assert( pOld==pIndex || pOld==0 ); @@ -624,8 +619,7 @@ void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char *zTabName){ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); testcase( zTabName[0]==0 ); /* Zero-length table names are allowed */ pDb = &db->aDb[iDb]; - p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, - sqlite3Strlen30(zTabName),0); + p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0); sqlite3DeleteTable(db, p); db->flags |= SQLITE_InternChanges; } @@ -1947,8 +1941,7 @@ void sqlite3EndTable( Table *pOld; Schema *pSchema = p->pSchema; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, - sqlite3Strlen30(p->zName),p); + pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, p); if( pOld ){ assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ db->mallocFailed = 1; @@ -2598,7 +2591,7 @@ void sqlite3CreateForeignKey( assert( sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, - pFKey->zTo, sqlite3Strlen30(pFKey->zTo), (void *)pFKey + pFKey->zTo, (void *)pFKey ); if( pNextTo==pFKey ){ db->mallocFailed = 1; @@ -3146,8 +3139,7 @@ Index *sqlite3CreateIndex( Index *p; assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); p = sqlite3HashInsert(&pIndex->pSchema->idxHash, - pIndex->zName, sqlite3Strlen30(pIndex->zName), - pIndex); + pIndex->zName, pIndex); if( p ){ assert( p==pIndex ); /* Malloc must have failed */ db->mallocFailed = 1; diff --git a/src/callback.c b/src/callback.c index 46fbe2c21a..63090899fb 100644 --- a/src/callback.c +++ b/src/callback.c @@ -154,11 +154,11 @@ static CollSeq *findCollSeqEntry( int create /* Create a new entry if true */ ){ CollSeq *pColl; - int nName = sqlite3Strlen30(zName); - pColl = sqlite3HashFind(&db->aCollSeq, zName, nName); + pColl = sqlite3HashFind(&db->aCollSeq, zName); if( 0==pColl && create ){ - pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1 ); + int nName = sqlite3Strlen30(zName); + pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1); if( pColl ){ CollSeq *pDel = 0; pColl[0].zName = (char*)&pColl[3]; @@ -169,7 +169,7 @@ static CollSeq *findCollSeqEntry( pColl[2].enc = SQLITE_UTF16BE; memcpy(pColl[0].zName, zName, nName); pColl[0].zName[nName] = 0; - pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, nName, pColl); + pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, pColl); /* If a malloc() failure occurred in sqlite3HashInsert(), it will ** return the pColl pointer to be deleted (because it wasn't added diff --git a/src/fkey.c b/src/fkey.c index 50c10da822..415f35d2f8 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -659,8 +659,7 @@ static void fkScanChildren( ** table). */ FKey *sqlite3FkReferences(Table *pTab){ - int nName = sqlite3Strlen30(pTab->zName); - return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName, nName); + return (FKey *)sqlite3HashFind(&pTab->pSchema->fkeyHash, pTab->zName); } /* @@ -1338,7 +1337,7 @@ void sqlite3FkDelete(sqlite3 *db, Table *pTab){ }else{ void *p = (void *)pFKey->pNextTo; const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); - sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, sqlite3Strlen30(z), p); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, p); } if( pFKey->pNextTo ){ pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; diff --git a/src/hash.c b/src/hash.c index f9901fee8f..b5886e0641 100644 --- a/src/hash.c +++ b/src/hash.c @@ -52,12 +52,11 @@ void sqlite3HashClear(Hash *pH){ /* ** The hashing function. */ -static unsigned int strHash(const char *z, int nKey){ +static unsigned int strHash(const char *z){ unsigned int h = 0; - assert( nKey>=0 ); - while( nKey > 0 ){ - h = (h<<3) ^ h ^ sqlite3UpperToLower[(unsigned char)*z++]; - nKey--; + unsigned char c; + while( (c = (unsigned char)*z++)!=0 ){ + h = (h<<3) ^ h ^ sqlite3UpperToLower[c]; } return h; } @@ -129,7 +128,7 @@ static int rehash(Hash *pH, unsigned int new_size){ pH->htsize = new_size = sqlite3MallocSize(new_ht)/sizeof(struct _ht); memset(new_ht, 0, new_size*sizeof(struct _ht)); for(elem=pH->first, pH->first=0; elem; elem = next_elem){ - unsigned int h = strHash(elem->pKey, elem->nKey) % new_size; + unsigned int h = strHash(elem->pKey) % new_size; next_elem = elem->next; insertElement(pH, &new_ht[h], elem); } @@ -137,28 +136,33 @@ static int rehash(Hash *pH, unsigned int new_size){ } /* This function (for internal use only) locates an element in an -** hash table that matches the given key. The hash for this key has -** already been computed and is passed as the 4th parameter. +** hash table that matches the given key. The hash for this key is +** also computed and returned in the *pH parameter. */ -static HashElem *findElementGivenHash( +static HashElem *findElementWithHash( const Hash *pH, /* The pH to be searched */ const char *pKey, /* The key we are searching for */ - int nKey, /* Bytes in key (not counting zero terminator) */ - unsigned int h /* The hash for this key. */ + unsigned int *pHash /* Write the hash value here */ ){ HashElem *elem; /* Used to loop thru the element list */ int count; /* Number of elements left to test */ + unsigned int h; /* The computed hash */ if( pH->ht ){ - struct _ht *pEntry = &pH->ht[h]; + struct _ht *pEntry; + h = strHash(pKey) % pH->htsize; + pEntry = &pH->ht[h]; elem = pEntry->chain; count = pEntry->count; }else{ + h = 0; elem = pH->first; count = pH->count; } - while( count-- && ALWAYS(elem) ){ - if( elem->nKey==nKey && sqlite3StrNICmp(elem->pKey,pKey,nKey)==0 ){ + *pHash = h; + while( count-- ){ + assert( elem!=0 ); + if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ return elem; } elem = elem->next; @@ -201,26 +205,20 @@ static void removeElementGivenHash( } /* Attempt to locate an element of the hash table pH with a key -** that matches pKey,nKey. Return the data for this element if it is +** that matches pKey. Return the data for this element if it is ** found, or NULL if there is no match. */ -void *sqlite3HashFind(const Hash *pH, const char *pKey, int nKey){ +void *sqlite3HashFind(const Hash *pH, const char *pKey){ HashElem *elem; /* The element that matches key */ unsigned int h; /* A hash on key */ assert( pH!=0 ); assert( pKey!=0 ); - assert( nKey>=0 ); - if( pH->ht ){ - h = strHash(pKey, nKey) % pH->htsize; - }else{ - h = 0; - } - elem = findElementGivenHash(pH, pKey, nKey, h); + elem = findElementWithHash(pH, pKey, &h); return elem ? elem->data : 0; } -/* Insert an element into the hash table pH. The key is pKey,nKey +/* Insert an element into the hash table pH. The key is pKey ** and the data is "data". ** ** If no element exists with a matching key, then a new @@ -234,20 +232,14 @@ void *sqlite3HashFind(const Hash *pH, const char *pKey, int nKey){ ** If the "data" parameter to this function is NULL, then the ** element corresponding to "key" is removed from the hash table. */ -void *sqlite3HashInsert(Hash *pH, const char *pKey, int nKey, void *data){ +void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ unsigned int h; /* the hash of the key modulo hash table size */ HashElem *elem; /* Used to loop thru the element list */ HashElem *new_elem; /* New element added to the pH */ assert( pH!=0 ); assert( pKey!=0 ); - assert( nKey>=0 ); - if( pH->htsize ){ - h = strHash(pKey, nKey) % pH->htsize; - }else{ - h = 0; - } - elem = findElementGivenHash(pH,pKey,nKey,h); + elem = findElementWithHash(pH,pKey,&h); if( elem ){ void *old_data = elem->data; if( data==0 ){ @@ -255,7 +247,6 @@ void *sqlite3HashInsert(Hash *pH, const char *pKey, int nKey, void *data){ }else{ elem->data = data; elem->pKey = pKey; - assert(nKey==elem->nKey); } return old_data; } @@ -263,19 +254,14 @@ void *sqlite3HashInsert(Hash *pH, const char *pKey, int nKey, void *data){ new_elem = (HashElem*)sqlite3Malloc( sizeof(HashElem) ); if( new_elem==0 ) return data; new_elem->pKey = pKey; - new_elem->nKey = nKey; new_elem->data = data; pH->count++; if( pH->count>=10 && pH->count > 2*pH->htsize ){ if( rehash(pH, pH->count*2) ){ assert( pH->htsize>0 ); - h = strHash(pKey, nKey) % pH->htsize; + h = strHash(pKey) % pH->htsize; } } - if( pH->ht ){ - insertElement(pH, &pH->ht[h], new_elem); - }else{ - insertElement(pH, 0, new_elem); - } + insertElement(pH, pH->ht ? &pH->ht[h] : 0, new_elem); return 0; } diff --git a/src/hash.h b/src/hash.h index 82b7c58c71..6dfa4e035c 100644 --- a/src/hash.h +++ b/src/hash.h @@ -59,15 +59,15 @@ struct Hash { struct HashElem { HashElem *next, *prev; /* Next and previous elements in the table */ void *data; /* Data associated with this element */ - const char *pKey; int nKey; /* Key associated with this element */ + const char *pKey; /* Key associated with this element */ }; /* ** Access routines. To delete, insert a NULL pointer. */ void sqlite3HashInit(Hash*); -void *sqlite3HashInsert(Hash*, const char *pKey, int nKey, void *pData); -void *sqlite3HashFind(const Hash*, const char *pKey, int nKey); +void *sqlite3HashInsert(Hash*, const char *pKey, void *pData); +void *sqlite3HashFind(const Hash*, const char *pKey); void sqlite3HashClear(Hash*); /* diff --git a/src/main.c b/src/main.c index cea72829df..2472321759 100644 --- a/src/main.c +++ b/src/main.c @@ -1998,7 +1998,6 @@ static int createCollation( ){ CollSeq *pColl; int enc2; - int nName = sqlite3Strlen30(zName); assert( sqlite3_mutex_held(db->mutex) ); @@ -2037,7 +2036,7 @@ static int createCollation( ** to be called. */ if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ - CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName, nName); + CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); int j; for(j=0; j<3; j++){ CollSeq *p = &aColl[j]; diff --git a/src/trigger.c b/src/trigger.c index 01f7b21f74..fc32a663bf 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -180,8 +180,7 @@ void sqlite3BeginTrigger( goto trigger_cleanup; } assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash), - zName, sqlite3Strlen30(zName)) ){ + if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash),zName) ){ if( !noErr ){ sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); }else{ @@ -324,13 +323,12 @@ void sqlite3FinishTrigger( Trigger *pLink = pTrig; Hash *pHash = &db->aDb[iDb].pSchema->trigHash; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - pTrig = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), pTrig); + pTrig = sqlite3HashInsert(pHash, zName, pTrig); if( pTrig ){ db->mallocFailed = 1; }else if( pLink->pSchema==pLink->pTabSchema ){ Table *pTab; - int n = sqlite3Strlen30(pLink->table); - pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table, n); + pTab = sqlite3HashFind(&pLink->pTabSchema->tblHash, pLink->table); assert( pTab!=0 ); pLink->pNext = pTab->pTrigger; pTab->pTrigger = pLink; @@ -489,7 +487,6 @@ void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){ int i; const char *zDb; const char *zName; - int nName; sqlite3 *db = pParse->db; if( db->mallocFailed ) goto drop_trigger_cleanup; @@ -500,13 +497,12 @@ void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr){ assert( pName->nSrc==1 ); zDb = pName->a[0].zDatabase; zName = pName->a[0].zName; - nName = sqlite3Strlen30(zName); assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ if( zDb && sqlite3StrICmp(db->aDb[j].zName, zDb) ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); - pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName, nName); + pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName); if( pTrigger ) break; } if( !pTrigger ){ @@ -529,8 +525,7 @@ drop_trigger_cleanup: ** is set on. */ static Table *tableOfTrigger(Trigger *pTrigger){ - int n = sqlite3Strlen30(pTrigger->table); - return sqlite3HashFind(&pTrigger->pTabSchema->tblHash, pTrigger->table, n); + return sqlite3HashFind(&pTrigger->pTabSchema->tblHash, pTrigger->table); } @@ -602,7 +597,7 @@ void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const char *zName){ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); pHash = &(db->aDb[iDb].pSchema->trigHash); - pTrigger = sqlite3HashInsert(pHash, zName, sqlite3Strlen30(zName), 0); + pTrigger = sqlite3HashInsert(pHash, zName, 0); if( ALWAYS(pTrigger) ){ if( pTrigger->pSchema==pTrigger->pTabSchema ){ Table *pTab = tableOfTrigger(pTrigger); diff --git a/src/vtab.c b/src/vtab.c index ca0db214cc..ad18af48a4 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -43,7 +43,7 @@ static int createModule( sqlite3_mutex_enter(db->mutex); nName = sqlite3Strlen30(zName); - if( sqlite3HashFind(&db->aModule, zName, nName) ){ + if( sqlite3HashFind(&db->aModule, zName) ){ rc = SQLITE_MISUSE_BKPT; }else{ Module *pMod; @@ -56,7 +56,7 @@ static int createModule( pMod->pModule = pModule; pMod->pAux = pAux; pMod->xDestroy = xDestroy; - pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,nName,(void*)pMod); + pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); assert( pDel==0 || pDel==pMod ); if( pDel ){ db->mallocFailed = 1; @@ -425,9 +425,8 @@ void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ Table *pOld; Schema *pSchema = pTab->pSchema; const char *zName = pTab->zName; - int nName = sqlite3Strlen30(zName); assert( sqlite3SchemaMutexHeld(db, 0, pSchema) ); - pOld = sqlite3HashInsert(&pSchema->tblHash, zName, nName, pTab); + pOld = sqlite3HashInsert(&pSchema->tblHash, zName, pTab); if( pOld ){ db->mallocFailed = 1; assert( pTab==pOld ); /* Malloc must have failed inside HashInsert() */ @@ -593,7 +592,7 @@ int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ /* Locate the required virtual table module */ zMod = pTab->azModuleArg[0]; - pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod); if( !pMod ){ const char *zModule = pTab->azModuleArg[0]; @@ -661,7 +660,7 @@ int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, char **pzErr){ /* Locate the required virtual table module */ zMod = pTab->azModuleArg[0]; - pMod = (Module*)sqlite3HashFind(&db->aModule, zMod, sqlite3Strlen30(zMod)); + pMod = (Module*)sqlite3HashFind(&db->aModule, zMod); /* If the module has been registered and includes a Create method, ** invoke it now. If the module has not been registered, return an From 5ab567078a924d9eeff3616433532dd2905a21c7 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Aug 2014 11:11:30 +0000 Subject: [PATCH 246/710] Enhance the spellfix extension with the ability to specify a rowid when inserting new rows. FossilOrigin-Name: 369c480cda6fa66394b995346bbf51f3298446e1 --- ext/misc/spellfix.c | 22 ++++++++++++++++------ manifest | 14 +++++++------- manifest.uuid | 2 +- test/spellfix.test | 16 ++++++++++++++++ 4 files changed, 40 insertions(+), 14 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index 2e6743e4f7..2a26e08391 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -2736,12 +2736,22 @@ static int spellfix1Update( return SQLITE_NOMEM; } if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ - spellfix1DbExec(&rc, db, - "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) " - "VALUES(%d,%d,%Q,%Q,%Q)", - p->zDbName, p->zTableName, - iRank, iLang, zWord, zK1, zK2 - ); + if( sqlite3_value_type(argv[1])==SQLITE_NULL ){ + spellfix1DbExec(&rc, db, + "INSERT INTO \"%w\".\"%w_vocab\"(rank,langid,word,k1,k2) " + "VALUES(%d,%d,%Q,%Q,%Q)", + p->zDbName, p->zTableName, + iRank, iLang, zWord, zK1, zK2 + ); + }else{ + newRowid = sqlite3_value_int64(argv[1]); + spellfix1DbExec(&rc, db, + "INSERT INTO \"%w\".\"%w_vocab\"(id,rank,langid,word,k1,k2) " + "VALUES(%lld,%d,%d,%Q,%Q,%Q)", + p->zDbName, p->zTableName, + newRowid, iRank, iLang, zWord, zK1, zK2 + ); + } *pRowid = sqlite3_last_insert_rowid(db); }else{ rowid = sqlite3_value_int64(argv[0]); diff --git a/manifest b/manifest index bcba60c6c2..c3d516552f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\sinterface\sto\sthe\ssymbol\stable,\ssaving\s600\sbytes\sof\scode\sspace. -D 2014-08-21T20:26:37.728 +C Enhance\sthe\sspellfix\sextension\swith\sthe\sability\sto\sspecify\sa\srowid\swhen\ninserting\snew\srows. +D 2014-08-22T11:11:30.047 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -116,7 +116,7 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a -F ext/misc/spellfix.c cb016c2dab951ffd7b819a7bc8a750ebd6c26c0f +F ext/misc/spellfix.c 56739fab8c2ed6a9e2dac5592a88d281a999c43b F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e @@ -847,7 +847,7 @@ F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b F test/speedtest1.c d29c8048beb7ea9254191f3fde9414709166a920 -F test/spellfix.test 61309f5efbec53603b3f86457d34a504f80abafe +F test/spellfix.test 24f676831acddd2f4056a598fd731a72c6311f49 F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7029b3404d3f5f698a496934f3a3f2972051b257 -R 818f77c3be876451b2dad41de8a59565 +P 14b0f561fe15622b61c6676c9c455dca6b9ba5f0 +R e9738e2157edd7ced31f8162753e2f5b U drh -Z cb34ea9e2178e12926edef43b644d982 +Z 9671128edfd421e5d7e75c6453894b1f diff --git a/manifest.uuid b/manifest.uuid index 98b1fd2ac2..85d3cf892f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14b0f561fe15622b61c6676c9c455dca6b9ba5f0 \ No newline at end of file +369c480cda6fa66394b995346bbf51f3298446e1 \ No newline at end of file diff --git a/test/spellfix.test b/test/spellfix.test index 21383c2d33..954bdb21f1 100644 --- a/test/spellfix.test +++ b/test/spellfix.test @@ -124,6 +124,22 @@ do_execsql_test 1.23 { SELECT next_char('ab','vocab2','w',null,'binary'); } {c} +do_execsql_test 1.30 { + SELECT rowid FROM t1 WHERE word='rabbit'; +} {2} +do_execsql_test 1.31 { + UPDATE t1 SET rowid=2000 WHERE word='rabbit'; + SELECT rowid FROM t1 WHERE word='rabbit'; +} {2000} +do_execsql_test 1.32 { + INSERT INTO t1(rowid, word) VALUES(3000,'melody'); + SELECT rowid, word, matchlen FROM t1 WHERE word MATCH 'melotti' + ORDER BY score LIMIT 3; +} {3000 melody 6} +do_test 1.33 { + catchsql {INSERT INTO t1(rowid, word) VALUES(3000,'garden');} +} {1 {constraint failed}} + do_execsql_test 2.1 { CREATE VIRTUAL TABLE t2 USING spellfix1; INSERT INTO t2 (word, soundslike) VALUES('school', 'skuul'); From 3f5b199eb596a6e8031647d93e00451ade43806e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Aug 2014 13:22:32 +0000 Subject: [PATCH 247/710] Change a while-loop into a do-loop in sqlite3VdbeSerialPut() for a small size reduction and performance improvement. FossilOrigin-Name: 750bb0a0960606ab24037e0992e9f7a17524cc3e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 7 ++++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c3d516552f..0e059ae46c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sspellfix\sextension\swith\sthe\sability\sto\sspecify\sa\srowid\swhen\ninserting\snew\srows. -D 2014-08-22T11:11:30.047 +C Change\sa\swhile-loop\sinto\sa\sdo-loop\sin\ssqlite3VdbeSerialPut()\sfor\sa\ssmall\nsize\sreduction\sand\sperformance\simprovement. +D 2014-08-22T13:22:32.819 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,7 +288,7 @@ F src/vdbe.c f7f4066e4d6e3858878d76ce9288ea603e12ddf6 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 7858d7e7cd23267d3fbca18e3a28cce8e0d162a8 -F src/vdbeaux.c 25d62ef82cf1be2a1255eacac636fa0d943d8b3d +F src/vdbeaux.c 9c9571706aaf0e5debab5b01629ef569cde40920 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 14b0f561fe15622b61c6676c9c455dca6b9ba5f0 -R e9738e2157edd7ced31f8162753e2f5b +P 369c480cda6fa66394b995346bbf51f3298446e1 +R 9c1a6f648ddd3ddf03093ccb99facf2b U drh -Z 9671128edfd421e5d7e75c6453894b1f +Z 453b7d6a328e84b291043b75c1b3327d diff --git a/manifest.uuid b/manifest.uuid index 85d3cf892f..6ea1746008 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -369c480cda6fa66394b995346bbf51f3298446e1 \ No newline at end of file +750bb0a0960606ab24037e0992e9f7a17524cc3e \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index fb3f7c3a8c..83c6e1f65a 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2933,10 +2933,11 @@ u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ v = pMem->u.i; } len = i = sqlite3VdbeSerialTypeLen(serial_type); - while( i-- ){ - buf[i] = (u8)(v&0xFF); + assert( i>0 ); + do{ + buf[--i] = (u8)(v&0xFF); v >>= 8; - } + }while( i ); return len; } From 14a924a5cd77e353ebae67d3b4bc7e64b7bd715a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Aug 2014 14:34:05 +0000 Subject: [PATCH 248/710] Get the sqlite3VdbeSerialGet() routine to run faster by avoiding the use of local variables. FossilOrigin-Name: 8267d82174099e548a4f78d06af0c6324c89b83d --- manifest | 17 ++++++---- manifest.uuid | 2 +- src/sqliteInt.h | 12 +++++++ src/vdbeaux.c | 89 +++++++++++++++++++++++++++---------------------- 4 files changed, 73 insertions(+), 47 deletions(-) diff --git a/manifest b/manifest index 0e059ae46c..66e4c0b1b6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sa\swhile-loop\sinto\sa\sdo-loop\sin\ssqlite3VdbeSerialPut()\sfor\sa\ssmall\nsize\sreduction\sand\sperformance\simprovement. -D 2014-08-22T13:22:32.819 +C Get\sthe\ssqlite3VdbeSerialGet()\sroutine\sto\srun\sfaster\sby\savoiding\sthe\suse\nof\slocal\svariables. +D 2014-08-22T14:34:05.936 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/shell.c 728d2226594d356bf4fbdbdfd08538fd78fd06f3 F src/sqlite.h.in ed9d35990c61f0388ca6405706455c4095310553 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 641f8fbb65ca2084c8df95b525f6f82c7a1e91ae +F src/sqliteInt.h 99bd20e5a12dce7ba290e938123d27e87b5eae27 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -288,7 +288,7 @@ F src/vdbe.c f7f4066e4d6e3858878d76ce9288ea603e12ddf6 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 7858d7e7cd23267d3fbca18e3a28cce8e0d162a8 -F src/vdbeaux.c 9c9571706aaf0e5debab5b01629ef569cde40920 +F src/vdbeaux.c f83d5c265aea19d2e49ba018beaf99acff934020 F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 @@ -1188,7 +1188,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 369c480cda6fa66394b995346bbf51f3298446e1 -R 9c1a6f648ddd3ddf03093ccb99facf2b +P 750bb0a0960606ab24037e0992e9f7a17524cc3e +R 6bc6a7681a89a137c23142249db6cf86 +T *branch * experimental +T *sym-experimental * +T -sym-trunk * U drh -Z 453b7d6a328e84b291043b75c1b3327d +Z 612ca7a9b67c6d8d8dbe2b7affb6d3eb diff --git a/manifest.uuid b/manifest.uuid index 6ea1746008..9538f73f10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -750bb0a0960606ab24037e0992e9f7a17524cc3e \ No newline at end of file +8267d82174099e548a4f78d06af0c6324c89b83d \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ee52487d66..5202beff78 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -153,6 +153,18 @@ # define SQLITE_PTR_TO_INT(X) ((int)(X)) #endif +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define SQLITE_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) +# define SQLITE_NOINLINE __declspec(noinline) +#else +# define SQLITE_NOINLINE +#endif + /* ** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. ** 0 means mutexes are permanently disable and the library is never diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 83c6e1f65a..d338806962 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2965,14 +2965,55 @@ u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ /* ** Deserialize the data blob pointed to by buf as serial type serial_type ** and store the result in pMem. Return the number of bytes read. +** +** This function is implemented as two separate routines for performance. +** The few cases that require local variables are broken out into a separate +** routine so that in most cases the overhead of moving the stack pointer +** is avoided. */ -u32 sqlite3VdbeSerialGet( +static u32 SQLITE_NOINLINE serialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ Mem *pMem /* Memory cell to write value into */ ){ u64 x; - u32 y; + u32 y = FOUR_BYTE_UINT(buf); + if( serial_type==4 ){ + pMem->u.i = (i64)*(int*)&y; + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + return 4; + } + x = (((u64)y)<<32)|FOUR_BYTE_UINT(buf+4); + if( serial_type==6 ){ + pMem->u.i = *(i64*)&x; + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + }else{ +#if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) + /* Verify that integers and floating point values use the same + ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is + ** defined that 64-bit floating point values really are mixed + ** endian. + */ + static const u64 t1 = ((u64)0x3ff00000)<<32; + static const double r1 = 1.0; + u64 t2 = t1; + swapMixedEndianFloat(t2); + assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); +#endif + assert( sizeof(x)==8 && sizeof(pMem->r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->r, &x, sizeof(x)); + pMem->flags = sqlite3IsNaN(pMem->r) ? MEM_Null : MEM_Real; + } + return 8; +} +u32 sqlite3VdbeSerialGet( + const unsigned char *buf, /* Buffer to deserialize from */ + u32 serial_type, /* Serial type to deserialize */ + Mem *pMem /* Memory cell to write value into */ +){ switch( serial_type ){ case 10: /* Reserved for future use */ case 11: /* Reserved for future use */ @@ -2998,47 +3039,19 @@ u32 sqlite3VdbeSerialGet( testcase( pMem->u.i<0 ); return 3; } - case 4: { /* 4-byte signed integer */ - y = FOUR_BYTE_UINT(buf); - pMem->u.i = (i64)*(int*)&y; - pMem->flags = MEM_Int; - testcase( pMem->u.i<0 ); - return 4; - } case 5: { /* 6-byte signed integer */ pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 6; } + case 4: /* 4-byte signed integer */ case 6: /* 8-byte signed integer */ case 7: { /* IEEE floating point */ -#if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) - /* Verify that integers and floating point values use the same - ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is - ** defined that 64-bit floating point values really are mixed - ** endian. - */ - static const u64 t1 = ((u64)0x3ff00000)<<32; - static const double r1 = 1.0; - u64 t2 = t1; - swapMixedEndianFloat(t2); - assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); -#endif - x = FOUR_BYTE_UINT(buf); - y = FOUR_BYTE_UINT(buf+4); - x = (x<<32) | y; - if( serial_type==6 ){ - pMem->u.i = *(i64*)&x; - pMem->flags = MEM_Int; - testcase( pMem->u.i<0 ); - }else{ - assert( sizeof(x)==8 && sizeof(pMem->r)==8 ); - swapMixedEndianFloat(x); - memcpy(&pMem->r, &x, sizeof(x)); - pMem->flags = sqlite3IsNaN(pMem->r) ? MEM_Null : MEM_Real; - } - return 8; + /* These three cases require local variables, so do them in a + ** separate routine to avoid having to move the frame pointer in + ** the common case */ + return serialGet(buf,serial_type,pMem); } case 8: /* Integer 0 */ case 9: { /* Integer 1 */ @@ -3048,17 +3061,15 @@ u32 sqlite3VdbeSerialGet( } default: { static const u16 aFlag[] = { MEM_Blob|MEM_Ephem, MEM_Str|MEM_Ephem }; - u32 len = (serial_type-12)/2; pMem->z = (char *)buf; - pMem->n = len; + pMem->n = (serial_type-12)/2; pMem->xDel = 0; pMem->flags = aFlag[serial_type&1]; - return len; + return pMem->n; } } return 0; } - /* ** This routine is used to allocate sufficient space for an UnpackedRecord ** structure large enough to be used with sqlite3VdbeRecordUnpack() if From 8932becbef8458f91b9316249e2ac9a3d24ed816 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Aug 2014 14:56:13 +0000 Subject: [PATCH 249/710] Handle the 4-byte integer case in the stackless routine. FossilOrigin-Name: 3f55484e81000c75e231f5580632a68e782ded4f --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/vdbeaux.c | 25 ++++++++++++------------- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 66e4c0b1b6..2e7b46977a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\ssqlite3VdbeSerialGet()\sroutine\sto\srun\sfaster\sby\savoiding\sthe\suse\nof\slocal\svariables. -D 2014-08-22T14:34:05.936 +C Handle\sthe\s4-byte\sinteger\scase\sin\sthe\sstackless\sroutine. +D 2014-08-22T14:56:13.720 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,7 +288,7 @@ F src/vdbe.c f7f4066e4d6e3858878d76ce9288ea603e12ddf6 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 7858d7e7cd23267d3fbca18e3a28cce8e0d162a8 -F src/vdbeaux.c f83d5c265aea19d2e49ba018beaf99acff934020 +F src/vdbeaux.c d0b20a85d1ab8c951e5c8b2400a45252d6d2750c F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 @@ -1188,10 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 750bb0a0960606ab24037e0992e9f7a17524cc3e -R 6bc6a7681a89a137c23142249db6cf86 -T *branch * experimental -T *sym-experimental * -T -sym-trunk * +P 8267d82174099e548a4f78d06af0c6324c89b83d +R 3e2300d1a615c65baf7081c06688a5fd U drh -Z 612ca7a9b67c6d8d8dbe2b7affb6d3eb +Z 9f4fad73eafc2d2adf27281640f7d1a1 diff --git a/manifest.uuid b/manifest.uuid index 9538f73f10..301ab7c9f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8267d82174099e548a4f78d06af0c6324c89b83d \ No newline at end of file +3f55484e81000c75e231f5580632a68e782ded4f \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index d338806962..513481bf00 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2961,6 +2961,7 @@ u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ #define TWO_BYTE_INT(x) (256*(i8)((x)[0])|(x)[1]) #define THREE_BYTE_INT(x) (65536*(i8)((x)[0])|((x)[1]<<8)|(x)[2]) #define FOUR_BYTE_UINT(x) (((u32)(x)[0]<<24)|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) +#define FOUR_BYTE_INT(x) (16777216*(i8)((x)[0])|((x)[1]<<16)|((x)[2]<<8)|(x)[3]) /* ** Deserialize the data blob pointed to by buf as serial type serial_type @@ -2976,15 +2977,9 @@ static u32 SQLITE_NOINLINE serialGet( u32 serial_type, /* Serial type to deserialize */ Mem *pMem /* Memory cell to write value into */ ){ - u64 x; - u32 y = FOUR_BYTE_UINT(buf); - if( serial_type==4 ){ - pMem->u.i = (i64)*(int*)&y; - pMem->flags = MEM_Int; - testcase( pMem->u.i<0 ); - return 4; - } - x = (((u64)y)<<32)|FOUR_BYTE_UINT(buf+4); + u64 x = FOUR_BYTE_UINT(buf); + u32 y = FOUR_BYTE_UINT(buf+4); + x = (x<<32) + y; if( serial_type==6 ){ pMem->u.i = *(i64*)&x; pMem->flags = MEM_Int; @@ -3039,18 +3034,22 @@ u32 sqlite3VdbeSerialGet( testcase( pMem->u.i<0 ); return 3; } + case 4: { /* 4-byte signed integer */ + pMem->u.i = FOUR_BYTE_INT(buf); + pMem->flags = MEM_Int; + testcase( pMem->u.i<0 ); + return 4; + } case 5: { /* 6-byte signed integer */ pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 6; } - case 4: /* 4-byte signed integer */ case 6: /* 8-byte signed integer */ case 7: { /* IEEE floating point */ - /* These three cases require local variables, so do them in a - ** separate routine to avoid having to move the frame pointer in - ** the common case */ + /* These use local variables, so do them in a separate routine + ** to avoid having to move the frame pointer in the common case */ return serialGet(buf,serial_type,pMem); } case 8: /* Integer 0 */ From 172087fb733f54d025fe6a112898cdcabada7e47 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Aug 2014 15:40:20 +0000 Subject: [PATCH 250/710] Performance improvement in the printf() logic by avoiding unnecessary stack pointer movement. FossilOrigin-Name: f7f2160db014f0ae11ad13c8ad70ad3444124e3e --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/printf.c | 10 +++++----- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index ceff9d8346..5d853a3262 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\sperformance\sand\sreduce\sthe\ssize\sof\sthe\ssqlite3VdbeSerialGet()\nroutine\sby\savoiding\sthe\suse\sof\sstack. -D 2014-08-22T15:19:59.113 +C Performance\simprovement\sin\sthe\sprintf()\slogic\sby\savoiding\sunnecessary\sstack\npointer\smovement. +D 2014-08-22T15:40:20.728 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,7 +219,7 @@ F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c F src/pragma.c d10ef67c4de79f78188b965b4b7988aff1d66f2e F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 -F src/printf.c af06f66927919730f03479fed6ae9854f73419f4 +F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be @@ -1188,8 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 750bb0a0960606ab24037e0992e9f7a17524cc3e 3f55484e81000c75e231f5580632a68e782ded4f -R 3e2300d1a615c65baf7081c06688a5fd -T +closed 3f55484e81000c75e231f5580632a68e782ded4f +P ebc10e46c15017d7cd232b5f4f3ef67ef740d87f +R 200d7eb44581fc5fc17c5cd337adb810 U drh -Z 0797c3bb55e4f93e4c54043857e85ca3 +Z f2b0948089d8fe6b11165afcc09478da diff --git a/manifest.uuid b/manifest.uuid index 3b49bd178d..a75c3b426e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ebc10e46c15017d7cd232b5f4f3ef67ef740d87f \ No newline at end of file +f7f2160db014f0ae11ad13c8ad70ad3444124e3e \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 37910804d2..72ace932ba 100644 --- a/src/printf.c +++ b/src/printf.c @@ -784,7 +784,7 @@ void sqlite3AppendSpace(StrAccum *p, int N){ ** work (enlarging the buffer) using tail recursion, so that the ** sqlite3StrAccumAppend() routine can use fast calling semantics. */ -static void enlargeAndAppend(StrAccum *p, const char *z, int N){ +static void SQLITE_NOINLINE enlargeAndAppend(StrAccum *p, const char *z, int N){ N = sqlite3StrAccumEnlarge(p, N); if( N>0 ){ memcpy(&p->zText[p->nChar], z, N); @@ -803,11 +803,11 @@ void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ assert( p->accError==0 || p->nAlloc==0 ); if( p->nChar+N >= p->nAlloc ){ enlargeAndAppend(p,z,N); - return; + }else{ + assert( p->zText ); + p->nChar += N; + memcpy(&p->zText[p->nChar-N], z, N); } - assert( p->zText ); - memcpy(&p->zText[p->nChar], z, N); - p->nChar += N; } /* From 13f40da31d9692a96dcab00f85afee9fd68bae12 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Aug 2014 18:00:11 +0000 Subject: [PATCH 251/710] Split the sqlite3Error() routine into sqlite3Error() and sqlite3ErrorWithMsg(), for a slight size reduction and performance increase. FossilOrigin-Name: cf561d1f0bb60b3d638632d20bd686dda4fa4a04 --- manifest | 36 ++++++++++++++++++------------------ manifest.uuid | 2 +- src/backup.c | 12 ++++++------ src/legacy.c | 6 +++--- src/loadext.c | 2 +- src/main.c | 26 +++++++++++++------------- src/malloc.c | 2 +- src/notify.c | 2 +- src/prepare.c | 8 ++++---- src/sqliteInt.h | 3 ++- src/util.c | 23 ++++++++++++++++------- src/vdbeapi.c | 10 +++++----- src/vdbeaux.c | 4 ++-- src/vdbeblob.c | 6 +++--- src/vtab.c | 6 +++--- 15 files changed, 79 insertions(+), 69 deletions(-) diff --git a/manifest b/manifest index 5d853a3262..2ec516f887 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovement\sin\sthe\sprintf()\slogic\sby\savoiding\sunnecessary\sstack\npointer\smovement. -D 2014-08-22T15:40:20.728 +C Split\sthe\ssqlite3Error()\sroutine\sinto\ssqlite3Error()\sand\nsqlite3ErrorWithMsg(),\sfor\sa\sslight\ssize\sreduction\sand\sperformance\sincrease. +D 2014-08-22T18:00:11.621 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -165,7 +165,7 @@ F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 F src/analyze.c f98a351908da29f7b44741cfeb9eb20dda648ba0 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 -F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 +F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 4195fed5741b4dbcc9831b623aec487258f3e62d @@ -187,11 +187,11 @@ F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c d1a104e67b33314d4cc5c1356147446086ab9fc8 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d -F src/legacy.c febc2a9e7ad6c1a6191c7b5b9170b325d263f343 +F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b -F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 1e5d34fb6dee85019b4bcc44e8576457b5075174 -F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be +F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab +F src/main.c 900dd06e41d22795cbb23ab0240397f1e2901bf7 +F src/malloc.c 0a88a97fc5ae621ca9659d38b080e0b9ddbb80ad F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f @@ -203,7 +203,7 @@ F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 F src/mutex_w32.c 06bfff9a3a83b53389a51a967643db3967032e1e -F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 +F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 @@ -218,7 +218,7 @@ F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c F src/pragma.c d10ef67c4de79f78188b965b4b7988aff1d66f2e -F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 +F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 @@ -228,7 +228,7 @@ F src/shell.c 728d2226594d356bf4fbdbdfd08538fd78fd06f3 F src/sqlite.h.in ed9d35990c61f0388ca6405706455c4095310553 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 99bd20e5a12dce7ba290e938123d27e87b5eae27 +F src/sqliteInt.h 937869c407c61026443c879e8c90a9dff05d2d27 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -282,18 +282,18 @@ F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 4bddd12803275aa98f1c7ce0118fceb02b2167f6 F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 -F src/util.c 3076bdd51cdbf60a6e2e57fada745be37133c73e +F src/util.c 524127b3c330b9f490a505ff6eb7f80dfc873a3a F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c f7f4066e4d6e3858878d76ce9288ea603e12ddf6 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df -F src/vdbeapi.c 7858d7e7cd23267d3fbca18e3a28cce8e0d162a8 -F src/vdbeaux.c d0b20a85d1ab8c951e5c8b2400a45252d6d2750c -F src/vdbeblob.c 9205ce9d3b064d9600f8418a897fc88b5687d9ac +F src/vdbeapi.c 49b8d2943d02d276b4efef114578251a3277f47d +F src/vdbeaux.c f7fdc59e2eefa6fc4ba7324b03ea6f8f66e98d62 +F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 -F src/vtab.c 180bfc5e69c92f2014c094bc49a66e8c37c188ac +F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ebc10e46c15017d7cd232b5f4f3ef67ef740d87f -R 200d7eb44581fc5fc17c5cd337adb810 +P f7f2160db014f0ae11ad13c8ad70ad3444124e3e +R 668369bbbf2d81407f36345226c5d2a9 U drh -Z f2b0948089d8fe6b11165afcc09478da +Z bddb035a8072a327798c78fbe23bcf26 diff --git a/manifest.uuid b/manifest.uuid index a75c3b426e..5842d66545 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f7f2160db014f0ae11ad13c8ad70ad3444124e3e \ No newline at end of file +cf561d1f0bb60b3d638632d20bd686dda4fa4a04 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 4a6bc7493c..92c6334bde 100644 --- a/src/backup.c +++ b/src/backup.c @@ -87,12 +87,12 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ int rc = 0; pParse = sqlite3StackAllocZero(pErrorDb, sizeof(*pParse)); if( pParse==0 ){ - sqlite3Error(pErrorDb, SQLITE_NOMEM, "out of memory"); + sqlite3ErrorWithMsg(pErrorDb, SQLITE_NOMEM, "out of memory"); rc = SQLITE_NOMEM; }else{ pParse->db = pDb; if( sqlite3OpenTempDatabase(pParse) ){ - sqlite3Error(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); + sqlite3ErrorWithMsg(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); rc = SQLITE_ERROR; } sqlite3DbFree(pErrorDb, pParse->zErrMsg); @@ -105,7 +105,7 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ } if( i<0 ){ - sqlite3Error(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb); + sqlite3ErrorWithMsg(pErrorDb, SQLITE_ERROR, "unknown database %s", zDb); return 0; } @@ -150,7 +150,7 @@ sqlite3_backup *sqlite3_backup_init( sqlite3_mutex_enter(pDestDb->mutex); if( pSrcDb==pDestDb ){ - sqlite3Error( + sqlite3ErrorWithMsg( pDestDb, SQLITE_ERROR, "source and destination must be distinct" ); p = 0; @@ -161,7 +161,7 @@ sqlite3_backup *sqlite3_backup_init( ** sqlite3_backup_finish(). */ p = (sqlite3_backup *)sqlite3MallocZero(sizeof(sqlite3_backup)); if( !p ){ - sqlite3Error(pDestDb, SQLITE_NOMEM, 0); + sqlite3Error(pDestDb, SQLITE_NOMEM); } } @@ -602,7 +602,7 @@ int sqlite3_backup_finish(sqlite3_backup *p){ /* Set the error code of the destination database handle. */ rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; if( p->pDestDb ){ - sqlite3Error(p->pDestDb, rc, 0); + sqlite3Error(p->pDestDb, rc); /* Exit the mutexes and free the backup context structure. */ sqlite3LeaveMutexAndCloseZombie(p->pDestDb); diff --git a/src/legacy.c b/src/legacy.c index 1913f0b5af..b8cb90d707 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -44,7 +44,7 @@ int sqlite3_exec( if( zSql==0 ) zSql = ""; sqlite3_mutex_enter(db->mutex); - sqlite3Error(db, SQLITE_OK, 0); + sqlite3Error(db, SQLITE_OK); while( rc==SQLITE_OK && zSql[0] ){ int nCol; char **azVals = 0; @@ -102,7 +102,7 @@ int sqlite3_exec( rc = SQLITE_ABORT; sqlite3VdbeFinalize((Vdbe *)pStmt); pStmt = 0; - sqlite3Error(db, SQLITE_ABORT, 0); + sqlite3Error(db, SQLITE_ABORT); goto exec_out; } } @@ -132,7 +132,7 @@ exec_out: memcpy(*pzErrMsg, sqlite3_errmsg(db), nErrMsg); }else{ rc = SQLITE_NOMEM; - sqlite3Error(db, SQLITE_NOMEM, 0); + sqlite3Error(db, SQLITE_NOMEM); } }else if( pzErrMsg ){ *pzErrMsg = 0; diff --git a/src/loadext.c b/src/loadext.c index 828e865b61..05045dedb3 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -749,7 +749,7 @@ void sqlite3AutoLoadExtensions(sqlite3 *db){ sqlite3_mutex_leave(mutex); zErrmsg = 0; if( xInit && (rc = xInit(db, &zErrmsg, &sqlite3Apis))!=0 ){ - sqlite3Error(db, rc, + sqlite3ErrorWithMsg(db, rc, "automatic extension loading failed: %s", zErrmsg); go = 0; } diff --git a/src/main.c b/src/main.c index 2472321759..219a8bec92 100644 --- a/src/main.c +++ b/src/main.c @@ -852,7 +852,7 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ ** SQLITE_BUSY if the connection can not be closed immediately. */ if( !forceZombie && connectionIsBusy(db) ){ - sqlite3Error(db, SQLITE_BUSY, "unable to close due to unfinalized " + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to close due to unfinalized " "statements or unfinished backups"); sqlite3_mutex_leave(db->mutex); return SQLITE_BUSY; @@ -982,7 +982,7 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ sqlite3HashClear(&db->aModule); #endif - sqlite3Error(db, SQLITE_OK, 0); /* Deallocates any cached error strings. */ + sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ sqlite3ValueFree(db->pErr); sqlite3CloseExtensions(db); @@ -1415,7 +1415,7 @@ int sqlite3CreateFunc( p = sqlite3FindFunction(db, zFunctionName, nName, nArg, (u8)enc, 0); if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ if( db->nVdbeActive ){ - sqlite3Error(db, SQLITE_BUSY, + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify user-function due to active statements"); assert( !db->mallocFailed ); return SQLITE_BUSY; @@ -1753,10 +1753,10 @@ int sqlite3_wal_checkpoint_v2( } if( iDb<0 ){ rc = SQLITE_ERROR; - sqlite3Error(db, SQLITE_ERROR, "unknown database: %s", zDb); + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "unknown database: %s", zDb); }else{ rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt); - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); } rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); @@ -1911,7 +1911,7 @@ const void *sqlite3_errmsg16(sqlite3 *db){ }else{ z = sqlite3_value_text16(db->pErr); if( z==0 ){ - sqlite3Error(db, db->errCode, sqlite3ErrStr(db->errCode)); + sqlite3ErrorWithMsg(db, db->errCode, sqlite3ErrStr(db->errCode)); z = sqlite3_value_text16(db->pErr); } /* A malloc() may have failed within the call to sqlite3_value_text16() @@ -2022,7 +2022,7 @@ static int createCollation( pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); if( pColl && pColl->xCmp ){ if( db->nVdbeActive ){ - sqlite3Error(db, SQLITE_BUSY, + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify collation sequence due to active statements"); return SQLITE_BUSY; } @@ -2056,7 +2056,7 @@ static int createCollation( pColl->pUser = pCtx; pColl->xDel = xDel; pColl->enc = (u8)(enc2 | (enc & SQLITE_UTF16_ALIGNED)); - sqlite3Error(db, SQLITE_OK, 0); + sqlite3Error(db, SQLITE_OK); return SQLITE_OK; } @@ -2541,7 +2541,7 @@ static int openDatabase( rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM ) db->mallocFailed = 1; - sqlite3Error(db, rc, zErrMsg ? "%s" : 0, zErrMsg); + sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); sqlite3_free(zErrMsg); goto opendb_out; } @@ -2553,7 +2553,7 @@ static int openDatabase( if( rc==SQLITE_IOERR_NOMEM ){ rc = SQLITE_NOMEM; } - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); goto opendb_out; } db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); @@ -2577,7 +2577,7 @@ static int openDatabase( ** database schema yet. This is delayed until the first time the database ** is accessed. */ - sqlite3Error(db, SQLITE_OK, 0); + sqlite3Error(db, SQLITE_OK); sqlite3RegisterBuiltinFunctions(db); /* Load automatic extensions - extensions that have been registered @@ -2634,7 +2634,7 @@ static int openDatabase( SQLITE_DEFAULT_LOCKING_MODE); #endif - if( rc ) sqlite3Error(db, rc, 0); + if( rc ) sqlite3Error(db, rc); /* Enable the lookaside-malloc subsystem */ setupLookaside(db, 0, sqlite3GlobalConfig.szLookaside, @@ -2996,7 +2996,7 @@ error_out: zColumnName); rc = SQLITE_ERROR; } - sqlite3Error(db, rc, (zErrMsg?"%s":0), zErrMsg); + sqlite3ErrorWithMsg(db, rc, (zErrMsg?"%s":0), zErrMsg); sqlite3DbFree(db, zErrMsg); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); diff --git a/src/malloc.c b/src/malloc.c index 9c11d07767..9fb4303979 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -776,7 +776,7 @@ int sqlite3ApiExit(sqlite3* db, int rc){ */ assert( !db || sqlite3_mutex_held(db->mutex) ); if( db && (db->mallocFailed || rc==SQLITE_IOERR_NOMEM) ){ - sqlite3Error(db, SQLITE_NOMEM, 0); + sqlite3Error(db, SQLITE_NOMEM); db->mallocFailed = 0; rc = SQLITE_NOMEM; } diff --git a/src/notify.c b/src/notify.c index fcab5bfaf0..8137226f35 100644 --- a/src/notify.c +++ b/src/notify.c @@ -184,7 +184,7 @@ int sqlite3_unlock_notify( leaveMutex(); assert( !db->mallocFailed ); - sqlite3Error(db, rc, (rc?"database is deadlocked":0)); + sqlite3ErrorWithMsg(db, rc, (rc?"database is deadlocked":0)); sqlite3_mutex_leave(db->mutex); return rc; } diff --git a/src/prepare.c b/src/prepare.c index c7ba53a1f5..5b92e88513 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -593,7 +593,7 @@ static int sqlite3Prepare( rc = sqlite3BtreeSchemaLocked(pBt); if( rc ){ const char *zDb = db->aDb[i].zName; - sqlite3Error(db, rc, "database schema is locked: %s", zDb); + sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); testcase( db->flags & SQLITE_ReadUncommitted ); goto end_prepare; } @@ -610,7 +610,7 @@ static int sqlite3Prepare( testcase( nBytes==mxLen ); testcase( nBytes==mxLen+1 ); if( nBytes>mxLen ){ - sqlite3Error(db, SQLITE_TOOBIG, "statement too long"); + sqlite3ErrorWithMsg(db, SQLITE_TOOBIG, "statement too long"); rc = sqlite3ApiExit(db, SQLITE_TOOBIG); goto end_prepare; } @@ -677,10 +677,10 @@ static int sqlite3Prepare( } if( zErrMsg ){ - sqlite3Error(db, rc, "%s", zErrMsg); + sqlite3ErrorWithMsg(db, rc, "%s", zErrMsg); sqlite3DbFree(db, zErrMsg); }else{ - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); } /* Delete any TriggerPrg structures allocated while parsing this statement. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5202beff78..65dd7a9279 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3350,7 +3350,8 @@ int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); char sqlite3ExprAffinity(Expr *pExpr); int sqlite3Atoi64(const char*, i64*, int, u8); int sqlite3DecOrHexToI64(const char*, i64*); -void sqlite3Error(sqlite3*, int, const char*,...); +void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); +void sqlite3Error(sqlite3*,int); void *sqlite3HexToBlob(sqlite3*, const char *z, int n); u8 sqlite3HexToInt(int h); int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); diff --git a/src/util.c b/src/util.c index 619af7f758..1c75e4cfbe 100644 --- a/src/util.c +++ b/src/util.c @@ -111,6 +111,15 @@ int sqlite3Strlen30(const char *z){ return 0x3fffffff & (int)(z2 - z); } +/* +** Set the current error code to err_code and clear any prior error message. +*/ +void sqlite3Error(sqlite3 *db, int err_code){ + assert( db!=0 ); + db->errCode = err_code; + if( db->pErr ) sqlite3ValueSetNull(db->pErr); +} + /* ** Set the most recent error code and error string for the sqlite ** handle "db". The error code is set to "err_code". @@ -132,18 +141,18 @@ int sqlite3Strlen30(const char *z){ ** should be called with err_code set to SQLITE_OK and zFormat set ** to NULL. */ -void sqlite3Error(sqlite3 *db, int err_code, const char *zFormat, ...){ +void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *zFormat, ...){ assert( db!=0 ); db->errCode = err_code; - if( zFormat && (db->pErr || (db->pErr = sqlite3ValueNew(db))!=0) ){ + if( zFormat==0 ){ + sqlite3Error(db, err_code); + }else if( db->pErr || (db->pErr = sqlite3ValueNew(db))!=0 ){ char *z; va_list ap; va_start(ap, zFormat); z = sqlite3VMPrintf(db, zFormat, ap); va_end(ap); sqlite3ValueSetStr(db->pErr, -1, z, SQLITE_UTF8, SQLITE_DYNAMIC); - }else if( db->pErr ){ - sqlite3ValueSetNull(db->pErr); } } @@ -157,12 +166,12 @@ void sqlite3Error(sqlite3 *db, int err_code, const char *zFormat, ...){ ** %T Insert a token ** %S Insert the first element of a SrcList ** -** This function should be used to report any error that occurs whilst +** This function should be used to report any error that occurs while ** compiling an SQL statement (i.e. within sqlite3_prepare()). The ** last thing the sqlite3_prepare() function does is copy the error ** stored by this function into the database handle using sqlite3Error(). -** Function sqlite3Error() should be used during statement execution -** (sqlite3_step() etc.). +** Functions sqlite3Error() or sqlite3ErrorWithMsg() should be used +** during statement execution (sqlite3_step() etc.). */ void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ char *zMsg; diff --git a/src/vdbeapi.c b/src/vdbeapi.c index f6cc2d8c05..675361013a 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -770,7 +770,7 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ }else{ if( pVm && ALWAYS(pVm->db) ){ sqlite3_mutex_enter(pVm->db->mutex); - sqlite3Error(pVm->db, SQLITE_RANGE, 0); + sqlite3Error(pVm->db, SQLITE_RANGE); } pOut = (Mem*)columnNullValue(); } @@ -1035,14 +1035,14 @@ static int vdbeUnbind(Vdbe *p, int i){ } sqlite3_mutex_enter(p->db->mutex); if( p->magic!=VDBE_MAGIC_RUN || p->pc>=0 ){ - sqlite3Error(p->db, SQLITE_MISUSE, 0); + sqlite3Error(p->db, SQLITE_MISUSE); sqlite3_mutex_leave(p->db->mutex); sqlite3_log(SQLITE_MISUSE, "bind on a busy prepared statement: [%s]", p->zSql); return SQLITE_MISUSE_BKPT; } if( i<1 || i>p->nVar ){ - sqlite3Error(p->db, SQLITE_RANGE, 0); + sqlite3Error(p->db, SQLITE_RANGE); sqlite3_mutex_leave(p->db->mutex); return SQLITE_RANGE; } @@ -1050,7 +1050,7 @@ static int vdbeUnbind(Vdbe *p, int i){ pVar = &p->aVar[i]; sqlite3VdbeMemRelease(pVar); pVar->flags = MEM_Null; - sqlite3Error(p->db, SQLITE_OK, 0); + sqlite3Error(p->db, SQLITE_OK); /* If the bit corresponding to this variable in Vdbe.expmask is set, then ** binding a new value to this variable invalidates the current query plan. @@ -1092,7 +1092,7 @@ static int bindText( if( rc==SQLITE_OK && encoding!=0 ){ rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); } - sqlite3Error(p->db, rc, 0); + sqlite3Error(p->db, rc); rc = sqlite3ApiExit(p->db, rc); } sqlite3_mutex_leave(p->db->mutex); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 513481bf00..58f39a48a8 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2497,7 +2497,7 @@ int sqlite3VdbeTransferError(Vdbe *p){ db->mallocFailed = mallocFailed; db->errCode = rc; }else{ - sqlite3Error(db, rc, 0); + sqlite3Error(db, rc); } return rc; } @@ -2560,7 +2560,7 @@ int sqlite3VdbeReset(Vdbe *p){ ** to sqlite3_step(). For consistency (since sqlite3_step() was ** called), set the database error in this case as well. */ - sqlite3Error(db, p->rc, p->zErrMsg ? "%s" : 0, p->zErrMsg); + sqlite3ErrorWithMsg(db, p->rc, p->zErrMsg ? "%s" : 0, p->zErrMsg); sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; } diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 083f3f413c..71bd8816d5 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -318,7 +318,7 @@ blob_open_out: if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); sqlite3DbFree(db, pBlob); } - sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); sqlite3ParserReset(pParse); sqlite3StackFree(db, pParse); @@ -371,7 +371,7 @@ static int blobReadWrite( if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ /* Request is out of range. Return a transient error. */ rc = SQLITE_ERROR; - sqlite3Error(db, SQLITE_ERROR, 0); + sqlite3Error(db, SQLITE_ERROR); }else if( v==0 ){ /* If there is no statement handle, then the blob-handle has ** already been invalidated. Return SQLITE_ABORT in this case. @@ -451,7 +451,7 @@ int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ char *zErr; rc = blobSeekToRow(p, iRow, &zErr); if( rc!=SQLITE_OK ){ - sqlite3Error(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); } assert( rc!=SQLITE_SCHEMA ); diff --git a/src/vtab.c b/src/vtab.c index ad18af48a4..c7a8a5a33f 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -699,7 +699,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ sqlite3_mutex_enter(db->mutex); if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ - sqlite3Error(db, SQLITE_MISUSE, 0); + sqlite3Error(db, SQLITE_MISUSE); sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } @@ -727,7 +727,7 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ } db->pVtabCtx->pTab = 0; }else{ - sqlite3Error(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); sqlite3DbFree(db, zErr); rc = SQLITE_ERROR; } @@ -1088,7 +1088,7 @@ int sqlite3_vtab_config(sqlite3 *db, int op, ...){ } va_end(ap); - if( rc!=SQLITE_OK ) sqlite3Error(db, rc, 0); + if( rc!=SQLITE_OK ) sqlite3Error(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } From 2f2b2b85800866c9cebde588d09706feed6462eb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Aug 2014 18:48:25 +0000 Subject: [PATCH 252/710] Performance enhancement in sqlite3PutVarint(). FossilOrigin-Name: a929be551924144c9bc7aab608404d59e479abb5 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqliteInt.h | 25 +++++-------------------- src/util.c | 27 ++++++++------------------- src/vdbemem.c | 2 +- 5 files changed, 23 insertions(+), 49 deletions(-) diff --git a/manifest b/manifest index 2ec516f887..4718b15953 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Split\sthe\ssqlite3Error()\sroutine\sinto\ssqlite3Error()\sand\nsqlite3ErrorWithMsg(),\sfor\sa\sslight\ssize\sreduction\sand\sperformance\sincrease. -D 2014-08-22T18:00:11.621 +C Performance\senhancement\sin\ssqlite3PutVarint(). +D 2014-08-22T18:48:25.634 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/shell.c 728d2226594d356bf4fbdbdfd08538fd78fd06f3 F src/sqlite.h.in ed9d35990c61f0388ca6405706455c4095310553 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 937869c407c61026443c879e8c90a9dff05d2d27 +F src/sqliteInt.h d8a9be2aa123a78c90ad4aba09b23e7dd3f8cc9f F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -282,7 +282,7 @@ F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 4bddd12803275aa98f1c7ce0118fceb02b2167f6 F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 -F src/util.c 524127b3c330b9f490a505ff6eb7f80dfc873a3a +F src/util.c 068dcd26354a3898ccc64ad5c4bdb95a7a15d33a F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c f7f4066e4d6e3858878d76ce9288ea603e12ddf6 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 @@ -290,7 +290,7 @@ F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 49b8d2943d02d276b4efef114578251a3277f47d F src/vdbeaux.c f7fdc59e2eefa6fc4ba7324b03ea6f8f66e98d62 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c d90a1e8acf8b63dc9d14cbbea12bfec6cec31394 +F src/vdbemem.c 4a64659ed8e4c3b18a9238e038145ab1bdcd146f F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f7f2160db014f0ae11ad13c8ad70ad3444124e3e -R 668369bbbf2d81407f36345226c5d2a9 +P cf561d1f0bb60b3d638632d20bd686dda4fa4a04 +R 892fbe852e37f91de1b9f5afa35c0502 U drh -Z bddb035a8072a327798c78fbe23bcf26 +Z c70614e9322ab39ef4e9fbe07eb13b05 diff --git a/manifest.uuid b/manifest.uuid index 5842d66545..12c08a2c09 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cf561d1f0bb60b3d638632d20bd686dda4fa4a04 \ No newline at end of file +a929be551924144c9bc7aab608404d59e479abb5 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 65dd7a9279..a534c5cebc 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3307,38 +3307,23 @@ u64 sqlite3LogEstToInt(LogEst); /* ** Routines to read and write variable-length integers. These used to ** be defined locally, but now we use the varint routines in the util.c -** file. Code should use the MACRO forms below, as the Varint32 versions -** are coded to assume the single byte case is already handled (which -** the MACRO form does). +** file. */ int sqlite3PutVarint(unsigned char*, u64); -int sqlite3PutVarint32(unsigned char*, u32); u8 sqlite3GetVarint(const unsigned char *, u64 *); u8 sqlite3GetVarint32(const unsigned char *, u32 *); int sqlite3VarintLen(u64 v); /* -** The header of a record consists of a sequence variable-length integers. -** These integers are almost always small and are encoded as a single byte. -** The following macros take advantage this fact to provide a fast encode -** and decode of the integers in a record header. It is faster for the common -** case where the integer is a single byte. It is a little slower when the -** integer is two or more bytes. But overall it is faster. -** -** The following expressions are equivalent: -** -** x = sqlite3GetVarint32( A, &B ); -** x = sqlite3PutVarint32( A, B ); -** -** x = getVarint32( A, B ); -** x = putVarint32( A, B ); -** +** The common case is for a varint to be a single byte. They following +** macros handle the common case without a procedure call, but then call +** the procedure for larger varints. */ #define getVarint32(A,B) \ (u8)((*(A)<(u8)0x80)?((B)=(u32)*(A)),1:sqlite3GetVarint32((A),(u32 *)&(B))) #define putVarint32(A,B) \ (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1:\ - sqlite3PutVarint32((A),(B))) + sqlite3PutVarint((A),(B))) #define getVarint sqlite3GetVarint #define putVarint sqlite3PutVarint diff --git a/src/util.c b/src/util.c index 1c75e4cfbe..839a4a4636 100644 --- a/src/util.c +++ b/src/util.c @@ -708,7 +708,7 @@ int sqlite3Atoi(const char *z){ ** bit clear. Except, if we get to the 9th byte, it stores the full ** 8 bits and is the last byte. */ -int sqlite3PutVarint(unsigned char *p, u64 v){ +static int SQLITE_NOINLINE putVarint64(unsigned char *p, u64 v){ int i, j, n; u8 buf[10]; if( v & (((u64)0xff000000)<<32) ){ @@ -732,28 +732,17 @@ int sqlite3PutVarint(unsigned char *p, u64 v){ } return n; } - -/* -** This routine is a faster version of sqlite3PutVarint() that only -** works for 32-bit positive integers and which is optimized for -** the common case of small integers. A MACRO version, putVarint32, -** is provided which inlines the single-byte case. All code should use -** the MACRO version as this function assumes the single-byte case has -** already been handled. -*/ -int sqlite3PutVarint32(unsigned char *p, u32 v){ -#ifndef putVarint32 - if( (v & ~0x7f)==0 ){ - p[0] = v; +int sqlite3PutVarint(unsigned char *p, u64 v){ + if( v<=0x7f ){ + p[0] = v&0x7f; return 1; } -#endif - if( (v & ~0x3fff)==0 ){ - p[0] = (u8)((v>>7) | 0x80); - p[1] = (u8)(v & 0x7f); + if( v<=0x3fff ){ + p[0] = ((v>>7)&0x7f)|0x80; + p[1] = v&0x7f; return 2; } - return sqlite3PutVarint(p, v); + return putVarint64(p,v); } /* diff --git a/src/vdbemem.c b/src/vdbemem.c index cf44aa7e2d..08cf40eaa8 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1130,7 +1130,7 @@ static void recordFunc( sqlite3_result_error_nomem(context); }else{ aRet[0] = nSerial+1; - sqlite3PutVarint(&aRet[1], iSerial); + putVarint32(&aRet[1], iSerial); sqlite3VdbeSerialPut(&aRet[1+nSerial], argv[0], iSerial); sqlite3_result_blob(context, aRet, nRet, SQLITE_TRANSIENT); sqlite3DbFree(db, aRet); From 31753c833df675335f2303841c0ae89489aed85a Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 22 Aug 2014 19:12:16 +0000 Subject: [PATCH 253/710] Revise #ifdef ordering to prefer the ANSI version of GetVersionEx, when available. FossilOrigin-Name: 9fe0f0754c063c17fffa9d4814b7180397cf5226 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/os_win.c | 14 +++++++------- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 8ca8db3ef3..8d07b759fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\ssqlite3_win32_is_nt(),\sassume\sWinRT\sis\sNT-based. -D 2014-08-21T19:11:17.859 +C Revise\s#ifdef\sordering\sto\sprefer\sthe\sANSI\sversion\sof\sGetVersionEx,\swhen\savailable. +D 2014-08-22T19:12:16.818 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -209,7 +209,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bd7df3094a60915c148517504c76df4fca24e542 -F src/os_win.c 817005660016dcaee908b44e4437c000f3666b4c +F src/os_win.c b979aabc18857645ea3bdc7eaa38df1c5678fae0 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1188,10 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7029b3404d3f5f698a496934f3a3f2972051b257 -R 2bf9f3345de9d3c6f464f51024249e55 -T *branch * winrt -T *sym-winrt * -T -sym-trunk * +P 2f59e71fbf31d2bd788d11197943b2ff3f4046e1 +R 01ec23265c3130f55ace673019fe8500 U mistachkin -Z 677c2dec084bbb63edeb1740996a4b29 +Z 9a72fd0590798f8454637a5afefb97ac diff --git a/manifest.uuid b/manifest.uuid index a2db9504d9..acdb35ace2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f59e71fbf31d2bd788d11197943b2ff3f4046e1 \ No newline at end of file +9fe0f0754c063c17fffa9d4814b7180397cf5226 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 17613d253f..8ff25a139f 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1325,18 +1325,18 @@ int sqlite3_win32_is_nt(void){ return 1; #elif defined(SQLITE_WIN32_GETVERSIONEX) && SQLITE_WIN32_GETVERSIONEX if( osInterlockedCompareExchange(&sqlite3_os_type, 0, 0)==0 ){ -#if defined(SQLITE_WIN32_HAS_WIDE) - OSVERSIONINFOW sInfo; - sInfo.dwOSVersionInfoSize = sizeof(sInfo); - osGetVersionExW(&sInfo); - osInterlockedCompareExchange(&sqlite3_os_type, - (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); -#elif defined(SQLITE_WIN32_HAS_ANSI) +#if defined(SQLITE_WIN32_HAS_ANSI) OSVERSIONINFOA sInfo; sInfo.dwOSVersionInfoSize = sizeof(sInfo); osGetVersionExA(&sInfo); osInterlockedCompareExchange(&sqlite3_os_type, (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); +#elif defined(SQLITE_WIN32_HAS_WIDE) + OSVERSIONINFOW sInfo; + sInfo.dwOSVersionInfoSize = sizeof(sInfo); + osGetVersionExW(&sInfo); + osInterlockedCompareExchange(&sqlite3_os_type, + (sInfo.dwPlatformId == VER_PLATFORM_WIN32_NT) ? 2 : 1, 0); #endif } return osInterlockedCompareExchange(&sqlite3_os_type, 2, 2)==2; From a8dcba9199cef12aa979e6be7c98c50d5a777c9e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Aug 2014 20:35:29 +0000 Subject: [PATCH 254/710] Combine the pcacheAddToDirtyList() and pcacheRemoveFromDirtyList() routines into a single pcacheManageDirtyList() routine. The resulting binary code is slightly faster and a few bytes smaller. FossilOrigin-Name: 6bcf1af6a48dbda5ac6f6b3b02810bdfc4730000 --- manifest | 12 ++-- manifest.uuid | 2 +- src/pcache.c | 150 ++++++++++++++++++++++++-------------------------- 3 files changed, 80 insertions(+), 84 deletions(-) diff --git a/manifest b/manifest index 4718b15953..63a01156f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\senhancement\sin\ssqlite3PutVarint(). -D 2014-08-22T18:48:25.634 +C Combine\sthe\spcacheAddToDirtyList()\sand\spcacheRemoveFromDirtyList()\sroutines\ninto\sa\ssingle\spcacheManageDirtyList()\sroutine.\s\sThe\sresulting\sbinary\scode\sis\nslightly\sfaster\sand\sa\sfew\sbytes\ssmaller. +D 2014-08-22T20:35:29.948 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 -F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 +F src/pcache.c da602c5447051705cab41604bf3276815eb569d0 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c F src/pragma.c d10ef67c4de79f78188b965b4b7988aff1d66f2e @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cf561d1f0bb60b3d638632d20bd686dda4fa4a04 -R 892fbe852e37f91de1b9f5afa35c0502 +P a929be551924144c9bc7aab608404d59e479abb5 +R 2ac5f67b7ba10391163889b9ff4c4145 U drh -Z c70614e9322ab39ef4e9fbe07eb13b05 +Z 6bffc4c72a94895b60e52defb6944efd diff --git a/manifest.uuid b/manifest.uuid index 12c08a2c09..da73897dba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a929be551924144c9bc7aab608404d59e479abb5 \ No newline at end of file +6bcf1af6a48dbda5ac6f6b3b02810bdfc4730000 \ No newline at end of file diff --git a/src/pcache.c b/src/pcache.c index e18bf93be0..2e4b5d78b2 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -62,71 +62,73 @@ static int pcacheCheckSynced(PCache *pCache){ } #endif /* !NDEBUG && SQLITE_ENABLE_EXPENSIVE_ASSERT */ -/* -** Remove page pPage from the list of dirty pages. -*/ -static void pcacheRemoveFromDirtyList(PgHdr *pPage){ - PCache *p = pPage->pCache; - - assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); - assert( pPage->pDirtyPrev || pPage==p->pDirty ); - - /* Update the PCache1.pSynced variable if necessary. */ - if( p->pSynced==pPage ){ - PgHdr *pSynced = pPage->pDirtyPrev; - while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){ - pSynced = pSynced->pDirtyPrev; - } - p->pSynced = pSynced; - } - - if( pPage->pDirtyNext ){ - pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; - }else{ - assert( pPage==p->pDirtyTail ); - p->pDirtyTail = pPage->pDirtyPrev; - } - if( pPage->pDirtyPrev ){ - pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; - }else{ - assert( pPage==p->pDirty ); - p->pDirty = pPage->pDirtyNext; - if( p->pDirty==0 && p->bPurgeable ){ - assert( p->eCreate==1 ); - p->eCreate = 2; - } - } - pPage->pDirtyNext = 0; - pPage->pDirtyPrev = 0; - - expensive_assert( pcacheCheckSynced(p) ); -} +/* Allowed values for second argument to pcacheManageDirtyList() */ +#define PCACHE_DIRTYLIST_REMOVE 1 /* Remove pPage from dirty list */ +#define PCACHE_DIRTYLIST_ADD 2 /* Add pPage to the dirty list */ +#define PCACHE_DIRTYLIST_FRONT 3 /* Move pPage to the front of the list */ /* -** Add page pPage to the head of the dirty list (PCache1.pDirty is set to -** pPage). +** Manage pPage's participation on the dirty list. Bits of the addRemove +** argument determines what operation to do. The 0x01 bit means first +** remove pPage from the dirty list. The 0x02 means add pPage back to +** the dirty list. Doing both moves pPage to the front of the dirty list. */ -static void pcacheAddToDirtyList(PgHdr *pPage){ +static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ PCache *p = pPage->pCache; - assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); - - pPage->pDirtyNext = p->pDirty; - if( pPage->pDirtyNext ){ - assert( pPage->pDirtyNext->pDirtyPrev==0 ); - pPage->pDirtyNext->pDirtyPrev = pPage; - }else if( p->bPurgeable ){ - assert( p->eCreate==2 ); - p->eCreate = 1; + if( addRemove & PCACHE_DIRTYLIST_REMOVE ){ + assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); + assert( pPage->pDirtyPrev || pPage==p->pDirty ); + + /* Update the PCache1.pSynced variable if necessary. */ + if( p->pSynced==pPage ){ + PgHdr *pSynced = pPage->pDirtyPrev; + while( pSynced && (pSynced->flags&PGHDR_NEED_SYNC) ){ + pSynced = pSynced->pDirtyPrev; + } + p->pSynced = pSynced; + } + + if( pPage->pDirtyNext ){ + pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; + }else{ + assert( pPage==p->pDirtyTail ); + p->pDirtyTail = pPage->pDirtyPrev; + } + if( pPage->pDirtyPrev ){ + pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; + }else{ + assert( pPage==p->pDirty ); + p->pDirty = pPage->pDirtyNext; + if( p->pDirty==0 && p->bPurgeable ){ + assert( p->eCreate==1 ); + p->eCreate = 2; + } + } + pPage->pDirtyNext = 0; + pPage->pDirtyPrev = 0; + expensive_assert( pcacheCheckSynced(p) ); } - p->pDirty = pPage; - if( !p->pDirtyTail ){ - p->pDirtyTail = pPage; + if( addRemove & PCACHE_DIRTYLIST_ADD ){ + assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); + + pPage->pDirtyNext = p->pDirty; + if( pPage->pDirtyNext ){ + assert( pPage->pDirtyNext->pDirtyPrev==0 ); + pPage->pDirtyNext->pDirtyPrev = pPage; + }else if( p->bPurgeable ){ + assert( p->eCreate==2 ); + p->eCreate = 1; + } + p->pDirty = pPage; + if( !p->pDirtyTail ){ + p->pDirtyTail = pPage; + } + if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ + p->pSynced = pPage; + } + expensive_assert( pcacheCheckSynced(p) ); } - if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ - p->pSynced = pPage; - } - expensive_assert( pcacheCheckSynced(p) ); } /* @@ -134,12 +136,11 @@ static void pcacheAddToDirtyList(PgHdr *pPage){ ** being used for an in-memory database, this function is a no-op. */ static void pcacheUnpin(PgHdr *p){ - PCache *pCache = p->pCache; - if( pCache->bPurgeable ){ + if( p->pCache->bPurgeable ){ if( p->pgno==1 ){ - pCache->pPage1 = 0; + p->pCache->pPage1 = 0; } - sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, p->pPage, 0); + sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 0); } } @@ -332,18 +333,16 @@ int sqlite3PcacheFetch( ** Decrement the reference count on a page. If the page is clean and the ** reference count drops to 0, then it is made elible for recycling. */ -void sqlite3PcacheRelease(PgHdr *p){ +void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ assert( p->nRef>0 ); p->nRef--; if( p->nRef==0 ){ - PCache *pCache = p->pCache; - pCache->nRef--; + p->pCache->nRef--; if( (p->flags&PGHDR_DIRTY)==0 ){ pcacheUnpin(p); }else{ /* Move the page to the head of the dirty list. */ - pcacheRemoveFromDirtyList(p); - pcacheAddToDirtyList(p); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); } } } @@ -362,17 +361,15 @@ void sqlite3PcacheRef(PgHdr *p){ ** page pointed to by p is invalid. */ void sqlite3PcacheDrop(PgHdr *p){ - PCache *pCache; assert( p->nRef==1 ); if( p->flags&PGHDR_DIRTY ){ - pcacheRemoveFromDirtyList(p); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); } - pCache = p->pCache; - pCache->nRef--; + p->pCache->nRef--; if( p->pgno==1 ){ - pCache->pPage1 = 0; + p->pCache->pPage1 = 0; } - sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, p->pPage, 1); + sqlite3GlobalConfig.pcache2.xUnpin(p->pCache->pCache, p->pPage, 1); } /* @@ -384,7 +381,7 @@ void sqlite3PcacheMakeDirty(PgHdr *p){ assert( p->nRef>0 ); if( 0==(p->flags & PGHDR_DIRTY) ){ p->flags |= PGHDR_DIRTY; - pcacheAddToDirtyList( p); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); } } @@ -394,7 +391,7 @@ void sqlite3PcacheMakeDirty(PgHdr *p){ */ void sqlite3PcacheMakeClean(PgHdr *p){ if( (p->flags & PGHDR_DIRTY) ){ - pcacheRemoveFromDirtyList(p); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC); if( p->nRef==0 ){ pcacheUnpin(p); @@ -433,8 +430,7 @@ void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ - pcacheRemoveFromDirtyList(p); - pcacheAddToDirtyList(p); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); } } From 75e2a2d362175e222873a90d1d356bac92b31037 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Aug 2014 21:58:10 +0000 Subject: [PATCH 255/710] Performance enhancements in the b-tree mutex logic. FossilOrigin-Name: 8914530644f938a7a98e25ea1fb0bca1f9d79101 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btmutex.c | 19 ++++++++++++++++--- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 63a01156f0..812c653a73 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Combine\sthe\spcacheAddToDirtyList()\sand\spcacheRemoveFromDirtyList()\sroutines\ninto\sa\ssingle\spcacheManageDirtyList()\sroutine.\s\sThe\sresulting\sbinary\scode\sis\nslightly\sfaster\sand\sa\sfew\sbytes\ssmaller. -D 2014-08-22T20:35:29.948 +C Performance\senhancements\sin\sthe\sb-tree\smutex\slogic. +D 2014-08-22T21:58:10.354 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb -F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 +F src/btmutex.c ec9d3f1295dafeb278c3830211cc5584132468f4 F src/btree.c 4195fed5741b4dbcc9831b623aec487258f3e62d F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a929be551924144c9bc7aab608404d59e479abb5 -R 2ac5f67b7ba10391163889b9ff4c4145 +P 6bcf1af6a48dbda5ac6f6b3b02810bdfc4730000 +R 24d5ffe2f1d9679a613707454c70f45a U drh -Z 6bffc4c72a94895b60e52defb6944efd +Z 6801ae93f83fb9ed7a247305de8bd03a diff --git a/manifest.uuid b/manifest.uuid index da73897dba..5367fb5584 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6bcf1af6a48dbda5ac6f6b3b02810bdfc4730000 \ No newline at end of file +8914530644f938a7a98e25ea1fb0bca1f9d79101 \ No newline at end of file diff --git a/src/btmutex.c b/src/btmutex.c index d87d4d5fee..9672687fe0 100644 --- a/src/btmutex.c +++ b/src/btmutex.c @@ -38,7 +38,7 @@ static void lockBtreeMutex(Btree *p){ ** Release the BtShared mutex associated with B-Tree handle p and ** clear the p->locked boolean. */ -static void unlockBtreeMutex(Btree *p){ +static void SQLITE_NOINLINE unlockBtreeMutex(Btree *p){ BtShared *pBt = p->pBt; assert( p->locked==1 ); assert( sqlite3_mutex_held(pBt->mutex) ); @@ -49,6 +49,9 @@ static void unlockBtreeMutex(Btree *p){ p->locked = 0; } +/* Forward reference */ +static void SQLITE_NOINLINE btreeLockCarefully(Btree *p); + /* ** Enter a mutex on the given BTree object. ** @@ -66,8 +69,6 @@ static void unlockBtreeMutex(Btree *p){ ** subsequent Btrees that desire a lock. */ void sqlite3BtreeEnter(Btree *p){ - Btree *pLater; - /* Some basic sanity checking on the Btree. The list of Btrees ** connected by pNext and pPrev should be in sorted order by ** Btree.pBt value. All elements of the list should belong to @@ -92,6 +93,17 @@ void sqlite3BtreeEnter(Btree *p){ if( !p->sharable ) return; p->wantToLock++; if( p->locked ) return; + btreeLockCarefully(p); +} + +/* This is a helper function for sqlite3BtreeLock(). By moving +** complex, but seldom used logic, out of sqlite3BtreeLock() and +** into this routine, we avoid unnecessary stack pointer changes +** and thus help the sqlite3BtreeLock() routine to run much faster +** in the common case. +*/ +static void SQLITE_NOINLINE btreeLockCarefully(Btree *p){ + Btree *pLater; /* In most cases, we should be able to acquire the lock we ** want without having to go throught the ascending lock @@ -124,6 +136,7 @@ void sqlite3BtreeEnter(Btree *p){ } } + /* ** Exit the recursive mutex on a Btree. */ From 637f3d83b9455eeb87bce698877f9d8c03ec912b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Aug 2014 22:26:07 +0000 Subject: [PATCH 256/710] Factor the saveAllCursors() routine of btree.c into two separate routines, for a noticable performance improvement. FossilOrigin-Name: 3eb084390382c108e9b0ff0b29dede58ebb149bc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 35 +++++++++++++++++++++++++++++++---- 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 812c653a73..a0fd1d8b2e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\senhancements\sin\sthe\sb-tree\smutex\slogic. -D 2014-08-22T21:58:10.354 +C Factor\sthe\ssaveAllCursors()\sroutine\sof\sbtree.c\sinto\stwo\sseparate\sroutines,\nfor\sa\snoticable\sperformance\simprovement. +D 2014-08-22T22:26:07.834 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c ec9d3f1295dafeb278c3830211cc5584132468f4 -F src/btree.c 4195fed5741b4dbcc9831b623aec487258f3e62d +F src/btree.c 776885dfef3033af7e8eabbf004481f219870a1d F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 058e3aadb1376521ff291735237edf4c10f438fb @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6bcf1af6a48dbda5ac6f6b3b02810bdfc4730000 -R 24d5ffe2f1d9679a613707454c70f45a +P 8914530644f938a7a98e25ea1fb0bca1f9d79101 +R 036983fb95199f1ba06f1397f389a453 U drh -Z 6801ae93f83fb9ed7a247305de8bd03a +Z 9ea20fed62648c22294e9e322338cabb diff --git a/manifest.uuid b/manifest.uuid index 5367fb5584..334a3ae4c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8914530644f938a7a98e25ea1fb0bca1f9d79101 \ No newline at end of file +3eb084390382c108e9b0ff0b29dede58ebb149bc \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 56718b69d6..6da3715e30 100644 --- a/src/btree.c +++ b/src/btree.c @@ -629,16 +629,42 @@ static int saveCursorPosition(BtCursor *pCur){ return rc; } +/* Forward reference */ +static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*); + /* ** Save the positions of all cursors (except pExcept) that are open on -** the table with root-page iRoot. Usually, this is called just before cursor -** pExcept is used to modify the table (BtreeDelete() or BtreeInsert()). +** the table with root-page iRoot. "Saving the cursor position" means that +** the location in the btree is remembered in such a way that it can be +** moved back to the same spot after the btree has been modified. This +** routine is called just before cursor pExcept is used to modify the +** table, for example in BtreeDelete() or BtreeInsert(). +** +** Implementation note: This routine merely checks to see if any cursors +** need to be saved. It calls out to saveCursorsOnList() in the (unusual) +** event that cursors are in need to being saved. */ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ - BtCursor *p; assert( sqlite3_mutex_held(pBt->mutex) ); assert( pExcept==0 || pExcept->pBt==pBt ); + BtCursor *p; for(p=pBt->pCursor; p; p=p->pNext){ + if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ) break; + } + return p ? saveCursorsOnList(p, iRoot, pExcept) : SQLITE_OK; +} + +/* This helper routine to saveAllCursors does the actual work of saving +** the cursors if and when a cursor is found that actually requires saving. +** The common case is that no cursors need to be saved, so this routine is +** broken out from its caller to avoid unnecessary stack pointer movement. +*/ +static int SQLITE_NOINLINE saveCursorsOnList( + BtCursor *p, /* The first cursor that needs saving */ + Pgno iRoot, /* Only save cursor with this iRoot. Save all if zero */ + BtCursor *pExcept /* Do not save this cursor */ +){ + do{ if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ if( p->eState==CURSOR_VALID ){ int rc = saveCursorPosition(p); @@ -650,7 +676,8 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ btreeReleaseAllCursorPages(p); } } - } + p = p->pNext; + }while( p ); return SQLITE_OK; } From 6848dad89477be4146d535e7a2646300c472f429 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 22 Aug 2014 23:33:03 +0000 Subject: [PATCH 257/710] Another performance tweak: Split the sqlite3BtreeCursorHasMoved() routine into two with the second routine named sqlite3BtreeCursorRestore(). The first now only reports whether or not the cursor has moved and the second tries to restore the cursor. This allows the sqlite3VdbeCursorMoveto() routine to be refactored to avoid stack pointer movements, for a noticable performance gain. FossilOrigin-Name: ce123b5c592556a8cd38b01fcc91ba76231d3098 --- manifest | 16 ++++++------ manifest.uuid | 2 +- src/btree.c | 51 ++++++++++++++++++++++--------------- src/btree.h | 3 ++- src/vdbeaux.c | 69 ++++++++++++++++++++++++++++++++++----------------- 5 files changed, 88 insertions(+), 53 deletions(-) diff --git a/manifest b/manifest index a0fd1d8b2e..98cd57119e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Factor\sthe\ssaveAllCursors()\sroutine\sof\sbtree.c\sinto\stwo\sseparate\sroutines,\nfor\sa\snoticable\sperformance\simprovement. -D 2014-08-22T22:26:07.834 +C Another\sperformance\stweak:\sSplit\sthe\ssqlite3BtreeCursorHasMoved()\sroutine\s\ninto\stwo\swith\sthe\ssecond\sroutine\snamed\ssqlite3BtreeCursorRestore().\s\sThe\sfirst\nnow\sonly\sreports\swhether\sor\snot\sthe\scursor\shas\smoved\sand\sthe\ssecond\stries\sto\nrestore\sthe\scursor.\s\sThis\sallows\sthe\ssqlite3VdbeCursorMoveto()\sroutine\sto\sbe\nrefactored\sto\savoid\sstack\spointer\smovements,\sfor\sa\snoticable\sperformance\sgain. +D 2014-08-22T23:33:03.189 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,8 +168,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c ec9d3f1295dafeb278c3830211cc5584132468f4 -F src/btree.c 776885dfef3033af7e8eabbf004481f219870a1d -F src/btree.h 4245a349bfe09611d7ff887dbc3a80cee8b7955a +F src/btree.c c4b4e1c9524ba7b7dab840e6f8b29e2379a3bdd2 +F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 058e3aadb1376521ff291735237edf4c10f438fb F src/callback.c b97d0695ffcf6a8710ee445ffe56ee387d4d8a6f @@ -288,7 +288,7 @@ F src/vdbe.c f7f4066e4d6e3858878d76ce9288ea603e12ddf6 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df F src/vdbeapi.c 49b8d2943d02d276b4efef114578251a3277f47d -F src/vdbeaux.c f7fdc59e2eefa6fc4ba7324b03ea6f8f66e98d62 +F src/vdbeaux.c dba006f67c9fd1b1d07ee7fb0fb38aa1905161d1 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 4a64659ed8e4c3b18a9238e038145ab1bdcd146f F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8914530644f938a7a98e25ea1fb0bca1f9d79101 -R 036983fb95199f1ba06f1397f389a453 +P 3eb084390382c108e9b0ff0b29dede58ebb149bc +R 356d2e894dc7576ad768e53fab77ee5b U drh -Z 9ea20fed62648c22294e9e322338cabb +Z 78a9bc2b7c59c5e05ea01f499ba8646b diff --git a/manifest.uuid b/manifest.uuid index 334a3ae4c0..af6d676ce0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3eb084390382c108e9b0ff0b29dede58ebb149bc \ No newline at end of file +ce123b5c592556a8cd38b01fcc91ba76231d3098 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 6da3715e30..23de31db07 100644 --- a/src/btree.c +++ b/src/btree.c @@ -762,37 +762,48 @@ static int btreeRestoreCursorPosition(BtCursor *pCur){ SQLITE_OK) /* -** Determine whether or not a cursor has moved from the position it -** was last placed at. Cursors can move when the row they are pointing -** at is deleted out from under them. +** Determine whether or not a cursor has moved from the position where +** it was last placed, or has been invalidated for any other reason. +** Cursors can move when the row they are pointing at is deleted out +** from under them, for example. Cursor might also move if a btree +** is rebalanced. ** -** This routine returns an error code if something goes wrong. The -** integer *pHasMoved is set as follows: +** Calling this routine with a NULL cursor pointer returns false. ** -** 0: The cursor is unchanged -** 1: The cursor is still pointing at the same row, but the pointers -** returned by sqlite3BtreeKeyFetch() or sqlite3BtreeDataFetch() -** might now be invalid because of a balance() or other change to the -** b-tree. -** 2: The cursor is no longer pointing to the row. The row might have -** been deleted out from under the cursor. +** Use the separate sqlite3BtreeCursorRestore() routine to restore a cursor +** back to where it ought to be if this routine returns true. */ -int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ +int sqlite3BtreeCursorHasMoved(BtCursor *pCur){ + return pCur && pCur->eState!=CURSOR_VALID; +} + +/* +** This routine restores a cursor back to its original position after it +** has been moved by some outside activity (such as a btree rebalance or +** a row having been deleted out from under the cursor). +** +** On success, the *pDifferentRow parameter is false if the cursor is left +** pointing at exactly the same row. *pDifferntRow is the row the cursor +** was pointing to has been deleted, forcing the cursor to point to some +** nearby row. +** +** This routine should only be called for a cursor that just returned +** TRUE from sqlite3BtreeCursorHasMoved(). +*/ +int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow){ int rc; - if( pCur->eState==CURSOR_VALID ){ - *pHasMoved = 0; - return SQLITE_OK; - } + assert( pCur!=0 ); + assert( pCur->eState!=CURSOR_VALID ); rc = restoreCursorPosition(pCur); if( rc ){ - *pHasMoved = 2; + *pDifferentRow = 1; return rc; } if( pCur->eState!=CURSOR_VALID || NEVER(pCur->skipNext!=0) ){ - *pHasMoved = 2; + *pDifferentRow = 1; }else{ - *pHasMoved = 1; + *pDifferentRow = 0; } return SQLITE_OK; } diff --git a/src/btree.h b/src/btree.h index 7118534f93..38abdca1a2 100644 --- a/src/btree.h +++ b/src/btree.h @@ -169,7 +169,8 @@ int sqlite3BtreeMovetoUnpacked( int bias, int *pRes ); -int sqlite3BtreeCursorHasMoved(BtCursor*, int*); +int sqlite3BtreeCursorHasMoved(BtCursor*); +int sqlite3BtreeCursorRestore(BtCursor*, int*); int sqlite3BtreeDelete(BtCursor*); int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, const void *pData, int nData, diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 58f39a48a8..2037121ddc 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2713,6 +2713,48 @@ void sqlite3VdbeDelete(Vdbe *p){ sqlite3DbFree(db, p); } +/* +** The cursor "p" has a pending seek operation that has not yet been +** carried out. Seek the cursor now. If an error occurs, return +** the appropriate error code. +*/ +static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ + int res, rc; +#ifdef SQLITE_TEST + extern int sqlite3_search_count; +#endif + assert( p->deferredMoveto ); + assert( p->isTable ); + rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); + if( rc ) return rc; + p->lastRowid = p->movetoTarget; + if( res!=0 ) return SQLITE_CORRUPT_BKPT; + p->rowidIsValid = 1; +#ifdef SQLITE_TEST + sqlite3_search_count++; +#endif + p->deferredMoveto = 0; + p->cacheStatus = CACHE_STALE; + return SQLITE_OK; +} + +/* +** Something has moved cursor "p" out of place. Maybe the row it was +** pointed to was deleted out from under it. Or maybe the btree was +** rebalanced. Whatever the cause, try to restore "p" to the place it +** is suppose to be pointing. If the row was deleted out from under the +** cursor, set the cursor to point to a NULL row. +*/ +static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ + int isDifferentRow, rc; + assert( p->pCursor!=0 ); + assert( sqlite3BtreeCursorHasMoved(p->pCursor) ); + rc = sqlite3BtreeCursorRestore(p->pCursor, &isDifferentRow); + p->cacheStatus = CACHE_STALE; + if( isDifferentRow ) p->nullRow = 1; + return rc; +} + /* ** Make sure the cursor p is ready to read or write the row to which it ** was last positioned. Return an error code if an OOM fault or I/O error @@ -2728,29 +2770,10 @@ void sqlite3VdbeDelete(Vdbe *p){ */ int sqlite3VdbeCursorMoveto(VdbeCursor *p){ if( p->deferredMoveto ){ - int res, rc; -#ifdef SQLITE_TEST - extern int sqlite3_search_count; -#endif - assert( p->isTable ); - rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); - if( rc ) return rc; - p->lastRowid = p->movetoTarget; - if( res!=0 ) return SQLITE_CORRUPT_BKPT; - p->rowidIsValid = 1; -#ifdef SQLITE_TEST - sqlite3_search_count++; -#endif - p->deferredMoveto = 0; - p->cacheStatus = CACHE_STALE; - }else if( p->pCursor ){ - int hasMoved; - int rc = sqlite3BtreeCursorHasMoved(p->pCursor, &hasMoved); - if( rc ) return rc; - if( hasMoved ){ - p->cacheStatus = CACHE_STALE; - if( hasMoved==2 ) p->nullRow = 1; - } + return handleDeferredMoveto(p); + } + if( sqlite3BtreeCursorHasMoved(p->pCursor) ){ + return handleMovedCursor(p); } return SQLITE_OK; } From bd9507c8daa292989b382628cb7fedba18dc2fb3 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Aug 2014 17:21:37 +0000 Subject: [PATCH 258/710] Performance optimization in the applyAffinity() logic inside the VDBE. FossilOrigin-Name: 25f2246be404f38b4f8dd70397cd1454d46358c4 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 34 +++++++++++++++++++++++----------- src/vdbeInt.h | 2 +- src/vdbemem.c | 13 +++++++------ 5 files changed, 40 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 98cd57119e..8450ec16e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\sperformance\stweak:\sSplit\sthe\ssqlite3BtreeCursorHasMoved()\sroutine\s\ninto\stwo\swith\sthe\ssecond\sroutine\snamed\ssqlite3BtreeCursorRestore().\s\sThe\sfirst\nnow\sonly\sreports\swhether\sor\snot\sthe\scursor\shas\smoved\sand\sthe\ssecond\stries\sto\nrestore\sthe\scursor.\s\sThis\sallows\sthe\ssqlite3VdbeCursorMoveto()\sroutine\sto\sbe\nrefactored\sto\savoid\sstack\spointer\smovements,\sfor\sa\snoticable\sperformance\sgain. -D 2014-08-22T23:33:03.189 +C Performance\soptimization\sin\sthe\sapplyAffinity()\slogic\sinside\sthe\sVDBE. +D 2014-08-23T17:21:37.343 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,13 +284,13 @@ F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 068dcd26354a3898ccc64ad5c4bdb95a7a15d33a F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c f7f4066e4d6e3858878d76ce9288ea603e12ddf6 +F src/vdbe.c 76f9fc30dc28751900e85f615e29cdf89c069a77 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 -F src/vdbeInt.h f5513f2b5ac1e2c5128996c7ea23add256a301df +F src/vdbeInt.h 764a055bc9a3e61a30ba37cd4c1826f62bcf8759 F src/vdbeapi.c 49b8d2943d02d276b4efef114578251a3277f47d F src/vdbeaux.c dba006f67c9fd1b1d07ee7fb0fb38aa1905161d1 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 4a64659ed8e4c3b18a9238e038145ab1bdcd146f +F src/vdbemem.c f2e162888521a9af35d608ac4c13db4991dfb417 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3eb084390382c108e9b0ff0b29dede58ebb149bc -R 356d2e894dc7576ad768e53fab77ee5b +P ce123b5c592556a8cd38b01fcc91ba76231d3098 +R f59cf04b4ec2e84886c139f6982d2702 U drh -Z 78a9bc2b7c59c5e05ea01f499ba8646b +Z 3f5bd9d4c135b090a54f8be497682e06 diff --git a/manifest.uuid b/manifest.uuid index af6d676ce0..4ee0b9c481 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ce123b5c592556a8cd38b01fcc91ba76231d3098 \ No newline at end of file +25f2246be404f38b4f8dd70397cd1454d46358c4 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 61adb9cccd..13a076cacb 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -146,7 +146,7 @@ int sqlite3_found_count = 0; ** already. Return non-zero if a malloc() fails. */ #define Stringify(P, enc) \ - if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc)) \ + if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc,0)) \ { goto no_mem; } /* @@ -228,8 +228,17 @@ static VdbeCursor *allocateCursor( ** do so without loss of information. In other words, if the string ** looks like a number, convert it into a number. If it does not ** look like a number, leave it alone. +** +** If the bTryForInt flag is true, then extra effort is made to give +** an integer representation. Strings that look like floating point +** values but which have no fractional component (example: '48.00') +** will have a MEM_Int representation when bTryForInt is true. +** +** If bTryForInt is false, then if the input string contains a decimal +** point or exponential notation, the result is only MEM_Real, even +** if there is an exact integer representation of the quantity. */ -static void applyNumericAffinity(Mem *pRec){ +static void applyNumericAffinity(Mem *pRec, int bTryForInt){ double rValue; i64 iValue; u8 enc = pRec->enc; @@ -241,10 +250,9 @@ static void applyNumericAffinity(Mem *pRec){ }else{ pRec->r = rValue; pRec->flags |= MEM_Real; + if( bTryForInt ) sqlite3VdbeIntegerAffinity(pRec); } } -#define ApplyNumericAffinity(X) \ - if(((X)->flags&(MEM_Real|MEM_Int))==0){applyNumericAffinity(X);} /* ** Processing is determine by the affinity parameter: @@ -275,15 +283,17 @@ static void applyAffinity( ** representation. */ if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){ - sqlite3VdbeMemStringify(pRec, enc); + sqlite3VdbeMemStringify(pRec, enc, 1); } - pRec->flags &= ~(MEM_Real|MEM_Int); }else if( affinity!=SQLITE_AFF_NONE ){ assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL || affinity==SQLITE_AFF_NUMERIC ); - ApplyNumericAffinity(pRec); - if( pRec->flags & MEM_Real ){ - sqlite3VdbeIntegerAffinity(pRec); + if( (pRec->flags & MEM_Int)==0 ){ + if( (pRec->flags & MEM_Real)==0 ){ + applyNumericAffinity(pRec,1); + }else{ + sqlite3VdbeIntegerAffinity(pRec); + } } } } @@ -298,7 +308,7 @@ int sqlite3_value_numeric_type(sqlite3_value *pVal){ int eType = sqlite3_value_type(pVal); if( eType==SQLITE_TEXT ){ Mem *pMem = (Mem*)pVal; - applyNumericAffinity(pMem); + applyNumericAffinity(pMem, 0); eType = sqlite3_value_type(pVal); } return eType; @@ -3603,7 +3613,9 @@ case OP_SeekGT: { /* jump, in3 */ ** blob, or NULL. But it needs to be an integer before we can do ** the seek, so covert it. */ pIn3 = &aMem[pOp->p3]; - ApplyNumericAffinity(pIn3); + if( (pIn3->flags & (MEM_Int|MEM_Real))==0 ){ + applyNumericAffinity(pIn3, 0); + } iKey = sqlite3VdbeIntValue(pIn3); pC->rowidIsValid = 0; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 141573eef4..ba44b61e91 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -418,7 +418,7 @@ void sqlite3VdbeMemSetNull(Mem*); void sqlite3VdbeMemSetZeroBlob(Mem*,int); void sqlite3VdbeMemSetRowSet(Mem*); int sqlite3VdbeMemMakeWriteable(Mem*); -int sqlite3VdbeMemStringify(Mem*, int); +int sqlite3VdbeMemStringify(Mem*, u8, u8); i64 sqlite3VdbeIntValue(Mem*); int sqlite3VdbeMemIntegerify(Mem*); double sqlite3VdbeRealValue(Mem*); diff --git a/src/vdbemem.c b/src/vdbemem.c index 08cf40eaa8..572d486fc4 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -223,7 +223,8 @@ int sqlite3VdbeMemNulTerminate(Mem *pMem){ ** are converted using sqlite3_snprintf(). Converting a BLOB to a string ** is a no-op. ** -** Existing representations MEM_Int and MEM_Real are *not* invalidated. +** Existing representations MEM_Int and MEM_Real are invalidated if +** bForce is true but are retained if bForce is false. ** ** A MEM_Null value will never be passed to this function. This function is ** used for converting values to text for returning to the user (i.e. via @@ -231,8 +232,7 @@ int sqlite3VdbeMemNulTerminate(Mem *pMem){ ** keys are strings. In the former case a NULL pointer is returned the ** user and the later is an internal programming error. */ -int sqlite3VdbeMemStringify(Mem *pMem, int enc){ - int rc = SQLITE_OK; +int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ int fg = pMem->flags; const int nByte = 32; @@ -248,7 +248,7 @@ int sqlite3VdbeMemStringify(Mem *pMem, int enc){ return SQLITE_NOMEM; } - /* For a Real or Integer, use sqlite3_mprintf() to produce the UTF-8 + /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8 ** string representation of the value. Then, if the required encoding ** is UTF-16le or UTF-16be do a translation. ** @@ -263,8 +263,9 @@ int sqlite3VdbeMemStringify(Mem *pMem, int enc){ pMem->n = sqlite3Strlen30(pMem->z); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; + if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real); sqlite3VdbeChangeEncoding(pMem, enc); - return rc; + return SQLITE_OK; } /* @@ -877,7 +878,7 @@ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-31275-44060 */ }else{ assert( (pVal->flags&MEM_Blob)==0 ); - sqlite3VdbeMemStringify(pVal, enc); + sqlite3VdbeMemStringify(pVal, enc, 0); assert( 0==(1&SQLITE_PTR_TO_INT(pVal->z)) ); } assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 From f1a89ede4a392fec648f6a51f8aa0a3f5216cd4e Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Aug 2014 17:41:15 +0000 Subject: [PATCH 259/710] Improved performance in the type handling of arithmetic operators in the VDBE. FossilOrigin-Name: 0c0a603950c97837442d82886f947aab0acbd805 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 26 +++++++++++++++++++------- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 8450ec16e4..a5fad99ba9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sin\sthe\sapplyAffinity()\slogic\sinside\sthe\sVDBE. -D 2014-08-23T17:21:37.343 +C Improved\sperformance\sin\sthe\stype\shandling\sof\sarithmetic\soperators\sin\sthe\sVDBE. +D 2014-08-23T17:41:15.456 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,7 +284,7 @@ F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 F src/util.c 068dcd26354a3898ccc64ad5c4bdb95a7a15d33a F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 76f9fc30dc28751900e85f615e29cdf89c069a77 +F src/vdbe.c 52ee5d589cbb171a8b096f210b69deb4a33c4369 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h 764a055bc9a3e61a30ba37cd4c1826f62bcf8759 F src/vdbeapi.c 49b8d2943d02d276b4efef114578251a3277f47d @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ce123b5c592556a8cd38b01fcc91ba76231d3098 -R f59cf04b4ec2e84886c139f6982d2702 +P 25f2246be404f38b4f8dd70397cd1454d46358c4 +R 2a69d28bf2271ac25d420aa80c42027b U drh -Z 3f5bd9d4c135b090a54f8be497682e06 +Z c64b6b94b93d1da5e9e0a75cc412b115 diff --git a/manifest.uuid b/manifest.uuid index 4ee0b9c481..53a34bdd8d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -25f2246be404f38b4f8dd70397cd1454d46358c4 \ No newline at end of file +0c0a603950c97837442d82886f947aab0acbd805 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 13a076cacb..2e797eeaf5 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -326,6 +326,24 @@ void sqlite3ValueApplyAffinity( applyAffinity((Mem *)pVal, affinity, enc); } +/* +** pMem currently only holds a string type (or maybe a BLOB that we can +** interpret as a string if we want to). Compute its corresponding +** numeric type, if has one. Set the pMem->r and pMem->u.i fields +** accordingly. +*/ +static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ + assert( (pMem->flags & (MEM_Int|MEM_Real))==0 ); + assert( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ); + if( sqlite3AtoF(pMem->z, &pMem->r, pMem->n, pMem->enc)==0 ){ + return 0; + } + if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){ + return MEM_Int; + } + return MEM_Real; +} + /* ** Return the numeric type for pMem, either MEM_Int or MEM_Real or both or ** none. @@ -338,13 +356,7 @@ static u16 numericType(Mem *pMem){ return pMem->flags & (MEM_Int|MEM_Real); } if( pMem->flags & (MEM_Str|MEM_Blob) ){ - if( sqlite3AtoF(pMem->z, &pMem->r, pMem->n, pMem->enc)==0 ){ - return 0; - } - if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){ - return MEM_Int; - } - return MEM_Real; + return computeNumericType(pMem); } return 0; } From 9de4a17185a18927a72a7278fc70b600e46c653e Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Aug 2014 18:17:19 +0000 Subject: [PATCH 260/710] Make the implementation of the sqlite3_aggregate_context() interface faster for second an subsequent invocations. This helps all aggregate functions to perform better. FossilOrigin-Name: 802148f3110462eac939d53ce08eb9a2f6aac739 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeapi.c | 42 ++++++++++++++++++++++++++---------------- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index a5fad99ba9..aa0c254b02 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sperformance\sin\sthe\stype\shandling\sof\sarithmetic\soperators\sin\sthe\sVDBE. -D 2014-08-23T17:41:15.456 +C Make\sthe\simplementation\sof\sthe\ssqlite3_aggregate_context()\sinterface\sfaster\nfor\ssecond\san\ssubsequent\sinvocations.\s\sThis\shelps\sall\saggregate\sfunctions\sto\nperform\sbetter. +D 2014-08-23T18:17:19.059 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -287,7 +287,7 @@ F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 52ee5d589cbb171a8b096f210b69deb4a33c4369 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h 764a055bc9a3e61a30ba37cd4c1826f62bcf8759 -F src/vdbeapi.c 49b8d2943d02d276b4efef114578251a3277f47d +F src/vdbeapi.c cda974083d7597f807640d344ffcf76d872201ce F src/vdbeaux.c dba006f67c9fd1b1d07ee7fb0fb38aa1905161d1 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c f2e162888521a9af35d608ac4c13db4991dfb417 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 25f2246be404f38b4f8dd70397cd1454d46358c4 -R 2a69d28bf2271ac25d420aa80c42027b +P 0c0a603950c97837442d82886f947aab0acbd805 +R d4bdc3de1168bf2d862dde32e02d949b U drh -Z c64b6b94b93d1da5e9e0a75cc412b115 +Z 5d66d6e81701eb9b183824f31680ac42 diff --git a/manifest.uuid b/manifest.uuid index 53a34bdd8d..5b8e019c7e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c0a603950c97837442d82886f947aab0acbd805 \ No newline at end of file +802148f3110462eac939d53ce08eb9a2f6aac739 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 675361013a..b2d8059153 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -606,32 +606,42 @@ void sqlite3InvalidFunction( sqlite3_free(zErr); } +/* +** Create a new aggregate context for p and return a pointer to +** its pMem->z element. +*/ +static SQLITE_NOINLINE void *createAggContext(sqlite3_context *p, int nByte){ + Mem *pMem = p->pMem; + assert( (pMem->flags & MEM_Agg)==0 ); + if( nByte<=0 ){ + sqlite3VdbeMemReleaseExternal(pMem); + pMem->flags = MEM_Null; + pMem->z = 0; + }else{ + sqlite3VdbeMemGrow(pMem, nByte, 0); + pMem->flags = MEM_Agg; + pMem->u.pDef = p->pFunc; + if( pMem->z ){ + memset(pMem->z, 0, nByte); + } + } + return (void*)pMem->z; +} + /* ** Allocate or return the aggregate context for a user function. A new ** context is allocated on the first call. Subsequent calls return the ** same context that was returned on prior calls. */ void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ - Mem *pMem; assert( p && p->pFunc && p->pFunc->xStep ); assert( sqlite3_mutex_held(p->s.db->mutex) ); - pMem = p->pMem; testcase( nByte<0 ); - if( (pMem->flags & MEM_Agg)==0 ){ - if( nByte<=0 ){ - sqlite3VdbeMemReleaseExternal(pMem); - pMem->flags = MEM_Null; - pMem->z = 0; - }else{ - sqlite3VdbeMemGrow(pMem, nByte, 0); - pMem->flags = MEM_Agg; - pMem->u.pDef = p->pFunc; - if( pMem->z ){ - memset(pMem->z, 0, nByte); - } - } + if( (p->pMem->flags & MEM_Agg)==0 ){ + return createAggContext(p, nByte); + }else{ + return (void*)p->pMem->z; } - return (void*)pMem->z; } /* From 3ccd5bf89e57d67b1ee89e7cf9ad3bc5989b0339 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Aug 2014 19:04:55 +0000 Subject: [PATCH 261/710] Changes to sqlite3ScratchMalloc() that make the entire memory allocation interface a little faster and about 100 bytes smaller. FossilOrigin-Name: f83daa16f65ef35062412e88c214852a4aeb3da2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/malloc.c | 14 ++++++-------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index aa0c254b02..91d323beb8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\simplementation\sof\sthe\ssqlite3_aggregate_context()\sinterface\sfaster\nfor\ssecond\san\ssubsequent\sinvocations.\s\sThis\shelps\sall\saggregate\sfunctions\sto\nperform\sbetter. -D 2014-08-23T18:17:19.059 +C Changes\sto\ssqlite3ScratchMalloc()\sthat\smake\sthe\sentire\smemory\sallocation\ninterface\sa\slittle\sfaster\sand\sabout\s100\sbytes\ssmaller. +D 2014-08-23T19:04:55.170 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -191,7 +191,7 @@ F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab F src/main.c 900dd06e41d22795cbb23ab0240397f1e2901bf7 -F src/malloc.c 0a88a97fc5ae621ca9659d38b080e0b9ddbb80ad +F src/malloc.c 7f26fcedc42b93bb49c4c12a77aa0d891182392e F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0c0a603950c97837442d82886f947aab0acbd805 -R d4bdc3de1168bf2d862dde32e02d949b +P 802148f3110462eac939d53ce08eb9a2f6aac739 +R f2effcc85284cd0fae163c222d4987e9 U drh -Z 5d66d6e81701eb9b183824f31680ac42 +Z d2f180c60ae7591aa14ffa4ab8835b4a diff --git a/manifest.uuid b/manifest.uuid index 5b8e019c7e..233ca94126 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -802148f3110462eac939d53ce08eb9a2f6aac739 \ No newline at end of file +f83daa16f65ef35062412e88c214852a4aeb3da2 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 9fb4303979..f3b317da5f 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -352,22 +352,20 @@ void *sqlite3ScratchMalloc(int n){ assert( n>0 ); sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ p = mem0.pScratchFree; mem0.pScratchFree = mem0.pScratchFree->pNext; mem0.nScratchFree--; sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_USED, 1); - sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); sqlite3_mutex_leave(mem0.mutex); }else{ - if( sqlite3GlobalConfig.bMemstat ){ - sqlite3StatusSet(SQLITE_STATUS_SCRATCH_SIZE, n); - n = mallocWithAlarm(n, &p); - if( p ) sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, n); + sqlite3_mutex_leave(mem0.mutex); + p = sqlite3Malloc(n); + if( sqlite3GlobalConfig.bMemstat && p ){ + sqlite3_mutex_enter(mem0.mutex); + sqlite3StatusAdd(SQLITE_STATUS_SCRATCH_OVERFLOW, sqlite3MallocSize(p)); sqlite3_mutex_leave(mem0.mutex); - }else{ - sqlite3_mutex_leave(mem0.mutex); - p = sqlite3GlobalConfig.m.xMalloc(n); } sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH); } From 3bdffddc418b3b202d48eff685780eba2af24465 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Aug 2014 19:08:09 +0000 Subject: [PATCH 262/710] Fix a variable-declaration after code problem in btree.c. Harmless in GCC and CLANG but unacceptable for MSVC. FossilOrigin-Name: 45abd5c0bad2847861f3b26a7040490aa9bb1332 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 91d323beb8..f86063d01b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\ssqlite3ScratchMalloc()\sthat\smake\sthe\sentire\smemory\sallocation\ninterface\sa\slittle\sfaster\sand\sabout\s100\sbytes\ssmaller. -D 2014-08-23T19:04:55.170 +C Fix\sa\svariable-declaration\safter\scode\sproblem\sin\sbtree.c.\s\sHarmless\sin\nGCC\sand\sCLANG\sbut\sunacceptable\sfor\sMSVC. +D 2014-08-23T19:08:09.445 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c ec9d3f1295dafeb278c3830211cc5584132468f4 -F src/btree.c c4b4e1c9524ba7b7dab840e6f8b29e2379a3bdd2 +F src/btree.c 4737cb5bdb2eb8989cb292f6ff921f7ff45f0c46 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 058e3aadb1376521ff291735237edf4c10f438fb @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 802148f3110462eac939d53ce08eb9a2f6aac739 -R f2effcc85284cd0fae163c222d4987e9 +P f83daa16f65ef35062412e88c214852a4aeb3da2 +R 964769625a1b08555f23e3f40e4e1b01 U drh -Z d2f180c60ae7591aa14ffa4ab8835b4a +Z 8bbf37cac5a7f4ae4955d0f942660ce9 diff --git a/manifest.uuid b/manifest.uuid index 233ca94126..948c00a194 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f83daa16f65ef35062412e88c214852a4aeb3da2 \ No newline at end of file +45abd5c0bad2847861f3b26a7040490aa9bb1332 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 23de31db07..384bab218c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -645,9 +645,9 @@ static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*); ** event that cursors are in need to being saved. */ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ + BtCursor *p; assert( sqlite3_mutex_held(pBt->mutex) ); assert( pExcept==0 || pExcept->pBt==pBt ); - BtCursor *p; for(p=pBt->pCursor; p; p=p->pNext){ if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ) break; } From b4586f1254df07a7eccba2101ae4a95dfe2ca34b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Aug 2014 19:42:06 +0000 Subject: [PATCH 263/710] Another memory allocator performance optimization. FossilOrigin-Name: 6da6f46d0c43e3b68c21f514ddf8ee663c20f249 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/malloc.c | 10 +++++++++- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f86063d01b..0f001ae399 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\svariable-declaration\safter\scode\sproblem\sin\sbtree.c.\s\sHarmless\sin\nGCC\sand\sCLANG\sbut\sunacceptable\sfor\sMSVC. -D 2014-08-23T19:08:09.445 +C Another\smemory\sallocator\sperformance\soptimization. +D 2014-08-23T19:42:06.737 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -191,7 +191,7 @@ F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab F src/main.c 900dd06e41d22795cbb23ab0240397f1e2901bf7 -F src/malloc.c 7f26fcedc42b93bb49c4c12a77aa0d891182392e +F src/malloc.c 06f8507831d04d2830237bf0c9f2fd04623f9868 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f83daa16f65ef35062412e88c214852a4aeb3da2 -R 964769625a1b08555f23e3f40e4e1b01 +P 45abd5c0bad2847861f3b26a7040490aa9bb1332 +R 863f9a5c456ee79d86d610ec7db424be U drh -Z 8bbf37cac5a7f4ae4955d0f942660ce9 +Z 2076d0986bc3817907a655695f4a8b4e diff --git a/manifest.uuid b/manifest.uuid index 948c00a194..7e569bfcb7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -45abd5c0bad2847861f3b26a7040490aa9bb1332 \ No newline at end of file +6da6f46d0c43e3b68c21f514ddf8ee663c20f249 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index f3b317da5f..1b219604bc 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -477,6 +477,14 @@ void sqlite3_free(void *p){ } } +/* +** Add the size of memory allocation "p" to the count in +** *db->pnBytesFreed. +*/ +static SQLITE_NOINLINE void measureAllocationSize(sqlite3 *db, void *p){ + *db->pnBytesFreed += sqlite3DbMallocSize(db,p); +} + /* ** Free memory that might be associated with a particular database ** connection. @@ -486,7 +494,7 @@ void sqlite3DbFree(sqlite3 *db, void *p){ if( p==0 ) return; if( db ){ if( db->pnBytesFreed ){ - *db->pnBytesFreed += sqlite3DbMallocSize(db, p); + measureAllocationSize(db, p); return; } if( isLookaside(db, p) ){ From b50c65d561532c689a894112459be6e48f9ae366 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Aug 2014 20:25:53 +0000 Subject: [PATCH 264/710] Faster implementation of the sqlite3ApiExit() routine. FossilOrigin-Name: bd41d394d48516eb7d8ddc46abdcb427aa80173e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/malloc.c | 17 ++++++++++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0f001ae399..1044726bb0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Another\smemory\sallocator\sperformance\soptimization. -D 2014-08-23T19:42:06.737 +C Faster\simplementation\sof\sthe\ssqlite3ApiExit()\sroutine. +D 2014-08-23T20:25:53.209 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -191,7 +191,7 @@ F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab F src/main.c 900dd06e41d22795cbb23ab0240397f1e2901bf7 -F src/malloc.c 06f8507831d04d2830237bf0c9f2fd04623f9868 +F src/malloc.c 954de5f998c23237e04474a3f2159bf483bba65a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 45abd5c0bad2847861f3b26a7040490aa9bb1332 -R 863f9a5c456ee79d86d610ec7db424be +P 6da6f46d0c43e3b68c21f514ddf8ee663c20f249 +R cfcb5b6ff857caed22b759c12472ea06 U drh -Z 2076d0986bc3817907a655695f4a8b4e +Z c1d7ee14996f329ac13456e7bf7bb3df diff --git a/manifest.uuid b/manifest.uuid index 7e569bfcb7..215297c16b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6da6f46d0c43e3b68c21f514ddf8ee663c20f249 \ No newline at end of file +bd41d394d48516eb7d8ddc46abdcb427aa80173e \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 1b219604bc..b4b70350f4 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -761,6 +761,14 @@ void sqlite3SetString(char **pz, sqlite3 *db, const char *zFormat, ...){ *pz = z; } +/* +** Take actions at the end of an API call to indicate an OOM error +*/ +static SQLITE_NOINLINE int apiOomError(sqlite3 *db){ + db->mallocFailed = 0; + sqlite3Error(db, SQLITE_NOMEM); + return SQLITE_NOMEM; +} /* ** This function must be called before exiting any API function (i.e. @@ -781,10 +789,9 @@ int sqlite3ApiExit(sqlite3* db, int rc){ ** is unsafe, as is the call to sqlite3Error(). */ assert( !db || sqlite3_mutex_held(db->mutex) ); - if( db && (db->mallocFailed || rc==SQLITE_IOERR_NOMEM) ){ - sqlite3Error(db, SQLITE_NOMEM); - db->mallocFailed = 0; - rc = SQLITE_NOMEM; + if( db==0 ) return rc & 0xff; + if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){ + return apiOomError(db); } - return rc & (db ? db->errMask : 0xff); + return rc & db->errMask; } From efbf04458341672c9200d51b1fd7d485ddd09ce3 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 23 Aug 2014 23:15:31 +0000 Subject: [PATCH 265/710] Faster implementation of pcache1Fetch() FossilOrigin-Name: 0371cc3bb07448bcd64fd671f3e71bb7f30deb4d --- manifest | 12 +-- manifest.uuid | 2 +- src/pcache1.c | 210 ++++++++++++++++++++++++++------------------------ 3 files changed, 116 insertions(+), 108 deletions(-) diff --git a/manifest b/manifest index 1044726bb0..6e1bca5a72 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Faster\simplementation\sof\sthe\ssqlite3ApiExit()\sroutine. -D 2014-08-23T20:25:53.209 +C Faster\simplementation\sof\spcache1Fetch() +D 2014-08-23T23:15:31.233 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -216,7 +216,7 @@ F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c da602c5447051705cab41604bf3276815eb569d0 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 -F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c +F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c F src/pragma.c d10ef67c4de79f78188b965b4b7988aff1d66f2e F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6da6f46d0c43e3b68c21f514ddf8ee663c20f249 -R cfcb5b6ff857caed22b759c12472ea06 +P bd41d394d48516eb7d8ddc46abdcb427aa80173e +R d2491d65f44f8709d1d156c840c25703 U drh -Z c1d7ee14996f329ac13456e7bf7bb3df +Z 15bb2365f5f8da3e5de34ee820ad0674 diff --git a/manifest.uuid b/manifest.uuid index 215297c16b..bd2aebab20 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bd41d394d48516eb7d8ddc46abdcb427aa80173e \ No newline at end of file +0371cc3bb07448bcd64fd671f3e71bb7f30deb4d \ No newline at end of file diff --git a/src/pcache1.c b/src/pcache1.c index 1644b0693c..82015befce 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -383,7 +383,7 @@ static int pcache1UnderMemoryPressure(PCache1 *pCache){ ** ** The PCache mutex must be held when this function is called. */ -static int pcache1ResizeHash(PCache1 *p){ +static void pcache1ResizeHash(PCache1 *p){ PgHdr1 **apNew; unsigned int nNew; unsigned int i; @@ -415,8 +415,6 @@ static int pcache1ResizeHash(PCache1 *p){ p->apHash = apNew; p->nHash = nNew; } - - return (p->apHash ? SQLITE_OK : SQLITE_NOMEM); } /* @@ -551,6 +549,9 @@ static void pcache1Shutdown(void *NotUsed){ memset(&pcache1, 0, sizeof(pcache1)); } +/* forward declaration */ +static void pcache1Destroy(sqlite3_pcache *p); + /* ** Implementation of the sqlite3_pcache.xCreate method. ** @@ -595,12 +596,17 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ pCache->szPage = szPage; pCache->szExtra = szExtra; pCache->bPurgeable = (bPurgeable ? 1 : 0); + pcache1EnterMutex(pGroup); + pcache1ResizeHash(pCache); if( bPurgeable ){ pCache->nMin = 10; - pcache1EnterMutex(pGroup); pGroup->nMinPage += pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; - pcache1LeaveMutex(pGroup); + } + pcache1LeaveMutex(pGroup); + if( pCache->nHash==0 ){ + pcache1Destroy((sqlite3_pcache*)pCache); + pCache = 0; } } return (sqlite3_pcache *)pCache; @@ -656,6 +662,95 @@ static int pcache1Pagecount(sqlite3_pcache *p){ return n; } + +/* +** Implement steps 3, 4, and 5 of the pcache1Fetch() algorithm described +** in the header of the pcache1Fetch() procedure. +** +** This steps are broken out into a separate procedure because they are +** usually not needed, and by avoiding the stack initialization required +** for these steps, the main pcache1Fetch() procedure can run faster. +*/ +static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( + PCache1 *pCache, + unsigned int iKey, + int createFlag +){ + unsigned int nPinned; + PGroup *pGroup = pCache->pGroup; + PgHdr1 *pPage = 0; + + /* Step 3: Abort if createFlag is 1 but the cache is nearly full */ + assert( pCache->nPage >= pCache->nRecyclable ); + nPinned = pCache->nPage - pCache->nRecyclable; + assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage ); + assert( pCache->n90pct == pCache->nMax*9/10 ); + if( createFlag==1 && ( + nPinned>=pGroup->mxPinned + || nPinned>=pCache->n90pct + || pcache1UnderMemoryPressure(pCache) + )){ + return 0; + } + + if( pCache->nPage>=pCache->nHash ) pcache1ResizeHash(pCache); + assert( pCache->nHash>0 && pCache->apHash ); + + /* Step 4. Try to recycle a page. */ + if( pCache->bPurgeable && pGroup->pLruTail && ( + (pCache->nPage+1>=pCache->nMax) + || pGroup->nCurrentPage>=pGroup->nMaxPage + || pcache1UnderMemoryPressure(pCache) + )){ + PCache1 *pOther; + pPage = pGroup->pLruTail; + assert( pPage->isPinned==0 ); + pcache1RemoveFromHash(pPage); + pcache1PinPage(pPage); + pOther = pPage->pCache; + + /* We want to verify that szPage and szExtra are the same for pOther + ** and pCache. Assert that we can verify this by comparing sums. */ + assert( (pCache->szPage & (pCache->szPage-1))==0 && pCache->szPage>=512 ); + assert( pCache->szExtra<512 ); + assert( (pOther->szPage & (pOther->szPage-1))==0 && pOther->szPage>=512 ); + assert( pOther->szExtra<512 ); + + if( pOther->szPage+pOther->szExtra != pCache->szPage+pCache->szExtra ){ + pcache1FreePage(pPage); + pPage = 0; + }else{ + pGroup->nCurrentPage -= (pOther->bPurgeable - pCache->bPurgeable); + } + } + + /* Step 5. If a usable page buffer has still not been found, + ** attempt to allocate a new one. + */ + if( !pPage ){ + if( createFlag==1 ) sqlite3BeginBenignMalloc(); + pPage = pcache1AllocPage(pCache); + if( createFlag==1 ) sqlite3EndBenignMalloc(); + } + + if( pPage ){ + unsigned int h = iKey % pCache->nHash; + pCache->nPage++; + pPage->iKey = iKey; + pPage->pNext = pCache->apHash[h]; + pPage->pCache = pCache; + pPage->pLruPrev = 0; + pPage->pLruNext = 0; + pPage->isPinned = 1; + *(void **)pPage->page.pExtra = 0; + pCache->apHash[h] = pPage; + if( iKey>pCache->iMaxKey ){ + pCache->iMaxKey = iKey; + } + } + return pPage; +} + /* ** Implementation of the sqlite3_pcache.xFetch method. ** @@ -715,9 +810,7 @@ static sqlite3_pcache_page *pcache1Fetch( unsigned int iKey, int createFlag ){ - unsigned int nPinned; PCache1 *pCache = (PCache1 *)p; - PGroup *pGroup; PgHdr1 *pPage = 0; assert( offsetof(PgHdr1,page)==0 ); @@ -725,107 +818,22 @@ static sqlite3_pcache_page *pcache1Fetch( assert( pCache->bPurgeable || pCache->nMin==0 ); assert( pCache->bPurgeable==0 || pCache->nMin==10 ); assert( pCache->nMin==0 || pCache->bPurgeable ); - pcache1EnterMutex(pGroup = pCache->pGroup); + assert( pCache->nHash>0 ); + pcache1EnterMutex(pCache->pGroup); /* Step 1: Search the hash table for an existing entry. */ - if( pCache->nHash>0 ){ - unsigned int h = iKey % pCache->nHash; - for(pPage=pCache->apHash[h]; pPage&&pPage->iKey!=iKey; pPage=pPage->pNext); - } + pPage = pCache->apHash[iKey % pCache->nHash]; + while( pPage && pPage->iKey!=iKey ){ pPage = pPage->pNext; } /* Step 2: Abort if no existing page is found and createFlag is 0 */ if( pPage ){ if( !pPage->isPinned ) pcache1PinPage(pPage); - goto fetch_out; + }else if( createFlag ){ + /* Steps 3, 4, and 5 implemented by this subroutine */ + pPage = pcache1FetchStage2(pCache, iKey, createFlag); } - if( createFlag==0 ){ - goto fetch_out; - } - - /* The pGroup local variable will normally be initialized by the - ** pcache1EnterMutex() macro above. But if SQLITE_MUTEX_OMIT is defined, - ** then pcache1EnterMutex() is a no-op, so we have to initialize the - ** local variable here. Delaying the initialization of pGroup is an - ** optimization: The common case is to exit the module before reaching - ** this point. - */ -#ifdef SQLITE_MUTEX_OMIT - pGroup = pCache->pGroup; -#endif - - /* Step 3: Abort if createFlag is 1 but the cache is nearly full */ - assert( pCache->nPage >= pCache->nRecyclable ); - nPinned = pCache->nPage - pCache->nRecyclable; - assert( pGroup->mxPinned == pGroup->nMaxPage + 10 - pGroup->nMinPage ); - assert( pCache->n90pct == pCache->nMax*9/10 ); - if( createFlag==1 && ( - nPinned>=pGroup->mxPinned - || nPinned>=pCache->n90pct - || pcache1UnderMemoryPressure(pCache) - )){ - goto fetch_out; - } - - if( pCache->nPage>=pCache->nHash && pcache1ResizeHash(pCache) ){ - goto fetch_out; - } - assert( pCache->nHash>0 && pCache->apHash ); - - /* Step 4. Try to recycle a page. */ - if( pCache->bPurgeable && pGroup->pLruTail && ( - (pCache->nPage+1>=pCache->nMax) - || pGroup->nCurrentPage>=pGroup->nMaxPage - || pcache1UnderMemoryPressure(pCache) - )){ - PCache1 *pOther; - pPage = pGroup->pLruTail; - assert( pPage->isPinned==0 ); - pcache1RemoveFromHash(pPage); - pcache1PinPage(pPage); - pOther = pPage->pCache; - - /* We want to verify that szPage and szExtra are the same for pOther - ** and pCache. Assert that we can verify this by comparing sums. */ - assert( (pCache->szPage & (pCache->szPage-1))==0 && pCache->szPage>=512 ); - assert( pCache->szExtra<512 ); - assert( (pOther->szPage & (pOther->szPage-1))==0 && pOther->szPage>=512 ); - assert( pOther->szExtra<512 ); - - if( pOther->szPage+pOther->szExtra != pCache->szPage+pCache->szExtra ){ - pcache1FreePage(pPage); - pPage = 0; - }else{ - pGroup->nCurrentPage -= (pOther->bPurgeable - pCache->bPurgeable); - } - } - - /* Step 5. If a usable page buffer has still not been found, - ** attempt to allocate a new one. - */ - if( !pPage ){ - if( createFlag==1 ) sqlite3BeginBenignMalloc(); - pPage = pcache1AllocPage(pCache); - if( createFlag==1 ) sqlite3EndBenignMalloc(); - } - - if( pPage ){ - unsigned int h = iKey % pCache->nHash; - pCache->nPage++; - pPage->iKey = iKey; - pPage->pNext = pCache->apHash[h]; - pPage->pCache = pCache; - pPage->pLruPrev = 0; - pPage->pLruNext = 0; - pPage->isPinned = 1; - *(void **)pPage->page.pExtra = 0; - pCache->apHash[h] = pPage; - } - -fetch_out: - if( pPage && iKey>pCache->iMaxKey ){ - pCache->iMaxKey = iKey; - } - pcache1LeaveMutex(pGroup); + assert( pPage==0 || pCache->iMaxKey>=iKey ); + pcache1LeaveMutex(pCache->pGroup); return (sqlite3_pcache_page*)pPage; } From f063e08fd93c4f00cb44c03bad01e811d76ddb6e Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 24 Aug 2014 01:32:43 +0000 Subject: [PATCH 266/710] Patch the sqlite3PagerWrite() method in the Pager to run a bit faster. FossilOrigin-Name: c63311e2f3344363a5ed99838fb5850004eaee30 --- manifest | 12 ++-- manifest.uuid | 2 +- src/pager.c | 185 ++++++++++++++++++++++++++------------------------ 3 files changed, 105 insertions(+), 94 deletions(-) diff --git a/manifest b/manifest index 6e1bca5a72..a21c0da13e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Faster\simplementation\sof\spcache1Fetch() -D 2014-08-23T23:15:31.233 +C Patch\sthe\ssqlite3PagerWrite()\smethod\sin\sthe\sPager\sto\srun\sa\sbit\sfaster. +D 2014-08-24T01:32:43.379 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -211,7 +211,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bd7df3094a60915c148517504c76df4fca24e542 F src/os_win.c d067fce558a5032e6e6afe62899e5397bf63cf3e F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c f6bb1fa6cdf2062f2d8aec3e64db302bca519ab8 +F src/pager.c c831b0df879114915c08dc5a0188f9f22c48403b F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c da602c5447051705cab41604bf3276815eb569d0 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bd41d394d48516eb7d8ddc46abdcb427aa80173e -R d2491d65f44f8709d1d156c840c25703 +P 0371cc3bb07448bcd64fd671f3e71bb7f30deb4d +R 2fe545d9c9446b3adf94c91625adc119 U drh -Z 15bb2365f5f8da3e5de34ee820ad0674 +Z 79e99766e8b4bdebd57118e507ef67dc diff --git a/manifest.uuid b/manifest.uuid index bd2aebab20..308d8ad8ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0371cc3bb07448bcd64fd671f3e71bb7f30deb4d \ No newline at end of file +c63311e2f3344363a5ed99838fb5850004eaee30 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index a77dcecc4c..4a178a2985 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5772,6 +5772,97 @@ static int pager_write(PgHdr *pPg){ return rc; } +/* +** This is a variant of sqlite3PagerWrite() that runs when the sector size +** is larger than the page size. SQLite makes the (reasonable) assumption that +** all bytes of a sector are written together by hardware. Hence, all bytes of +** a sector need to be journalled in case of a power loss in the middle of +** a write. +** +** Usually, the sector size is less than or equal to the page size, in which +** case pages can be individually written. This routine only runs in the exceptional +** case where the page size is smaller than the sector size. +*/ +static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ + int rc = SQLITE_OK; /* Return code */ + Pgno nPageCount; /* Total number of pages in database file */ + Pgno pg1; /* First page of the sector pPg is located on. */ + int nPage = 0; /* Number of pages starting at pg1 to journal */ + int ii; /* Loop counter */ + int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ + Pager *pPager = pPg->pPager; /* The pager that owns pPg */ + Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); + + /* Set the doNotSpill NOSYNC bit to 1. This is because we cannot allow + ** a journal header to be written between the pages journaled by + ** this function. + */ + assert( !MEMDB ); + assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)==0 ); + pPager->doNotSpill |= SPILLFLAG_NOSYNC; + + /* This trick assumes that both the page-size and sector-size are + ** an integer power of 2. It sets variable pg1 to the identifier + ** of the first page of the sector pPg is located on. + */ + pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; + + nPageCount = pPager->dbSize; + if( pPg->pgno>nPageCount ){ + nPage = (pPg->pgno - pg1)+1; + }else if( (pg1+nPagePerSector-1)>nPageCount ){ + nPage = nPageCount+1-pg1; + }else{ + nPage = nPagePerSector; + } + assert(nPage>0); + assert(pg1<=pPg->pgno); + assert((pg1+nPage)>pPg->pgno); + + for(ii=0; iipgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ + if( pg!=PAGER_MJ_PGNO(pPager) ){ + rc = sqlite3PagerGet(pPager, pg, &pPage); + if( rc==SQLITE_OK ){ + rc = pager_write(pPage); + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnrefNotNull(pPage); + } + } + }else if( (pPage = pager_lookup(pPager, pg))!=0 ){ + if( pPage->flags&PGHDR_NEED_SYNC ){ + needSync = 1; + } + sqlite3PagerUnrefNotNull(pPage); + } + } + + /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages + ** starting at pg1, then it needs to be set for all of them. Because + ** writing to any of these nPage pages may damage the others, the + ** journal file must contain sync()ed copies of all of them + ** before any of them can be written out to the database file. + */ + if( rc==SQLITE_OK && needSync ){ + assert( !MEMDB ); + for(ii=0; iiflags |= PGHDR_NEED_SYNC; + sqlite3PagerUnrefNotNull(pPage); + } + } + } + + assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)!=0 ); + pPager->doNotSpill &= ~SPILLFLAG_NOSYNC; + return rc; +} + /* ** Mark a data page as writeable. This routine must be called before ** making changes to a page. The caller must check the return value @@ -5786,96 +5877,16 @@ static int pager_write(PgHdr *pPg){ ** If an error occurs, SQLITE_NOMEM or an IO error code is returned ** as appropriate. Otherwise, SQLITE_OK. */ -int sqlite3PagerWrite(DbPage *pDbPage){ - int rc = SQLITE_OK; - - PgHdr *pPg = pDbPage; - Pager *pPager = pPg->pPager; - +int sqlite3PagerWrite(PgHdr *pPg){ assert( (pPg->flags & PGHDR_MMAP)==0 ); - assert( pPager->eState>=PAGER_WRITER_LOCKED ); - assert( pPager->eState!=PAGER_ERROR ); - assert( assert_pager_state(pPager) ); - - if( pPager->sectorSize > (u32)pPager->pageSize ){ - Pgno nPageCount; /* Total number of pages in database file */ - Pgno pg1; /* First page of the sector pPg is located on. */ - int nPage = 0; /* Number of pages starting at pg1 to journal */ - int ii; /* Loop counter */ - int needSync = 0; /* True if any page has PGHDR_NEED_SYNC */ - Pgno nPagePerSector = (pPager->sectorSize/pPager->pageSize); - - /* Set the doNotSpill NOSYNC bit to 1. This is because we cannot allow - ** a journal header to be written between the pages journaled by - ** this function. - */ - assert( !MEMDB ); - assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)==0 ); - pPager->doNotSpill |= SPILLFLAG_NOSYNC; - - /* This trick assumes that both the page-size and sector-size are - ** an integer power of 2. It sets variable pg1 to the identifier - ** of the first page of the sector pPg is located on. - */ - pg1 = ((pPg->pgno-1) & ~(nPagePerSector-1)) + 1; - - nPageCount = pPager->dbSize; - if( pPg->pgno>nPageCount ){ - nPage = (pPg->pgno - pg1)+1; - }else if( (pg1+nPagePerSector-1)>nPageCount ){ - nPage = nPageCount+1-pg1; - }else{ - nPage = nPagePerSector; - } - assert(nPage>0); - assert(pg1<=pPg->pgno); - assert((pg1+nPage)>pPg->pgno); - - for(ii=0; iipgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ - if( pg!=PAGER_MJ_PGNO(pPager) ){ - rc = sqlite3PagerGet(pPager, pg, &pPage); - if( rc==SQLITE_OK ){ - rc = pager_write(pPage); - if( pPage->flags&PGHDR_NEED_SYNC ){ - needSync = 1; - } - sqlite3PagerUnrefNotNull(pPage); - } - } - }else if( (pPage = pager_lookup(pPager, pg))!=0 ){ - if( pPage->flags&PGHDR_NEED_SYNC ){ - needSync = 1; - } - sqlite3PagerUnrefNotNull(pPage); - } - } - - /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages - ** starting at pg1, then it needs to be set for all of them. Because - ** writing to any of these nPage pages may damage the others, the - ** journal file must contain sync()ed copies of all of them - ** before any of them can be written out to the database file. - */ - if( rc==SQLITE_OK && needSync ){ - assert( !MEMDB ); - for(ii=0; iiflags |= PGHDR_NEED_SYNC; - sqlite3PagerUnrefNotNull(pPage); - } - } - } - - assert( (pPager->doNotSpill & SPILLFLAG_NOSYNC)!=0 ); - pPager->doNotSpill &= ~SPILLFLAG_NOSYNC; + assert( pPg->pPager->eState>=PAGER_WRITER_LOCKED ); + assert( pPg->pPager->eState!=PAGER_ERROR ); + assert( assert_pager_state(pPg->pPager) ); + if( pPg->pPager->sectorSize > (u32)pPg->pPager->pageSize ){ + return pagerWriteLargeSector(pPg); }else{ - rc = pager_write(pDbPage); + return pager_write(pPg); } - return rc; } /* From 4274dae9e9971ce49bda44989be0e3aff4e1b865 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 24 Aug 2014 02:53:23 +0000 Subject: [PATCH 267/710] The sqlite3VdbeChangeEncoding() routine goes about 3x faster if the sqlite3VdbeMemTranslate() subroutine is not inlined. FossilOrigin-Name: 0c7e1b875a14ff9d71af7bb125a0272a23d57353 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/utf.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a21c0da13e..bb740b999a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Patch\sthe\ssqlite3PagerWrite()\smethod\sin\sthe\sPager\sto\srun\sa\sbit\sfaster. -D 2014-08-24T01:32:43.379 +C The\ssqlite3VdbeChangeEncoding()\sroutine\sgoes\sabout\s3x\sfaster\sif\sthe\nsqlite3VdbeMemTranslate()\ssubroutine\sis\snot\sinlined. +D 2014-08-24T02:53:23.646 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -281,7 +281,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec F src/trigger.c 4bddd12803275aa98f1c7ce0118fceb02b2167f6 F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 -F src/utf.c a0314e637768a030e6e84a957d0c4f6ba910cc05 +F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 F src/util.c 068dcd26354a3898ccc64ad5c4bdb95a7a15d33a F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 52ee5d589cbb171a8b096f210b69deb4a33c4369 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0371cc3bb07448bcd64fd671f3e71bb7f30deb4d -R 2fe545d9c9446b3adf94c91625adc119 +P c63311e2f3344363a5ed99838fb5850004eaee30 +R 195c0a9c31ce9f43b0d460b5ca896167 U drh -Z 79e99766e8b4bdebd57118e507ef67dc +Z 7685d4da1bf506f63008f6a3d5e2d0bd diff --git a/manifest.uuid b/manifest.uuid index 308d8ad8ef..de00fe8cee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c63311e2f3344363a5ed99838fb5850004eaee30 \ No newline at end of file +0c7e1b875a14ff9d71af7bb125a0272a23d57353 \ No newline at end of file diff --git a/src/utf.c b/src/utf.c index 97898746a2..557f3a95e4 100644 --- a/src/utf.c +++ b/src/utf.c @@ -199,7 +199,7 @@ u32 sqlite3Utf8Read( ** desiredEnc. It is an error if the string is already of the desired ** encoding, or if *pMem does not contain a string value. */ -int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ +SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ int len; /* Maximum length of output string in bytes */ unsigned char *zOut; /* Output buffer */ unsigned char *zIn; /* Input iterator */ From 12b7c7d8c397ff07626d720d9c2b68701b53fe0b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Aug 2014 11:20:27 +0000 Subject: [PATCH 268/710] Change the name of the VdbeMemRelease() macro to VdbeMemReleaseExtern() to more accurately reflect what it does. Performance enhancement to the sqlite3VdbeMemRelease() function. FossilOrigin-Name: 3ca5846da7da5e08192a4c96288197be3b7ab6f7 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 8 ++++---- src/vdbeInt.h | 2 +- src/vdbemem.c | 39 ++++++++++++++++++++++++++++++--------- 5 files changed, 44 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index bb740b999a..bfef72eb6a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\ssqlite3VdbeChangeEncoding()\sroutine\sgoes\sabout\s3x\sfaster\sif\sthe\nsqlite3VdbeMemTranslate()\ssubroutine\sis\snot\sinlined. -D 2014-08-24T02:53:23.646 +C Change\sthe\sname\sof\sthe\sVdbeMemRelease()\smacro\sto\sVdbeMemReleaseExtern()\sto\nmore\saccurately\sreflect\swhat\sit\sdoes.\s\sPerformance\senhancement\sto\sthe\nsqlite3VdbeMemRelease()\sfunction. +D 2014-08-25T11:20:27.189 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,13 +284,13 @@ F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 F src/util.c 068dcd26354a3898ccc64ad5c4bdb95a7a15d33a F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 52ee5d589cbb171a8b096f210b69deb4a33c4369 +F src/vdbe.c 0fe4b47668b36a50bd9f7fd7b15cbeeb69d54b37 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 -F src/vdbeInt.h 764a055bc9a3e61a30ba37cd4c1826f62bcf8759 +F src/vdbeInt.h 20056cd59ff93ef9eb91009ece726d65dd7ed322 F src/vdbeapi.c cda974083d7597f807640d344ffcf76d872201ce F src/vdbeaux.c dba006f67c9fd1b1d07ee7fb0fb38aa1905161d1 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c f2e162888521a9af35d608ac4c13db4991dfb417 +F src/vdbemem.c 4c9d686da474957d2e78834f13cc5f141fe6b87f F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c63311e2f3344363a5ed99838fb5850004eaee30 -R 195c0a9c31ce9f43b0d460b5ca896167 +P 0c7e1b875a14ff9d71af7bb125a0272a23d57353 +R 3449942ddb4a22bfe8e7f1684bd27e8b U drh -Z 7685d4da1bf506f63008f6a3d5e2d0bd +Z c7041d8c52f909d88b499b4a5f3227aa diff --git a/manifest.uuid b/manifest.uuid index de00fe8cee..725161693a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0c7e1b875a14ff9d71af7bb125a0272a23d57353 \ No newline at end of file +3ca5846da7da5e08192a4c96288197be3b7ab6f7 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 2e797eeaf5..72a1552ce1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -640,7 +640,7 @@ int sqlite3VdbeExec( assert( pOp->p2<=(p->nMem-p->nCursor) ); pOut = &aMem[pOp->p2]; memAboutToChange(p, pOut); - VdbeMemRelease(pOut); + VdbeMemReleaseExtern(pOut); pOut->flags = MEM_Int; } @@ -1079,7 +1079,7 @@ case OP_Null: { /* out2-prerelease */ while( cnt>0 ){ pOut++; memAboutToChange(p, pOut); - VdbeMemRelease(pOut); + VdbeMemReleaseExtern(pOut); pOut->flags = nullFlag; cnt--; } @@ -1165,7 +1165,7 @@ case OP_Move: { assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); - VdbeMemRelease(pOut); + VdbeMemReleaseExtern(pOut); zMalloc = pOut->zMalloc; memcpy(pOut, pIn1, sizeof(Mem)); #ifdef SQLITE_DEBUG @@ -2538,7 +2538,7 @@ case OP_Column: { if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ - VdbeMemRelease(pDest); + VdbeMemReleaseExtern(pDest); sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], aType[p2], pDest); }else{ /* This branch happens only when content is on overflow pages */ diff --git a/src/vdbeInt.h b/src/vdbeInt.h index ba44b61e91..384c5eb37b 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -430,7 +430,7 @@ void sqlite3VdbeMemRelease(Mem *p); void sqlite3VdbeMemReleaseExternal(Mem *p); #define VdbeMemDynamic(X) \ (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) -#define VdbeMemRelease(X) \ +#define VdbeMemReleaseExtern(X) \ if( VdbeMemDynamic(X) ) sqlite3VdbeMemReleaseExternal(X); int sqlite3VdbeMemFinalize(Mem*, FuncDef*); const char *sqlite3OpcodeName(int); diff --git a/src/vdbemem.c b/src/vdbemem.c index 572d486fc4..55e212d8ca 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -121,7 +121,7 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); } if( pMem->zMalloc==0 ){ - VdbeMemRelease(pMem); + VdbeMemReleaseExtern(pMem); pMem->z = 0; pMem->flags = MEM_Null; return SQLITE_NOMEM; @@ -300,6 +300,9 @@ int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ ** If the memory cell contains a string value that must be freed by ** invoking an external callback, free it now. Calling this function ** does not free any Mem.zMalloc buffer. +** +** The VdbeMemReleaseExtern() macro invokes this routine if only if there +** is work for this routine to do. */ void sqlite3VdbeMemReleaseExternal(Mem *p){ assert( p->db==0 || sqlite3_mutex_held(p->db->mutex) ); @@ -319,6 +322,25 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ } } +/* +** Release memory held by the Mem p, both external memory cleared +** by p->xDel and memory in p->zMalloc. +** +** This is a helper routine invoked by sqlite3VdbeMemRelease() in +** the uncommon case when there really is memory in p that is +** need of freeing. +*/ +static SQLITE_NOINLINE void vdbeMemRelease(Mem *p){ + if( VdbeMemDynamic(p) ){ + sqlite3VdbeMemReleaseExternal(p); + } + if( p->zMalloc ){ + sqlite3DbFree(p->db, p->zMalloc); + p->zMalloc = 0; + } + p->z = 0; +} + /* ** Release any memory held by the Mem. This may leave the Mem in an ** inconsistent state, for example with (Mem.z==0) and @@ -326,13 +348,12 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ */ void sqlite3VdbeMemRelease(Mem *p){ assert( sqlite3VdbeCheckMemInvariants(p) ); - VdbeMemRelease(p); - if( p->zMalloc ){ - sqlite3DbFree(p->db, p->zMalloc); - p->zMalloc = 0; + if( VdbeMemDynamic(p) || p->zMalloc ){ + vdbeMemRelease(p); + }else{ + p->z = 0; } - p->z = 0; - assert( p->xDel==0 ); /* Zeroed by VdbeMemRelease() above */ + assert( p->xDel==0 ); } /* @@ -638,7 +659,7 @@ void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ */ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( (pFrom->flags & MEM_RowSet)==0 ); - VdbeMemRelease(pTo); + VdbeMemReleaseExtern(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->xDel = 0; if( (pFrom->flags&MEM_Static)==0 ){ @@ -656,7 +677,7 @@ int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; assert( (pFrom->flags & MEM_RowSet)==0 ); - VdbeMemRelease(pTo); + VdbeMemReleaseExtern(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; pTo->xDel = 0; From c137807a87d4754885c83dc7660026ca6c0bb8af Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Aug 2014 11:33:41 +0000 Subject: [PATCH 269/710] Remove the pager_lookup() function since it is redundant with sqlite3PagerLookup(). FossilOrigin-Name: 54164ce47cfc3ad5dd8797114e4ba78811f23bef --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 26 +++++--------------------- 3 files changed, 12 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index bfef72eb6a..3c6bec475e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sname\sof\sthe\sVdbeMemRelease()\smacro\sto\sVdbeMemReleaseExtern()\sto\nmore\saccurately\sreflect\swhat\sit\sdoes.\s\sPerformance\senhancement\sto\sthe\nsqlite3VdbeMemRelease()\sfunction. -D 2014-08-25T11:20:27.189 +C Remove\sthe\spager_lookup()\sfunction\ssince\sit\sis\sredundant\swith\s\nsqlite3PagerLookup(). +D 2014-08-25T11:33:41.726 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -211,7 +211,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bd7df3094a60915c148517504c76df4fca24e542 F src/os_win.c d067fce558a5032e6e6afe62899e5397bf63cf3e F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c c831b0df879114915c08dc5a0188f9f22c48403b +F src/pager.c 53cc5e9d73afb74add79f49755c8ee240fbdbef7 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c da602c5447051705cab41604bf3276815eb569d0 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0c7e1b875a14ff9d71af7bb125a0272a23d57353 -R 3449942ddb4a22bfe8e7f1684bd27e8b +P 3ca5846da7da5e08192a4c96288197be3b7ab6f7 +R e183e353a27fb2d0c63a06eeb0b66e97 U drh -Z c7041d8c52f909d88b499b4a5f3227aa +Z 12acde8fd951a315f106f1112e06c5c6 diff --git a/manifest.uuid b/manifest.uuid index 725161693a..698dca573d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ca5846da7da5e08192a4c96288197be3b7ab6f7 \ No newline at end of file +54164ce47cfc3ad5dd8797114e4ba78811f23bef \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 4a178a2985..8930ce862c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1677,21 +1677,6 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ return rc; } -/* -** Find a page in the hash table given its page number. Return -** a pointer to the page or NULL if the requested page is not -** already in memory. -*/ -static PgHdr *pager_lookup(Pager *pPager, Pgno pgno){ - PgHdr *p = 0; /* Return value */ - - /* It is not possible for a call to PcacheFetch() with createFlag==0 to - ** fail, since no attempt to allocate dynamic memory will be made. - */ - (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &p); - return p; -} - /* ** Discard the entire contents of the in-memory page-cache. */ @@ -1984,7 +1969,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ #ifdef SQLITE_CHECK_PAGES sqlite3PcacheIterateDirty(pPager->pPCache, pager_set_pagehash); if( pPager->dbSize==0 && sqlite3PcacheRefCount(pPager->pPCache)>0 ){ - PgHdr *p = pager_lookup(pPager, 1); + PgHdr *p = sqlite3PagerLookup(pPager, 1); if( p ){ p->pageHash = 0; sqlite3PagerUnrefNotNull(p); @@ -2263,7 +2248,7 @@ static int pager_playback_one_page( if( pagerUseWal(pPager) ){ pPg = 0; }else{ - pPg = pager_lookup(pPager, pgno); + pPg = sqlite3PagerLookup(pPager, pgno); } assert( pPg || !MEMDB ); assert( pPager->eState!=PAGER_OPEN || pPg==0 ); @@ -5434,7 +5419,6 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ assert( pPager!=0 ); assert( pgno!=0 ); assert( pPager->pPCache!=0 ); - assert( pPager->eState>=PAGER_READER && pPager->eState!=PAGER_ERROR ); sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); return pPg; } @@ -5833,7 +5817,7 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ sqlite3PagerUnrefNotNull(pPage); } } - }else if( (pPage = pager_lookup(pPager, pg))!=0 ){ + }else if( (pPage = sqlite3PagerLookup(pPager, pg))!=0 ){ if( pPage->flags&PGHDR_NEED_SYNC ){ needSync = 1; } @@ -5850,7 +5834,7 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ if( rc==SQLITE_OK && needSync ){ assert( !MEMDB ); for(ii=0; iiflags |= PGHDR_NEED_SYNC; sqlite3PagerUnrefNotNull(pPage); @@ -6782,7 +6766,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** for the page moved there. */ pPg->flags &= ~PGHDR_NEED_SYNC; - pPgOld = pager_lookup(pPager, pgno); + pPgOld = sqlite3PagerLookup(pPager, pgno); assert( !pPgOld || pPgOld->nRef==1 ); if( pPgOld ){ pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); From 034596153de60aca340675354c9d10959f187baa Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Aug 2014 15:13:22 +0000 Subject: [PATCH 270/710] Query or change the maximum number of worker threads allowed on each database connection separately using the "PRAGMA threads" command. FossilOrigin-Name: 29c5e8a7c9d7ce349a1e1d72082d23450e877b45 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/pragma.c | 44 +++++++++++++++++++++++++++++++++++--------- src/shell.c | 4 ++-- src/sqliteInt.h | 1 + src/vdbesort.c | 4 +++- tool/mkpragmatab.tcl | 2 ++ 7 files changed, 54 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index d18353dedb..22dd6d848a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\srecent\sperformance\senhancements\simplemented\son\strunk\sinto\sthe\nthreads\sbranch. -D 2014-08-25T13:27:02.036 +C Query\sor\schange\sthe\smaximum\snumber\sof\sworker\sthreads\sallowed\son\seach\ndatabase\sconnection\sseparately\susing\sthe\s"PRAGMA\sthreads"\scommand. +D 2014-08-25T15:13:22.455 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,18 +217,18 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c da602c5447051705cab41604bf3276815eb569d0 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c -F src/pragma.c d10ef67c4de79f78188b965b4b7988aff1d66f2e +F src/pragma.c 4ed8bc86d1a9ee336e66bc441b30abb702f4de33 F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c f8b0b6c43bee15f4e239ead1c9c9e3009e507e39 -F src/shell.c 200772eebd7b0fcf2072fc3dbd95d694389d5efa +F src/shell.c 6dab215a30f7ca4d5fc31338c44b007bb6ef0dee F src/sqlite.h.in fef15a64d1358f5c365bd3f46f4c1915d5a5e5f0 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 80d0bb053ef728896142c4e808bfcdc49494e9be +F src/sqliteInt.h d8eb2d4d4ce26365dc9d49efee3bc880618e87c2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -292,7 +292,7 @@ F src/vdbeapi.c cda974083d7597f807640d344ffcf76d872201ce F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 4c9d686da474957d2e78834f13cc5f141fe6b87f -F src/vdbesort.c b9a830685826c057cfd41993902a5afc6fe436e1 +F src/vdbesort.c 3a76f51efdf0790fd3d26afabdd81e0a21f52ae7 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1158,7 +1158,7 @@ F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh 5dc5010e2e748a9e1bba67baca5956a2c2deda7b F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460 +F tool/mkpragmatab.tcl cce51d8f60c7f145d8fccabe6b5dfdedf31c5f5c F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 88a1e3b0c769773fb7a9ebb363ffc603a4ac21d8 F tool/mksqlite3c.tcl e72c0c97fe1a105fa9616483e652949be2199fe6 @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 05807c4122505567ec64fb2d142077f48a0a10b1 54164ce47cfc3ad5dd8797114e4ba78811f23bef -R c157557aea90a1674d041ac0f1de2f68 +P dfdc900f5d1a31ee5c5f35a630c4a8253e69093b +R 4c161b4070e5a7022237a28172d6e331 U drh -Z 10ad80e0aa78a5fbf32d4d843030da4f +Z 1c68bfb86ec54e7d60991aea6df8a67a diff --git a/manifest.uuid b/manifest.uuid index 8fecff4bd8..645cc6429f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dfdc900f5d1a31ee5c5f35a630c4a8253e69093b \ No newline at end of file +29c5e8a7c9d7ce349a1e1d72082d23450e877b45 \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 9ed5e13eb0..897d3b5d24 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -61,14 +61,15 @@ #define PragTyp_TABLE_INFO 30 #define PragTyp_TEMP_STORE 31 #define PragTyp_TEMP_STORE_DIRECTORY 32 -#define PragTyp_WAL_AUTOCHECKPOINT 33 -#define PragTyp_WAL_CHECKPOINT 34 -#define PragTyp_ACTIVATE_EXTENSIONS 35 -#define PragTyp_HEXKEY 36 -#define PragTyp_KEY 37 -#define PragTyp_REKEY 38 -#define PragTyp_LOCK_STATUS 39 -#define PragTyp_PARSER_TRACE 40 +#define PragTyp_THREADS 33 +#define PragTyp_WAL_AUTOCHECKPOINT 34 +#define PragTyp_WAL_CHECKPOINT 35 +#define PragTyp_ACTIVATE_EXTENSIONS 36 +#define PragTyp_HEXKEY 37 +#define PragTyp_KEY 38 +#define PragTyp_REKEY 39 +#define PragTyp_LOCK_STATUS 40 +#define PragTyp_PARSER_TRACE 41 #define PragFlag_NeedSchema 0x01 static const struct sPragmaNames { const char *const zName; /* Name of pragma */ @@ -418,6 +419,10 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif + { /* zName: */ "threads", + /* ePragTyp: */ PragTyp_THREADS, + /* ePragFlag: */ 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) { /* zName: */ "user_version", /* ePragTyp: */ PragTyp_HEADER_VALUE, @@ -465,7 +470,7 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 56 on by default, 69 total. */ +/* Number of pragmas: 57 on by default, 70 total. */ /* End of the automatically generated pragma table. ***************************************************************************/ @@ -2273,6 +2278,27 @@ void sqlite3Pragma( break; } + /* + ** PRAGMA threads + ** PRAGMA threads = N + ** + ** Configure the maximum number of worker threads. Return the new + ** maximum, which might be less than requested. + */ + case PragTyp_THREADS: { + sqlite3_int64 N; + if( sqlite3GlobalConfig.bCoreMutex + && zRight + && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK + && N>=0 + ){ + if( N>sqlite3GlobalConfig.nWorker ) N = sqlite3GlobalConfig.nWorker; + db->mxWorker = N&0xff; + } + returnSingleInt(pParse, "soft_heap_limit", db->mxWorker); + break; + } + #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* ** Report the current state of file logs for all databases diff --git a/src/shell.c b/src/shell.c index f2a275fe39..0357c1a082 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3817,10 +3817,10 @@ static void main_init(ShellState *data) { data->showHeader = 0; sqlite3_config(SQLITE_CONFIG_URI, 1); sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); + sqlite3_config(SQLITE_CONFIG_MULTITHREAD); + sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 64); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); - sqlite3_config(SQLITE_CONFIG_MULTITHREAD); - sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 4); } /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f3d49a7b10..ce123b395f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1012,6 +1012,7 @@ struct sqlite3 { u8 suppressErr; /* Do not issue error messages if true */ u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ + u8 mxWorker; /* Maximum number of worker threads */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ diff --git a/src/vdbesort.c b/src/vdbesort.c index 6915a2a9b9..e71b19fc11 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -795,8 +795,10 @@ int sqlite3VdbeSorterInit( int rc = SQLITE_OK; #if SQLITE_MAX_WORKER_THREADS==0 # define nWorker 0 +#elif SQLITE_MAX_WORKER_THREADS>=SORTER_MAX_MERGE_COUNT + int nWorker = MIN(SORTER_MAX_MERGE_COUNT-1, db->mxWorker); #else - int nWorker = (sqlite3GlobalConfig.bCoreMutex?sqlite3GlobalConfig.nWorker:0); + int nWorker = db->mxWorker; #endif assert( pCsr->pKeyInfo && pCsr->pBt==0 ); diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 28a1e468b8..aa7c8078c5 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -294,6 +294,8 @@ set pragma_def { IF: defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) NAME: soft_heap_limit + + NAME: threads } fconfigure stdout -translation lf set name {} From f741e0491e0a1ac499983f5d709c08c334c18783 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 25 Aug 2014 18:29:38 +0000 Subject: [PATCH 271/710] In cases where stat4 data is available but cannot be used because the rhs of a range constraint is too complex a expression, fall back to using the default estimates for number of rows scanned. FossilOrigin-Name: e06dc6f0c35f87c44292c71677111b74f073a5c4 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 6 +++--- test/analyze9.test | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 3c6bec475e..1e3b094e1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\spager_lookup()\sfunction\ssince\sit\sis\sredundant\swith\s\nsqlite3PagerLookup(). -D 2014-08-25T11:33:41.726 +C In\scases\swhere\sstat4\sdata\sis\savailable\sbut\scannot\sbe\sused\sbecause\sthe\srhs\sof\sa\srange\sconstraint\sis\stoo\scomplex\sa\sexpression,\sfall\sback\sto\susing\sthe\sdefault\sestimates\sfor\snumber\sof\srows\sscanned. +D 2014-08-25T18:29:38.998 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -297,7 +297,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 4c499d185827a492643cf017ae5e3aa0523f9f18 +F src/where.c 4e2770a1914b8ce30f3e44ad954b720eca3b5efd F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -317,7 +317,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test f1c552ce39cca4ec922a7e4e0e5d0203d6b3281f F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test 3ef1b471247308e710a794b6e50a6ab536c5604b +F test/analyze9.test 72795c8113604b5dcd47a1498a61d6d7fb5d041a F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/analyzeC.test 555a6cc388b9818b6eda6df816f01ce0a75d3a93 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3ca5846da7da5e08192a4c96288197be3b7ab6f7 -R e183e353a27fb2d0c63a06eeb0b66e97 -U drh -Z 12acde8fd951a315f106f1112e06c5c6 +P 54164ce47cfc3ad5dd8797114e4ba78811f23bef +R 2f8cc3122e94f945065c591b14dad548 +U dan +Z ef14939a1b480c479e36ded151638132 diff --git a/manifest.uuid b/manifest.uuid index 698dca573d..0ea03bbedf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54164ce47cfc3ad5dd8797114e4ba78811f23bef \ No newline at end of file +e06dc6f0c35f87c44292c71677111b74f073a5c4 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6a4299cc9a..4ba8de6e41 100644 --- a/src/where.c +++ b/src/where.c @@ -2220,6 +2220,7 @@ static int whereRangeScanEst( iNew = a[0] + ((pLower->eOperator & WO_GT) ? a[1] : 0); if( iNew>iLower ) iLower = iNew; nOut--; + pLower = 0; } } @@ -2235,6 +2236,7 @@ static int whereRangeScanEst( iNew = a[0] + ((pUpper->eOperator & WO_LE) ? a[1] : 0); if( iNewnOut = (LogEst)nOut; WHERETRACE(0x10, ("range scan regions: %u..%u est=%d\n", (u32)iLower, (u32)iUpper, nOut)); - return SQLITE_OK; } }else{ int bDone = 0; @@ -2262,8 +2262,8 @@ static int whereRangeScanEst( #else UNUSED_PARAMETER(pParse); UNUSED_PARAMETER(pBuilder); -#endif assert( pLower || pUpper ); +#endif assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); diff --git a/test/analyze9.test b/test/analyze9.test index 0d72658df6..8572cbea00 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -1088,4 +1088,50 @@ foreach {tn where eqp} { do_eqp_test 24.$tn "SeLeCt * FROM t5 WHERE $where" $eqp } +#------------------------------------------------------------------------- +# Test that if stat4 data is available but cannot be used because the +# rhs of a range constraint is a complex expression, the default estimates +# are used instead. +ifcapable stat4&&cte { + do_execsql_test 25.1 { + CREATE TABLE t6(a, b); + WITH ints(i,j) AS ( + SELECT 1,1 UNION ALL SELECT i+1,j+1 FROM ints WHERE i<100 + ) INSERT INTO t6 SELECT * FROM ints; + CREATE INDEX aa ON t6(a); + CREATE INDEX bb ON t6(b); + ANALYZE; + } + + # Term (b? AND b=? term. Better than + # (a<20) but not as good as (a<10). + do_eqp_test 25.4.1 { + SELECT * FROM t6 WHERE a < 10 AND (b BETWEEN ? AND 60) + } { + 0 0 0 {SEARCH TABLE t6 USING INDEX aa (a? AND b Date: Mon, 25 Aug 2014 20:11:52 +0000 Subject: [PATCH 272/710] Allow CAST expressions and unary "+" operators to be used in the DEFAULT argument of an ALTER TABLE ADD COLUMN and to be understand on the RHS of range constraints interpreted by STAT3/4. This involves a rewrite of the implementation of the CAST operator. FossilOrigin-Name: 91d8a8d0b792ea5c4fe68fd9caaf3345eddea486 --- manifest | 22 +++++------ manifest.uuid | 2 +- src/expr.c | 17 +------- src/vdbe.c | 100 ++++++----------------------------------------- src/vdbeInt.h | 1 + src/vdbemem.c | 57 ++++++++++++++++++++++++++- test/alter4.test | 2 +- 7 files changed, 84 insertions(+), 117 deletions(-) diff --git a/manifest b/manifest index 1e3b094e1d..f27497a3eb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\scases\swhere\sstat4\sdata\sis\savailable\sbut\scannot\sbe\sused\sbecause\sthe\srhs\sof\sa\srange\sconstraint\sis\stoo\scomplex\sa\sexpression,\sfall\sback\sto\susing\sthe\sdefault\sestimates\sfor\snumber\sof\srows\sscanned. -D 2014-08-25T18:29:38.998 +C Allow\sCAST\sexpressions\sand\sunary\s"+"\soperators\sto\sbe\sused\sin\sthe\sDEFAULT\nargument\sof\san\sALTER\sTABLE\sADD\sCOLUMN\sand\sto\sbe\sunderstand\son\sthe\sRHS\sof\nrange\sconstraints\sinterpreted\sby\sSTAT3/4.\s\sThis\sinvolves\sa\srewrite\sof\sthe\nimplementation\sof\sthe\sCAST\soperator. +D 2014-08-25T20:11:52.974 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 5adcd322c6b08fc25d215d780ca62cebce66304d -F src/expr.c f749009cf4a8534efb5e0d5cd7c9fb1fb0f2836c +F src/expr.c 358634f4ddeeb4e69643cb6db5819104a7834c60 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8d81a780ad78d16ec9082585758a8f1d6bf02ca3 F src/func.c bbb724b74ed96ca42675a7274646a71dd52bcda7 @@ -284,13 +284,13 @@ F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 F src/util.c 068dcd26354a3898ccc64ad5c4bdb95a7a15d33a F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 0fe4b47668b36a50bd9f7fd7b15cbeeb69d54b37 +F src/vdbe.c 2b3420d22410089b95a1555872dbc35183927a25 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 -F src/vdbeInt.h 20056cd59ff93ef9eb91009ece726d65dd7ed322 +F src/vdbeInt.h df58400454823954cfb241e5858f07f37fc1fd78 F src/vdbeapi.c cda974083d7597f807640d344ffcf76d872201ce F src/vdbeaux.c dba006f67c9fd1b1d07ee7fb0fb38aa1905161d1 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 4c9d686da474957d2e78834f13cc5f141fe6b87f +F src/vdbemem.c 5b5e296ac25f7458b6496fbee2756a087e8d569d F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -307,7 +307,7 @@ F test/all.test 6ff7b43c2b4b905c74dc4a813d201d0fa64c5783 F test/alter.test 547dc2d292644301ac9a7dda22b319b74f9c08d2 F test/alter2.test 7ea05c7d92ac99349a802ef7ada17294dd647060 F test/alter3.test 49c9d9fba2b8fcdce2dedeca97bbf1f369cc548d -F test/alter4.test d6c011fa0d6227abba762498cafbb607c9609e93 +F test/alter4.test c461150723ac957f3b2214aa0b11552cd72023ec F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/amatch1.test b5ae7065f042b7f4c1c922933f4700add50cdb9f F test/analyze.test 1772936d66471c65221e437b6d1999c3a03166c4 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 54164ce47cfc3ad5dd8797114e4ba78811f23bef -R 2f8cc3122e94f945065c591b14dad548 -U dan -Z ef14939a1b480c479e36ded151638132 +P e06dc6f0c35f87c44292c71677111b74f073a5c4 +R 9f2d5499974a5fb7517d1c1150241c64 +U drh +Z 9fb020703498fe8e63fa568755e6ee4e diff --git a/manifest.uuid b/manifest.uuid index 0ea03bbedf..c3a49f3b80 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e06dc6f0c35f87c44292c71677111b74f073a5c4 \ No newline at end of file +91d8a8d0b792ea5c4fe68fd9caaf3345eddea486 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 0d2292e943..fabdae2fcf 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2595,26 +2595,13 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ - int aff, to_op; inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - aff = sqlite3AffinityType(pExpr->u.zToken, 0); - to_op = aff - SQLITE_AFF_TEXT + OP_ToText; - assert( to_op==OP_ToText || aff!=SQLITE_AFF_TEXT ); - assert( to_op==OP_ToBlob || aff!=SQLITE_AFF_NONE ); - assert( to_op==OP_ToNumeric || aff!=SQLITE_AFF_NUMERIC ); - assert( to_op==OP_ToInt || aff!=SQLITE_AFF_INTEGER ); - assert( to_op==OP_ToReal || aff!=SQLITE_AFF_REAL ); - testcase( to_op==OP_ToText ); - testcase( to_op==OP_ToBlob ); - testcase( to_op==OP_ToNumeric ); - testcase( to_op==OP_ToInt ); - testcase( to_op==OP_ToReal ); if( inReg!=target ){ sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); inReg = target; } - sqlite3VdbeAddOp1(v, to_op, inReg); + sqlite3VdbeAddOp2(v, OP_Cast, target, + sqlite3AffinityType(pExpr->u.zToken, 0)); testcase( usedAsColumnCache(pParse, inReg, inReg) ); sqlite3ExprCacheAffinityChange(pParse, inReg, 1); break; diff --git a/src/vdbe.c b/src/vdbe.c index 72a1552ce1..ec2b64f770 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1768,106 +1768,30 @@ case OP_RealAffinity: { /* in1 */ #endif #ifndef SQLITE_OMIT_CAST -/* Opcode: ToText P1 * * * * +/* Opcode: Cast P1 P2 * * * ** -** Force the value in register P1 to be text. -** If the value is numeric, convert it to a string using the -** equivalent of sprintf(). Blob values are unchanged and -** are afterwards simply interpreted as text. +** Force the value in register P1 to be the type defined by P2. +** +**
      +**
    • TEXT +**
    • BLOB +**
    • NUMERIC +**
    • INTEGER +**
    • REAL +**
    ** ** A NULL value is not changed by this routine. It remains NULL. */ -case OP_ToText: { /* same as TK_TO_TEXT, in1 */ +case OP_Cast: { /* in1 */ pIn1 = &aMem[pOp->p1]; memAboutToChange(p, pIn1); - if( pIn1->flags & MEM_Null ) break; - assert( MEM_Str==(MEM_Blob>>3) ); - pIn1->flags |= (pIn1->flags&MEM_Blob)>>3; - applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); rc = ExpandBlob(pIn1); - assert( pIn1->flags & MEM_Str || db->mallocFailed ); - pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + sqlite3VdbeMemCast(pIn1, pOp->p2, encoding); UPDATE_MAX_BLOBSIZE(pIn1); break; } - -/* Opcode: ToBlob P1 * * * * -** -** Force the value in register P1 to be a BLOB. -** If the value is numeric, convert it to a string first. -** Strings are simply reinterpreted as blobs with no change -** to the underlying data. -** -** A NULL value is not changed by this routine. It remains NULL. -*/ -case OP_ToBlob: { /* same as TK_TO_BLOB, in1 */ - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Null ) break; - if( (pIn1->flags & MEM_Blob)==0 ){ - applyAffinity(pIn1, SQLITE_AFF_TEXT, encoding); - assert( pIn1->flags & MEM_Str || db->mallocFailed ); - MemSetTypeFlag(pIn1, MEM_Blob); - }else{ - pIn1->flags &= ~(MEM_TypeMask&~MEM_Blob); - } - UPDATE_MAX_BLOBSIZE(pIn1); - break; -} - -/* Opcode: ToNumeric P1 * * * * -** -** Force the value in register P1 to be numeric (either an -** integer or a floating-point number.) -** If the value is text or blob, try to convert it to an using the -** equivalent of atoi() or atof() and store 0 if no such conversion -** is possible. -** -** A NULL value is not changed by this routine. It remains NULL. -*/ -case OP_ToNumeric: { /* same as TK_TO_NUMERIC, in1 */ - pIn1 = &aMem[pOp->p1]; - sqlite3VdbeMemNumerify(pIn1); - break; -} #endif /* SQLITE_OMIT_CAST */ -/* Opcode: ToInt P1 * * * * -** -** Force the value in register P1 to be an integer. If -** The value is currently a real number, drop its fractional part. -** If the value is text or blob, try to convert it to an integer using the -** equivalent of atoi() and store 0 if no such conversion is possible. -** -** A NULL value is not changed by this routine. It remains NULL. -*/ -case OP_ToInt: { /* same as TK_TO_INT, in1 */ - pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & MEM_Null)==0 ){ - sqlite3VdbeMemIntegerify(pIn1); - } - break; -} - -#if !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) -/* Opcode: ToReal P1 * * * * -** -** Force the value in register P1 to be a floating point number. -** If The value is currently an integer, convert it. -** If the value is text or blob, try to convert it to an integer using the -** equivalent of atoi() and store 0.0 if no such conversion is possible. -** -** A NULL value is not changed by this routine. It remains NULL. -*/ -case OP_ToReal: { /* same as TK_TO_REAL, in1 */ - pIn1 = &aMem[pOp->p1]; - memAboutToChange(p, pIn1); - if( (pIn1->flags & MEM_Null)==0 ){ - sqlite3VdbeMemRealify(pIn1); - } - break; -} -#endif /* !defined(SQLITE_OMIT_CAST) && !defined(SQLITE_OMIT_FLOATING_POINT) */ - /* Opcode: Lt P1 P2 P3 P4 P5 ** Synopsis: if r[P1]flags & MEM_Null ) return; + switch( aff ){ + case SQLITE_AFF_NONE: { /* Really a cast to BLOB */ + if( (pMem->flags & MEM_Blob)==0 ){ + sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); + assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); + MemSetTypeFlag(pMem, MEM_Blob); + }else{ + pMem->flags &= ~(MEM_TypeMask&~MEM_Blob); + } + break; + } + case SQLITE_AFF_NUMERIC: { + sqlite3VdbeMemNumerify(pMem); + break; + } + case SQLITE_AFF_INTEGER: { + sqlite3VdbeMemIntegerify(pMem); + break; + } + case SQLITE_AFF_REAL: { + sqlite3VdbeMemRealify(pMem); + break; + } + default: { + assert( aff==SQLITE_AFF_TEXT ); + assert( MEM_Str==(MEM_Blob>>3) ); + pMem->flags |= (pMem->flags&MEM_Blob)>>3; + sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); + assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); + pMem->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); + break; + } + } +} + + /* ** Delete any previous value and set the value stored in *pMem to NULL. */ @@ -1015,9 +1060,19 @@ static int valueFromExpr( *ppVal = 0; return SQLITE_OK; } - op = pExpr->op; + while( (op = pExpr->op)==TK_UPLUS ) pExpr = pExpr->pLeft; if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; + if( op==TK_CAST ){ + u8 aff = sqlite3AffinityType(pExpr->u.zToken,0); + rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); + if( rc==SQLITE_OK && *ppVal ){ + sqlite3VdbeMemCast(*ppVal, aff, SQLITE_UTF8); + sqlite3ValueApplyAffinity(*ppVal, affinity, SQLITE_UTF8); + } + return rc; + } + /* Handle negative integers in a single step. This is needed in the ** case when the value is -9223372036854775808. */ diff --git a/test/alter4.test b/test/alter4.test index eaad37e001..ac39d614a5 100644 --- a/test/alter4.test +++ b/test/alter4.test @@ -145,7 +145,7 @@ do_test alter4-2.6 { } {1 {Cannot add a column with non-constant default}} do_test alter4-2.7 { catchsql { - alter table t1 add column d default (-+1); + alter table t1 add column d default (-5+1); } } {1 {Cannot add a column with non-constant default}} do_test alter4-2.99 { From 21b0e733540b21b64fd90312f3a8741da8060793 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Aug 2014 20:21:07 +0000 Subject: [PATCH 273/710] Test cases added for using unary "+" and CAST operators on the RHS of range constraints and verifying that STAT3/4 can use those constraints. FossilOrigin-Name: 42505e5a810832442699ca54a46637c50e7f9e71 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/analyzeA.test | 23 +++++++++++++++++++++-- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f27497a3eb..761ead9728 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sCAST\sexpressions\sand\sunary\s"+"\soperators\sto\sbe\sused\sin\sthe\sDEFAULT\nargument\sof\san\sALTER\sTABLE\sADD\sCOLUMN\sand\sto\sbe\sunderstand\son\sthe\sRHS\sof\nrange\sconstraints\sinterpreted\sby\sSTAT3/4.\s\sThis\sinvolves\sa\srewrite\sof\sthe\nimplementation\sof\sthe\sCAST\soperator. -D 2014-08-25T20:11:52.974 +C Test\scases\sadded\sfor\susing\sunary\s"+"\sand\sCAST\soperators\son\sthe\sRHS\sof\srange\nconstraints\sand\sverifying\sthat\sSTAT3/4\scan\suse\sthose\sconstraints. +D 2014-08-25T20:21:07.683 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -318,7 +318,7 @@ F test/analyze6.test f1c552ce39cca4ec922a7e4e0e5d0203d6b3281f F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 F test/analyze9.test 72795c8113604b5dcd47a1498a61d6d7fb5d041a -F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 +F test/analyzeA.test 3335697f6700c7052295cfd0067fc5b2aacddf9a F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/analyzeC.test 555a6cc388b9818b6eda6df816f01ce0a75d3a93 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e06dc6f0c35f87c44292c71677111b74f073a5c4 -R 9f2d5499974a5fb7517d1c1150241c64 +P 91d8a8d0b792ea5c4fe68fd9caaf3345eddea486 +R 12fb957b1fcac5280a480acdbd82e25d U drh -Z 9fb020703498fe8e63fa568755e6ee4e +Z dc13f49ba965d3549e28167f01462351 diff --git a/manifest.uuid b/manifest.uuid index c3a49f3b80..edf7f89be2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91d8a8d0b792ea5c4fe68fd9caaf3345eddea486 \ No newline at end of file +42505e5a810832442699ca54a46637c50e7f9e71 \ No newline at end of file diff --git a/test/analyzeA.test b/test/analyzeA.test index d9ca2c0f3b..a2da10edff 100644 --- a/test/analyzeA.test +++ b/test/analyzeA.test @@ -117,7 +117,7 @@ foreach {tn analyze_cmd} { } { reset_db do_test 1.$tn.1 { - execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT, c INT) } for {set i 0} {$i < 100} {incr i} { set c [expr int(pow(1.1,$i)/100)] set b [expr 125 - int(pow(1.1,99-$i))/100] @@ -161,7 +161,26 @@ foreach {tn analyze_cmd} { do_eqp_test 1.$tn.3.6 { SELECT * FROM t1 WHERE b BETWEEN 75 AND 125 AND c BETWEEN 75 AND 125 } {0 0 0 {SEARCH TABLE t1 USING INDEX t1c (c>? AND c? AND b? AND b? AND c? AND c Date: Mon, 25 Aug 2014 21:11:01 +0000 Subject: [PATCH 274/710] Minor changes to the CAST logic to make it more testable. FossilOrigin-Name: 1ad70ec550c004160d9c0c57e6c416812cdead5e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 761ead9728..192b84c230 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scases\sadded\sfor\susing\sunary\s"+"\sand\sCAST\soperators\son\sthe\sRHS\sof\srange\nconstraints\sand\sverifying\sthat\sSTAT3/4\scan\suse\sthose\sconstraints. -D 2014-08-25T20:21:07.683 +C Minor\schanges\sto\sthe\sCAST\slogic\sto\smake\sit\smore\stestable. +D 2014-08-25T21:11:01.892 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vdbeInt.h df58400454823954cfb241e5858f07f37fc1fd78 F src/vdbeapi.c cda974083d7597f807640d344ffcf76d872201ce F src/vdbeaux.c dba006f67c9fd1b1d07ee7fb0fb38aa1905161d1 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 5b5e296ac25f7458b6496fbee2756a087e8d569d +F src/vdbemem.c 4e08ea087aea367dae7c45129b75487e0056e819 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 91d8a8d0b792ea5c4fe68fd9caaf3345eddea486 -R 12fb957b1fcac5280a480acdbd82e25d +P 42505e5a810832442699ca54a46637c50e7f9e71 +R 64d7800c0b3c56b546428f1dfa98a8bd U drh -Z dc13f49ba965d3549e28167f01462351 +Z 2062d1773620f0a49c6fdbe873cdd5f6 diff --git a/manifest.uuid b/manifest.uuid index edf7f89be2..e6aa85bdb1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42505e5a810832442699ca54a46637c50e7f9e71 \ No newline at end of file +1ad70ec550c004160d9c0c57e6c416812cdead5e \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 22e213898a..e4012593da 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -409,7 +409,6 @@ i64 sqlite3VdbeIntValue(Mem *pMem){ }else if( flags & (MEM_Str|MEM_Blob) ){ i64 value = 0; assert( pMem->z || pMem->n==0 ); - testcase( pMem->z==0 ); sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); return value; }else{ @@ -1066,7 +1065,8 @@ static int valueFromExpr( if( op==TK_CAST ){ u8 aff = sqlite3AffinityType(pExpr->u.zToken,0); rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); - if( rc==SQLITE_OK && *ppVal ){ + testcase( rc!=SQLITE_OK ); + if( *ppVal ){ sqlite3VdbeMemCast(*ppVal, aff, SQLITE_UTF8); sqlite3ValueApplyAffinity(*ppVal, affinity, SQLITE_UTF8); } From 05bbb2e824df2af559aaf301a4a44f16c080ffc8 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Aug 2014 22:37:19 +0000 Subject: [PATCH 275/710] Add an assert() and five testcase() macros to the OP_Cast opcode implementation to help verify that it is fully tested. FossilOrigin-Name: af364cce9da0961593ef876b646197f82df08ad5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 192b84c230..a75c4cd6f7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\schanges\sto\sthe\sCAST\slogic\sto\smake\sit\smore\stestable. -D 2014-08-25T21:11:01.892 +C Add\san\sassert()\sand\sfive\stestcase()\smacros\sto\sthe\sOP_Cast\sopcode\simplementation\nto\shelp\sverify\sthat\sit\sis\sfully\stested. +D 2014-08-25T22:37:19.150 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,7 +284,7 @@ F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 F src/util.c 068dcd26354a3898ccc64ad5c4bdb95a7a15d33a F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 2b3420d22410089b95a1555872dbc35183927a25 +F src/vdbe.c 2f8fbc520cac2f5bacc43de2aeed6a4880c9cb57 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h df58400454823954cfb241e5858f07f37fc1fd78 F src/vdbeapi.c cda974083d7597f807640d344ffcf76d872201ce @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 42505e5a810832442699ca54a46637c50e7f9e71 -R 64d7800c0b3c56b546428f1dfa98a8bd +P 1ad70ec550c004160d9c0c57e6c416812cdead5e +R b751a4133861a7b502aa405ba96eea49 U drh -Z 2062d1773620f0a49c6fdbe873cdd5f6 +Z 0bd8a75e39f468a6e4b9b4599a579487 diff --git a/manifest.uuid b/manifest.uuid index e6aa85bdb1..c36f309914 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1ad70ec550c004160d9c0c57e6c416812cdead5e \ No newline at end of file +af364cce9da0961593ef876b646197f82df08ad5 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index ec2b64f770..5d440d825b 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1783,6 +1783,12 @@ case OP_RealAffinity: { /* in1 */ ** A NULL value is not changed by this routine. It remains NULL. */ case OP_Cast: { /* in1 */ + assert( pOp->p2>=SQLITE_AFF_TEXT && pOp->p2<=SQLITE_AFF_REAL ); + testcase( pOp->p2==SQLITE_AFF_TEXT ); + testcase( pOp->p2==SQLITE_AFF_NONE ); + testcase( pOp->p2==SQLITE_AFF_NUMERIC ); + testcase( pOp->p2==SQLITE_AFF_INTEGER ); + testcase( pOp->p2==SQLITE_AFF_REAL ); pIn1 = &aMem[pOp->p1]; memAboutToChange(p, pIn1); rc = ExpandBlob(pIn1); From 028696c4cc1f4bddcd376d3a290415f7eb67526e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 25 Aug 2014 23:44:44 +0000 Subject: [PATCH 276/710] Remove the SQLITE_CONFIG_WORKER_THREADS configuration parameter. The number of worker threads in the sorter is now determined only by the PRAGMA threads=N setting. FossilOrigin-Name: e3305d4b4efcbe06945ce7f6ec0f2e864244aaf9 --- manifest | 34 +++++++++++++++++----------------- manifest.uuid | 2 +- src/global.c | 1 - src/main.c | 9 --------- src/pragma.c | 2 +- src/shell.c | 1 - src/sqlite.h.in | 11 ----------- src/sqliteInt.h | 4 ++-- src/test_malloc.c | 26 -------------------------- src/vdbesort.c | 12 +++++++++--- test/sort.test | 6 ++---- test/sort2.test | 10 +--------- test/sort4.test | 11 +---------- test/sortfault.test | 6 +----- 14 files changed, 35 insertions(+), 100 deletions(-) diff --git a/manifest b/manifest index 84d67d6062..9edec3f675 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sCAST\soperator\senhancements\sfrom\strunk. -D 2014-08-25T22:43:17.516 +C Remove\sthe\sSQLITE_CONFIG_WORKER_THREADS\sconfiguration\sparameter.\s\sThe\snumber\nof\sworker\sthreads\sin\sthe\ssorter\sis\snow\sdetermined\sonly\sby\sthe\nPRAGMA\sthreads=N\ssetting. +D 2014-08-25T23:44:44.281 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/expr.c 358634f4ddeeb4e69643cb6db5819104a7834c60 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8d81a780ad78d16ec9082585758a8f1d6bf02ca3 F src/func.c bbb724b74ed96ca42675a7274646a71dd52bcda7 -F src/global.c 77ec119d6f6453039c2820336af8e8f804f20acf +F src/global.c 1e4bd956dc2f608f87d2a929abc4a20db65f30e4 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -190,7 +190,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab -F src/main.c ce41520e565eb8ef09824fa9778a72364291d371 +F src/main.c f1726e704941d365ce2846161e93ba689a245845 F src/malloc.c 954de5f998c23237e04474a3f2159bf483bba65a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -217,18 +217,18 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c da602c5447051705cab41604bf3276815eb569d0 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c -F src/pragma.c 4ed8bc86d1a9ee336e66bc441b30abb702f4de33 +F src/pragma.c 33971fcaa7c13b84b1a0f2e813f4a3ab4d745ede F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c f8b0b6c43bee15f4e239ead1c9c9e3009e507e39 -F src/shell.c 6dab215a30f7ca4d5fc31338c44b007bb6ef0dee -F src/sqlite.h.in fef15a64d1358f5c365bd3f46f4c1915d5a5e5f0 +F src/shell.c 88378cef39aba4b4a1df82793dcb1daf9276bb81 +F src/sqlite.h.in aa2cc1405cb999c9d73484e0686f7b869b430ba3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h d8eb2d4d4ce26365dc9d49efee3bc880618e87c2 +F src/sqliteInt.h 43419afaed8cd3bf99df06d38952a52f827217b9 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -257,7 +257,7 @@ F src/test_intarray.c db4614c2262a06abc4409dc048d59c580c38320f F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 -F src/test_malloc.c 27047a841f5bff1cb638123806a2c30714771307 +F src/test_malloc.c 5368fb1de77246da1ae0ff59cba0d30cb0e5812f F src/test_multiplex.c ca90057438b63bf0840ebb84d0ef050624519a76 F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f @@ -292,7 +292,7 @@ F src/vdbeapi.c cda974083d7597f807640d344ffcf76d872201ce F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 4e08ea087aea367dae7c45129b75487e0056e819 -F src/vdbesort.c 3a76f51efdf0790fd3d26afabdd81e0a21f52ae7 +F src/vdbesort.c 50fe3442f41dbfe0b37d2e8b55e7460244015533 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -838,11 +838,11 @@ F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test d8b9692b702745a0e41c23f9da6beac81df01196 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 688468cef8c9a66fcc1d54235de8e4deac745690 -F test/sort2.test c5e25eb674689e291d06b5209fe8d337ae0ec010 +F test/sort.test 15e1d3014abc3f6d4357ed81b93b82117aefd235 +F test/sort2.test 269f4f50c6e468cc32b302ae7ff0add8338ec6de F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 -F test/sort4.test 971452fd4e2928e6fc05c3868396ad7d5f9ce2ad -F test/sortfault.test 1a12b6e27d475f50658a8164aaa34f0080a86b36 +F test/sort4.test 6c37d85f7cd28d50cce222fcab84ccd771e105cb +F test/sortfault.test b8e35177f97438b930ee87c9419ca2599e8073e1 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb F test/speed1p.test b180e98609c7677382cf618c0ec9b69f789033a8 @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 29c5e8a7c9d7ce349a1e1d72082d23450e877b45 af364cce9da0961593ef876b646197f82df08ad5 -R c7c8cecd9a8ffa9d596fa5137e696964 +P 6c8f86e4e08d5d57e21496277613e0f9dcc06514 +R 428258d1143faa4ab393e38d0d72251a U drh -Z 1d9d719c4b1a959a79787aab5421ad60 +Z 37ad2e5d491e7e49897b74763238faf5 diff --git a/manifest.uuid b/manifest.uuid index 25b89b4eae..a22d912847 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c8f86e4e08d5d57e21496277613e0f9dcc06514 \ No newline at end of file +e3305d4b4efcbe06945ce7f6ec0f2e864244aaf9 \ No newline at end of file diff --git a/src/global.c b/src/global.c index f152c3b5d0..2c14b58abd 100644 --- a/src/global.c +++ b/src/global.c @@ -167,7 +167,6 @@ SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* nPage */ 0, /* mxParserStack */ 0, /* sharedCacheEnabled */ - SQLITE_DEFAULT_WORKER_THREADS, /* nWorker */ /* All the rest should always be initialized to zero */ 0, /* isInit */ 0, /* inProgress */ diff --git a/src/main.c b/src/main.c index 8a6456d685..98bd82a298 100644 --- a/src/main.c +++ b/src/main.c @@ -515,15 +515,6 @@ int sqlite3_config(int op, ...){ } #endif - case SQLITE_CONFIG_WORKER_THREADS: { -#if SQLITE_MAX_WORKER_THREADS>0 - int n = va_arg(ap, int); - if( n>SQLITE_MAX_WORKER_THREADS ) n = SQLITE_MAX_WORKER_THREADS; - if( n>=0 ) sqlite3GlobalConfig.nWorker = n; -#endif - break; - } - default: { rc = SQLITE_ERROR; break; diff --git a/src/pragma.c b/src/pragma.c index 897d3b5d24..8421042e00 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2292,7 +2292,7 @@ void sqlite3Pragma( && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK && N>=0 ){ - if( N>sqlite3GlobalConfig.nWorker ) N = sqlite3GlobalConfig.nWorker; + if( N>SQLITE_MAX_WORKER_THREADS ) N = SQLITE_MAX_WORKER_THREADS; db->mxWorker = N&0xff; } returnSingleInt(pParse, "soft_heap_limit", db->mxWorker); diff --git a/src/shell.c b/src/shell.c index 0357c1a082..9c235414af 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3818,7 +3818,6 @@ static void main_init(ShellState *data) { sqlite3_config(SQLITE_CONFIG_URI, 1); sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); sqlite3_config(SQLITE_CONFIG_MULTITHREAD); - sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, 64); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index be0ed74565..f8ea7ad623 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1718,16 +1718,6 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** -** -** [[SQLITE_CONFIG_WORKER_THREADS]] -**
    SQLITE_CONFIG_WORKER_THREADS -**
    ^SQLITE_CONFIG_WORKER_THREADS takes a single argument of type int. -** It is used to set the number of background worker threads that may be -** launched when sorting large amounts of data. A value of 0 means launch -** no background threads at all. The maximum number of background threads -** allowed is configured at build-time by the SQLITE_MAX_WORKER_THREADS -** pre-processor option. -** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -1752,7 +1742,6 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ #define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ -#define SQLITE_CONFIG_WORKER_THREADS 24 /* int nWorker */ /* ** CAPI3REF: Database Connection Configuration Options diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ce123b395f..b48d5cf795 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -441,9 +441,10 @@ */ #if SQLITE_TEMP_STORE==3 # undef SQLITE_MAX_WORKER_THREADS +# define SQLITE_MAX_WORKER_THREADS 0 #endif #ifndef SQLITE_MAX_WORKER_THREADS -# define SQLITE_MAX_WORKER_THREADS 0 +# define SQLITE_MAX_WORKER_THREADS 4 #endif #ifndef SQLITE_DEFAULT_WORKER_THREADS # define SQLITE_DEFAULT_WORKER_THREADS 0 @@ -2764,7 +2765,6 @@ struct Sqlite3Config { int nPage; /* Number of pages in pPage[] */ int mxParserStack; /* maximum depth of the parser stack */ int sharedCacheEnabled; /* true if shared-cache mode enabled */ - int nWorker; /* Number of worker threads to use */ /* The above might be initialized to non-zero. The following need to always ** initially be zero, however. */ int isInit; /* True after initialization has finished */ diff --git a/src/test_malloc.c b/src/test_malloc.c index 6ac030f23c..900a8ac40c 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -1253,31 +1253,6 @@ static int test_config_cis( return TCL_OK; } -/* -** Usage: sqlite3_config_worker_threads N -*/ -static int test_config_worker_threads( - void * clientData, - Tcl_Interp *interp, - int objc, - Tcl_Obj *CONST objv[] -){ - int rc; - int nThread; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "N"); - return TCL_ERROR; - } - if( Tcl_GetIntFromObj(interp, objv[1], &nThread) ){ - return TCL_ERROR; - } - - rc = sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, nThread); - Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); - - return TCL_OK; -} /* ** Usage: sqlite3_dump_memsys3 FILENAME @@ -1532,7 +1507,6 @@ int Sqlitetest_malloc_Init(Tcl_Interp *interp){ { "sqlite3_config_error", test_config_error ,0 }, { "sqlite3_config_uri", test_config_uri ,0 }, { "sqlite3_config_cis", test_config_cis ,0 }, - { "sqlite3_config_worker_threads", test_config_worker_threads ,0 }, { "sqlite3_db_config_lookaside",test_db_config_lookaside ,0 }, { "sqlite3_dump_memsys3", test_dump_memsys3 ,3 }, { "sqlite3_dump_memsys5", test_dump_memsys3 ,5 }, diff --git a/src/vdbesort.c b/src/vdbesort.c index e71b19fc11..cdbf6e0252 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -795,10 +795,16 @@ int sqlite3VdbeSorterInit( int rc = SQLITE_OK; #if SQLITE_MAX_WORKER_THREADS==0 # define nWorker 0 -#elif SQLITE_MAX_WORKER_THREADS>=SORTER_MAX_MERGE_COUNT - int nWorker = MIN(SORTER_MAX_MERGE_COUNT-1, db->mxWorker); #else - int nWorker = db->mxWorker; + int nWorker = sqlite3TempInMemory(db) ? 0 : db->mxWorker ; +#endif + + /* Do not allow the total number of threads (main thread + all workers) + ** to exceed the maximum merge count */ +#if SQLITE_MAX_WORKER_THREADS>=SORTER_MAX_MERGE_COUNT + if( nWorker>=SORTER_MAX_MERGE_COUNT ){ + nWorker = SORTER_MAX_MERGE_COUNT-1; + } #endif assert( pCsr->pKeyInfo && pCsr->pBt==0 ); diff --git a/test/sort.test b/test/sort.test index e75740e9c3..1c89552bb1 100644 --- a/test/sort.test +++ b/test/sort.test @@ -544,7 +544,6 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap softheaplimit} { } { db close sqlite3_shutdown - sqlite3_config_worker_threads $nWorker if {$coremutex} { sqlite3_config multithread } else { @@ -556,7 +555,8 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap softheaplimit} { reset_db sqlite3_test_control SQLITE_TESTCTRL_SORTER_MMAP db $mmap_limit - execsql "PRAGMA temp_store = $tmpstore" + execsql "PRAGMA temp_store = $tmpstore; PRAGMA threads = $nWorker" + set ten [string repeat X 10300] set one [string repeat y 200] @@ -602,7 +602,6 @@ foreach {tn mmap_limit nWorker tmpstore coremutex fakeheap softheaplimit} { db close sqlite3_shutdown -sqlite3_config_worker_threads 0 set t(0) singlethread set t(1) multithread set t(2) serialized @@ -637,4 +636,3 @@ do_execsql_test 17.1 { } {} finish_test - diff --git a/test/sort2.test b/test/sort2.test index e4e40dab74..29001f0099 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -22,9 +22,8 @@ foreach {tn script} { 1 { } 2 { catch { db close } - sqlite3_shutdown - sqlite3_config_worker_threads 7 reset_db + catch { db eval {PRAGMA threads=7} } } } { @@ -76,13 +75,6 @@ foreach {tn script} { } { 200000 100 200000 100 200000 100 200000 100 200000 100 } - - db close - sqlite3_shutdown - sqlite3_config_worker_threads 0 - sqlite3_initialize - } finish_test - diff --git a/test/sort4.test b/test/sort4.test index 4e8336cd84..01fcbfee95 100644 --- a/test/sort4.test +++ b/test/sort4.test @@ -19,11 +19,7 @@ source $testdir/tester.tcl set testprefix sort4 # Configure the sorter to use 3 background threads. -catch { db close } -sqlite3_shutdown -sqlite3_config_worker_threads 3 -sqlite3_initialize -reset_db +db eval {PRAGMA threads=3} # Minimum number of seconds to run for. If the value is 0, each test # is run exactly once. Otherwise, tests are repeated until the timeout @@ -190,9 +186,4 @@ for {set t 2} {1} {incr tn} { do_test "$testprefix-([expr $iTimeLimit-$iNow] seconds remain)" {} {} } -catch { db close } -sqlite3_shutdown -sqlite3_config_worker_threads 0 -sqlite3_initialize finish_test - diff --git a/test/sortfault.test b/test/sortfault.test index 4c199ab212..a1983ac1c0 100644 --- a/test/sortfault.test +++ b/test/sortfault.test @@ -30,9 +30,7 @@ foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap lookaside} { } { if {$sqlite_options(threadsafe)} { set threadsmode singlethread } - catch { db close } - sqlite3_shutdown - sqlite3_config_worker_threads $nWorker + db eval "PRAGMA threads=$nWorker" sqlite3_config $threadsmode if { $lookaside } { sqlite3_config_lookaside 100 500 @@ -110,7 +108,6 @@ foreach {tn mmap_limit nWorker tmpstore threadsmode fakeheap lookaside} { catch { db close } sqlite3_shutdown -sqlite3_config_worker_threads 0 set t(0) singlethread set t(1) multithread set t(2) serialized @@ -166,4 +163,3 @@ do_faultsim_test 5.1 -faults oom* -body { } finish_test - From c3031c61ef0b010c61c7b6c592f30cea634d2037 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 26 Aug 2014 15:06:49 +0000 Subject: [PATCH 277/710] Change the page cache so that a new sqlite3_pcache object is allocated as soon as the page cache is opened, not delayed until the first fetch request. This give a noticable performance boost. The interface between pager and the page cache has changed slightly, which might break ZIPVFS. FossilOrigin-Name: f1f94a971e031e784f8c30a6faf829df58709329 --- manifest | 16 ++++----- manifest.uuid | 2 +- src/pager.c | 25 ++++++------- src/pcache.c | 98 ++++++++++++++++++++++----------------------------- src/pcache.h | 4 +-- 5 files changed, 67 insertions(+), 78 deletions(-) diff --git a/manifest b/manifest index a75c4cd6f7..80f1fa7f2b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sassert()\sand\sfive\stestcase()\smacros\sto\sthe\sOP_Cast\sopcode\simplementation\nto\shelp\sverify\sthat\sit\sis\sfully\stested. -D 2014-08-25T22:37:19.150 +C Change\sthe\spage\scache\sso\sthat\sa\snew\ssqlite3_pcache\sobject\sis\sallocated\sas\nsoon\sas\sthe\spage\scache\sis\sopened,\snot\sdelayed\suntil\sthe\sfirst\sfetch\srequest.\nThis\sgive\sa\snoticable\sperformance\sboost.\s\sThe\sinterface\sbetween\spager\sand\nthe\spage\scache\shas\schanged\sslightly,\swhich\smight\sbreak\sZIPVFS. +D 2014-08-26T15:06:49.829 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -211,11 +211,11 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bd7df3094a60915c148517504c76df4fca24e542 F src/os_win.c d067fce558a5032e6e6afe62899e5397bf63cf3e F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 53cc5e9d73afb74add79f49755c8ee240fbdbef7 +F src/pager.c 27fb89e62e0ccf10218805ed31315ccb2d56c0ce F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 -F src/pcache.c da602c5447051705cab41604bf3276815eb569d0 -F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 +F src/pcache.c c216e4077449be57e9752a348490ffa467b85599 +F src/pcache.h 80a9c3f7d7b7080388e8654717cb45e7b99f14a6 F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c F src/pragma.c d10ef67c4de79f78188b965b4b7988aff1d66f2e F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1ad70ec550c004160d9c0c57e6c416812cdead5e -R b751a4133861a7b502aa405ba96eea49 +P af364cce9da0961593ef876b646197f82df08ad5 +R e52a11dddfc2629a8399ec2b801365c3 U drh -Z 0bd8a75e39f468a6e4b9b4599a579487 +Z 1a77b3226c1f56b49ad44674617b7aa6 diff --git a/manifest.uuid b/manifest.uuid index c36f309914..ea22adc369 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -af364cce9da0961593ef876b646197f82df08ad5 \ No newline at end of file +f1f94a971e031e784f8c30a6faf829df58709329 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 8930ce862c..246371dbd7 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3622,7 +3622,7 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ pPager->pageSize = pageSize; sqlite3PageFree(pPager->pTmpSpace); pPager->pTmpSpace = pNew; - sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); + rc = sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); } } @@ -4385,7 +4385,7 @@ static int pagerStress(void *p, PgHdr *pPg){ ** ** Spilling is also prohibited when in an error state since that could ** lead to database corruption. In the current implementaton it - ** is impossible for sqlite3PcacheFetch() to be called with createFlag==1 + ** is impossible for sqlite3PcacheFetch() to be called with createFlag==3 ** while in the error state, hence it is impossible for this routine to ** be called in the error state. Nevertheless, we include a NEVER() ** test for the error state as a safeguard against future changes. @@ -4721,22 +4721,23 @@ act_like_temp_file: testcase( rc!=SQLITE_OK ); } - /* If an error occurred in either of the blocks above, free the - ** Pager structure and close the file. + /* Initialize the PCache object. */ + if( rc==SQLITE_OK ){ + assert( nExtra<1000 ); + nExtra = ROUND8(nExtra); + rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, + !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); + } + + /* If an error occurred above, free the Pager structure and close the file. */ if( rc!=SQLITE_OK ){ - assert( !pPager->pTmpSpace ); sqlite3OsClose(pPager->fd); + sqlite3PageFree(pPager->pTmpSpace); sqlite3_free(pPager); return rc; } - /* Initialize the PCache object. */ - assert( nExtra<1000 ); - nExtra = ROUND8(nExtra); - sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, - !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); - PAGERTRACE(("OPEN %d %s\n", FILEHANDLEID(pPager->fd), pPager->zFilename)); IOTRACE(("OPEN %p %s\n", pPager, pPager->zFilename)) @@ -5318,7 +5319,7 @@ int sqlite3PagerAcquire( } } - rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 1, ppPage); + rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 3, ppPage); } if( rc!=SQLITE_OK ){ diff --git a/src/pcache.c b/src/pcache.c index 2e4b5d78b2..5ca552e356 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -144,6 +144,17 @@ static void pcacheUnpin(PgHdr *p){ } } +/* +** Compute the number of pages of cache requested. +*/ +static int numberOfCachePages(PCache *p){ + if( p->szCache>=0 ){ + return p->szCache; + }else{ + return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); + } +} + /*************************************************** General Interfaces ****** ** ** Initialize and shutdown the page cache subsystem. Neither of these @@ -176,7 +187,7 @@ int sqlite3PcacheSize(void){ return sizeof(PCache); } ** The caller discovers how much space needs to be allocated by ** calling sqlite3PcacheSize(). */ -void sqlite3PcacheOpen( +int sqlite3PcacheOpen( int szPage, /* Size of every page */ int szExtra, /* Extra space associated with each page */ int bPurgeable, /* True if pages are on backing store */ @@ -185,38 +196,37 @@ void sqlite3PcacheOpen( PCache *p /* Preallocated space for the PCache */ ){ memset(p, 0, sizeof(PCache)); - p->szPage = szPage; + p->szPage = 1; p->szExtra = szExtra; p->bPurgeable = bPurgeable; p->eCreate = 2; p->xStress = xStress; p->pStress = pStress; p->szCache = 100; + return sqlite3PcacheSetPageSize(p, szPage); } /* ** Change the page size for PCache object. The caller must ensure that there ** are no outstanding page references when this function is called. */ -void sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ +int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ assert( pCache->nRef==0 && pCache->pDirty==0 ); - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); - pCache->pCache = 0; + if( pCache->szPage ){ + sqlite3_pcache *pNew; + pNew = sqlite3GlobalConfig.pcache2.xCreate( + szPage, pCache->szExtra + sizeof(PgHdr), pCache->bPurgeable + ); + if( pNew==0 ) return SQLITE_NOMEM; + sqlite3GlobalConfig.pcache2.xCachesize(pNew, numberOfCachePages(pCache)); + if( pCache->pCache ){ + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); + } + pCache->pCache = pNew; pCache->pPage1 = 0; + pCache->szPage = szPage; } - pCache->szPage = szPage; -} - -/* -** Compute the number of pages of cache requested. -*/ -static int numberOfCachePages(PCache *p){ - if( p->szCache>=0 ){ - return p->szCache; - }else{ - return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); - } + return SQLITE_OK; } /* @@ -233,28 +243,10 @@ int sqlite3PcacheFetch( int eCreate; assert( pCache!=0 ); - assert( createFlag==1 || createFlag==0 ); + assert( pCache->pCache!=0 ); + assert( createFlag==3 || createFlag==0 ); assert( pgno>0 ); - /* If the pluggable cache (sqlite3_pcache*) has not been allocated, - ** allocate it now. - */ - if( !pCache->pCache ){ - sqlite3_pcache *p; - if( !createFlag ){ - *ppPage = 0; - return SQLITE_OK; - } - p = sqlite3GlobalConfig.pcache2.xCreate( - pCache->szPage, pCache->szExtra + sizeof(PgHdr), pCache->bPurgeable - ); - if( !p ){ - return SQLITE_NOMEM; - } - sqlite3GlobalConfig.pcache2.xCachesize(p, numberOfCachePages(pCache)); - pCache->pCache = p; - } - /* eCreate defines what to do if the page does not exist. ** 0 Do not allocate a new page. (createFlag==0) ** 1 Allocate a new page if doing so is inexpensive. @@ -262,8 +254,10 @@ int sqlite3PcacheFetch( ** 2 Allocate a new page even it doing so is difficult. ** (createFlag==1 AND !(bPurgeable AND pDirty) */ - eCreate = createFlag==0 ? 0 : pCache->eCreate; - assert( (createFlag*(1+(!pCache->bPurgeable||!pCache->pDirty)))==eCreate ); + eCreate = createFlag & pCache->eCreate; + assert( eCreate==0 || eCreate==1 || eCreate==2 ); + assert( createFlag==0 || pCache->eCreate==eCreate ); + assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); if( !pPage && eCreate==1 ){ PgHdr *pPg; @@ -471,9 +465,8 @@ void sqlite3PcacheTruncate(PCache *pCache, Pgno pgno){ ** Close a cache. */ void sqlite3PcacheClose(PCache *pCache){ - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); - } + assert( pCache->pCache!=0 ); + sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); } /* @@ -582,11 +575,8 @@ int sqlite3PcachePageRefcount(PgHdr *p){ ** Return the total number of pages in the cache. */ int sqlite3PcachePagecount(PCache *pCache){ - int nPage = 0; - if( pCache->pCache ){ - nPage = sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache); - } - return nPage; + assert( pCache->pCache!=0 ); + return sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache); } #ifdef SQLITE_TEST @@ -602,20 +592,18 @@ int sqlite3PcacheGetCachesize(PCache *pCache){ ** Set the suggested cache-size value. */ void sqlite3PcacheSetCachesize(PCache *pCache, int mxPage){ + assert( pCache->pCache!=0 ); pCache->szCache = mxPage; - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache2.xCachesize(pCache->pCache, - numberOfCachePages(pCache)); - } + sqlite3GlobalConfig.pcache2.xCachesize(pCache->pCache, + numberOfCachePages(pCache)); } /* ** Free up as much memory as possible from the page cache. */ void sqlite3PcacheShrink(PCache *pCache){ - if( pCache->pCache ){ - sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); - } + assert( pCache->pCache!=0 ); + sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); } #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) diff --git a/src/pcache.h b/src/pcache.h index f4d4ad71c1..11ffa9f39d 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -68,7 +68,7 @@ void sqlite3PCacheBufferSetup(void *, int sz, int n); ** Under memory stress, invoke xStress to try to make pages clean. ** Only clean and unpinned pages can be reclaimed. */ -void sqlite3PcacheOpen( +int sqlite3PcacheOpen( int szPage, /* Size of every page */ int szExtra, /* Extra space associated with each page */ int bPurgeable, /* True if pages are on backing store */ @@ -78,7 +78,7 @@ void sqlite3PcacheOpen( ); /* Modify the page-size after the cache has been created. */ -void sqlite3PcacheSetPageSize(PCache *, int); +int sqlite3PcacheSetPageSize(PCache *, int); /* Return the size in bytes of a PCache object. Used to preallocate ** storage space. From b63388b6a4accfa45200e313534a044409a5b6d8 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 27 Aug 2014 00:50:11 +0000 Subject: [PATCH 278/710] Performance enhancement in sqlite3VdbeMemNulTerminate(). FossilOrigin-Name: f94cacc393e895522b92c9717c53357afc918d60 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 24 +++++++++++++++++------- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 80f1fa7f2b..5a5dcce860 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\spage\scache\sso\sthat\sa\snew\ssqlite3_pcache\sobject\sis\sallocated\sas\nsoon\sas\sthe\spage\scache\sis\sopened,\snot\sdelayed\suntil\sthe\sfirst\sfetch\srequest.\nThis\sgive\sa\snoticable\sperformance\sboost.\s\sThe\sinterface\sbetween\spager\sand\nthe\spage\scache\shas\schanged\sslightly,\swhich\smight\sbreak\sZIPVFS. -D 2014-08-26T15:06:49.829 +C Performance\senhancement\sin\ssqlite3VdbeMemNulTerminate(). +D 2014-08-27T00:50:11.231 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vdbeInt.h df58400454823954cfb241e5858f07f37fc1fd78 F src/vdbeapi.c cda974083d7597f807640d344ffcf76d872201ce F src/vdbeaux.c dba006f67c9fd1b1d07ee7fb0fb38aa1905161d1 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 4e08ea087aea367dae7c45129b75487e0056e819 +F src/vdbemem.c 39cde2d8ddaa391055885caba9cb137ec2750474 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P af364cce9da0961593ef876b646197f82df08ad5 -R e52a11dddfc2629a8399ec2b801365c3 +P f1f94a971e031e784f8c30a6faf829df58709329 +R a4b7581e490aa7d71b634e963210e5ae U drh -Z 1a77b3226c1f56b49ad44674617b7aa6 +Z d9bb31c0fee81f84beaf0ed2a801edba diff --git a/manifest.uuid b/manifest.uuid index ea22adc369..44b2a27745 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1f94a971e031e784f8c30a6faf829df58709329 \ No newline at end of file +f94cacc393e895522b92c9717c53357afc918d60 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index e4012593da..8457aa0da6 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -200,15 +200,11 @@ int sqlite3VdbeMemExpandBlob(Mem *pMem){ } #endif - /* -** Make sure the given Mem is \u0000 terminated. +** It is already known that pMem contains an unterminated string. +** Add the zero terminator. */ -int sqlite3VdbeMemNulTerminate(Mem *pMem){ - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - if( (pMem->flags & MEM_Term)!=0 || (pMem->flags & MEM_Str)==0 ){ - return SQLITE_OK; /* Nothing to do */ - } +static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ return SQLITE_NOMEM; } @@ -218,6 +214,20 @@ int sqlite3VdbeMemNulTerminate(Mem *pMem){ return SQLITE_OK; } +/* +** Make sure the given Mem is \u0000 terminated. +*/ +int sqlite3VdbeMemNulTerminate(Mem *pMem){ + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + testcase( (pMem->flags & (MEM_Term|MEM_Str))==(MEM_Term|MEM_Str) ); + testcase( (pMem->flags & (MEM_Term|MEM_Str))==0 ); + if( (pMem->flags & (MEM_Term|MEM_Str))!=MEM_Str ){ + return SQLITE_OK; /* Nothing to do */ + }else{ + return vdbeMemAddTerminator(pMem); + } +} + /* ** Add MEM_Str to the set of representations for the given Mem. Numbers ** are converted using sqlite3_snprintf(). Converting a BLOB to a string From 6c9f8e67de8a87a792499b1af81772f084ea2e1e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 27 Aug 2014 03:28:50 +0000 Subject: [PATCH 279/710] Factor out the exception paths from sqlite3ValueToText() into a separate function so that the main routine is much faster for the common case of no required type or encoding conversions. FossilOrigin-Name: 1624916c6e9bc5dbcfa146b316a99ac8fecb13a9 --- manifest | 12 ++++----- manifest.uuid | 2 +- src/vdbemem.c | 69 ++++++++++++++++++++++++++++++++------------------- 3 files changed, 50 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 5a5dcce860..9a75dd0703 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\senhancement\sin\ssqlite3VdbeMemNulTerminate(). -D 2014-08-27T00:50:11.231 +C Factor\sout\sthe\sexception\spaths\sfrom\ssqlite3ValueToText()\sinto\sa\sseparate\nfunction\sso\sthat\sthe\smain\sroutine\sis\smuch\sfaster\sfor\sthe\scommon\scase\sof\nno\srequired\stype\sor\sencoding\sconversions. +D 2014-08-27T03:28:50.316 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vdbeInt.h df58400454823954cfb241e5858f07f37fc1fd78 F src/vdbeapi.c cda974083d7597f807640d344ffcf76d872201ce F src/vdbeaux.c dba006f67c9fd1b1d07ee7fb0fb38aa1905161d1 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 39cde2d8ddaa391055885caba9cb137ec2750474 +F src/vdbemem.c 9fc61d0860fd23399715fc436b2645154df08ce8 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f1f94a971e031e784f8c30a6faf829df58709329 -R a4b7581e490aa7d71b634e963210e5ae +P f94cacc393e895522b92c9717c53357afc918d60 +R 0d26b1eecc3f76fc31005918debb0d62 U drh -Z d9bb31c0fee81f84beaf0ed2a801edba +Z 03245edbdb5b11cf6bf42be492d2d8e5 diff --git a/manifest.uuid b/manifest.uuid index 44b2a27745..40c815eed9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f94cacc393e895522b92c9717c53357afc918d60 \ No newline at end of file +1624916c6e9bc5dbcfa146b316a99ac8fecb13a9 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 8457aa0da6..a76bd48865 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -919,6 +919,45 @@ int sqlite3VdbeMemFromBtree( return rc; } +/* +** The pVal argument is known to be a value other than NULL. +** Convert it into a string with encoding enc and return a pointer +** to a zero-terminated version of that string. +*/ +SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ + assert( pVal!=0 ); + assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); + assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); + assert( (pVal->flags & MEM_RowSet)==0 ); + assert( (pVal->flags & (MEM_Null))==0 ); + if( pVal->flags & (MEM_Blob|MEM_Str) ){ + pVal->flags |= MEM_Str; + if( pVal->flags & MEM_Zero ){ + sqlite3VdbeMemExpandBlob(pVal); + } + if( pVal->enc != (enc & ~SQLITE_UTF16_ALIGNED) ){ + sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); + } + if( (enc & SQLITE_UTF16_ALIGNED)!=0 && 1==(1&SQLITE_PTR_TO_INT(pVal->z)) ){ + assert( (pVal->flags & (MEM_Ephem|MEM_Static))!=0 ); + if( sqlite3VdbeMemMakeWriteable(pVal)!=SQLITE_OK ){ + return 0; + } + } + sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-31275-44060 */ + }else{ + sqlite3VdbeMemStringify(pVal, enc, 0); + assert( 0==(1&SQLITE_PTR_TO_INT(pVal->z)) ); + } + assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 + || pVal->db->mallocFailed ); + if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ + return pVal->z; + }else{ + return 0; + } +} + /* This function is only available internally, it is not part of the ** external API. It works in a similar way to sqlite3_value_text(), ** except the data returned is in the encoding specified by the second @@ -931,38 +970,16 @@ int sqlite3VdbeMemFromBtree( */ const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ if( !pVal ) return 0; - assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); assert( (pVal->flags & MEM_RowSet)==0 ); - + if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){ + return pVal->z; + } if( pVal->flags&MEM_Null ){ return 0; } - assert( (MEM_Blob>>3) == MEM_Str ); - pVal->flags |= (pVal->flags & MEM_Blob)>>3; - ExpandBlob(pVal); - if( pVal->flags&MEM_Str ){ - sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); - if( (enc & SQLITE_UTF16_ALIGNED)!=0 && 1==(1&SQLITE_PTR_TO_INT(pVal->z)) ){ - assert( (pVal->flags & (MEM_Ephem|MEM_Static))!=0 ); - if( sqlite3VdbeMemMakeWriteable(pVal)!=SQLITE_OK ){ - return 0; - } - } - sqlite3VdbeMemNulTerminate(pVal); /* IMP: R-31275-44060 */ - }else{ - assert( (pVal->flags&MEM_Blob)==0 ); - sqlite3VdbeMemStringify(pVal, enc, 0); - assert( 0==(1&SQLITE_PTR_TO_INT(pVal->z)) ); - } - assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 - || pVal->db->mallocFailed ); - if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ - return pVal->z; - }else{ - return 0; - } + return valueToText(pVal, enc); } /* From 9bd038f12ee1c8c3a5675ec5d3673d4678a7b112 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 27 Aug 2014 14:14:06 +0000 Subject: [PATCH 280/710] In the sqlite3_context object, keep a pointer to the result value rather than storing the result value in the sqlite3_context object and using memcpy() to move the value back into its register after the function returns. This runs faster and saves over 500 bytes of code space. FossilOrigin-Name: 6c1ee3e388eb110de815270467b1e50592c0ba6c --- manifest | 18 +++++------ manifest.uuid | 2 +- src/vdbe.c | 82 +++++++++++++-------------------------------------- src/vdbeInt.h | 2 +- src/vdbeapi.c | 68 +++++++++++++++++++++--------------------- src/vdbemem.c | 29 ++++++++++++++---- 6 files changed, 88 insertions(+), 113 deletions(-) diff --git a/manifest b/manifest index 9a75dd0703..c45e2fe4fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Factor\sout\sthe\sexception\spaths\sfrom\ssqlite3ValueToText()\sinto\sa\sseparate\nfunction\sso\sthat\sthe\smain\sroutine\sis\smuch\sfaster\sfor\sthe\scommon\scase\sof\nno\srequired\stype\sor\sencoding\sconversions. -D 2014-08-27T03:28:50.316 +C In\sthe\ssqlite3_context\sobject,\skeep\sa\spointer\sto\sthe\sresult\svalue\srather\nthan\sstoring\sthe\sresult\svalue\sin\sthe\ssqlite3_context\sobject\sand\susing\nmemcpy()\sto\smove\sthe\svalue\sback\sinto\sits\sregister\nafter\sthe\sfunction\sreturns.\s\sThis\sruns\sfaster\sand\ssaves\sover\s500\sbytes\nof\scode\sspace. +D 2014-08-27T14:14:06.320 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,13 +284,13 @@ F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 F src/util.c 068dcd26354a3898ccc64ad5c4bdb95a7a15d33a F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 2f8fbc520cac2f5bacc43de2aeed6a4880c9cb57 +F src/vdbe.c 8b6638420347e01f7be6c276726441579a407be0 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 -F src/vdbeInt.h df58400454823954cfb241e5858f07f37fc1fd78 -F src/vdbeapi.c cda974083d7597f807640d344ffcf76d872201ce +F src/vdbeInt.h 4653bb420abb7acdc215659cdcedd3a59f336191 +F src/vdbeapi.c 09677a53dd8c71bcd670b0bd073bb9aefa02b441 F src/vdbeaux.c dba006f67c9fd1b1d07ee7fb0fb38aa1905161d1 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 9fc61d0860fd23399715fc436b2645154df08ce8 +F src/vdbemem.c 921d5468a68ac06f369810992e84ca22cc730a62 F src/vdbesort.c f7f5563bf7d4695ca8f3203f3bf9de96d04ed0b3 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f94cacc393e895522b92c9717c53357afc918d60 -R 0d26b1eecc3f76fc31005918debb0d62 +P 1624916c6e9bc5dbcfa146b316a99ac8fecb13a9 +R d81ba05103ba02a6e3d94271217446d4 U drh -Z 03245edbdb5b11cf6bf42be492d2d8e5 +Z 3300f3f8d44a1fbcab3339e234a49760 diff --git a/manifest.uuid b/manifest.uuid index 40c815eed9..057a225c61 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1624916c6e9bc5dbcfa146b316a99ac8fecb13a9 \ No newline at end of file +6c1ee3e388eb110de815270467b1e50592c0ba6c \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 5d440d825b..b8c1258f50 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1545,8 +1545,8 @@ case OP_Function: { apVal = p->apArg; assert( apVal || n==0 ); assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); - pOut = &aMem[pOp->p3]; - memAboutToChange(p, pOut); + ctx.pOut = &aMem[pOp->p3]; + memAboutToChange(p, ctx.pOut); assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); @@ -1562,16 +1562,7 @@ case OP_Function: { ctx.pFunc = pOp->p4.pFunc; ctx.iOp = pc; ctx.pVdbe = p; - - /* The output cell may already have a buffer allocated. Move - ** the pointer to ctx.s so in case the user-function can use - ** the already allocated buffer instead of allocating a new one. - */ - memcpy(&ctx.s, pOut, sizeof(Mem)); - pOut->flags = MEM_Null; - pOut->xDel = 0; - pOut->zMalloc = 0; - MemSetTypeFlag(&ctx.s, MEM_Null); + MemSetTypeFlag(ctx.pOut, MEM_Null); ctx.fErrorOrAux = 0; if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ @@ -1584,43 +1575,23 @@ case OP_Function: { (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ lastRowid = db->lastRowid; - if( db->mallocFailed ){ - /* Even though a malloc() has failed, the implementation of the - ** user function may have called an sqlite3_result_XXX() function - ** to return a value. The following call releases any resources - ** associated with such a value. - */ - sqlite3VdbeMemRelease(&ctx.s); - goto no_mem; - } - /* If the function returned an error, throw an exception */ if( ctx.fErrorOrAux ){ if( ctx.isError ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s)); + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(ctx.pOut)); rc = ctx.isError; } sqlite3VdbeDeleteAuxData(p, pc, pOp->p1); } /* Copy the result of the function into register P3 */ - sqlite3VdbeChangeEncoding(&ctx.s, encoding); - assert( pOut->flags==MEM_Null ); - memcpy(pOut, &ctx.s, sizeof(Mem)); - if( sqlite3VdbeMemTooBig(pOut) ){ + sqlite3VdbeChangeEncoding(ctx.pOut, encoding); + if( sqlite3VdbeMemTooBig(ctx.pOut) ){ goto too_big; } -#if 0 - /* The app-defined function has done something that as caused this - ** statement to expire. (Perhaps the function called sqlite3_exec() - ** with a CREATE TABLE statement.) - */ - if( p->expired ) rc = SQLITE_ABORT; -#endif - - REGISTER_TRACE(pOp->p3, pOut); - UPDATE_MAX_BLOBSIZE(pOut); + REGISTER_TRACE(pOp->p3, ctx.pOut); + UPDATE_MAX_BLOBSIZE(ctx.pOut); break; } @@ -5610,6 +5581,7 @@ case OP_AggStep: { int i; Mem *pMem; Mem *pRec; + Mem t; sqlite3_context ctx; sqlite3_value **apVal; @@ -5627,11 +5599,12 @@ case OP_AggStep: { assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); ctx.pMem = pMem = &aMem[pOp->p3]; pMem->n++; - ctx.s.flags = MEM_Null; - ctx.s.z = 0; - ctx.s.zMalloc = 0; - ctx.s.xDel = 0; - ctx.s.db = db; + t.flags = MEM_Null; + t.z = 0; + t.zMalloc = 0; + t.xDel = 0; + t.db = db; + ctx.pOut = &t; ctx.isError = 0; ctx.pColl = 0; ctx.skipFlag = 0; @@ -5643,7 +5616,7 @@ case OP_AggStep: { } (ctx.pFunc->xStep)(&ctx, n, apVal); /* IMP: R-24505-23230 */ if( ctx.isError ){ - sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&ctx.s)); + sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&t)); rc = ctx.isError; } if( ctx.skipFlag ){ @@ -5651,9 +5624,7 @@ case OP_AggStep: { i = pOp[-1].p1; if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1); } - - sqlite3VdbeMemRelease(&ctx.s); - + sqlite3VdbeMemRelease(&t); break; } @@ -6103,27 +6074,14 @@ case OP_VColumn: { pModule = pVtab->pModule; assert( pModule->xColumn ); memset(&sContext, 0, sizeof(sContext)); - - /* The output cell may already have a buffer allocated. Move - ** the current contents to sContext.s so in case the user-function - ** can use the already allocated buffer instead of allocating a - ** new one. - */ - sqlite3VdbeMemMove(&sContext.s, pDest); - MemSetTypeFlag(&sContext.s, MEM_Null); - + sContext.pOut = pDest; + MemSetTypeFlag(pDest, MEM_Null); rc = pModule->xColumn(pCur->pVtabCursor, &sContext, pOp->p2); sqlite3VtabImportErrmsg(p, pVtab); if( sContext.isError ){ rc = sContext.isError; } - - /* Copy the result of the function to the P3 register. We - ** do this regardless of whether or not an error occurred to ensure any - ** dynamic allocation in sContext.s (a Mem struct) is released. - */ - sqlite3VdbeChangeEncoding(&sContext.s, encoding); - sqlite3VdbeMemMove(pDest, &sContext.s); + sqlite3VdbeChangeEncoding(pDest, encoding); REGISTER_TRACE(pOp->p3, pDest); UPDATE_MAX_BLOBSIZE(pDest); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 3f84ba8455..a96fa4e3c5 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -266,8 +266,8 @@ struct AuxData { ** (Mem) which are only defined there. */ struct sqlite3_context { + Mem *pOut; /* The return value is stored here */ FuncDef *pFunc; /* Pointer to function information. MUST BE FIRST */ - Mem s; /* The return value is stored here */ Mem *pMem; /* Memory cell used to store aggregate context */ CollSeq *pColl; /* Collating sequence */ Vdbe *pVdbe; /* The VM that owns this context */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index b2d8059153..c4a2eebe08 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -223,7 +223,7 @@ static void setResultStrOrError( u8 enc, /* Encoding of z. 0 for BLOBs */ void (*xDel)(void*) /* Destructor function */ ){ - if( sqlite3VdbeMemSetStr(&pCtx->s, z, n, enc, xDel)==SQLITE_TOOBIG ){ + if( sqlite3VdbeMemSetStr(pCtx->pOut, z, n, enc, xDel)==SQLITE_TOOBIG ){ sqlite3_result_error_toobig(pCtx); } } @@ -234,38 +234,38 @@ void sqlite3_result_blob( void (*xDel)(void *) ){ assert( n>=0 ); - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, 0, xDel); } void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetDouble(&pCtx->s, rVal); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetDouble(pCtx->pOut, rVal); } void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; pCtx->fErrorOrAux = 1; - sqlite3VdbeMemSetStr(&pCtx->s, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); + sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); } #ifndef SQLITE_OMIT_UTF16 void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; pCtx->fErrorOrAux = 1; - sqlite3VdbeMemSetStr(&pCtx->s, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); + sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); } #endif void sqlite3_result_int(sqlite3_context *pCtx, int iVal){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetInt64(&pCtx->s, (i64)iVal); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetInt64(pCtx->pOut, (i64)iVal); } void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetInt64(&pCtx->s, iVal); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetInt64(pCtx->pOut, iVal); } void sqlite3_result_null(sqlite3_context *pCtx){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetNull(&pCtx->s); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetNull(pCtx->pOut); } void sqlite3_result_text( sqlite3_context *pCtx, @@ -273,7 +273,7 @@ void sqlite3_result_text( int n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); } #ifndef SQLITE_OMIT_UTF16 @@ -283,7 +283,7 @@ void sqlite3_result_text16( int n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); } void sqlite3_result_text16be( @@ -292,7 +292,7 @@ void sqlite3_result_text16be( int n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); } void sqlite3_result_text16le( @@ -301,43 +301,43 @@ void sqlite3_result_text16le( int n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); } #endif /* SQLITE_OMIT_UTF16 */ void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemCopy(&pCtx->s, pValue); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemCopy(pCtx->pOut, pValue); } void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetZeroBlob(&pCtx->s, n); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetZeroBlob(pCtx->pOut, n); } void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ pCtx->isError = errCode; pCtx->fErrorOrAux = 1; - if( pCtx->s.flags & MEM_Null ){ - sqlite3VdbeMemSetStr(&pCtx->s, sqlite3ErrStr(errCode), -1, + if( pCtx->pOut->flags & MEM_Null ){ + sqlite3VdbeMemSetStr(pCtx->pOut, sqlite3ErrStr(errCode), -1, SQLITE_UTF8, SQLITE_STATIC); } } /* Force an SQLITE_TOOBIG error. */ void sqlite3_result_error_toobig(sqlite3_context *pCtx){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_TOOBIG; pCtx->fErrorOrAux = 1; - sqlite3VdbeMemSetStr(&pCtx->s, "string or blob too big", -1, + sqlite3VdbeMemSetStr(pCtx->pOut, "string or blob too big", -1, SQLITE_UTF8, SQLITE_STATIC); } /* An SQLITE_NOMEM error. */ void sqlite3_result_error_nomem(sqlite3_context *pCtx){ - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); - sqlite3VdbeMemSetNull(&pCtx->s); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + sqlite3VdbeMemSetNull(pCtx->pOut); pCtx->isError = SQLITE_NOMEM; pCtx->fErrorOrAux = 1; - pCtx->s.db->mallocFailed = 1; + pCtx->pOut->db->mallocFailed = 1; } /* @@ -568,7 +568,7 @@ void *sqlite3_user_data(sqlite3_context *p){ */ sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ assert( p && p->pFunc ); - return p->s.db; + return p->pOut->db; } /* @@ -578,7 +578,7 @@ sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context *p){ Vdbe *v = p->pVdbe; int rc; if( v->iCurrentTime==0 ){ - rc = sqlite3OsCurrentTimeInt64(p->s.db->pVfs, &v->iCurrentTime); + rc = sqlite3OsCurrentTimeInt64(p->pOut->db->pVfs, &v->iCurrentTime); if( rc ) v->iCurrentTime = 0; } return v->iCurrentTime; @@ -635,7 +635,7 @@ static SQLITE_NOINLINE void *createAggContext(sqlite3_context *p, int nByte){ */ void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ assert( p && p->pFunc && p->pFunc->xStep ); - assert( sqlite3_mutex_held(p->s.db->mutex) ); + assert( sqlite3_mutex_held(p->pOut->db->mutex) ); testcase( nByte<0 ); if( (p->pMem->flags & MEM_Agg)==0 ){ return createAggContext(p, nByte); @@ -651,7 +651,7 @@ void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ AuxData *pAuxData; - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); for(pAuxData=pCtx->pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){ if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break; } @@ -673,7 +673,7 @@ void sqlite3_set_auxdata( AuxData *pAuxData; Vdbe *pVdbe = pCtx->pVdbe; - assert( sqlite3_mutex_held(pCtx->s.db->mutex) ); + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); if( iArg<0 ) goto failed; for(pAuxData=pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){ diff --git a/src/vdbemem.c b/src/vdbemem.c index a76bd48865..95e23c61a7 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -290,17 +290,20 @@ int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ int rc = SQLITE_OK; if( ALWAYS(pFunc && pFunc->xFinalize) ){ sqlite3_context ctx; + Mem t; assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); memset(&ctx, 0, sizeof(ctx)); - ctx.s.flags = MEM_Null; - ctx.s.db = pMem->db; + memset(&t, 0, sizeof(t)); + t.flags = MEM_Null; + t.db = pMem->db; + ctx.pOut = &t; ctx.pMem = pMem; ctx.pFunc = pFunc; pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ assert( 0==(pMem->flags&MEM_Dyn) && !pMem->xDel ); sqlite3DbFree(pMem->db, pMem->zMalloc); - memcpy(pMem, &ctx.s, sizeof(ctx.s)); + memcpy(pMem, &t, sizeof(t)); rc = ctx.isError; } return rc; @@ -615,14 +618,28 @@ void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ #endif } +/* +** The pMem is known to contain content that needs to be destroyed prior +** to a value change. So invoke the destructor, then set the value to +** a 64-bit integer. +*/ +static SQLITE_NOINLINE void vdbeReleaseAndSetInt64(Mem *pMem, i64 val){ + sqlite3VdbeMemReleaseExternal(pMem); + pMem->u.i = val; + pMem->flags = MEM_Int; +} + /* ** Delete any previous value and set the value stored in *pMem to val, ** manifest type INTEGER. */ void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ - sqlite3VdbeMemRelease(pMem); - pMem->u.i = val; - pMem->flags = MEM_Int; + if( VdbeMemDynamic(pMem) ){ + vdbeReleaseAndSetInt64(pMem, val); + }else{ + pMem->u.i = val; + pMem->flags = MEM_Int; + } } #ifndef SQLITE_OMIT_FLOATING_POINT From 0829169f1e13563c761a75e9665a074cf1f5b263 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 27 Aug 2014 17:37:20 +0000 Subject: [PATCH 281/710] When determining whether or not a partial index is usable, do not assume that the cursor number assigned to each table in the query is the same as its index in the FROM clause. Fix for ticket [98d973b8f5]. FossilOrigin-Name: fcebca166f15431764b82a8b267f11d28386e975 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 3 ++- test/index7.test | 30 ++++++++++++++++++++++++++++++ 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c45e2fe4fc..284a4c0be0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\ssqlite3_context\sobject,\skeep\sa\spointer\sto\sthe\sresult\svalue\srather\nthan\sstoring\sthe\sresult\svalue\sin\sthe\ssqlite3_context\sobject\sand\susing\nmemcpy()\sto\smove\sthe\svalue\sback\sinto\sits\sregister\nafter\sthe\sfunction\sreturns.\s\sThis\sruns\sfaster\sand\ssaves\sover\s500\sbytes\nof\scode\sspace. -D 2014-08-27T14:14:06.320 +C When\sdetermining\swhether\sor\snot\sa\spartial\sindex\sis\susable,\sdo\snot\sassume\sthat\sthe\scursor\snumber\sassigned\sto\seach\stable\sin\sthe\squery\sis\sthe\ssame\sas\sits\sindex\sin\sthe\sFROM\sclause.\sFix\sfor\sticket\s[98d973b8f5]. +D 2014-08-27T17:37:20.006 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -297,7 +297,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 4e2770a1914b8ce30f3e44ad954b720eca3b5efd +F src/where.c b80cb72645390ba6d8b715a47f5696d27b4fc6c9 F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -626,7 +626,7 @@ F test/index3.test 55a90cff99834305e8141df7afaef39674b57062 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33 F test/index6.test fb370966ac3cd0989053dd5385757b5c3e24ab6a -F test/index7.test a3baf9a625bda7fd49471e99aeae04095fbfeecf +F test/index7.test 4a1d3707e95067dfa069786baab124adedd55945 F test/indexedby.test b2f22f3e693a53813aa3f50b812eb609ba6df1ec F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1624916c6e9bc5dbcfa146b316a99ac8fecb13a9 -R d81ba05103ba02a6e3d94271217446d4 -U drh -Z 3300f3f8d44a1fbcab3339e234a49760 +P 6c1ee3e388eb110de815270467b1e50592c0ba6c +R 464ad42e9750481b657eb9d5845e92c2 +U dan +Z e49f678a3d5ba39b1ed4a3e5b185dc37 diff --git a/manifest.uuid b/manifest.uuid index 057a225c61..52c4bd20c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c1ee3e388eb110de815270467b1e50592c0ba6c \ No newline at end of file +fcebca166f15431764b82a8b267f11d28386e975 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 4ba8de6e41..22dfd0d0fa 100644 --- a/src/where.c +++ b/src/where.c @@ -4713,7 +4713,8 @@ static int whereLoopAddBtree( */ for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ if( pProbe->pPartIdxWhere!=0 - && !whereUsablePartialIndex(pNew->iTab, pWC, pProbe->pPartIdxWhere) ){ + && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){ + testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ continue; /* Partial index inappropriate for this query */ } rSize = pProbe->aiRowLogEst[0]; diff --git a/test/index7.test b/test/index7.test index 1c81f6024b..49b64b7c5a 100644 --- a/test/index7.test +++ b/test/index7.test @@ -248,4 +248,34 @@ do_execsql_test index7-5.0 { SELECT stat+0 FROM sqlite_stat1 WHERE idx='t3b'; } {6 6} +# Verify that the problem identified by ticket [] has been fixed. +# +do_execsql_test index7-6.1 { + CREATE TABLE t5(a, b); + CREATE TABLE t4(c, d); + INSERT INTO t5 VALUES(1, 'xyz'); + INSERT INTO t4 VALUES('abc', 'not xyz'); + SELECT * FROM (SELECT * FROM t5 WHERE a=1 AND b='xyz'), t4 WHERE c='abc'; +} { + 1 xyz abc {not xyz} +} +do_execsql_test index7-6.2 { + CREATE INDEX i4 ON t4(c) WHERE d='xyz'; + SELECT * FROM (SELECT * FROM t5 WHERE a=1 AND b='xyz'), t4 WHERE c='abc'; +} { + 1 xyz abc {not xyz} +} +do_execsql_test index7-6.3 { + CREATE VIEW v4 AS SELECT * FROM t4; + INSERT INTO t4 VALUES('def', 'xyz'); + SELECT * FROM v4 WHERE d='xyz' AND c='def' +} { + def xyz +} +do_eqp_test index7-6.4 { + SELECT * FROM v4 WHERE d='xyz' AND c='def' +} { + 0 0 0 {SEARCH TABLE t4 USING INDEX i4 (c=?)} +} + finish_test From 54ae4e31797abf465ffae8f990d3b81ec42d9749 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 27 Aug 2014 17:48:15 +0000 Subject: [PATCH 282/710] Add a missing ticket number to a comment in index7.test. FossilOrigin-Name: d8b1c4336145d436241863c3525530e24a24799b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/index7.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 284a4c0be0..1287aaa01c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sdetermining\swhether\sor\snot\sa\spartial\sindex\sis\susable,\sdo\snot\sassume\sthat\sthe\scursor\snumber\sassigned\sto\seach\stable\sin\sthe\squery\sis\sthe\ssame\sas\sits\sindex\sin\sthe\sFROM\sclause.\sFix\sfor\sticket\s[98d973b8f5]. -D 2014-08-27T17:37:20.006 +C Add\sa\smissing\sticket\snumber\sto\sa\scomment\sin\sindex7.test. +D 2014-08-27T17:48:15.250 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -626,7 +626,7 @@ F test/index3.test 55a90cff99834305e8141df7afaef39674b57062 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33 F test/index6.test fb370966ac3cd0989053dd5385757b5c3e24ab6a -F test/index7.test 4a1d3707e95067dfa069786baab124adedd55945 +F test/index7.test 917cf1e1c7439bb155abbeabec511b28945e157b F test/indexedby.test b2f22f3e693a53813aa3f50b812eb609ba6df1ec F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6c1ee3e388eb110de815270467b1e50592c0ba6c -R 464ad42e9750481b657eb9d5845e92c2 +P fcebca166f15431764b82a8b267f11d28386e975 +R 37c899ad123d2de7fd7d4d94d2ae021b U dan -Z e49f678a3d5ba39b1ed4a3e5b185dc37 +Z ddd6cebe79a234ecfb1f478d7979e62b diff --git a/manifest.uuid b/manifest.uuid index 52c4bd20c2..d17b998878 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fcebca166f15431764b82a8b267f11d28386e975 \ No newline at end of file +d8b1c4336145d436241863c3525530e24a24799b \ No newline at end of file diff --git a/test/index7.test b/test/index7.test index 49b64b7c5a..0f341a3129 100644 --- a/test/index7.test +++ b/test/index7.test @@ -248,7 +248,7 @@ do_execsql_test index7-5.0 { SELECT stat+0 FROM sqlite_stat1 WHERE idx='t3b'; } {6 6} -# Verify that the problem identified by ticket [] has been fixed. +# Verify that the problem identified by ticket [98d973b8f5] has been fixed. # do_execsql_test index7-6.1 { CREATE TABLE t5(a, b); From a1dc42aa9102467cf6ea3bb0aca128ef87c15b9b Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 27 Aug 2014 17:53:40 +0000 Subject: [PATCH 283/710] Add a VDBE synopsis comment for clarification. FossilOrigin-Name: 029a6dc744c24e7be482298c678af8a115d6a87b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1287aaa01c..3418dea1f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smissing\sticket\snumber\sto\sa\scomment\sin\sindex7.test. -D 2014-08-27T17:48:15.250 +C Add\sa\sVDBE\ssynopsis\scomment\sfor\sclarification. +D 2014-08-27T17:53:40.315 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -284,7 +284,7 @@ F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 F src/util.c 068dcd26354a3898ccc64ad5c4bdb95a7a15d33a F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 8b6638420347e01f7be6c276726441579a407be0 +F src/vdbe.c fd193824d1cc4a71c631eb792ce837aab530210a F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h 4653bb420abb7acdc215659cdcedd3a59f336191 F src/vdbeapi.c 09677a53dd8c71bcd670b0bd073bb9aefa02b441 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fcebca166f15431764b82a8b267f11d28386e975 -R 37c899ad123d2de7fd7d4d94d2ae021b -U dan -Z ddd6cebe79a234ecfb1f478d7979e62b +P d8b1c4336145d436241863c3525530e24a24799b +R 34521287dadc9603bb93e0f806afdcf3 +U mistachkin +Z d8914cdf39bc88ab4bc7c63f9115cf85 diff --git a/manifest.uuid b/manifest.uuid index d17b998878..e87597da96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d8b1c4336145d436241863c3525530e24a24799b \ No newline at end of file +029a6dc744c24e7be482298c678af8a115d6a87b \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index b8c1258f50..e3168805be 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1740,6 +1740,7 @@ case OP_RealAffinity: { /* in1 */ #ifndef SQLITE_OMIT_CAST /* Opcode: Cast P1 P2 * * * +** Synopsis: affinity(r[P1]) ** ** Force the value in register P1 to be the type defined by P2. ** From bc59ac0e26692db0346e7eea7710bf5fa90a450c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 27 Aug 2014 23:18:01 +0000 Subject: [PATCH 284/710] Refactor the sqlite3PcacheFetch() routine into three separate routines, which are significantly faster overall and about 100 bytes smaller in size as well. FossilOrigin-Name: bdb6e4978d1a26d5f795262172605184264ede9c --- manifest | 18 ++--- manifest.uuid | 2 +- src/pager.c | 20 ++++-- src/pcache.c | 191 +++++++++++++++++++++++++++++++++----------------- src/pcache.h | 4 +- 5 files changed, 155 insertions(+), 80 deletions(-) diff --git a/manifest b/manifest index 3418dea1f4..3e0e010c17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sVDBE\ssynopsis\scomment\sfor\sclarification. -D 2014-08-27T17:53:40.315 +C Refactor\sthe\ssqlite3PcacheFetch()\sroutine\sinto\sthree\sseparate\sroutines,\nwhich\sare\ssignificantly\sfaster\soverall\sand\sabout\s100\sbytes\ssmaller\sin\nsize\sas\swell. +D 2014-08-27T23:18:01.324 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -211,11 +211,11 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c bd7df3094a60915c148517504c76df4fca24e542 F src/os_win.c d067fce558a5032e6e6afe62899e5397bf63cf3e F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 27fb89e62e0ccf10218805ed31315ccb2d56c0ce +F src/pager.c 3e732d2bbdd8d8d95fed0c5ae7e718d73153c4c5 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 -F src/pcache.c c216e4077449be57e9752a348490ffa467b85599 -F src/pcache.h 80a9c3f7d7b7080388e8654717cb45e7b99f14a6 +F src/pcache.c 3b3791297e8977002e56b4a9b8916f2039abad9b +F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c F src/pragma.c d10ef67c4de79f78188b965b4b7988aff1d66f2e F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d8b1c4336145d436241863c3525530e24a24799b -R 34521287dadc9603bb93e0f806afdcf3 -U mistachkin -Z d8914cdf39bc88ab4bc7c63f9115cf85 +P 029a6dc744c24e7be482298c678af8a115d6a87b +R dd4d11ad7a65bf573cc08ce3e7bd3aa7 +U drh +Z c0c64dee6940e13662884c6ed139ac1a diff --git a/manifest.uuid b/manifest.uuid index e87597da96..64c260eec0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -029a6dc744c24e7be482298c678af8a115d6a87b \ No newline at end of file +bdb6e4978d1a26d5f795262172605184264ede9c \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 246371dbd7..3ef54d98e0 100644 --- a/src/pager.c +++ b/src/pager.c @@ -5286,7 +5286,6 @@ int sqlite3PagerAcquire( if( pPager->errCode!=SQLITE_OK ){ rc = pPager->errCode; }else{ - if( bMmapOk && pagerUseWal(pPager) ){ rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); if( rc!=SQLITE_OK ) goto pager_acquire_err; @@ -5301,7 +5300,7 @@ int sqlite3PagerAcquire( if( rc==SQLITE_OK && pData ){ if( pPager->eState>PAGER_READER ){ - (void)sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); + pPg = sqlite3PagerLookup(pPager, pgno); } if( pPg==0 ){ rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); @@ -5319,7 +5318,16 @@ int sqlite3PagerAcquire( } } - rc = sqlite3PcacheFetch(pPager->pPCache, pgno, 3, ppPage); + { + sqlite3_pcache_page *pBase; + pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); + if( pBase==0 ){ + rc = sqlite3PcacheFetchStress(pPager->pPCache, pgno, &pBase); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + } + pPg = *ppPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pBase); + if( pPg==0 ) rc = SQLITE_NOMEM; + } } if( rc!=SQLITE_OK ){ @@ -5416,12 +5424,12 @@ pager_acquire_err: ** has ever happened. */ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ - PgHdr *pPg = 0; + sqlite3_pcache_page *pPage; assert( pPager!=0 ); assert( pgno!=0 ); assert( pPager->pPCache!=0 ); - sqlite3PcacheFetch(pPager->pPCache, pgno, 0, &pPg); - return pPg; + pPage = sqlite3PcacheFetch(pPager->pPCache, pgno, 0); + return sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pPage); } /* diff --git a/src/pcache.c b/src/pcache.c index 5ca552e356..eabfadd4b5 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -231,15 +231,33 @@ int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ /* ** Try to obtain a page from the cache. +** +** This routine returns a pointer to an sqlite3_pcache_page object if +** such an object is already in cache, or if a new one is created. +** This routine returns a NULL pointer if the object was not in cache +** and could not be created. +** +** The createFlags should be 0 to check for existing pages and should +** be 3 (not 1, but 3) to try to create a new page. +** +** If the createFlag is 0, then NULL is always returned if the page +** is not already in the cache. If createFlag is 1, then a new page +** is created only if that can be done without spilling dirty pages +** and without exceeding the cache size limit. +** +** The caller needs to invoke sqlite3PcacheFetchFinish() to properly +** initialize the sqlite3_pcache_page object and convert it into a +** PgHdr object. The sqlite3PcacheFetch() and sqlite3PcacheFetchFinish() +** routines are split this way for performance reasons. When separated +** they can both (usually) operate without having to push values to +** the stack on entry and pop them back off on exit, which saves a +** lot of pushing and popping. */ -int sqlite3PcacheFetch( +sqlite3_pcache_page *sqlite3PcacheFetch( PCache *pCache, /* Obtain the page from this cache */ Pgno pgno, /* Page number to obtain */ - int createFlag, /* If true, create page if it does not exist already */ - PgHdr **ppPage /* Write the page here */ + int createFlag /* If true, create page if it does not exist already */ ){ - sqlite3_pcache_page *pPage; - PgHdr *pPgHdr = 0; int eCreate; assert( pCache!=0 ); @@ -258,69 +276,116 @@ int sqlite3PcacheFetch( assert( eCreate==0 || eCreate==1 || eCreate==2 ); assert( createFlag==0 || pCache->eCreate==eCreate ); assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); - pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); - if( !pPage && eCreate==1 ){ - PgHdr *pPg; + return sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); +} - /* Find a dirty page to write-out and recycle. First try to find a - ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC - ** cleared), but if that is not possible settle for any other - ** unreferenced dirty page. - */ - expensive_assert( pcacheCheckSynced(pCache) ); - for(pPg=pCache->pSynced; - pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); - pPg=pPg->pDirtyPrev - ); - pCache->pSynced = pPg; - if( !pPg ){ - for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); - } - if( pPg ){ - int rc; +/* +** If the sqlite3PcacheFetch() routine is unable to allocate a new +** page because new clean pages are available for reuse and the cache +** size limit has been reached, then this routine can be invoked to +** try harder to allocate a page. This routine might invoke the stress +** callback to spill dirty pages to the journal. It will then try to +** allocate the new page and will only fail to allocate a new page on +** an OOM error. +** +** This routine should be invoked only after sqlite3PcacheFetch() fails. +*/ +int sqlite3PcacheFetchStress( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number to obtain */ + sqlite3_pcache_page **ppPage /* Write result here */ +){ + PgHdr *pPg; + if( pCache->eCreate==2 ) return 0; + + + /* Find a dirty page to write-out and recycle. First try to find a + ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC + ** cleared), but if that is not possible settle for any other + ** unreferenced dirty page. + */ + expensive_assert( pcacheCheckSynced(pCache) ); + for(pPg=pCache->pSynced; + pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); + pPg=pPg->pDirtyPrev + ); + pCache->pSynced = pPg; + if( !pPg ){ + for(pPg=pCache->pDirtyTail; pPg && pPg->nRef; pPg=pPg->pDirtyPrev); + } + if( pPg ){ + int rc; #ifdef SQLITE_LOG_CACHE_SPILL - sqlite3_log(SQLITE_FULL, - "spill page %d making room for %d - cache used: %d/%d", - pPg->pgno, pgno, - sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), - numberOfCachePages(pCache)); + sqlite3_log(SQLITE_FULL, + "spill page %d making room for %d - cache used: %d/%d", + pPg->pgno, pgno, + sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), + numberOfCachePages(pCache)); #endif - rc = pCache->xStress(pCache->pStress, pPg); - if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ - return rc; - } - } - - pPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); - } - - if( pPage ){ - pPgHdr = (PgHdr *)pPage->pExtra; - - if( !pPgHdr->pPage ){ - memset(pPgHdr, 0, sizeof(PgHdr)); - pPgHdr->pPage = pPage; - pPgHdr->pData = pPage->pBuf; - pPgHdr->pExtra = (void *)&pPgHdr[1]; - memset(pPgHdr->pExtra, 0, pCache->szExtra); - pPgHdr->pCache = pCache; - pPgHdr->pgno = pgno; - } - assert( pPgHdr->pCache==pCache ); - assert( pPgHdr->pgno==pgno ); - assert( pPgHdr->pData==pPage->pBuf ); - assert( pPgHdr->pExtra==(void *)&pPgHdr[1] ); - - if( 0==pPgHdr->nRef ){ - pCache->nRef++; - } - pPgHdr->nRef++; - if( pgno==1 ){ - pCache->pPage1 = pPgHdr; + rc = pCache->xStress(pCache->pStress, pPg); + if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ + return rc; } } - *ppPage = pPgHdr; - return (pPgHdr==0 && eCreate) ? SQLITE_NOMEM : SQLITE_OK; + *ppPage = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, 2); + return *ppPage==0 ? SQLITE_NOMEM : SQLITE_OK; +} + +/* +** This is a helper routine for sqlite3PcacheFetchFinish() +** +** In the uncommon case where the page being fetched has not been +** initialized, this routine is invoked to do the initialization. +** This routine is broken out into a separate function since it +** requires extra stack manipulation that can be avoided in the common +** case. +*/ +static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number obtained */ + sqlite3_pcache_page *pPage /* Page obtained by prior PcacheFetch() call */ +){ + PgHdr *pPgHdr; + assert( pPage!=0 ); + pPgHdr = (PgHdr*)pPage->pExtra; + assert( pPgHdr->pPage==0 ); + memset(pPgHdr, 0, sizeof(PgHdr)); + pPgHdr->pPage = pPage; + pPgHdr->pData = pPage->pBuf; + pPgHdr->pExtra = (void *)&pPgHdr[1]; + memset(pPgHdr->pExtra, 0, pCache->szExtra); + pPgHdr->pCache = pCache; + pPgHdr->pgno = pgno; + return sqlite3PcacheFetchFinish(pCache,pgno,pPage); +} + +/* +** This routine converts the sqlite3_pcache_page object returned by +** sqlite3PcacheFetch() into an initialized PgHdr object. This routine +** must be called after sqlite3PcacheFetch() in order to get a usable +** result. +*/ +PgHdr *sqlite3PcacheFetchFinish( + PCache *pCache, /* Obtain the page from this cache */ + Pgno pgno, /* Page number obtained */ + sqlite3_pcache_page *pPage /* Page obtained by prior PcacheFetch() call */ +){ + PgHdr *pPgHdr; + + if( pPage==0 ) return 0; + pPgHdr = (PgHdr *)pPage->pExtra; + + if( !pPgHdr->pPage ){ + return pcacheFetchFinishWithInit(pCache, pgno, pPage); + } + if( 0==pPgHdr->nRef ){ + pCache->nRef++; + } + pPgHdr->nRef++; + if( pgno==1 ){ + pCache->pPage1 = pPgHdr; + } + return pPgHdr; } /* diff --git a/src/pcache.h b/src/pcache.h index 11ffa9f39d..dd9bfc7451 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -88,7 +88,9 @@ int sqlite3PcacheSize(void); /* One release per successful fetch. Page is pinned until released. ** Reference counted. */ -int sqlite3PcacheFetch(PCache*, Pgno, int createFlag, PgHdr**); +sqlite3_pcache_page *sqlite3PcacheFetch(PCache*, Pgno, int createFlag); +int sqlite3PcacheFetchStress(PCache*, Pgno, sqlite3_pcache_page**); +PgHdr *sqlite3PcacheFetchFinish(PCache*, Pgno, sqlite3_pcache_page *pPage); void sqlite3PcacheRelease(PgHdr*); void sqlite3PcacheDrop(PgHdr*); /* Remove page from cache */ From 5418b1298faf903bee8aeb55abfba88754f4aee9 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 28 Aug 2014 13:42:13 +0000 Subject: [PATCH 285/710] Fix a formatting error ("%d" needed in place of "%g") on a WHERETRACE macro inside of the query planner. This fix applies to debugging logic only. FossilOrigin-Name: c931ca2b7763cf894ece26ca4775d638876107e9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3e0e010c17..acc19cfe1f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\ssqlite3PcacheFetch()\sroutine\sinto\sthree\sseparate\sroutines,\nwhich\sare\ssignificantly\sfaster\soverall\sand\sabout\s100\sbytes\ssmaller\sin\nsize\sas\swell. -D 2014-08-27T23:18:01.324 +C Fix\sa\sformatting\serror\s("%d"\sneeded\sin\splace\sof\s"%g")\son\sa\sWHERETRACE\nmacro\sinside\sof\sthe\squery\splanner.\s\sThis\sfix\sapplies\sto\sdebugging\slogic\nonly. +D 2014-08-28T13:42:13.503 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -297,7 +297,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c b80cb72645390ba6d8b715a47f5696d27b4fc6c9 +F src/where.c 2fd5cfaf260d020e87053be8fb3741782c941adc F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 029a6dc744c24e7be482298c678af8a115d6a87b -R dd4d11ad7a65bf573cc08ce3e7bd3aa7 +P bdb6e4978d1a26d5f795262172605184264ede9c +R 34004330a355f02374768ce4f339cc8f U drh -Z c0c64dee6940e13662884c6ed139ac1a +Z aa80dcf0253dbc225958d679762190fc diff --git a/manifest.uuid b/manifest.uuid index 64c260eec0..dda8ba50bd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bdb6e4978d1a26d5f795262172605184264ede9c \ No newline at end of file +c931ca2b7763cf894ece26ca4775d638876107e9 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 22dfd0d0fa..656b6ea590 100644 --- a/src/where.c +++ b/src/where.c @@ -2390,7 +2390,7 @@ static int whereInScanEst( if( rc==SQLITE_OK ){ if( nRowEst > nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; - WHERETRACE(0x10,("IN row estimate: est=%g\n", nRowEst)); + WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); } assert( pBuilder->nRecValid==nRecValid ); return rc; From 7fa0631ece34c849a55f413d9753fa4ad0671d36 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 28 Aug 2014 16:01:31 +0000 Subject: [PATCH 286/710] Fix a problem causing an inaccurate stat4-based estimate for the number of rows visited by a range scan. FossilOrigin-Name: 052d89b554a947b7e420b7feb6650f5069b7cefa --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/where.c | 1 + test/whereJ.test | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index acc19cfe1f..de99e3bdbb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sformatting\serror\s("%d"\sneeded\sin\splace\sof\s"%g")\son\sa\sWHERETRACE\nmacro\sinside\sof\sthe\squery\splanner.\s\sThis\sfix\sapplies\sto\sdebugging\slogic\nonly. -D 2014-08-28T13:42:13.503 +C Fix\sa\sproblem\scausing\san\sinaccurate\sstat4-based\sestimate\sfor\sthe\snumber\sof\srows\svisited\sby\sa\srange\sscan. +D 2014-08-28T16:01:31.625 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -297,7 +297,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 2fd5cfaf260d020e87053be8fb3741782c941adc +F src/where.c 70dc8408930f9361d7b6240bf61c08ac3990ee4b F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1120,7 +1120,7 @@ F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test 69f5ec4b15760a8c860f80e2d55525669390aab3 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test 1d89199697919d4930be05a71e7fe620f114e622 -F test/whereJ.test 35a40a50d0e13aa6b0de7cc5d4b204e5f9f9669f +F test/whereJ.test 8880784c211c459595f734a35bcc5f2061fce987 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1188,7 +1188,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bdb6e4978d1a26d5f795262172605184264ede9c -R 34004330a355f02374768ce4f339cc8f -U drh -Z aa80dcf0253dbc225958d679762190fc +P c931ca2b7763cf894ece26ca4775d638876107e9 +R ef66ab622eee0820f789fdd0466c8ee4 +T *branch * stat4-experimental +T *sym-stat4-experimental * +T -sym-trunk * +U dan +Z 96900af81552b8a268140227f697d961 diff --git a/manifest.uuid b/manifest.uuid index dda8ba50bd..029666a0e8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c931ca2b7763cf894ece26ca4775d638876107e9 \ No newline at end of file +052d89b554a947b7e420b7feb6650f5069b7cefa \ No newline at end of file diff --git a/src/where.c b/src/where.c index 656b6ea590..64b3286086 100644 --- a/src/where.c +++ b/src/where.c @@ -2191,6 +2191,7 @@ static int whereRangeScanEst( tRowcnt iLower; tRowcnt iUpper; + if( pRec ) pRec->nField = pBuilder->nRecValid; if( nEq==p->nKeyCol ){ aff = SQLITE_AFF_INTEGER; }else{ diff --git a/test/whereJ.test b/test/whereJ.test index 5209f16193..7c37321cbf 100644 --- a/test/whereJ.test +++ b/test/whereJ.test @@ -371,5 +371,52 @@ do_execsql_test whereJ-2.2 { ORDER BY t4.x; } {~/SCAN/} +############################################################################ + +ifcapable stat4 { + # Create and populate table. + do_execsql_test 3.1 { CREATE TABLE t1(a, b, c) } + for {set i 0} {$i < 32} {incr i 2} { + for {set x 0} {$x < 100} {incr x} { + execsql { INSERT INTO t1 VALUES($i, $x, $c) } + incr c + } + execsql { INSERT INTO t1 VALUES($i+1, 5, $c) } + incr c + } + + do_execsql_test 3.2 { + SELECT a, count(*) FROM t1 GROUP BY a HAVING a < 8; + } { + 0 100 1 1 2 100 3 1 4 100 5 1 6 100 7 1 + } + + do_execsql_test 3.3 { + CREATE INDEX idx_ab ON t1(a, b); + CREATE INDEX idx_c ON t1(c); + ANALYZE; + } {} + + # This one should use index "idx_c". + do_eqp_test 3.4 { + SELECT * FROM t1 WHERE + a = 4 AND b BETWEEN 20 AND 80 -- Matches 80 rows + AND + c BETWEEN 150 AND 160 -- Matches 10 rows + } { + 0 0 0 {SEARCH TABLE t1 USING INDEX idx_c (c>? AND c? AND b Date: Thu, 28 Aug 2014 17:20:37 +0000 Subject: [PATCH 287/710] Add a testcase() macro to help verfity the row estimation logic. FossilOrigin-Name: 6ba609522ea7325341fad93bdb5f869c6506dea4 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/where.c | 5 ++++- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index de99e3bdbb..8348ae4636 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scausing\san\sinaccurate\sstat4-based\sestimate\sfor\sthe\snumber\sof\srows\svisited\sby\sa\srange\sscan. -D 2014-08-28T16:01:31.625 +C Add\sa\stestcase()\smacro\sto\shelp\sverfity\sthe\srow\sestimation\slogic. +D 2014-08-28T17:20:37.257 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -297,7 +297,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 70dc8408930f9361d7b6240bf61c08ac3990ee4b +F src/where.c 8214c30123537c5ae49f5775f7298dbb1c64d676 F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1188,10 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c931ca2b7763cf894ece26ca4775d638876107e9 -R ef66ab622eee0820f789fdd0466c8ee4 -T *branch * stat4-experimental -T *sym-stat4-experimental * -T -sym-trunk * -U dan -Z 96900af81552b8a268140227f697d961 +P 052d89b554a947b7e420b7feb6650f5069b7cefa +R 3ae9c7f00ac4ed418fd1e37d2965b964 +U drh +Z 6043bd4cd21e295f0b7cecd3f4e6f8d8 diff --git a/manifest.uuid b/manifest.uuid index 029666a0e8..0e4f3928fc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -052d89b554a947b7e420b7feb6650f5069b7cefa \ No newline at end of file +6ba609522ea7325341fad93bdb5f869c6506dea4 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 64b3286086..e5f61c0edd 100644 --- a/src/where.c +++ b/src/where.c @@ -2191,7 +2191,10 @@ static int whereRangeScanEst( tRowcnt iLower; tRowcnt iUpper; - if( pRec ) pRec->nField = pBuilder->nRecValid; + if( pRec ){ + testcase( pRec->nField!=pBuilder->nRecValid ); + pRec->nField = pBuilder->nRecValid; + } if( nEq==p->nKeyCol ){ aff = SQLITE_AFF_INTEGER; }else{ From ae914d783a2db649b1b4505cf7314e5dafa7a199 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 28 Aug 2014 19:38:22 +0000 Subject: [PATCH 288/710] Improved WHERETRACE messages for the estimated output row reductions from range scans. FossilOrigin-Name: fdd478bb11eb9e244a7a1fb628ac27c53830fb24 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/where.c | 8 +++++++- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 0b3792092e..dbe5333a45 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scausing\san\sinaccurate\sstat4-based\sestimate\sfor\sthe\nnumber\sof\srows\svisited\sby\sa\srange\sscan. -D 2014-08-28T17:30:28.372 +C Improved\sWHERETRACE\smessages\sfor\sthe\sestimated\soutput\srow\sreductions\sfrom\nrange\sscans. +D 2014-08-28T19:38:22.884 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -297,7 +297,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 8214c30123537c5ae49f5775f7298dbb1c64d676 +F src/where.c d9eae96b2cbbe4842eac3ee156ccd1b933d802c4 F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1188,8 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c931ca2b7763cf894ece26ca4775d638876107e9 6ba609522ea7325341fad93bdb5f869c6506dea4 -R 3ae9c7f00ac4ed418fd1e37d2965b964 -T +closed 6ba609522ea7325341fad93bdb5f869c6506dea4 +P a9daf3ac444b896013f4054051937d49a597624a +R 9b7fb0a8b772f987f88c09068dd60142 U drh -Z 21e61dd5628a06df27222872476717f3 +Z 8017c3a896f88a1957938bddddebc44a diff --git a/manifest.uuid b/manifest.uuid index 21fb81eb35..9b52e2f1ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a9daf3ac444b896013f4054051937d49a597624a \ No newline at end of file +fdd478bb11eb9e244a7a1fb628ac27c53830fb24 \ No newline at end of file diff --git a/src/where.c b/src/where.c index e5f61c0edd..e1e1e1d528 100644 --- a/src/where.c +++ b/src/where.c @@ -2254,7 +2254,7 @@ static int whereRangeScanEst( if( nNewnOut>nOut ){ + WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + pLoop->nOut, nOut)); + } +#endif pLoop->nOut = (LogEst)nOut; return rc; } From 53932ce8fa8d095cd61b44aa1244af78a955ccf3 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 29 Aug 2014 12:29:39 +0000 Subject: [PATCH 289/710] Avoid an unnecessary OP_Move operation for expression subqueries. FossilOrigin-Name: 462f42af52f146fd328ddcbbe78c8444ef7bd2c3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 1 + src/select.c | 2 +- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index dbe5333a45..c4e73d59dd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sWHERETRACE\smessages\sfor\sthe\sestimated\soutput\srow\sreductions\sfrom\nrange\sscans. -D 2014-08-28T19:38:22.884 +C Avoid\san\sunnecessary\sOP_Move\soperation\sfor\sexpression\ssubqueries. +D 2014-08-29T12:29:39.634 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -177,7 +177,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 5adcd322c6b08fc25d215d780ca62cebce66304d -F src/expr.c 358634f4ddeeb4e69643cb6db5819104a7834c60 +F src/expr.c e1691ab0fe6be7247ef073b0038fb8ecd9944fad F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 8d81a780ad78d16ec9082585758a8f1d6bf02ca3 F src/func.c bbb724b74ed96ca42675a7274646a71dd52bcda7 @@ -223,7 +223,7 @@ F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c ea48e891406ccdf748f3eb02893e056d134a0fea +F src/select.c 1c4667571f2c9e339b5a5c5b152a9ea7b0bc4163 F src/shell.c 728d2226594d356bf4fbdbdfd08538fd78fd06f3 F src/sqlite.h.in ed9d35990c61f0388ca6405706455c4095310553 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a9daf3ac444b896013f4054051937d49a597624a -R 9b7fb0a8b772f987f88c09068dd60142 +P fdd478bb11eb9e244a7a1fb628ac27c53830fb24 +R ee023126020f0faa82dce1872da63eb6 U drh -Z 8017c3a896f88a1957938bddddebc44a +Z 1a15b0689431fa59072b2264b67b8606 diff --git a/manifest.uuid b/manifest.uuid index 9b52e2f1ba..380c902508 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fdd478bb11eb9e244a7a1fb628ac27c53830fb24 \ No newline at end of file +462f42af52f146fd328ddcbbe78c8444ef7bd2c3 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index fabdae2fcf..1a2465f7ef 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1912,6 +1912,7 @@ int sqlite3CodeSubselect( sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); if( pExpr->op==TK_SELECT ){ dest.eDest = SRT_Mem; + dest.iSdst = dest.iSDParm; sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iSDParm); VdbeComment((v, "Init subquery result")); }else{ diff --git a/src/select.c b/src/select.c index 932874d8fe..44b9cd034c 100644 --- a/src/select.c +++ b/src/select.c @@ -829,7 +829,7 @@ static void selectInnerLoop( if( pSort ){ pushOntoSorter(pParse, pSort, p, regResult); }else{ - sqlite3ExprCodeMove(pParse, regResult, iParm, 1); + assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ } break; From 111544cbb4e1226ca80a1670f72075f07f949968 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 29 Aug 2014 16:20:47 +0000 Subject: [PATCH 290/710] Add SQLITE_LIMIT_WORKER_THREADS for controlling the maximum number of worker threads. FossilOrigin-Name: 1b598c68f32db635d1cea1373bedc434aa60cf08 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/main.c | 8 +++++++- src/pragma.c | 9 ++++----- src/sqlite.h.in | 5 +++++ src/sqliteInt.h | 3 +-- src/vdbesort.c | 11 ++++++++++- 7 files changed, 38 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index b491b9fbcb..6f1298ff91 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\sperformance\senhancements\sfrom\strunk\sonto\sthe\sthreads\sbranch. -D 2014-08-29T14:40:07.113 +C Add\sSQLITE_LIMIT_WORKER_THREADS\sfor\scontrolling\sthe\smaximum\snumber\sof\nworker\sthreads. +D 2014-08-29T16:20:47.382 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -190,7 +190,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab -F src/main.c f1726e704941d365ce2846161e93ba689a245845 +F src/main.c e71d7c8b415e3b54d729aabb876f39f7aa35b000 F src/malloc.c 954de5f998c23237e04474a3f2159bf483bba65a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -217,7 +217,7 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 3b3791297e8977002e56b4a9b8916f2039abad9b F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c -F src/pragma.c 33971fcaa7c13b84b1a0f2e813f4a3ab4d745ede +F src/pragma.c 14bcdb504128a476cce5bbc086d5226c5e46c225 F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -225,10 +225,10 @@ F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 89e569b263535662f54b537eb9118b2c554ae7aa F src/shell.c 88378cef39aba4b4a1df82793dcb1daf9276bb81 -F src/sqlite.h.in aa2cc1405cb999c9d73484e0686f7b869b430ba3 +F src/sqlite.h.in 74b42237f0d2b010779cc1b1a00190452b31a2ec F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 43419afaed8cd3bf99df06d38952a52f827217b9 +F src/sqliteInt.h e539938b3504520da5cefe225712452abea6795a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -292,7 +292,7 @@ F src/vdbeapi.c 09677a53dd8c71bcd670b0bd073bb9aefa02b441 F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 921d5468a68ac06f369810992e84ca22cc730a62 -F src/vdbesort.c 50fe3442f41dbfe0b37d2e8b55e7460244015533 +F src/vdbesort.c f92628f3d5d4432f751b15a5f39bacc3c6a64a03 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e3305d4b4efcbe06945ce7f6ec0f2e864244aaf9 2a74129a21f9745f1363f844807e2d10201a3f40 -R a4f057fd137875212defd2b87d648d95 +P 35c44a3c73e2e8b14ff194c41986f4bdb9dfe737 +R 2f78b359e3468cdedd78b18f8317c2eb U drh -Z c730a4dfee48578be19b0ff5cd3b7719 +Z bfedd71a211c8cc0f7a49234bc820538 diff --git a/manifest.uuid b/manifest.uuid index 7f66b19a61..cd1149dbb0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35c44a3c73e2e8b14ff194c41986f4bdb9dfe737 \ No newline at end of file +1b598c68f32db635d1cea1373bedc434aa60cf08 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 98bd82a298..5681ba5abc 100644 --- a/src/main.c +++ b/src/main.c @@ -2078,6 +2078,7 @@ static const int aHardLimit[] = { SQLITE_MAX_LIKE_PATTERN_LENGTH, SQLITE_MAX_VARIABLE_NUMBER, /* IMP: R-38091-32352 */ SQLITE_MAX_TRIGGER_DEPTH, + SQLITE_MAX_WORKER_THREADS, }; /* @@ -2113,6 +2114,9 @@ static const int aHardLimit[] = { #if SQLITE_MAX_TRIGGER_DEPTH<1 # error SQLITE_MAX_TRIGGER_DEPTH must be at least 1 #endif +#if SQLITE_MAX_WORKER_THREADS<0 || SQLITE_MAX_WORKER_THREADS>50 +# error SQLITE_MAX_WORKER_THREADS must be between 0 and 50 +#endif /* @@ -2146,7 +2150,8 @@ int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ SQLITE_MAX_LIKE_PATTERN_LENGTH ); assert( aHardLimit[SQLITE_LIMIT_VARIABLE_NUMBER]==SQLITE_MAX_VARIABLE_NUMBER); assert( aHardLimit[SQLITE_LIMIT_TRIGGER_DEPTH]==SQLITE_MAX_TRIGGER_DEPTH ); - assert( SQLITE_LIMIT_TRIGGER_DEPTH==(SQLITE_N_LIMIT-1) ); + assert( aHardLimit[SQLITE_LIMIT_WORKER_THREADS]==SQLITE_MAX_WORKER_THREADS ); + assert( SQLITE_LIMIT_WORKER_THREADS==(SQLITE_N_LIMIT-1) ); if( limitId<0 || limitId>=SQLITE_N_LIMIT ){ @@ -2493,6 +2498,7 @@ static int openDatabase( assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = 0; db->autoCommit = 1; db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; diff --git a/src/pragma.c b/src/pragma.c index 8421042e00..12446125fb 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2287,15 +2287,14 @@ void sqlite3Pragma( */ case PragTyp_THREADS: { sqlite3_int64 N; - if( sqlite3GlobalConfig.bCoreMutex - && zRight + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK && N>=0 ){ - if( N>SQLITE_MAX_WORKER_THREADS ) N = SQLITE_MAX_WORKER_THREADS; - db->mxWorker = N&0xff; + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); } - returnSingleInt(pParse, "soft_heap_limit", db->mxWorker); + returnSingleInt(pParse, "threads", + sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); break; } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f8ea7ad623..103bb0a80c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3073,6 +3073,10 @@ int sqlite3_limit(sqlite3*, int id, int newVal); ** ** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
    SQLITE_LIMIT_TRIGGER_DEPTH
    **
    The maximum depth of recursion for triggers.
    )^ +** +** [[SQLITE_LIMIT_WORKER_THREADS]] ^(
    SQLITE_LIMIT_WORKER_THREADS
    +**
    The maximum number of separate worker threads that a single +** [database connection] may start to help it with a computation.
    )^ ** */ #define SQLITE_LIMIT_LENGTH 0 @@ -3086,6 +3090,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal); #define SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8 #define SQLITE_LIMIT_VARIABLE_NUMBER 9 #define SQLITE_LIMIT_TRIGGER_DEPTH 10 +#define SQLITE_LIMIT_WORKER_THREADS 11 /* ** CAPI3REF: Compiling An SQL Statement diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b48d5cf795..aea497b7a9 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -941,7 +941,7 @@ struct Schema { ** The number of different kinds of things that can be limited ** using the sqlite3_limit() interface. */ -#define SQLITE_N_LIMIT (SQLITE_LIMIT_TRIGGER_DEPTH+1) +#define SQLITE_N_LIMIT (SQLITE_LIMIT_WORKER_THREADS+1) /* ** Lookaside malloc is a set of fixed-size buffers that can be used @@ -1013,7 +1013,6 @@ struct sqlite3 { u8 suppressErr; /* Do not issue error messages if true */ u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ - u8 mxWorker; /* Maximum number of worker threads */ int nextPagesize; /* Pagesize after VACUUM if >0 */ u32 magic; /* Magic number for detect library misuse */ int nChange; /* Value returned by sqlite3_changes() */ diff --git a/src/vdbesort.c b/src/vdbesort.c index cdbf6e0252..158fa440fb 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -796,7 +796,16 @@ int sqlite3VdbeSorterInit( #if SQLITE_MAX_WORKER_THREADS==0 # define nWorker 0 #else - int nWorker = sqlite3TempInMemory(db) ? 0 : db->mxWorker ; + int nWorker; +#endif + + /* Initialize the upper limit on the number of worker threads */ +#if SQLITE_MAX_WORKER_THREADS>0 + if( sqlite3TempInMemory(db) || sqlite3GlobalConfig.bCoreMutex==0 ){ + nWorker = 0; + }else{ + nWorker = db->aLimit[SQLITE_LIMIT_WORKER_THREADS]; + } #endif /* Do not allow the total number of threads (main thread + all workers) From 43cbe1437891abfaf89ae72b85c7c3a0c6dbe45d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 29 Aug 2014 18:06:33 +0000 Subject: [PATCH 291/710] Fix the speedtest1.c test program to set the worker thread count using the threads pragma. FossilOrigin-Name: 2ab4b5adc60b52bf2d2b79968d226b8dd7d2ab3b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/speedtest1.c | 6 +----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 6f1298ff91..31a0c63cb4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sSQLITE_LIMIT_WORKER_THREADS\sfor\scontrolling\sthe\smaximum\snumber\sof\nworker\sthreads. -D 2014-08-29T16:20:47.382 +C Fix\sthe\sspeedtest1.c\stest\sprogram\sto\sset\sthe\sworker\sthread\scount\susing\nthe\sthreads\spragma. +D 2014-08-29T18:06:33.187 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -851,7 +851,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c f452891e50571627f7060c0e1262359127055717 +F test/speedtest1.c 83f6b3318f7ee60e52b978b5a5e5dd7e83dfb7ee F test/spellfix.test 24f676831acddd2f4056a598fd731a72c6311f49 F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 35c44a3c73e2e8b14ff194c41986f4bdb9dfe737 -R 2f78b359e3468cdedd78b18f8317c2eb +P 1b598c68f32db635d1cea1373bedc434aa60cf08 +R 0f5325539e5516baa4a3f930643f73d7 U drh -Z bfedd71a211c8cc0f7a49234bc820538 +Z 8e5082b548f72bf30dfd81cfdd246c1c diff --git a/manifest.uuid b/manifest.uuid index cd1149dbb0..8313e46779 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1b598c68f32db635d1cea1373bedc434aa60cf08 \ No newline at end of file +2ab4b5adc60b52bf2d2b79968d226b8dd7d2ab3b \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 5fb9917f9b..8589b16333 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -1278,11 +1278,6 @@ int main(int argc, char **argv){ rc = sqlite3_config(SQLITE_CONFIG_SCRATCH, pScratch, szScratch, nScratch); if( rc ) fatal_error("scratch configuration failed: %d\n", rc); } -#ifdef SQLITE_CONFIG_WORKER_THREADS - if( nThread>0 ){ - sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, nThread); - } -#endif if( nLook>0 ){ sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0); } @@ -1300,6 +1295,7 @@ int main(int argc, char **argv){ /* Set database connection options */ sqlite3_create_function(g.db, "random", 0, SQLITE_UTF8, 0, randomFunc, 0, 0); if( doTrace ) sqlite3_trace(g.db, traceCallback, 0); + speedtest1_exec("PRAGMA threads=%d", nThread); if( zKey ){ speedtest1_exec("PRAGMA key('%s')", zKey); } From 6b2129aaaf69c4318d4dc158b685e4522c4bf633 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 29 Aug 2014 19:06:07 +0000 Subject: [PATCH 292/710] Disable worker threads when SQLITE_THREADSAFE=0. Set the default compile-time maximum number of worker threads to 8 and honor the SQLITE_DEFAULT_WORKER_THREADS compile-time constant (which defaults to 0). FossilOrigin-Name: 33fa0410499900dd8beb44b9a8ffbd9f4b68c8d8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 2 +- src/sqliteInt.h | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 31a0c63cb4..6c7fe672c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sspeedtest1.c\stest\sprogram\sto\sset\sthe\sworker\sthread\scount\susing\nthe\sthreads\spragma. -D 2014-08-29T18:06:33.187 +C Disable\sworker\sthreads\swhen\sSQLITE_THREADSAFE=0.\s\sSet\sthe\sdefault\scompile-time\nmaximum\snumber\sof\sworker\sthreads\sto\s8\sand\shonor\sthe\nSQLITE_DEFAULT_WORKER_THREADS\scompile-time\sconstant\s(which\sdefaults\sto\s0). +D 2014-08-29T19:06:07.922 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -190,7 +190,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab -F src/main.c e71d7c8b415e3b54d729aabb876f39f7aa35b000 +F src/main.c d2ef03a45552e11813c68326d5edfda992e319d4 F src/malloc.c 954de5f998c23237e04474a3f2159bf483bba65a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -228,7 +228,7 @@ F src/shell.c 88378cef39aba4b4a1df82793dcb1daf9276bb81 F src/sqlite.h.in 74b42237f0d2b010779cc1b1a00190452b31a2ec F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h e539938b3504520da5cefe225712452abea6795a +F src/sqliteInt.h 6244ee9052752e26d1275ab20c9b774385aa57d2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1b598c68f32db635d1cea1373bedc434aa60cf08 -R 0f5325539e5516baa4a3f930643f73d7 +P 2ab4b5adc60b52bf2d2b79968d226b8dd7d2ab3b +R 97462c8661f494fef9aea0705c0547b9 U drh -Z 8e5082b548f72bf30dfd81cfdd246c1c +Z 369e082ce9db2aa4e0cc673a60a91a43 diff --git a/manifest.uuid b/manifest.uuid index 8313e46779..680e153df5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2ab4b5adc60b52bf2d2b79968d226b8dd7d2ab3b \ No newline at end of file +33fa0410499900dd8beb44b9a8ffbd9f4b68c8d8 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 5681ba5abc..fd7151b174 100644 --- a/src/main.c +++ b/src/main.c @@ -2498,7 +2498,7 @@ static int openDatabase( assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); - db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = 0; + db->aLimit[SQLITE_LIMIT_WORKER_THREADS] = SQLITE_DEFAULT_WORKER_THREADS; db->autoCommit = 1; db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index aea497b7a9..7fd999d9ee 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -439,12 +439,12 @@ ** SQLITE_TEMP_STORE is set to 3 (never use temporary files), set it ** to zero. */ -#if SQLITE_TEMP_STORE==3 +#if SQLITE_TEMP_STORE==3 || SQLITE_THREADSAFE==0 # undef SQLITE_MAX_WORKER_THREADS # define SQLITE_MAX_WORKER_THREADS 0 #endif #ifndef SQLITE_MAX_WORKER_THREADS -# define SQLITE_MAX_WORKER_THREADS 4 +# define SQLITE_MAX_WORKER_THREADS 8 #endif #ifndef SQLITE_DEFAULT_WORKER_THREADS # define SQLITE_DEFAULT_WORKER_THREADS 0 From 44dec8753890ee819da4136babe9dbaff43c5589 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 30 Aug 2014 15:49:25 +0000 Subject: [PATCH 293/710] In the command-line shell, added options --lookaside, --pagecache, and --scratch used to configure auxiliary memories. FossilOrigin-Name: f61db04be4d7fb21b7f721647c37c45e283ffbea --- manifest | 13 ++++---- manifest.uuid | 2 +- src/shell.c | 89 ++++++++++++++++++++++++++++++++++++++------------- 3 files changed, 74 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 6785ddc6c4..b65dc64d8c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Note\sa\sfault\safter\sall,\srather\sa\stesting\smistake.\nMove\sthis\schange\sback\sto\strunk.\nWas:\sAvoid\san\sunnecessary\sOP_Move\soperation\sfor\sexpression\ssubqueries. -D 2014-08-29T14:20:46.550 +C In\sthe\scommand-line\sshell,\sadded\soptions\s--lookaside,\s--pagecache,\sand\n--scratch\sused\sto\sconfigure\sauxiliary\smemories. +D 2014-08-30T15:49:25.923 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -224,7 +224,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 1c4667571f2c9e339b5a5c5b152a9ea7b0bc4163 -F src/shell.c 728d2226594d356bf4fbdbdfd08538fd78fd06f3 +F src/shell.c 10b3bbf929aca5f33815444f03c9eac17e3c6daf F src/sqlite.h.in ed9d35990c61f0388ca6405706455c4095310553 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1188,8 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fdd478bb11eb9e244a7a1fb628ac27c53830fb24 462f42af52f146fd328ddcbbe78c8444ef7bd2c3 -R ee023126020f0faa82dce1872da63eb6 -T +closed 462f42af52f146fd328ddcbbe78c8444ef7bd2c3 +P 2a74129a21f9745f1363f844807e2d10201a3f40 +R 3444e463c529c6a0145dadfcb31aa003 U drh -Z 4973cc497d9d5b46eb65808e822ee62c +Z 5a770ceaa0af97ce932adfe78ff7c68a diff --git a/manifest.uuid b/manifest.uuid index 39e2d4034e..ae67577cfc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2a74129a21f9745f1363f844807e2d10201a3f40 \ No newline at end of file +f61db04be4d7fb21b7f721647c37c45e283ffbea \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 68478a19ba..994675ba11 100644 --- a/src/shell.c +++ b/src/shell.c @@ -462,6 +462,7 @@ struct ShellState { int mode; /* An output mode setting */ int writableSchema; /* True if PRAGMA writable_schema=ON */ int showHeader; /* True to show column names in List or Column mode */ + unsigned shellFlgs; /* Various flags */ char *zDestTable; /* Name of destination table when MODE_Insert */ char separator[20]; /* Separator character for MODE_List */ char newline[20]; /* Record separator in MODE_Csv */ @@ -481,6 +482,13 @@ struct ShellState { int iIndent; /* Index of current op in aiIndent[] */ }; +/* +** These are the allowed shellFlgs values +*/ +#define SHFLG_Scratch 0x00001 /* The --scratch option is used */ +#define SHFLG_Pagecache 0x00002 /* The --pagecache option is used */ +#define SHFLG_Lookaside 0x00004 /* Lookaside memory is used */ + /* ** These are the allowed modes. */ @@ -1097,21 +1105,19 @@ static int display_stats( iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Number of Outstanding Allocations: %d (max %d)\n", iCur, iHiwtr); -/* -** Not currently used by the CLI. -** iHiwtr = iCur = -1; -** sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset); -** fprintf(pArg->out, "Number of Pcache Pages Used: %d (max %d) pages\n", iCur, iHiwtr); -*/ + if( pArg->shellFlgs & SHFLG_Pagecache ){ + iHiwtr = iCur = -1; + sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Number of Pcache Pages Used: %d (max %d) pages\n", iCur, iHiwtr); + } iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", iCur, iHiwtr); -/* -** Not currently used by the CLI. -** iHiwtr = iCur = -1; -** sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset); -** fprintf(pArg->out, "Number of Scratch Allocations Used: %d (max %d)\n", iCur, iHiwtr); -*/ + if( pArg->shellFlgs & SHFLG_Scratch ){ + iHiwtr = iCur = -1; + sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Number of Scratch Allocations Used: %d (max %d)\n", iCur, iHiwtr); + } iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", iCur, iHiwtr); @@ -1132,15 +1138,17 @@ static int display_stats( } if( pArg && pArg->out && db ){ - iHiwtr = iCur = -1; - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Successful lookaside attempts: %d\n", iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Lookaside failures due to size: %d\n", iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Lookaside failures due to OOM: %d\n", iHiwtr); + if( pArg->shellFlgs & SHFLG_Lookaside ){ + iHiwtr = iCur = -1; + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Successful lookaside attempts: %d\n", iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Lookaside failures due to size: %d\n", iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Lookaside failures due to OOM: %d\n", iHiwtr); + } iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Pager Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; @@ -3779,12 +3787,15 @@ static const char zOptions[] = " -interactive force interactive I/O\n" " -line set output mode to 'line'\n" " -list set output mode to 'list'\n" + " -lookaside SIZE N use N entries of SZ bytes for lookaside memory\n" " -mmap N default mmap size set to N\n" #ifdef SQLITE_ENABLE_MULTIPLEX " -multiplex enable the multiplexor VFS\n" #endif " -newline SEP set newline character(s) for CSV\n" " -nullvalue TEXT set text string for NULL values. Default ''\n" + " -pagecache SIZE N use N slots of SZ bytes each for page cache memory\n" + " -scratch SIZE N use N slots of SZ bytes each for scratch memory\n" " -separator SEP set output field separator. Default: '|'\n" " -stats print memory stats before each finalize\n" " -version show SQLite version\n" @@ -3815,11 +3826,12 @@ static void main_init(ShellState *data) { memcpy(data->separator,"|", 2); memcpy(data->newline,"\r\n", 3); data->showHeader = 0; + data->shellFlgs = SHFLG_Lookaside; sqlite3_config(SQLITE_CONFIG_URI, 1); sqlite3_config(SQLITE_CONFIG_LOG, shellLog, data); + sqlite3_config(SQLITE_CONFIG_MULTITHREAD); sqlite3_snprintf(sizeof(mainPrompt), mainPrompt,"sqlite> "); sqlite3_snprintf(sizeof(continuePrompt), continuePrompt," ...> "); - sqlite3_config(SQLITE_CONFIG_SINGLETHREAD); } /* @@ -3928,6 +3940,33 @@ int main(int argc, char **argv){ if( szHeap>0x7fff0000 ) szHeap = 0x7fff0000; sqlite3_config(SQLITE_CONFIG_HEAP, malloc((int)szHeap), (int)szHeap, 64); #endif + }else if( strcmp(z,"-scratch")==0 ){ + int n, sz; + sz = integerValue(cmdline_option_value(argc,argv,++i)); + if( sz>400000 ) sz = 400000; + if( sz<2500 ) sz = 2500; + n = integerValue(cmdline_option_value(argc,argv,++i)); + if( n>10 ) n = 10; + if( n<1 ) n = 1; + sqlite3_config(SQLITE_CONFIG_SCRATCH, malloc(n*sz+1), sz, n); + data.shellFlgs |= SHFLG_Scratch; + }else if( strcmp(z,"-pagecache")==0 ){ + int n, sz; + sz = integerValue(cmdline_option_value(argc,argv,++i)); + if( sz>70000 ) sz = 70000; + if( sz<800 ) sz = 800; + n = integerValue(cmdline_option_value(argc,argv,++i)); + if( n<10 ) n = 10; + sqlite3_config(SQLITE_CONFIG_PAGECACHE, malloc(n*sz+1), sz, n); + data.shellFlgs |= SHFLG_Pagecache; + }else if( strcmp(z,"-lookaside")==0 ){ + int n, sz; + sz = integerValue(cmdline_option_value(argc,argv,++i)); + if( sz<0 ) sz = 0; + n = integerValue(cmdline_option_value(argc,argv,++i)); + if( n<0 ) n = 0; + sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n); + if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside; #ifdef SQLITE_ENABLE_VFSTRACE }else if( strcmp(z,"-vfstrace")==0 ){ extern int vfstrace_register( @@ -4043,6 +4082,12 @@ int main(int argc, char **argv){ stdin_is_interactive = 0; }else if( strcmp(z,"-heap")==0 ){ i++; + }else if( strcmp(z,"-scratch")==0 ){ + i+=2; + }else if( strcmp(z,"-pagecache")==0 ){ + i+=2; + }else if( strcmp(z,"-lookaside")==0 ){ + i+=2; }else if( strcmp(z,"-mmap")==0 ){ i++; }else if( strcmp(z,"-vfs")==0 ){ From 31970cca38ca1b268e898f79490a143633ba5dd6 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 1 Sep 2014 01:16:49 +0000 Subject: [PATCH 294/710] Fix harmless compiler warnings for MSVC. FossilOrigin-Name: 3ef3246120d72dffe469733bb21667a548af0a44 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index b65dc64d8c..3d6a9f0403 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\scommand-line\sshell,\sadded\soptions\s--lookaside,\s--pagecache,\sand\n--scratch\sused\sto\sconfigure\sauxiliary\smemories. -D 2014-08-30T15:49:25.923 +C Fix\sharmless\scompiler\swarnings\sfor\sMSVC. +D 2014-09-01T01:16:49.641 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -224,7 +224,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 1c4667571f2c9e339b5a5c5b152a9ea7b0bc4163 -F src/shell.c 10b3bbf929aca5f33815444f03c9eac17e3c6daf +F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50 F src/sqlite.h.in ed9d35990c61f0388ca6405706455c4095310553 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2a74129a21f9745f1363f844807e2d10201a3f40 -R 3444e463c529c6a0145dadfcb31aa003 -U drh -Z 5a770ceaa0af97ce932adfe78ff7c68a +P f61db04be4d7fb21b7f721647c37c45e283ffbea +R fb90b1156ce26645b130e94c9528a286 +U mistachkin +Z ab89ed2842a9e03f7299dfc1fb42f5cb diff --git a/manifest.uuid b/manifest.uuid index ae67577cfc..0e29df6f4e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f61db04be4d7fb21b7f721647c37c45e283ffbea \ No newline at end of file +3ef3246120d72dffe469733bb21667a548af0a44 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 994675ba11..afe01ef1a1 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3942,28 +3942,28 @@ int main(int argc, char **argv){ #endif }else if( strcmp(z,"-scratch")==0 ){ int n, sz; - sz = integerValue(cmdline_option_value(argc,argv,++i)); + sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( sz>400000 ) sz = 400000; if( sz<2500 ) sz = 2500; - n = integerValue(cmdline_option_value(argc,argv,++i)); + n = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( n>10 ) n = 10; if( n<1 ) n = 1; sqlite3_config(SQLITE_CONFIG_SCRATCH, malloc(n*sz+1), sz, n); data.shellFlgs |= SHFLG_Scratch; }else if( strcmp(z,"-pagecache")==0 ){ int n, sz; - sz = integerValue(cmdline_option_value(argc,argv,++i)); + sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( sz>70000 ) sz = 70000; if( sz<800 ) sz = 800; - n = integerValue(cmdline_option_value(argc,argv,++i)); + n = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( n<10 ) n = 10; sqlite3_config(SQLITE_CONFIG_PAGECACHE, malloc(n*sz+1), sz, n); data.shellFlgs |= SHFLG_Pagecache; }else if( strcmp(z,"-lookaside")==0 ){ int n, sz; - sz = integerValue(cmdline_option_value(argc,argv,++i)); + sz = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( sz<0 ) sz = 0; - n = integerValue(cmdline_option_value(argc,argv,++i)); + n = (int)integerValue(cmdline_option_value(argc,argv,++i)); if( n<0 ) n = 0; sqlite3_config(SQLITE_CONFIG_LOOKASIDE, sz, n); if( sz*n==0 ) data.shellFlgs &= ~SHFLG_Lookaside; From ee6438df0b109a2137d19cad0072e848527c86cc Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Sep 2014 13:29:32 +0000 Subject: [PATCH 295/710] Micro-optimizations in sqlite3BtreeNext() and sqlite3BtreePrevious(). FossilOrigin-Name: 839c7996eecd5480152c514555b9aa1121a69ce0 --- manifest | 14 +++--- manifest.uuid | 2 +- src/btree.c | 123 +++++++++++++++++++++++++++++++------------------- 3 files changed, 84 insertions(+), 55 deletions(-) diff --git a/manifest b/manifest index 3d6a9f0403..ab5c37744c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings\sfor\sMSVC. -D 2014-09-01T01:16:49.641 +C Micro-optimizations\sin\ssqlite3BtreeNext()\sand\ssqlite3BtreePrevious(). +D 2014-09-01T13:29:32.250 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c ec9d3f1295dafeb278c3830211cc5584132468f4 -F src/btree.c 4737cb5bdb2eb8989cb292f6ff921f7ff45f0c46 +F src/btree.c 4d0427bab54229030fc5b8577d2e5ffdc8129030 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 F src/build.c 058e3aadb1376521ff291735237edf4c10f438fb @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f61db04be4d7fb21b7f721647c37c45e283ffbea -R fb90b1156ce26645b130e94c9528a286 -U mistachkin -Z ab89ed2842a9e03f7299dfc1fb42f5cb +P 3ef3246120d72dffe469733bb21667a548af0a44 +R 8a87472d819e62a6d7e88f3d5e77ba23 +U drh +Z 5b0c8a527e1dec5d8cc1bbd203422489 diff --git a/manifest.uuid b/manifest.uuid index 0e29df6f4e..1ed9d46abc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ef3246120d72dffe469733bb21667a548af0a44 \ No newline at end of file +839c7996eecd5480152c514555b9aa1121a69ce0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 384bab218c..eb98ff34db 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4513,17 +4513,16 @@ static int moveToRightmost(BtCursor *pCur){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ + while( !(pPage = pCur->apPage[pCur->iPage])->leaf ){ pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); pCur->aiIdx[pCur->iPage] = pPage->nCell; rc = moveToChild(pCur, pgno); + if( rc ) return rc; } - if( rc==SQLITE_OK ){ - pCur->aiIdx[pCur->iPage] = pPage->nCell-1; - pCur->info.nSize = 0; - pCur->curFlags &= ~BTCF_ValidNKey; - } - return rc; + pCur->aiIdx[pCur->iPage] = pPage->nCell-1; + assert( pCur->info.nSize==0 ); + assert( (pCur->curFlags & BTCF_ValidNKey)==0 ); + return SQLITE_OK; } /* Move the cursor to the first entry in the table. Return SQLITE_OK @@ -4843,6 +4842,12 @@ int sqlite3BtreeEof(BtCursor *pCur){ ** was already pointing to the last entry in the database before ** this routine was called, then set *pRes=1. ** +** The main entry point is sqlite3BtreeNext(). That routine is optimized +** for the common case of merely incrementing the cell counter BtCursor.aiIdx +** to the next cell on the current page. The (slower) btreeNext() helper +** routine is called when it is necessary to move to a different page or +** to restore the cursor. +** ** The calling function will set *pRes to 0 or 1. The initial *pRes value ** will be 1 if the cursor being stepped corresponds to an SQL index and ** if this routine could have been skipped if that SQL index had been @@ -4852,20 +4857,18 @@ int sqlite3BtreeEof(BtCursor *pCur){ ** SQLite btree implementation does not. (Note that the comdb2 btree ** implementation does use this hint, however.) */ -int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ +static SQLITE_NOINLINE int btreeNext(BtCursor *pCur, int *pRes){ int rc; int idx; MemPage *pPage; assert( cursorHoldsMutex(pCur) ); - assert( pRes!=0 ); - assert( *pRes==0 || *pRes==1 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + assert( *pRes==0 ); if( pCur->eState!=CURSOR_VALID ){ - invalidateOverflowCache(pCur); + assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); rc = restoreCursorPosition(pCur); if( rc!=SQLITE_OK ){ - *pRes = 0; return rc; } if( CURSOR_INVALID==pCur->eState ){ @@ -4877,7 +4880,6 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ pCur->eState = CURSOR_VALID; if( pCur->skipNext>0 ){ pCur->skipNext = 0; - *pRes = 0; return SQLITE_OK; } pCur->skipNext = 0; @@ -4895,18 +4897,11 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** page into more than one b-tree structure. */ testcase( idx>pPage->nCell ); - pCur->info.nSize = 0; - pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); if( idx>=pPage->nCell ){ if( !pPage->leaf ){ rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); - if( rc ){ - *pRes = 0; - return rc; - } - rc = moveToLeftmost(pCur); - *pRes = 0; - return rc; + if( rc ) return rc; + return moveToLeftmost(pCur); } do{ if( pCur->iPage==0 ){ @@ -4917,22 +4912,39 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ moveToParent(pCur); pPage = pCur->apPage[pCur->iPage]; }while( pCur->aiIdx[pCur->iPage]>=pPage->nCell ); - *pRes = 0; if( pPage->intKey ){ - rc = sqlite3BtreeNext(pCur, pRes); + return sqlite3BtreeNext(pCur, pRes); }else{ - rc = SQLITE_OK; + return SQLITE_OK; } - return rc; } - *pRes = 0; if( pPage->leaf ){ return SQLITE_OK; + }else{ + return moveToLeftmost(pCur); + } +} +int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ + MemPage *pPage; + assert( cursorHoldsMutex(pCur) ); + assert( pRes!=0 ); + assert( *pRes==0 || *pRes==1 ); + assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + *pRes = 0; + if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur, pRes); + pPage = pCur->apPage[pCur->iPage]; + if( (++pCur->aiIdx[pCur->iPage])>=pPage->nCell ){ + pCur->aiIdx[pCur->iPage]--; + return btreeNext(pCur, pRes); + } + if( pPage->leaf ){ + return SQLITE_OK; + }else{ + return moveToLeftmost(pCur); } - rc = moveToLeftmost(pCur); - return rc; } - /* ** Step the cursor to the back to the previous entry in the database. If @@ -4940,6 +4952,12 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** was already pointing to the first entry in the database before ** this routine was called, then set *pRes=1. ** +** The main entry point is sqlite3BtreePrevious(). That routine is optimized +** for the common case of merely decrementing the cell counter BtCursor.aiIdx +** to the previous cell on the current page. The (slower) btreePrevious() helper +** routine is called when it is necessary to move to a different page or +** to restore the cursor. +** ** The calling function will set *pRes to 0 or 1. The initial *pRes value ** will be 1 if the cursor being stepped corresponds to an SQL index and ** if this routine could have been skipped if that SQL index had been @@ -4949,22 +4967,21 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** SQLite btree implementation does not. (Note that the comdb2 btree ** implementation does use this hint, however.) */ -int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ +static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur, int *pRes){ int rc; MemPage *pPage; assert( cursorHoldsMutex(pCur) ); assert( pRes!=0 ); - assert( *pRes==0 || *pRes==1 ); + assert( *pRes==0 ); assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); - pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl); + assert( (pCur->curFlags & (BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey))==0 ); + assert( pCur->info.nSize==0 ); if( pCur->eState!=CURSOR_VALID ){ - if( ALWAYS(pCur->eState>=CURSOR_REQUIRESEEK) ){ - rc = btreeRestoreCursorPosition(pCur); - if( rc!=SQLITE_OK ){ - *pRes = 0; - return rc; - } + assert( pCur->eState>=CURSOR_REQUIRESEEK ); + rc = btreeRestoreCursorPosition(pCur); + if( rc!=SQLITE_OK ){ + return rc; } if( CURSOR_INVALID==pCur->eState ){ *pRes = 1; @@ -4975,7 +4992,6 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ pCur->eState = CURSOR_VALID; if( pCur->skipNext<0 ){ pCur->skipNext = 0; - *pRes = 0; return SQLITE_OK; } pCur->skipNext = 0; @@ -4987,10 +5003,7 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ if( !pPage->leaf ){ int idx = pCur->aiIdx[pCur->iPage]; rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); - if( rc ){ - *pRes = 0; - return rc; - } + if( rc ) return rc; rc = moveToRightmost(pCur); }else{ while( pCur->aiIdx[pCur->iPage]==0 ){ @@ -5001,8 +5014,8 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ } moveToParent(pCur); } - pCur->info.nSize = 0; - pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + assert( pCur->info.nSize==0 ); + assert( (pCur->curFlags & (BTCF_ValidNKey|BTCF_ValidOvfl))==0 ); pCur->aiIdx[pCur->iPage]--; pPage = pCur->apPage[pCur->iPage]; @@ -5012,9 +5025,25 @@ int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ rc = SQLITE_OK; } } - *pRes = 0; return rc; } +int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ + assert( cursorHoldsMutex(pCur) ); + assert( pRes!=0 ); + assert( *pRes==0 || *pRes==1 ); + assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + *pRes = 0; + pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey); + pCur->info.nSize = 0; + if( pCur->eState!=CURSOR_VALID + || pCur->aiIdx[pCur->iPage]==0 + || pCur->apPage[pCur->iPage]->leaf==0 + ){ + return btreePrevious(pCur, pRes); + } + pCur->aiIdx[pCur->iPage]--; + return SQLITE_OK; +} /* ** Allocate a new page from the database file. From 19541f3018295f89a57931917539d1ea2fa93eff Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Sep 2014 13:37:55 +0000 Subject: [PATCH 296/710] Attempt to make the xDelete method of the unix VFS more robust on VxWorks. FossilOrigin-Name: b0f6b91f36b503d8ba8d5257bb194f8c1afb4833 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ab5c37744c..5851cdd221 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Micro-optimizations\sin\ssqlite3BtreeNext()\sand\ssqlite3BtreePrevious(). -D 2014-09-01T13:29:32.250 +C Attempt\sto\smake\sthe\sxDelete\smethod\sof\sthe\sunix\sVFS\smore\srobust\son\sVxWorks. +D 2014-09-01T13:37:55.088 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5eb79e334a5de69c87740edd56af6527dd219308 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c bd7df3094a60915c148517504c76df4fca24e542 +F src/os_unix.c 8525ca79457c5b4673a5fda2774ee39fe155f40f F src/os_win.c d067fce558a5032e6e6afe62899e5397bf63cf3e F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 3e732d2bbdd8d8d95fed0c5ae7e718d73153c4c5 @@ -1188,7 +1188,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3ef3246120d72dffe469733bb21667a548af0a44 -R 8a87472d819e62a6d7e88f3d5e77ba23 +P 839c7996eecd5480152c514555b9aa1121a69ce0 +R aa6b1a81952b002633810b3bb14c199e U drh -Z 5b0c8a527e1dec5d8cc1bbd203422489 +Z 8d5a13ba98607c42fffb4d1d92841eac diff --git a/manifest.uuid b/manifest.uuid index 1ed9d46abc..9f24642c58 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -839c7996eecd5480152c514555b9aa1121a69ce0 \ No newline at end of file +b0f6b91f36b503d8ba8d5257bb194f8c1afb4833 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index b1a0bedcff..f63afc6bc5 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -5885,7 +5885,7 @@ static int unixDelete( if( osUnlink(zPath)==(-1) ){ if( errno==ENOENT #if OS_VXWORKS - || errno==0x380003 + || osAccess(zPath,0)!=0 #endif ){ rc = SQLITE_IOERR_DELETE_NOENT; From 54d75187463047a51f346bb329ce680e6e7d2e9a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Sep 2014 18:21:27 +0000 Subject: [PATCH 297/710] Tweak the documentation for SQLITE_LIMIT_WORKER_THREADS. No changes to executable code. FossilOrigin-Name: 672e7387b1bda8d007da7de4244226577d7ab2dc --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 4 ++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index cbe9919f0e..262cca2377 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\susing\sseparate\sworker\sthreads\sto\sspeed\slarge\ssorts.\nThe\sSQLITE_MAX_WORKER_THREADS\sand\sSQLITE_DEFAULT_WORKER_THREADS\scompile-time\noptions\sand\sthe\sSQLITE_LIMIT_WORKER_THREADS\sargument\sto\s\nsqlite3_limit()\sand\sthe\s"PRAGMA\sthreads=N"\spragma\sare\sadded. -D 2014-09-01T17:36:46.754 +C Tweak\sthe\sdocumentation\sfor\sSQLITE_LIMIT_WORKER_THREADS.\s\sNo\schanges\sto\nexecutable\scode. +D 2014-09-01T18:21:27.480 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -225,7 +225,7 @@ F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be F src/select.c 89e569b263535662f54b537eb9118b2c554ae7aa F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50 -F src/sqlite.h.in 74b42237f0d2b010779cc1b1a00190452b31a2ec +F src/sqlite.h.in 43852c8b68b4c579948cb37182918078836c5c06 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h 6244ee9052752e26d1275ab20c9b774385aa57d2 @@ -1193,8 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b0f6b91f36b503d8ba8d5257bb194f8c1afb4833 33fa0410499900dd8beb44b9a8ffbd9f4b68c8d8 -R 91d41afdb912bf253c979082c1abeeae -T +closed 33fa0410499900dd8beb44b9a8ffbd9f4b68c8d8 +P b1c0f0bc1bd8a3477cd7a7ab510f0442ac88b517 +R e35acb2d59911a863984f6391f72c8a4 U drh -Z 8ac04e78c2df72246f6a97d0d813c221 +Z 81654d2ebb78f72eec51a460f9ce0ce5 diff --git a/manifest.uuid b/manifest.uuid index 754fc696d7..870f68e991 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b1c0f0bc1bd8a3477cd7a7ab510f0442ac88b517 \ No newline at end of file +672e7387b1bda8d007da7de4244226577d7ab2dc \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 103bb0a80c..d446d2ea98 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3075,8 +3075,8 @@ int sqlite3_limit(sqlite3*, int id, int newVal); **
    The maximum depth of recursion for triggers.
    )^ ** ** [[SQLITE_LIMIT_WORKER_THREADS]] ^(
    SQLITE_LIMIT_WORKER_THREADS
    -**
    The maximum number of separate worker threads that a single -** [database connection] may start to help it with a computation.
    )^ +**
    The maximum number of auxiliary worker threads that a single +** [prepared statement] may start.
    )^ ** */ #define SQLITE_LIMIT_LENGTH 0 From 975e076681a671ccd828e8b02db84bfc388c9e98 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Sep 2014 22:34:54 +0000 Subject: [PATCH 298/710] Avoid a confusing (though correct) argument to the sqlite3_result_blob() function in the implementation of ANALYZE. FossilOrigin-Name: 4cae93f8ae8fb3fe38fd5dc7d3a5ea0d11552841 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 9b13d487a6..59d93adfce 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\ssqlite3_win32_is_nt(),\sassume\sWinRT\sis\sNT-based\sand\srevise\s#ifdef\sordering\sto\sprefer\sthe\sANSI\sversion\sof\sGetVersionEx,\swhen\savailable. -D 2014-09-01T19:29:19.886 +C Avoid\sa\sconfusing\s(though\scorrect)\sargument\sto\sthe\ssqlite3_result_blob()\nfunction\sin\sthe\simplementation\sof\sANALYZE. +D 2014-09-01T22:34:54.079 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c f98a351908da29f7b44741cfeb9eb20dda648ba0 +F src/analyze.c 75345fa58d5d044614bedff9d758e3483a4f4522 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 672e7387b1bda8d007da7de4244226577d7ab2dc 9fe0f0754c063c17fffa9d4814b7180397cf5226 -R d670693d3a9e7f5751de171ad69fa48b -U mistachkin -Z f220ace468e0f2c0622bd628ba90e47a +P be0a037244762cc0f5ff4a6d7822902f862767bd +R 5498da01c32f540309f235f11a91d5bf +U drh +Z 21cb2c2a001f5844bb20a699e6bc27e5 diff --git a/manifest.uuid b/manifest.uuid index 4ae70b9684..1f41bd67e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -be0a037244762cc0f5ff4a6d7822902f862767bd \ No newline at end of file +4cae93f8ae8fb3fe38fd5dc7d3a5ea0d11552841 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index f9c03dc848..693e2f5472 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -467,7 +467,7 @@ static void statInit( #endif /* Return a pointer to the allocated object to the caller */ - sqlite3_result_blob(context, p, sizeof(p), stat4Destructor); + sqlite3_result_blob(context, p, sizeof(*p), stat4Destructor); } static const FuncDef statInitFuncdef = { 2+IsStat34, /* nArg */ From f8ede57a6165889316f8a9df8ac1166c7a568100 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 1 Sep 2014 23:06:44 +0000 Subject: [PATCH 299/710] Update comments in the ANALYZE command that describe how the Stat4Accum objecct is passed around within the VDBE. No changes to functional code. FossilOrigin-Name: 9779c7a9eb1e2bd36e9286331a9314f064014d80 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 12 ++++++++---- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 59d93adfce..58fd42dc62 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\sconfusing\s(though\scorrect)\sargument\sto\sthe\ssqlite3_result_blob()\nfunction\sin\sthe\simplementation\sof\sANALYZE. -D 2014-09-01T22:34:54.079 +C Update\scomments\sin\sthe\sANALYZE\scommand\sthat\sdescribe\show\sthe\sStat4Accum\nobjecct\sis\spassed\saround\swithin\sthe\sVDBE.\s\sNo\schanges\sto\sfunctional\scode. +D 2014-09-01T23:06:44.401 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -162,7 +162,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c 75345fa58d5d044614bedff9d758e3483a4f4522 +F src/analyze.c f00f06e6ef66c61b41f154889fe7caf5ed55a0ce F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P be0a037244762cc0f5ff4a6d7822902f862767bd -R 5498da01c32f540309f235f11a91d5bf +P 4cae93f8ae8fb3fe38fd5dc7d3a5ea0d11552841 +R fe37d23fb1f96547febf2ff7c0ba49b9 U drh -Z 21cb2c2a001f5844bb20a699e6bc27e5 +Z 29e2a2847ff584a00093c5b89ffca704 diff --git a/manifest.uuid b/manifest.uuid index 1f41bd67e6..af2604f469 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4cae93f8ae8fb3fe38fd5dc7d3a5ea0d11552841 \ No newline at end of file +9779c7a9eb1e2bd36e9286331a9314f064014d80 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 693e2f5472..9920c32a80 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -387,8 +387,9 @@ static void stat4Destructor(void *pOld){ ** original WITHOUT ROWID table as N==K as a special case. ** ** This routine allocates the Stat4Accum object in heap memory. The return -** value is a pointer to the the Stat4Accum object encoded as a blob (i.e. -** the size of the blob is sizeof(void*) bytes). +** value is a pointer to the the Stat4Accum object. The datatype of the +** return value is BLOB, but it is really just a pointer to the Stat4Accum +** object. */ static void statInit( sqlite3_context *context, @@ -466,7 +467,10 @@ static void statInit( } #endif - /* Return a pointer to the allocated object to the caller */ + /* Return a pointer to the allocated object to the caller. Note that + ** only the pointer (the 2nd parameter) matters. The size of the object + ** (given by the 3rd parameter) is never used and can be any positive + ** value. */ sqlite3_result_blob(context, p, sizeof(*p), stat4Destructor); } static const FuncDef statInitFuncdef = { @@ -793,7 +797,7 @@ static const FuncDef statPushFuncdef = { ** Implementation of the stat_get(P,J) SQL function. This routine is ** used to query statistical information that has been gathered into ** the Stat4Accum object by prior calls to stat_push(). The P parameter -** is a BLOB which is decoded into a pointer to the Stat4Accum objects. +** has type BLOB but it is really just a pointer to the Stat4Accum object. ** The content to returned is determined by the parameter J ** which is one of the STAT_GET_xxxx values defined above. ** From 30c633a097241352a5232feb602bbfb330d7bcdf Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 5 Sep 2014 05:58:37 +0000 Subject: [PATCH 300/710] Fix harmless compiler warning. FossilOrigin-Name: 733119067757814609a9cea6b975818607bee4e3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 2 ++ 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 58fd42dc62..8ae6336224 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\scomments\sin\sthe\sANALYZE\scommand\sthat\sdescribe\show\sthe\sStat4Accum\nobjecct\sis\spassed\saround\swithin\sthe\sVDBE.\s\sNo\schanges\sto\sfunctional\scode. -D 2014-09-01T23:06:44.401 +C Fix\sharmless\scompiler\swarning. +D 2014-09-05T05:58:37.378 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -209,7 +209,7 @@ F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 8525ca79457c5b4673a5fda2774ee39fe155f40f -F src/os_win.c 2aa8aa7780d7cf03e912d2088ab2ec5c32f33dc5 +F src/os_win.c 3c9f7df710cb6c757b04b78bf3d98f03830e67b9 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 3e732d2bbdd8d8d95fed0c5ae7e718d73153c4c5 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4cae93f8ae8fb3fe38fd5dc7d3a5ea0d11552841 -R fe37d23fb1f96547febf2ff7c0ba49b9 -U drh -Z 29e2a2847ff584a00093c5b89ffca704 +P 9779c7a9eb1e2bd36e9286331a9314f064014d80 +R 05490459da8c4172c4abbda0933bfe68 +U mistachkin +Z b93a43f7e285c77b049b454cce5380d9 diff --git a/manifest.uuid b/manifest.uuid index af2604f469..fb28827b7e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9779c7a9eb1e2bd36e9286331a9314f064014d80 \ No newline at end of file +733119067757814609a9cea6b975818607bee4e3 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 46bf88e387..42d5106e5f 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1286,12 +1286,14 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #endif } +#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0 DWORD sqlite3Win32Wait(HANDLE hObject){ DWORD rc; while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, TRUE))==WAIT_IO_COMPLETION ){} return rc; } +#endif /* ** Return true (non-zero) if we are running under WinNT, Win2K, WinXP, From d8b77e20fc9bde3a55c1355705287cc23b203b95 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Sep 2014 01:35:57 +0000 Subject: [PATCH 301/710] Query planner heuristic update: When doing a full table scan on a table that has an equality constraint on an unindexed column, do not allow the estimated number of output rows to be greater than half the total number of rows in the table. FossilOrigin-Name: 73954f93c4c6f880c6e01d0d130e3fed40fd4106 --- manifest | 18 +-- manifest.uuid | 2 +- src/sqliteInt.h | 1 - src/where.c | 34 ++++-- test/whereJ.test | 303 ++++++++++++++++++++++++++++++++++++++++------- 5 files changed, 297 insertions(+), 61 deletions(-) diff --git a/manifest b/manifest index 8ae6336224..cf1f0a6433 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning. -D 2014-09-05T05:58:37.378 +C Query\splanner\sheuristic\supdate:\nWhen\sdoing\sa\sfull\stable\sscan\son\sa\stable\sthat\shas\san\sequality\sconstraint\son\nan\sunindexed\scolumn,\sdo\snot\sallow\sthe\sestimated\snumber\sof\soutput\srows\sto\nbe\sgreater\sthan\shalf\sthe\stotal\snumber\sof\srows\sin\sthe\stable. +D 2014-09-06T01:35:57.122 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50 F src/sqlite.h.in 43852c8b68b4c579948cb37182918078836c5c06 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 6244ee9052752e26d1275ab20c9b774385aa57d2 +F src/sqliteInt.h dbc9a4984a7e40f4334f8aefd3505f5aef4650a8 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -298,7 +298,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c d9eae96b2cbbe4842eac3ee156ccd1b933d802c4 +F src/where.c 738213f25bc74c0a7cc5427712c1ac7537270a3d F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1125,7 +1125,7 @@ F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test 69f5ec4b15760a8c860f80e2d55525669390aab3 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test 1d89199697919d4930be05a71e7fe620f114e622 -F test/whereJ.test 8880784c211c459595f734a35bcc5f2061fce987 +F test/whereJ.test 1c35169106be343b7aa1a2c1338322462f27901a F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9779c7a9eb1e2bd36e9286331a9314f064014d80 -R 05490459da8c4172c4abbda0933bfe68 -U mistachkin -Z b93a43f7e285c77b049b454cce5380d9 +P 733119067757814609a9cea6b975818607bee4e3 +R bfd7d8b21e261616a30a072bf666f263 +U drh +Z 6641f77f6592cd795a1a1b9c2011a8f5 diff --git a/manifest.uuid b/manifest.uuid index fb28827b7e..a84ea01b2d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -733119067757814609a9cea6b975818607bee4e3 \ No newline at end of file +73954f93c4c6f880c6e01d0d130e3fed40fd4106 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7fd999d9ee..e56da4edd0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1159,7 +1159,6 @@ struct sqlite3 { #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ #define SQLITE_Stat3 0x0800 /* Use the SQLITE_STAT3 table */ -#define SQLITE_AdjustOutEst 0x1000 /* Adjust output estimates using WHERE */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* diff --git a/src/where.c b/src/where.c index e1e1e1d528..49b16b345d 100644 --- a/src/where.c +++ b/src/where.c @@ -4221,14 +4221,16 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** the number of output rows by a factor of 10 and each additional term ** reduces the number of output rows by sqrt(2). */ -static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ +static void whereLoopOutputAdjust( + WhereClause *pWC, /* The WHERE clause */ + WhereLoop *pLoop, /* The loop to adjust downward */ + LogEst nRow /* Number of rows in the entire table */ +){ WhereTerm *pTerm, *pX; Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); int i, j; + int nEq = 0; /* Number of = constraints not within likely()/unlike() */ - if( !OptimizationEnabled(pWC->pWInfo->pParse->db, SQLITE_AdjustOutEst) ){ - return; - } for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; @@ -4240,9 +4242,21 @@ static void whereLoopOutputAdjust(WhereClause *pWC, WhereLoop *pLoop){ if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } if( j<0 ){ - pLoop->nOut += (pTerm->truthProb<=0 ? pTerm->truthProb : -1); + if( pTerm->truthProb<=0 ){ + pLoop->nOut += pTerm->truthProb; + }else{ + pLoop->nOut--; + if( pTerm->eOperator&WO_EQ ) nEq++; + } } } + /* TUNING: If there is at least one equality constraint in the WHERE + ** clause that does not have a likelihood() explicitly assigned to it + ** then do not let the estimated number of output rows exceed half + ** the number of rows in the table. */ + if( nEq && pLoop->nOut>nRow-10 ){ + pLoop->nOut = nRow - 10; + } } /* @@ -4288,6 +4302,7 @@ static int whereLoopAddBtreeIndex( LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ int rc = SQLITE_OK; /* Return code */ + LogEst rSize; /* Number of rows in the table */ LogEst rLogSize; /* Logarithm of table size */ WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ @@ -4317,7 +4332,8 @@ static int whereLoopAddBtreeIndex( saved_prereq = pNew->prereq; saved_nOut = pNew->nOut; pNew->rSetup = 0; - rLogSize = estLog(pProbe->aiRowLogEst[0]); + rSize = pProbe->aiRowLogEst[0]; + rLogSize = estLog(rSize); /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average @@ -4494,7 +4510,7 @@ static int whereLoopAddBtreeIndex( nOutUnadjusted = pNew->nOut; pNew->rRun += nInMul + nIn; pNew->nOut += nInMul + nIn; - whereLoopOutputAdjust(pBuilder->pWC, pNew); + whereLoopOutputAdjust(pBuilder->pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ @@ -4748,7 +4764,7 @@ static int whereLoopAddBtree( /* TUNING: Cost of full table scan is (N*3.0). */ pNew->rRun = rSize + 16; ApplyCostMultiplier(pNew->rRun, pTab->costMult); - whereLoopOutputAdjust(pWC, pNew); + whereLoopOutputAdjust(pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; if( rc ) break; @@ -4784,7 +4800,7 @@ static int whereLoopAddBtree( pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); } ApplyCostMultiplier(pNew->rRun, pTab->costMult); - whereLoopOutputAdjust(pWC, pNew); + whereLoopOutputAdjust(pWC, pNew, rSize); rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; if( rc ) break; diff --git a/test/whereJ.test b/test/whereJ.test index 7c37321cbf..fbc2e57a3f 100644 --- a/test/whereJ.test +++ b/test/whereJ.test @@ -373,50 +373,271 @@ do_execsql_test whereJ-2.2 { ############################################################################ -ifcapable stat4 { - # Create and populate table. - do_execsql_test 3.1 { CREATE TABLE t1(a, b, c) } - for {set i 0} {$i < 32} {incr i 2} { - for {set x 0} {$x < 100} {incr x} { - execsql { INSERT INTO t1 VALUES($i, $x, $c) } - incr c - } - execsql { INSERT INTO t1 VALUES($i+1, 5, $c) } +# Create and populate table. +do_execsql_test 3.1 { CREATE TABLE t1(a, b, c) } +for {set i 0} {$i < 32} {incr i 2} { + for {set x 0} {$x < 100} {incr x} { + execsql { INSERT INTO t1 VALUES($i, $x, $c) } incr c } - - do_execsql_test 3.2 { - SELECT a, count(*) FROM t1 GROUP BY a HAVING a < 8; - } { - 0 100 1 1 2 100 3 1 4 100 5 1 6 100 7 1 - } - - do_execsql_test 3.3 { - CREATE INDEX idx_ab ON t1(a, b); - CREATE INDEX idx_c ON t1(c); - ANALYZE; - } {} - - # This one should use index "idx_c". - do_eqp_test 3.4 { - SELECT * FROM t1 WHERE - a = 4 AND b BETWEEN 20 AND 80 -- Matches 80 rows - AND - c BETWEEN 150 AND 160 -- Matches 10 rows - } { - 0 0 0 {SEARCH TABLE t1 USING INDEX idx_c (c>? AND c? AND b? AND c? AND b Date: Sat, 6 Sep 2014 02:00:41 +0000 Subject: [PATCH 302/710] Fix a couple of typos in comments. No changes to code. FossilOrigin-Name: a758465e3cfa7e0cb8749d097cd6fb5f86b60955 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 2 +- test/whereJ.test | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index cf1f0a6433..8921cbf2bc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Query\splanner\sheuristic\supdate:\nWhen\sdoing\sa\sfull\stable\sscan\son\sa\stable\sthat\shas\san\sequality\sconstraint\son\nan\sunindexed\scolumn,\sdo\snot\sallow\sthe\sestimated\snumber\sof\soutput\srows\sto\nbe\sgreater\sthan\shalf\sthe\stotal\snumber\sof\srows\sin\sthe\stable. -D 2014-09-06T01:35:57.122 +C Fix\sa\scouple\sof\stypos\sin\scomments.\s\sNo\schanges\sto\scode. +D 2014-09-06T02:00:41.865 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -298,7 +298,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 738213f25bc74c0a7cc5427712c1ac7537270a3d +F src/where.c 27af96780ded74196df231849b0584db5dc67104 F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1125,7 +1125,7 @@ F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test 69f5ec4b15760a8c860f80e2d55525669390aab3 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test 1d89199697919d4930be05a71e7fe620f114e622 -F test/whereJ.test 1c35169106be343b7aa1a2c1338322462f27901a +F test/whereJ.test 63599653dfefe4e74ebb358db753417fe0aa8a49 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 733119067757814609a9cea6b975818607bee4e3 -R bfd7d8b21e261616a30a072bf666f263 -U drh -Z 6641f77f6592cd795a1a1b9c2011a8f5 +P 73954f93c4c6f880c6e01d0d130e3fed40fd4106 +R fbb4555bc7385b94054e7551d7f50581 +U mistachkin +Z 680a54f7aef088a3bc667976a8b39ee7 diff --git a/manifest.uuid b/manifest.uuid index a84ea01b2d..08d7f777bc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -73954f93c4c6f880c6e01d0d130e3fed40fd4106 \ No newline at end of file +a758465e3cfa7e0cb8749d097cd6fb5f86b60955 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 49b16b345d..921baa29eb 100644 --- a/src/where.c +++ b/src/where.c @@ -4229,7 +4229,7 @@ static void whereLoopOutputAdjust( WhereTerm *pTerm, *pX; Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); int i, j; - int nEq = 0; /* Number of = constraints not within likely()/unlike() */ + int nEq = 0; /* Number of = constraints not within likely()/unlikely() */ for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; diff --git a/test/whereJ.test b/test/whereJ.test index fbc2e57a3f..8431c3a4b0 100644 --- a/test/whereJ.test +++ b/test/whereJ.test @@ -418,7 +418,7 @@ do_eqp_test 3.5 { ########################################################################################### -# Reset the database and setup for a test case derived from an actual SQLite users +# Reset the database and setup for a test case derived from actual SQLite users # db close sqlite3 db test.db @@ -618,7 +618,7 @@ do_execsql_test 4.2 { ANALYZE sqlite_master; } {} -# The following query should do a ful ltable scan of cx in the outer loop. +# The following query should do a full table scan of cx in the outer loop. # It is not correct to search table px using indx p_pt in the outer loop # with cx in the middle loop. Test case from Bloomberg on 2014-09-05. # From 44ee1dc84393aaae0cf82f741eab77a7a1f11668 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Sep 2014 03:16:28 +0000 Subject: [PATCH 303/710] Add the sqlite3_memdebug_title_count global variable, used during debugging to count the number of invocations of test_memdebug_settitle. By examining this variable in the debugger after a segfault, one can then set a breakpoint on test_memdebug_settitle that will fire just before the problem. FossilOrigin-Name: 27e3ca3e0f1da54e3527704a8601bb9003b086bf --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test_malloc.c | 7 +++++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8921cbf2bc..2916276efe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sof\stypos\sin\scomments.\s\sNo\schanges\sto\scode. -D 2014-09-06T02:00:41.865 +C Add\sthe\ssqlite3_memdebug_title_count\sglobal\svariable,\sused\sduring\sdebugging\nto\scount\sthe\snumber\sof\sinvocations\sof\stest_memdebug_settitle.\s\sBy\sexamining\nthis\svariable\sin\sthe\sdebugger\safter\sa\ssegfault,\sone\scan\sthen\sset\sa\sbreakpoint\non\stest_memdebug_settitle\sthat\swill\sfire\sjust\sbefore\sthe\sproblem. +D 2014-09-06T03:16:28.654 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -257,7 +257,7 @@ F src/test_intarray.c db4614c2262a06abc4409dc048d59c580c38320f F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 -F src/test_malloc.c 5368fb1de77246da1ae0ff59cba0d30cb0e5812f +F src/test_malloc.c fdac9732bd5ad6ae21d6b36cc265cca467caefb1 F src/test_multiplex.c ca90057438b63bf0840ebb84d0ef050624519a76 F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 73954f93c4c6f880c6e01d0d130e3fed40fd4106 -R fbb4555bc7385b94054e7551d7f50581 -U mistachkin -Z 680a54f7aef088a3bc667976a8b39ee7 +P a758465e3cfa7e0cb8749d097cd6fb5f86b60955 +R 2db0351830e46d232702315b1a375c8e +U drh +Z b61df9da0225ca7d80ceab8d31d99aad diff --git a/manifest.uuid b/manifest.uuid index 08d7f777bc..48a730b1f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a758465e3cfa7e0cb8749d097cd6fb5f86b60955 \ No newline at end of file +27e3ca3e0f1da54e3527704a8601bb9003b086bf \ No newline at end of file diff --git a/src/test_malloc.c b/src/test_malloc.c index 900a8ac40c..b937cea7cf 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -696,6 +696,12 @@ static int test_memdebug_pending( return TCL_OK; } +/* +** The following global variable keeps track of the number of tests +** that have run. This variable is only useful when running in the +** debugger. +*/ +static int sqlite3_memdebug_title_count = 0; /* ** Usage: sqlite3_memdebug_settitle TITLE @@ -713,6 +719,7 @@ static int test_memdebug_settitle( int objc, Tcl_Obj *CONST objv[] ){ + sqlite3_memdebug_title_count++; if( objc!=2 ){ Tcl_WrongNumArgs(interp, 1, objv, "TITLE"); return TCL_ERROR; From 60da72741ab642bd590480a397dead6a581000d3 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Sep 2014 03:38:51 +0000 Subject: [PATCH 304/710] Do not require a page-size change if the attempt to change the page size failed due to an OOM error. FossilOrigin-Name: 4d4fb197dc438a486cf5d967cf435f1132902c63 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2916276efe..166446fe6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3_memdebug_title_count\sglobal\svariable,\sused\sduring\sdebugging\nto\scount\sthe\snumber\sof\sinvocations\sof\stest_memdebug_settitle.\s\sBy\sexamining\nthis\svariable\sin\sthe\sdebugger\safter\sa\ssegfault,\sone\scan\sthen\sset\sa\sbreakpoint\non\stest_memdebug_settitle\sthat\swill\sfire\sjust\sbefore\sthe\sproblem. -D 2014-09-06T03:16:28.654 +C Do\snot\srequire\sa\spage-size\schange\sif\sthe\sattempt\sto\schange\sthe\spage\ssize\nfailed\sdue\sto\san\sOOM\serror. +D 2014-09-06T03:38:51.792 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -211,7 +211,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 8525ca79457c5b4673a5fda2774ee39fe155f40f F src/os_win.c 3c9f7df710cb6c757b04b78bf3d98f03830e67b9 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 3e732d2bbdd8d8d95fed0c5ae7e718d73153c4c5 +F src/pager.c a5e1a498696c96799a84b1c0c7dcd8c6dda80620 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 3b3791297e8977002e56b4a9b8916f2039abad9b @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a758465e3cfa7e0cb8749d097cd6fb5f86b60955 -R 2db0351830e46d232702315b1a375c8e +P 27e3ca3e0f1da54e3527704a8601bb9003b086bf +R 3c6eecd4803891788e881a3031c570ff U drh -Z b61df9da0225ca7d80ceab8d31d99aad +Z b558ffb0132cb5c0f9f09b23281337a0 diff --git a/manifest.uuid b/manifest.uuid index 48a730b1f6..29e9f91571 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27e3ca3e0f1da54e3527704a8601bb9003b086bf \ No newline at end of file +4d4fb197dc438a486cf5d967cf435f1132902c63 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 3ef54d98e0..681968e3cd 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3618,12 +3618,14 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ if( rc==SQLITE_OK ){ pager_reset(pPager); - pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); - pPager->pageSize = pageSize; sqlite3PageFree(pPager->pTmpSpace); pPager->pTmpSpace = pNew; rc = sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); } + if( rc==SQLITE_OK ){ + pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); + pPager->pageSize = pageSize; + } } *pPageSize = pPager->pageSize; From 60ec914c74ec79e4b73c004d1c21330be7916c99 Mon Sep 17 00:00:00 2001 From: "peter.d.reid" Date: Sat, 6 Sep 2014 16:39:46 +0000 Subject: [PATCH 305/710] Fix typos in comments. No code changes. FossilOrigin-Name: e62aab5e9290503869e1f4d5e0fefd2b4dee0a69 --- manifest | 122 ++++++++++++++++++++++---------------------- manifest.uuid | 2 +- src/alter.c | 4 +- src/analyze.c | 6 +-- src/btmutex.c | 2 +- src/btree.c | 30 +++++------ src/btreeInt.h | 6 +-- src/build.c | 10 ++-- src/callback.c | 2 +- src/complete.c | 6 +-- src/date.c | 2 +- src/delete.c | 10 ++-- src/expr.c | 22 ++++---- src/fkey.c | 2 +- src/func.c | 4 +- src/global.c | 2 +- src/insert.c | 14 ++--- src/main.c | 6 +-- src/mem1.c | 2 +- src/mem5.c | 2 +- src/memjournal.c | 2 +- src/mutex.h | 2 +- src/os.h | 2 +- src/os_unix.c | 20 ++++---- src/os_win.c | 4 +- src/pager.c | 20 ++++---- src/pcache.c | 2 +- src/pcache1.c | 4 +- src/printf.c | 2 +- src/resolve.c | 2 +- src/rowset.c | 6 +-- src/select.c | 16 +++--- src/sqlite.h.in | 10 ++-- src/sqlite3ext.h | 4 +- src/sqliteInt.h | 8 +-- src/table.c | 2 +- src/tclsqlite.c | 2 +- src/test1.c | 4 +- src/test_intarray.c | 4 +- src/test_malloc.c | 4 +- src/test_schema.c | 2 +- src/tokenize.c | 2 +- src/trigger.c | 2 +- src/update.c | 10 ++-- src/util.c | 2 +- src/vacuum.c | 2 +- src/vdbe.c | 4 +- src/vdbeInt.h | 4 +- src/vdbeapi.c | 16 +++--- src/vdbeaux.c | 18 +++---- src/vdbemem.c | 8 +-- src/vdbesort.c | 2 +- src/vdbetrace.c | 2 +- src/wal.c | 12 ++--- src/walker.c | 2 +- src/where.c | 18 +++---- src/whereInt.h | 2 +- 57 files changed, 242 insertions(+), 242 deletions(-) diff --git a/manifest b/manifest index 166446fe6b..bfce404357 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\srequire\sa\spage-size\schange\sif\sthe\sattempt\sto\schange\sthe\spage\ssize\nfailed\sdue\sto\san\sOOM\serror. -D 2014-09-06T03:38:51.792 +C Fix\stypos\sin\scomments.\sNo\scode\schanges. +D 2014-09-06T16:39:46.963 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -161,79 +161,79 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a -F src/alter.c b00900877f766f116f9e16116f1ccacdc21d82f1 -F src/analyze.c f00f06e6ef66c61b41f154889fe7caf5ed55a0ce +F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb +F src/analyze.c 79383a54fee3b7f1fb03dd4c8c8115583f506de5 F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb -F src/btmutex.c ec9d3f1295dafeb278c3830211cc5584132468f4 -F src/btree.c 2a483a8045118faa99867a8679da42754b532318 +F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 +F src/btree.c b1c1cd1cc3ae2e433a23b9a6c9ab53805707d8cd F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 -F src/btreeInt.h cf180d86b2e9e418f638d65baa425c4c69c0e0e3 -F src/build.c c26b233dcdb1e2c8f468d49236c266f9f3de96d8 -F src/callback.c b97d0695ffcf6a8710ee445ffe56ee387d4d8a6f -F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac +F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc +F src/build.c 8cb237719c185eec7bd8449b2e747491ded11932 +F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 +F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a -F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 -F src/delete.c 5adcd322c6b08fc25d215d780ca62cebce66304d -F src/expr.c e1691ab0fe6be7247ef073b0038fb8ecd9944fad +F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 +F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f +F src/expr.c 441a7e24e2f7bea9475778fa8acce9e8a69ca8f0 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c 8d81a780ad78d16ec9082585758a8f1d6bf02ca3 -F src/func.c bbb724b74ed96ca42675a7274646a71dd52bcda7 -F src/global.c 1e4bd956dc2f608f87d2a929abc4a20db65f30e4 +F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 +F src/func.c 0517037766e18eff7dce298e6b3a8e6311df75ec +F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c d1a104e67b33314d4cc5c1356147446086ab9fc8 +F src/insert.c 0b073fade178d9dbd990bbb32b4438e50b884a06 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab -F src/main.c d2ef03a45552e11813c68326d5edfda992e319d4 +F src/main.c e48517e3da289d93ad86e8b7b4f68078df5e6e51 F src/malloc.c 954de5f998c23237e04474a3f2159bf483bba65a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 -F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b +F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 -F src/mem5.c 74670012946c4adc8a6ad84d03acc80959c3e529 -F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 +F src/mem5.c 61eeb90134f9a5be6c2e68d8daae7628b25953fb +F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c -F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea +F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 F src/mutex_w32.c 06bfff9a3a83b53389a51a967643db3967032e1e F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace -F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e +F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 8525ca79457c5b4673a5fda2774ee39fe155f40f -F src/os_win.c 3c9f7df710cb6c757b04b78bf3d98f03830e67b9 +F src/os_unix.c 576f95b5b02397044ebbcdc1027528ba1bb2d92e +F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c a5e1a498696c96799a84b1c0c7dcd8c6dda80620 +F src/pager.c 31da9594ad4c3b5851bb6fe1a95c33835ab7ddce F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 -F src/pcache.c 3b3791297e8977002e56b4a9b8916f2039abad9b +F src/pcache.c 2048affdb09a04478b5fc6e64cb1083078d369be F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a -F src/pcache1.c c5af6403a55178c9d1c09e4f77b0f9c88822762c +F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa F src/pragma.c 14bcdb504128a476cce5bbc086d5226c5e46c225 F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d -F src/printf.c 00986c86ddfffefc2fd3c73667ff51b3b9709c74 +F src/printf.c e74925089a85e3c9f0e315595f41c139d3d118c2 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c 0ea356d32a5e884add23d1b9b4e8736681dd5697 -F src/rowset.c a9c9aae3234b44a6d7c6f5a3cadf90dce1e627be -F src/select.c 89e569b263535662f54b537eb9118b2c554ae7aa +F src/resolve.c 0d1621e45fffe4b4396477cf46e41a84b0145ffb +F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e +F src/select.c b4457526cee73c0b69fad42f799f619b1d5a8a8a F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50 -F src/sqlite.h.in 43852c8b68b4c579948cb37182918078836c5c06 +F src/sqlite.h.in 64a77f2822f1325b12050972003184f99b655a0f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad -F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h dbc9a4984a7e40f4334f8aefd3505f5aef4650a8 +F src/sqlite3ext.h 1f40357fb9b12a80c5a3b2b109fd249b009213d4 +F src/sqliteInt.h 4d1b0488f097aa7be346176c4d5e3cc1e25d99da F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 -F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c 7d100e2e7aad614bb3d7026a41a0e3827dbaaebc -F src/test1.c 363a5089230a92cf0aaa7a2945da7f2bf3b0a8d3 +F src/table.c 4e28a53e66bad8d014a510ef0205f5497c712b08 +F src/tclsqlite.c 8d6d6833c0053f0b3b1aeb1c5c7a7eeff0ad4d3f +F src/test1.c 22bfe1ce9f2f3746d682093a475ec0a33e0e55d8 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -253,11 +253,11 @@ F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f F src/test_func.c d3013ce36f19ac72a99c73864930fd1fa41832f8 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 -F src/test_intarray.c db4614c2262a06abc4409dc048d59c580c38320f +F src/test_intarray.c 6c610a21ab8edde85a3a2c7f2b069244ecf4d834 F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 -F src/test_malloc.c fdac9732bd5ad6ae21d6b36cc265cca467caefb1 +F src/test_malloc.c ba34143f941a9d74b30bbffc8818389bb73a1ca2 F src/test_multiplex.c ca90057438b63bf0840ebb84d0ef050624519a76 F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f @@ -267,7 +267,7 @@ F src/test_pcache.c a5cd24730cb43c5b18629043314548c9169abb00 F src/test_quota.c 65f6348fec0f2b3020c907247fb47556b214abb9 F src/test_quota.h 2a8ad1952d1d2ca9af0ce0465e56e6c023b5e15d F src/test_rtree.c fdd8d29ca5165c7857987a2ba263fac5c69e231f -F src/test_schema.c cd12a2223c3a394f4d07bb93bdf6d344c5c121b6 +F src/test_schema.c 2bdba21b82f601da69793e1f1d11bf481a79b091 F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c c1c1bbedbcaf82b93d83e4f9dd990e62476a680e F src/test_stat.c 9898687a6c2beca733b0dd6fe19163d987826d31 @@ -279,27 +279,27 @@ F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 22dded4283dc4b25422f6444cdcb8d6b1ea0b5ff -F src/tokenize.c ae45399d6252b4d736af43bee1576ce7bff86aec -F src/trigger.c 4bddd12803275aa98f1c7ce0118fceb02b2167f6 -F src/update.c ea336ce7b8b3fc5e316ba8f082e6445babf81059 +F src/tokenize.c 722872c816887fd66931333c59570ebd9622a95f +F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f +F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 -F src/util.c 068dcd26354a3898ccc64ad5c4bdb95a7a15d33a -F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 90db7ad740b6d3f7ab446e6244dbc17ce495cca6 +F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 +F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a +F src/vdbe.c 9a45dcbcc967fc0cb9248c75ba245d1d47de3e78 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 -F src/vdbeInt.h cdc8e421f85beb1ac9b4669ec5beadab6faa15e0 -F src/vdbeapi.c 09677a53dd8c71bcd670b0bd073bb9aefa02b441 -F src/vdbeaux.c cef5d34a64ae3a65b56d96d3fd663246ec8e1c36 +F src/vdbeInt.h b4843c35c3ba533b69d4250f550b5bacf2fb013d +F src/vdbeapi.c e43a6b98f853d8064cc096e349ce47e63d4c72d2 +F src/vdbeaux.c 91fd1e0c54a765838dc61fcf79f31acce035ce38 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 921d5468a68ac06f369810992e84ca22cc730a62 -F src/vdbesort.c 02646a9f86421776ae5d7594f620f9ed669d3698 -F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 +F src/vdbemem.c dc36ea9fe26c25550c50085f388167086ef7d73a +F src/vdbesort.c ab39574ec6e0c6213bd2a5c09cca9f9f8ba98450 +F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f -F src/wal.c 264df50a1b33124130b23180ded2e2c5663c652a +F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 -F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 27af96780ded74196df231849b0584db5dc67104 -F src/whereInt.h 923820bee9726033a501a08d2fc69b9c1ee4feb3 +F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 +F src/where.c 839b5e1db2507e221ad1c308f148a8519ed750be +F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 27e3ca3e0f1da54e3527704a8601bb9003b086bf -R 3c6eecd4803891788e881a3031c570ff -U drh -Z b558ffb0132cb5c0f9f09b23281337a0 +P 4d4fb197dc438a486cf5d967cf435f1132902c63 +R db0dce11ff1f08f3ef74aeb9c5ce2829 +U peter.d.reid +Z 050bcae7bdbaf9a55cc282a84c1e1741 diff --git a/manifest.uuid b/manifest.uuid index 29e9f91571..3f8f17bbce 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d4fb197dc438a486cf5d967cf435f1132902c63 \ No newline at end of file +e62aab5e9290503869e1f4d5e0fefd2b4dee0a69 \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index 64204b7b24..dd060248b8 100644 --- a/src/alter.c +++ b/src/alter.c @@ -174,8 +174,8 @@ static void renameTriggerFunc( UNUSED_PARAMETER(NotUsed); /* The principle used to locate the table name in the CREATE TRIGGER - ** statement is that the table name is the first token that is immediatedly - ** preceded by either TK_ON or TK_DOT and immediatedly followed by one + ** statement is that the table name is the first token that is immediately + ** preceded by either TK_ON or TK_DOT and immediately followed by one ** of TK_WHEN, TK_BEGIN or TK_FOR. */ if( zSql ){ diff --git a/src/analyze.c b/src/analyze.c index 9920c32a80..d5a11a3ed6 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -35,7 +35,7 @@ ** not possible to enable both STAT3 and STAT4 at the same time. If they ** are both enabled, then STAT4 takes precedence. ** -** For most applications, sqlite_stat1 provides all the statisics required +** For most applications, sqlite_stat1 provides all the statistics required ** for the query planner to make good choices. ** ** Format of sqlite_stat1: @@ -387,7 +387,7 @@ static void stat4Destructor(void *pOld){ ** original WITHOUT ROWID table as N==K as a special case. ** ** This routine allocates the Stat4Accum object in heap memory. The return -** value is a pointer to the the Stat4Accum object. The datatype of the +** value is a pointer to the Stat4Accum object. The datatype of the ** return value is BLOB, but it is really just a pointer to the Stat4Accum ** object. */ @@ -1583,7 +1583,7 @@ static void initAvgEq(Index *pIdx){ /* Set nSum to the number of distinct (iCol+1) field prefixes that ** occur in the stat4 table for this index before pFinal. Set ** sumEq to the sum of the nEq values for column iCol for the same - ** set (adding the value only once where there exist dupicate + ** set (adding the value only once where there exist duplicate ** prefixes). */ for(i=0; i<(pIdx->nSample-1); i++){ if( aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] ){ diff --git a/src/btmutex.c b/src/btmutex.c index 9672687fe0..f9fe5b3dde 100644 --- a/src/btmutex.c +++ b/src/btmutex.c @@ -106,7 +106,7 @@ static void SQLITE_NOINLINE btreeLockCarefully(Btree *p){ Btree *pLater; /* In most cases, we should be able to acquire the lock we - ** want without having to go throught the ascending lock + ** want without having to go through the ascending lock ** procedure that follows. Just be sure not to block. */ if( sqlite3_mutex_try(p->pBt->mutex)==SQLITE_OK ){ diff --git a/src/btree.c b/src/btree.c index a04302225d..22880f8a54 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file implements a external (disk-based) database using BTrees. +** This file implements an external (disk-based) database using BTrees. ** See the header comment on "btreeInt.h" for additional information. ** Including a description of file format and an overview of operation. */ @@ -1144,7 +1144,7 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ */ static int defragmentPage(MemPage *pPage){ int i; /* Loop counter */ - int pc; /* Address of a i-th cell */ + int pc; /* Address of the i-th cell */ int hdr; /* Offset to the page header */ int size; /* Size of a cell */ int usableSize; /* Number of usable bytes on a page */ @@ -2601,7 +2601,7 @@ page1_init_failed: ** false then all cursors are counted. ** ** For the purposes of this routine, a cursor is any cursor that -** is capable of reading or writing to the databse. Cursors that +** is capable of reading or writing to the database. Cursors that ** have been tripped into the CURSOR_FAULT state are not counted. */ static int countValidCursors(BtShared *pBt, int wrOnly){ @@ -3065,7 +3065,7 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); ** calling this function again), return SQLITE_DONE. Or, if an error ** occurs, return some other error code. ** -** More specificly, this function attempts to re-organize the database so +** More specifically, this function attempts to re-organize the database so ** that the last page of the file currently in use is no longer in use. ** ** Parameter nFin is the number of pages that this database would contain @@ -3073,7 +3073,7 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); ** ** If the bCommit parameter is non-zero, this function assumes that the ** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE -** or an error. bCommit is passed true for an auto-vacuum-on-commmit +** or an error. bCommit is passed true for an auto-vacuum-on-commit ** operation, or false for an incremental vacuum. */ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ @@ -3540,7 +3540,7 @@ int sqlite3BtreeRollback(Btree *p, int tripCode){ } /* -** Start a statement subtransaction. The subtransaction can can be rolled +** Start a statement subtransaction. The subtransaction can be rolled ** back independently of the main transaction. You must start a transaction ** before starting a subtransaction. The subtransaction is ended automatically ** if the main transaction commits or rolls back. @@ -3774,7 +3774,7 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){ ** compiler to crash when getCellInfo() is implemented as a macro. ** But there is a measureable speed advantage to using the macro on gcc ** (when less compiler optimizations like -Os or -O0 are used and the -** compiler is not doing agressive inlining.) So we use a real function +** compiler is not doing aggressive inlining.) So we use a real function ** for MSVC and a macro for everything else. Ticket #2457. */ #ifndef NDEBUG @@ -3991,7 +3991,7 @@ static int copyPayload( ** ** If the current cursor entry uses one or more overflow pages and the ** eOp argument is not 2, this function may allocate space for and lazily -** popluates the overflow page-list cache array (BtCursor.aOverflow). +** populates the overflow page-list cache array (BtCursor.aOverflow). ** Subsequent calls use this cache to make seeking to the supplied offset ** more efficient. ** @@ -4193,7 +4193,7 @@ static int accessPayload( /* ** Read part of the key associated with cursor pCur. Exactly -** "amt" bytes will be transfered into pBuf[]. The transfer +** "amt" bytes will be transferred into pBuf[]. The transfer ** begins at "offset". ** ** The caller must ensure that pCur is pointing to a valid row @@ -5889,7 +5889,7 @@ static void insertCell( ** The cells are guaranteed to fit on the page. */ static void assemblePage( - MemPage *pPage, /* The page to be assemblied */ + MemPage *pPage, /* The page to be assembled */ int nCell, /* The number of cells to add to this page */ u8 **apCell, /* Pointers to cell bodies */ u16 *aSize /* Sizes of the cells */ @@ -6555,7 +6555,7 @@ static int balance_nonroot( } /* - ** Put the new pages in accending order. This helps to + ** Put the new pages in ascending order. This helps to ** keep entries in the disk file in order so that a scan ** of the table is a linear scan through the file. That ** in turn helps the operating system to deliver pages @@ -6950,7 +6950,7 @@ static int balance(BtCursor *pCur){ /* Call balance_quick() to create a new sibling of pPage on which ** to store the overflow cell. balance_quick() inserts a new cell ** into pParent, which may cause pParent overflow. If this - ** happens, the next interation of the do-loop will balance pParent + ** happens, the next iteration of the do-loop will balance pParent ** use either balance_nonroot() or balance_deeper(). Until this ** happens, the overflow cell is stored in the aBalanceQuickSpace[] ** buffer. @@ -7027,7 +7027,7 @@ static int balance(BtCursor *pCur){ ** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already ** been performed. seekResult is the search result returned (a negative ** number if pCur points at an entry that is smaller than (pKey, nKey), or -** a positive value if pCur points at an etry that is larger than +** a positive value if pCur points at an entry that is larger than ** (pKey, nKey)). ** ** If the seekResult parameter is non-zero, then the caller guarantees that @@ -7184,7 +7184,7 @@ end_insert: /* ** Delete the entry that the cursor is pointing to. The cursor -** is left pointing at a arbitrary location. +** is left pointing at an arbitrary location. */ int sqlite3BtreeDelete(BtCursor *pCur){ Btree *p = pCur->pBtree; @@ -7882,7 +7882,7 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ /* ** Add 1 to the reference count for page iPage. If this is the second ** reference to the page, add an error message to pCheck->zErrMsg. -** Return 1 if there are 2 ore more references to the page and 0 if +** Return 1 if there are 2 or more references to the page and 0 if ** if this is the first reference to the page. ** ** Also check that the page number is in bounds. diff --git a/src/btreeInt.h b/src/btreeInt.h index d1cdd46983..fbfe47f6bc 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file implements a external (disk-based) database using BTrees. +** This file implements an external (disk-based) database using BTrees. ** For a detailed discussion of BTrees, refer to ** ** Donald E. Knuth, THE ART OF COMPUTER PROGRAMMING, Volume 3: @@ -135,7 +135,7 @@ ** ** The flags define the format of this btree page. The leaf flag means that ** this page has no children. The zerodata flag means that this page carries -** only keys and no data. The intkey flag means that the key is a integer +** only keys and no data. The intkey flag means that the key is an integer ** which is stored in the key size entry of the cell header rather than in ** the payload area. ** @@ -544,7 +544,7 @@ struct BtCursor { ** seek the cursor to the saved position. ** ** CURSOR_FAULT: -** A unrecoverable error (an I/O error or a malloc failure) has occurred +** An unrecoverable error (an I/O error or a malloc failure) has occurred ** on a different connection that shares the BtShared cache with this ** cursor. The error has left the cache in an inconsistent state. ** Do nothing else with this cursor. Any attempt to use the cursor diff --git a/src/build.c b/src/build.c index 0921d6d25e..6d54befbcc 100644 --- a/src/build.c +++ b/src/build.c @@ -1619,7 +1619,7 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){ ** no rowid btree for a WITHOUT ROWID. Instead, the canonical ** data storage is a covering index btree. ** (2) Bypass the creation of the sqlite_master table entry -** for the PRIMARY KEY as the the primary key index is now +** for the PRIMARY KEY as the primary key index is now ** identified by the sqlite_master table entry of the table itself. ** (3) Set the Index.tnum of the PRIMARY KEY Index object in the ** schema to the rootpage from the main table. @@ -1640,7 +1640,7 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ Vdbe *v = pParse->pVdbe; /* Convert the OP_CreateTable opcode that would normally create the - ** root-page for the table into a OP_CreateIndex opcode. The index + ** root-page for the table into an OP_CreateIndex opcode. The index ** created will become the PRIMARY KEY index. */ if( pParse->addrCrTab ){ @@ -2654,7 +2654,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int iPartIdxLabel; /* Jump to this label to skip a row */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ - int regRecord; /* Register holding assemblied index record */ + int regRecord; /* Register holding assembled index record */ sqlite3 *db = pParse->db; /* The database connection */ int iDb = sqlite3SchemaToIndex(db, pIndex->pSchema); @@ -3254,7 +3254,7 @@ exit_create_index: ** Fill the Index.aiRowEst[] array with default information - information ** to be used when we have not run the ANALYZE command. ** -** aiRowEst[0] is suppose to contain the number of elements in the index. +** aiRowEst[0] is supposed to contain the number of elements in the index. ** Since we do not know, guess 1 million. aiRowEst[1] is an estimate of the ** number of rows in the table that match any particular value of the ** first column of the index. aiRowEst[2] is an estimate of the number @@ -3633,7 +3633,7 @@ void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ ** if this is the first term of the FROM clause. pTable and pDatabase ** are the name of the table and database named in the FROM clause term. ** pDatabase is NULL if the database name qualifier is missing - the -** usual case. If the term has a alias, then pAlias points to the +** usual case. If the term has an alias, then pAlias points to the ** alias token. If the term is a subquery, then pSubquery is the ** SELECT statement that the subquery encodes. The pTable and ** pDatabase parameters are NULL for subqueries. The pOn and pUsing diff --git a/src/callback.c b/src/callback.c index 63090899fb..cd213b4b28 100644 --- a/src/callback.c +++ b/src/callback.c @@ -142,7 +142,7 @@ int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ ** ** Each pointer stored in the sqlite3.aCollSeq hash table contains an ** array of three CollSeq structures. The first is the collation sequence -** prefferred for UTF-8, the second UTF-16le, and the third UTF-16be. +** preferred for UTF-8, the second UTF-16le, and the third UTF-16be. ** ** Stored immediately after the three collation sequences is a copy of ** the collation sequence name. A pointer to this string is stored in diff --git a/src/complete.c b/src/complete.c index 9e9140085c..6ab6f4a042 100644 --- a/src/complete.c +++ b/src/complete.c @@ -70,7 +70,7 @@ extern const char sqlite3IsEbcdicIdChar[]; ** a statement. ** ** (4) CREATE The keyword CREATE has been seen at the beginning of a -** statement, possibly preceeded by EXPLAIN and/or followed by +** statement, possibly preceded by EXPLAIN and/or followed by ** TEMP or TEMPORARY ** ** (5) TRIGGER We are in the middle of a trigger definition that must be @@ -80,7 +80,7 @@ extern const char sqlite3IsEbcdicIdChar[]; ** the end of a trigger definition. ** ** (7) END We've seen the ";END" of the ";END;" that occurs at the end -** of a trigger difinition. +** of a trigger definition. ** ** Transitions between states above are determined by tokens extracted ** from the input. The following tokens are significant: @@ -123,7 +123,7 @@ int sqlite3_complete(const char *zSql){ }; #else /* If triggers are not supported by this compile then the statement machine - ** used to detect the end of a statement is much simplier + ** used to detect the end of a statement is much simpler */ static const u8 trans[3][3] = { /* Token: */ diff --git a/src/date.c b/src/date.c index f8f4ee0a6b..11b04ea004 100644 --- a/src/date.c +++ b/src/date.c @@ -24,7 +24,7 @@ ** 1970-01-01 00:00:00 is JD 2440587.5 ** 2000-01-01 00:00:00 is JD 2451544.5 ** -** This implemention requires years to be expressed as a 4-digit number +** This implementation requires years to be expressed as a 4-digit number ** which means that only dates between 0000-01-01 and 9999-12-31 can ** be represented, even though julian day numbers allow a much wider ** range of dates. diff --git a/src/delete.c b/src/delete.c index af83903c42..b97407400b 100644 --- a/src/delete.c +++ b/src/delete.c @@ -90,7 +90,7 @@ void sqlite3MaterializeView( Parse *pParse, /* Parsing context */ Table *pView, /* View definition */ Expr *pWhere, /* Optional WHERE clause to be added */ - int iCur /* Cursor number for ephemerial table */ + int iCur /* Cursor number for ephemeral table */ ){ SelectDest dest; Select *pSel; @@ -248,7 +248,7 @@ void sqlite3DeleteFrom( int addrBypass = 0; /* Address of jump over the delete logic */ int addrLoop = 0; /* Top of the delete loop */ int addrDelete = 0; /* Jump directly to the delete logic */ - int addrEphOpen = 0; /* Instruction to open the Ephermeral table */ + int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ @@ -328,7 +328,7 @@ void sqlite3DeleteFrom( sqlite3BeginWriteOperation(pParse, 1, iDb); /* If we are trying to delete from a view, realize that view into - ** a ephemeral table. + ** an ephemeral table. */ #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ @@ -382,7 +382,7 @@ void sqlite3DeleteFrom( iRowSet = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); }else{ - /* For a WITHOUT ROWID table, create an ephermeral table used to + /* For a WITHOUT ROWID table, create an ephemeral table used to ** hold all primary keys for rows to be deleted. */ pPk = sqlite3PrimaryKeyIndex(pTab); assert( pPk!=0 ); @@ -557,7 +557,7 @@ delete_from_cleanup: return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file +** they may interfere with compilation of other functions in this file ** (or in another file, if this file becomes part of the amalgamation). */ #ifdef isView #undef isView diff --git a/src/expr.c b/src/expr.c index 1a2465f7ef..cd0983c579 100644 --- a/src/expr.c +++ b/src/expr.c @@ -22,7 +22,7 @@ ** affinity of that column is returned. Otherwise, 0x00 is returned, ** indicating no affinity for the expression. ** -** i.e. the WHERE clause expresssions in the following statements all +** i.e. the WHERE clause expressions in the following statements all ** have an affinity: ** ** CREATE TABLE t1(a); @@ -501,7 +501,7 @@ void sqlite3ExprAttachSubtrees( } /* -** Allocate a Expr node which joins as many as two subtrees. +** Allocate an Expr node which joins as many as two subtrees. ** ** One or both of the subtrees can be NULL. Return a pointer to the new ** Expr node. Or, if an OOM error occurs, set pParse->db->mallocFailed, @@ -611,7 +611,7 @@ Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ ** ** Wildcards of the form ":aaa", "@aaa", or "$aaa" are assigned the same number ** as the previous instance of the same wildcard. Or if this is the first -** instance of the wildcard, the next sequenial variable number is +** instance of the wildcard, the next sequential variable number is ** assigned. */ void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ @@ -746,7 +746,7 @@ static int exprStructSize(Expr *p){ ** During expression analysis, extra information is computed and moved into ** later parts of teh Expr object and that extra information might get chopped ** off if the expression is reduced. Note also that it does not work to -** make a EXPRDUP_REDUCE copy of a reduced expression. It is only legal +** make an EXPRDUP_REDUCE copy of a reduced expression. It is only legal ** to reduce a pristine expression tree from the parser. The implementation ** of dupedExprStructSize() contain multiple assert() statements that attempt ** to enforce this constraint. @@ -815,7 +815,7 @@ static int dupedExprSize(Expr *p, int flags){ ** is not NULL then *pzBuffer is assumed to point to a buffer large enough ** to store the copy of expression p, the copies of p->u.zToken ** (if applicable), and the copies of the p->pLeft and p->pRight expressions, -** if any. Before returning, *pzBuffer is set to the first byte passed the +** if any. Before returning, *pzBuffer is set to the first byte past the ** portion of the buffer copied into by this function. */ static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ @@ -1541,7 +1541,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** ** If the RHS of the IN operator is a list or a more complex subquery, then ** an ephemeral table might need to be generated from the RHS and then -** pX->iTable made to point to the ephermeral table instead of an +** pX->iTable made to point to the ephemeral table instead of an ** existing table. ** ** The inFlags parameter must contain exactly one of the bits @@ -1671,7 +1671,7 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ ** and IN_INDEX_NOOP is an allowed reply ** and the RHS of the IN operator is a list, not a subquery ** and the RHS is not contant or has two or fewer terms, - ** then it is not worth creating an ephermeral table to evaluate + ** then it is not worth creating an ephemeral table to evaluate ** the IN operator so return IN_INDEX_NOOP. */ if( eType==0 @@ -2758,7 +2758,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ } /* Attempt a direct implementation of the built-in COALESCE() and - ** IFNULL() functions. This avoids unnecessary evalation of + ** IFNULL() functions. This avoids unnecessary evaluation of ** arguments past the first non-NULL argument. */ if( pDef->funcFlags & SQLITE_FUNC_COALESCE ){ @@ -3197,7 +3197,7 @@ void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ } /* -** Generate code that evalutes the given expression and puts the result +** Generate code that evaluates the given expression and puts the result ** in register target. ** ** Also make a copy of the expression results into another "cache" register @@ -3552,7 +3552,7 @@ int sqlite3ExprCodeExprList( ** x>=y AND x<=z ** ** Code it as such, taking care to do the common subexpression -** elementation of x. +** elimination of x. */ static void exprCodeBetween( Parse *pParse, /* Parsing and code generating context */ @@ -4289,7 +4289,7 @@ int sqlite3GetTempReg(Parse *pParse){ ** purpose. ** ** If a register is currently being used by the column cache, then -** the dallocation is deferred until the column cache line that uses +** the deallocation is deferred until the column cache line that uses ** the register becomes stale. */ void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ diff --git a/src/fkey.c b/src/fkey.c index 415f35d2f8..e816bd95da 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -173,7 +173,7 @@ ** ** 4) No parent key columns were provided explicitly as part of the ** foreign key definition, and the PRIMARY KEY of the parent table -** consists of a a different number of columns to the child key in +** consists of a different number of columns to the child key in ** the child table. ** ** then non-zero is returned, and a "foreign key mismatch" error loaded diff --git a/src/func.c b/src/func.c index dbc8375541..e338ab842b 100644 --- a/src/func.c +++ b/src/func.c @@ -9,7 +9,7 @@ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains the C-language implementions for many of the SQL +** This file contains the C-language implementations for many of the SQL ** functions of SQLite. (Some function, and in particular the date and ** time functions, are implemented separately.) */ @@ -1638,7 +1638,7 @@ int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ } /* -** All all of the FuncDef structures in the aBuiltinFunc[] array above +** All of the FuncDef structures in the aBuiltinFunc[] array above ** to the global function hash table. This occurs at start-time (as ** a consequence of calling sqlite3_initialize()). ** diff --git a/src/global.c b/src/global.c index 2c14b58abd..22b990699b 100644 --- a/src/global.c +++ b/src/global.c @@ -10,7 +10,7 @@ ** ************************************************************************* ** -** This file contains definitions of global variables and contants. +** This file contains definitions of global variables and constants. */ #include "sqliteInt.h" diff --git a/src/insert.c b/src/insert.c index 3e6982d836..6a3ab8edae 100644 --- a/src/insert.c +++ b/src/insert.c @@ -410,7 +410,7 @@ static int xferOptimization( ** The 4th template is used if the insert statement takes its ** values from a SELECT but the data is being inserted into a table ** that is also read as part of the SELECT. In the third form, -** we have to use a intermediate table to store the results of +** we have to use an intermediate table to store the results of ** the select. The template is like this: ** ** X <- A @@ -575,7 +575,7 @@ void sqlite3Insert( regAutoinc = autoIncBegin(pParse, iDb, pTab); /* Allocate registers for holding the rowid of the new row, - ** the content of the new row, and the assemblied row record. + ** the content of the new row, and the assembled row record. */ regRowid = regIns = pParse->nMem+1; pParse->nMem += pTab->nCol + 1; @@ -1027,7 +1027,7 @@ insert_cleanup: } /* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file +** they may interfere with compilation of other functions in this file ** (or in another file, if this file becomes part of the amalgamation). */ #ifdef isView #undef isView @@ -1143,7 +1143,7 @@ void sqlite3GenerateConstraintChecks( int ix; /* Index loop counter */ int nCol; /* Number of columns */ int onError; /* Conflict resolution strategy */ - int j1; /* Addresss of jump instruction */ + int j1; /* Address of jump instruction */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ int ipkTop = 0; /* Top of the rowid change constraint check */ @@ -1547,7 +1547,7 @@ void sqlite3CompleteInsertion( Index *pIdx; /* An index being inserted or updated */ u8 pik_flags; /* flag values passed to the btree insert */ int regData; /* Content registers (after the rowid) */ - int regRec; /* Register holding assemblied record for the table */ + int regRec; /* Register holding assembled record for the table */ int i; /* Loop counter */ u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */ @@ -1672,7 +1672,7 @@ int sqlite3OpenTableAndIndices( ** The following global variable is incremented whenever the ** transfer optimization is used. This is used for testing ** purposes only - to make sure the transfer optimization really -** is happening when it is suppose to. +** is happening when it is supposed to. */ int sqlite3_xferopt_count; #endif /* SQLITE_TEST */ @@ -1739,7 +1739,7 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ ** INSERT INTO tab1 SELECT * FROM tab2; ** ** The xfer optimization transfers raw records from tab2 over to tab1. -** Columns are not decoded and reassemblied, which greatly improves +** Columns are not decoded and reassembled, which greatly improves ** performance. Raw index records are transferred in the same way. ** ** The xfer optimization is only attempted if tab1 and tab2 are compatible. diff --git a/src/main.c b/src/main.c index fd7151b174..977c786f38 100644 --- a/src/main.c +++ b/src/main.c @@ -2853,9 +2853,9 @@ int sqlite3_get_autocommit(sqlite3 *db){ } /* -** The following routines are subtitutes for constants SQLITE_CORRUPT, +** The following routines are substitutes for constants SQLITE_CORRUPT, ** SQLITE_MISUSE, SQLITE_CANTOPEN, SQLITE_IOERR and possibly other error -** constants. They server two purposes: +** constants. They serve two purposes: ** ** 1. Serve as a convenient place to set a breakpoint in a debugger ** to detect when version error conditions occurs. @@ -3169,7 +3169,7 @@ int sqlite3_test_control(int op, ...){ ** IMPORTANT: Changing the PENDING byte from 0x40000000 results in ** an incompatible database file format. Changing the PENDING byte ** while any database connection is open results in undefined and - ** dileterious behavior. + ** deleterious behavior. */ case SQLITE_TESTCTRL_PENDING_BYTE: { rc = PENDING_BYTE; diff --git a/src/mem1.c b/src/mem1.c index 6dbf1058ea..11fc1771ed 100644 --- a/src/mem1.c +++ b/src/mem1.c @@ -188,7 +188,7 @@ static int sqlite3MemSize(void *pPrior){ ** ** For this low-level interface, we know that pPrior!=0. Cases where ** pPrior==0 while have been intercepted by higher-level routine and -** redirected to xMalloc. Similarly, we know that nByte>0 becauses +** redirected to xMalloc. Similarly, we know that nByte>0 because ** cases where nByte<=0 will have been intercepted by higher-level ** routines and redirected to xFree. */ diff --git a/src/mem5.c b/src/mem5.c index 67615bb964..1479ddd0d0 100644 --- a/src/mem5.c +++ b/src/mem5.c @@ -28,7 +28,7 @@ ** 1. All memory allocations sizes are rounded up to a power of 2. ** ** 2. If two adjacent free blocks are the halves of a larger block, -** then the two blocks are coalesed into the single larger block. +** then the two blocks are coalesced into the single larger block. ** ** 3. New memory is allocated from the first available free block. ** diff --git a/src/memjournal.c b/src/memjournal.c index 65ed378b38..6452cecc37 100644 --- a/src/memjournal.c +++ b/src/memjournal.c @@ -26,7 +26,7 @@ typedef struct FileChunk FileChunk; ** ** The size chosen is a little less than a power of two. That way, ** the FileChunk object will have a size that almost exactly fills -** a power-of-two allocation. This mimimizes wasted space in power-of-two +** a power-of-two allocation. This minimizes wasted space in power-of-two ** memory allocators. */ #define JOURNAL_CHUNKSIZE ((int)(1024-sizeof(FileChunk*))) diff --git a/src/mutex.h b/src/mutex.h index 0978812252..03eb1faadb 100644 --- a/src/mutex.h +++ b/src/mutex.h @@ -25,7 +25,7 @@ ** Figure out what version of the code to use. The choices are ** ** SQLITE_MUTEX_OMIT No mutex logic. Not even stubs. The -** mutexes implemention cannot be overridden +** mutexes implementation cannot be overridden ** at start-time. ** ** SQLITE_MUTEX_NOOP For single-threaded applications. No diff --git a/src/os.h b/src/os.h index 3920a62ee3..2c1b86f913 100644 --- a/src/os.h +++ b/src/os.h @@ -120,7 +120,7 @@ ** shared locks begins at SHARED_FIRST. ** ** The same locking strategy and -** byte ranges are used for Unix. This leaves open the possiblity of having +** byte ranges are used for Unix. This leaves open the possibility of having ** clients on win95, winNT, and unix all talking to the same shared file ** and all locking correctly. To do so would require that samba (or whatever ** tool is being used for file sharing) implements locks correctly between diff --git a/src/os_unix.c b/src/os_unix.c index f63afc6bc5..c4f1d40d44 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -631,7 +631,7 @@ static int unixMutexHeld(void) { #if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) /* ** Helper function for printing out trace information from debugging -** binaries. This returns the string represetation of the supplied +** binaries. This returns the string representation of the supplied ** integer lock-type. */ static const char *azFileLock(int eFileLock){ @@ -3098,7 +3098,7 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ ** NB: If you define USE_PREAD or USE_PREAD64, then it might also ** be necessary to define _XOPEN_SOURCE to be 500. This varies from ** one system to another. Since SQLite does not define USE_PREAD -** any any form by default, we will not attempt to define _XOPEN_SOURCE. +** in any form by default, we will not attempt to define _XOPEN_SOURCE. ** See tickets #2741 and #2681. ** ** To avoid stomping the errno value on a failed read the lastErrno value @@ -3730,7 +3730,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } /* -** If *pArg is inititially negative then this is a query. Set *pArg to +** If *pArg is initially negative then this is a query. Set *pArg to ** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. ** ** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. @@ -3937,7 +3937,7 @@ static int unixSectorSize(sqlite3_file *id){ ** Return the device characteristics for the file. ** ** This VFS is set up to return SQLITE_IOCAP_POWERSAFE_OVERWRITE by default. -** However, that choice is contraversial since technically the underlying +** However, that choice is controversial since technically the underlying ** file system does not always provide powersafe overwrites. (In other ** words, after a power-loss event, parts of the file that were never ** written might end up being altered.) However, non-PSOW behavior is very, @@ -4909,7 +4909,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ ** looks at the filesystem type and tries to guess the best locking ** strategy from that. ** -** For finder-funtion F, two objects are created: +** For finder-function F, two objects are created: ** ** (1) The real finder-function named "FImpt()". ** @@ -5171,7 +5171,7 @@ static const sqlite3_io_methods #endif /* OS_VXWORKS && SQLITE_ENABLE_LOCKING_STYLE */ /* -** An abstract type for a pointer to a IO method finder function: +** An abstract type for a pointer to an IO method finder function: */ typedef const sqlite3_io_methods *(*finder_type)(const char*,unixFile*); @@ -5485,7 +5485,7 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** descriptor on the same path, fail, and return an error to SQLite. ** ** Even if a subsequent open() call does succeed, the consequences of - ** not searching for a resusable file descriptor are not dire. */ + ** not searching for a reusable file descriptor are not dire. */ if( 0==osStat(zPath, &sStat) ){ unixInodeInfo *pInode; @@ -5516,7 +5516,7 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** written to *pMode. If an IO error occurs, an SQLite error code is ** returned and the value of *pMode is not modified. ** -** In most cases cases, this routine sets *pMode to 0, which will become +** In most cases, this routine sets *pMode to 0, which will become ** an indication to robust_open() to create the file using ** SQLITE_DEFAULT_FILE_PERMISSIONS adjusted by the umask. ** But if the file being opened is a WAL or regular journal file, then @@ -6308,7 +6308,7 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** proxy path against the values stored in the conch. The conch file is ** stored in the same directory as the database file and the file name ** is patterned after the database file name as ".-conch". -** If the conch file does not exist, or it's contents do not match the +** If the conch file does not exist, or its contents do not match the ** host ID and/or proxy path, then the lock is escalated to an exclusive ** lock and the conch file contents is updated with the host ID and proxy ** path and the lock is downgraded to a shared lock again. If the conch @@ -6360,7 +6360,7 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** setting the environment variable SQLITE_FORCE_PROXY_LOCKING to 1 will ** force proxy locking to be used for every database file opened, and 0 ** will force automatic proxy locking to be disabled for all database -** files (explicity calling the SQLITE_SET_LOCKPROXYFILE pragma or +** files (explicitly calling the SQLITE_SET_LOCKPROXYFILE pragma or ** sqlite_file_control API is not affected by SQLITE_FORCE_PROXY_LOCKING). */ diff --git a/src/os_win.c b/src/os_win.c index 42d5106e5f..e12ce4e532 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3128,7 +3128,7 @@ static int winUnlock(sqlite3_file *id, int locktype){ } /* -** If *pArg is inititially negative then this is a query. Set *pArg to +** If *pArg is initially negative then this is a query. Set *pArg to ** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. ** ** If *pArg is 0 or 1, then clear or set the mask bit of pFile->ctrlFlags. @@ -4142,7 +4142,7 @@ static int winUnfetch(sqlite3_file *fd, i64 iOff, void *p){ }else{ /* FIXME: If Windows truly always prevents truncating or deleting a ** file while a mapping is held, then the following winUnmapfile() call - ** is unnecessary can can be omitted - potentially improving + ** is unnecessary can be omitted - potentially improving ** performance. */ winUnmapfile(pFd); } diff --git a/src/pager.c b/src/pager.c index 681968e3cd..8059bee055 100644 --- a/src/pager.c +++ b/src/pager.c @@ -76,12 +76,12 @@ ** Definition: Two databases (or the same database at two points it time) ** are said to be "logically equivalent" if they give the same answer to ** all queries. Note in particular the content of freelist leaf -** pages can be changed arbitarily without effecting the logical equivalence +** pages can be changed arbitrarily without affecting the logical equivalence ** of the database. ** ** (7) At any time, if any subset, including the empty set and the total set, ** of the unsynced changes to a rollback journal are removed and the -** journal is rolled back, the resulting database file will be logical +** journal is rolled back, the resulting database file will be logically ** equivalent to the database file at the beginning of the transaction. ** ** (8) When a transaction is rolled back, the xTruncate method of the VFS @@ -378,7 +378,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** The exception is when the database file is unlocked as the pager moves ** from ERROR to OPEN state. At this point there may be a hot-journal file -** in the file-system that needs to be rolled back (as part of a OPEN->SHARED +** in the file-system that needs to be rolled back (as part of an OPEN->SHARED ** transition, by the same pager or any other). If the call to xUnlock() ** fails at this point and the pager is left holding an EXCLUSIVE lock, this ** can confuse the call to xCheckReservedLock() call made later as part @@ -461,7 +461,7 @@ struct PagerSavepoint { #define SPILLFLAG_NOSYNC 0x04 /* Spill is ok, but do not sync */ /* -** A open page cache is an instance of struct Pager. A description of +** An open page cache is an instance of struct Pager. A description of ** some of the more important member variables follows: ** ** eState @@ -633,7 +633,7 @@ struct Pager { /************************************************************************** ** The following block contains those class members that change during - ** routine opertion. Class members not in this block are either fixed + ** routine operation. Class members not in this block are either fixed ** when the pager is first created or else only change when there is a ** significant mode change (such as changing the page_size, locking_mode, ** or the journal_mode). From another view, these class members describe @@ -2497,7 +2497,7 @@ delmaster_out: ** If the file on disk is currently larger than nPage pages, then use the VFS ** xTruncate() method to truncate it. ** -** Or, it might might be the case that the file on disk is smaller than +** Or, it might be the case that the file on disk is smaller than ** nPage pages. Some operating system implementations can get confused if ** you try to truncate a file to some size that is larger than it ** currently is, so detect this case and write a single zero byte to @@ -2556,7 +2556,7 @@ int sqlite3SectorSize(sqlite3_file *pFile){ /* ** Set the value of the Pager.sectorSize variable for the given ** pager based on the value returned by the xSectorSize method -** of the open database file. The sector size will be used used +** of the open database file. The sector size will be used ** to determine the size and alignment of journal header and ** master journal pointers within created journal files. ** @@ -3758,7 +3758,7 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ int rc; /* Return code */ /* Check that this is either a no-op (because the requested lock is - ** already held, or one of the transistions that the busy-handler + ** already held), or one of the transitions that the busy-handler ** may be invoked during, according to the comment above ** sqlite3PagerSetBusyhandler(). */ @@ -4386,7 +4386,7 @@ static int pagerStress(void *p, PgHdr *pPg){ ** a rollback or by user request, respectively. ** ** Spilling is also prohibited when in an error state since that could - ** lead to database corruption. In the current implementaton it + ** lead to database corruption. In the current implementation it ** is impossible for sqlite3PcacheFetch() to be called with createFlag==3 ** while in the error state, hence it is impossible for this routine to ** be called in the error state. Nevertheless, we include a NEVER() @@ -4926,7 +4926,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ *pExists = (first!=0); }else if( rc==SQLITE_CANTOPEN ){ /* If we cannot open the rollback journal file in order to see if - ** its has a zero header, that might be due to an I/O error, or + ** it has a zero header, that might be due to an I/O error, or ** it might be due to the race condition described above and in ** ticket #3883. Either way, assume that the journal is hot. ** This might be a false positive. But if it is, then the diff --git a/src/pcache.c b/src/pcache.c index eabfadd4b5..801df2b02a 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -390,7 +390,7 @@ PgHdr *sqlite3PcacheFetchFinish( /* ** Decrement the reference count on a page. If the page is clean and the -** reference count drops to 0, then it is made elible for recycling. +** reference count drops to 0, then it is made eligible for recycling. */ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ assert( p->nRef>0 ); diff --git a/src/pcache1.c b/src/pcache1.c index 82015befce..9d15e8514c 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -13,7 +13,7 @@ ** This file implements the default page cache implementation (the ** sqlite3_pcache interface). It also contains part of the implementation ** of the SQLITE_CONFIG_PAGECACHE and sqlite3_release_memory() features. -** If the default page cache implementation is overriden, then neither of +** If the default page cache implementation is overridden, then neither of ** these two features are available. */ @@ -25,7 +25,7 @@ typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; /* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set -** of one or more PCaches that are able to recycle each others unpinned +** of one or more PCaches that are able to recycle each other's unpinned ** pages when they are under memory pressure. A PGroup is an instance of ** the following object. ** diff --git a/src/printf.c b/src/printf.c index 72ace932ba..8e71ad8bc6 100644 --- a/src/printf.c +++ b/src/printf.c @@ -904,7 +904,7 @@ char *sqlite3MPrintf(sqlite3 *db, const char *zFormat, ...){ /* ** Like sqlite3MPrintf(), but call sqlite3DbFree() on zStr after formatting -** the string and before returnning. This routine is intended to be used +** the string and before returning. This routine is intended to be used ** to modify an existing string. For example: ** ** x = sqlite3MPrintf(db, x, "prefix %s suffix", x); diff --git a/src/resolve.c b/src/resolve.c index 935d311346..a77fd9d367 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -1118,7 +1118,7 @@ static int resolveOrderGroupBy( } /* -** Resolve names in the SELECT statement p and all of its descendents. +** Resolve names in the SELECT statement p and all of its descendants. */ static int resolveSelectStep(Walker *pWalker, Select *p){ NameContext *pOuterNC; /* Context that contains this SELECT */ diff --git a/src/rowset.c b/src/rowset.c index ba2e056bd4..ff5593892a 100644 --- a/src/rowset.c +++ b/src/rowset.c @@ -50,7 +50,7 @@ ** No INSERTs may occurs after a SMALLEST. An assertion will fail if ** that is attempted. ** -** The cost of an INSERT is roughly constant. (Sometime new memory +** The cost of an INSERT is roughly constant. (Sometimes new memory ** has to be allocated on an INSERT.) The cost of a TEST with a new ** batch number is O(NlogN) where N is the number of elements in the RowSet. ** The cost of a TEST using the same batch number is O(logN). The cost @@ -443,8 +443,8 @@ int sqlite3RowSetNext(RowSet *p, i64 *pRowid){ ** Check to see if element iRowid was inserted into the rowset as ** part of any insert batch prior to iBatch. Return 1 or 0. ** -** If this is the first test of a new batch and if there exist entires -** on pRowSet->pEntry, then sort those entires into the forest at +** If this is the first test of a new batch and if there exist entries +** on pRowSet->pEntry, then sort those entries into the forest at ** pRowSet->pForest so that they can be tested. */ int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 iRowid){ diff --git a/src/select.c b/src/select.c index 5508c2e694..31a70bff1b 100644 --- a/src/select.c +++ b/src/select.c @@ -1010,7 +1010,7 @@ int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } ** then the KeyInfo structure is appropriate for initializing a virtual ** index to implement a DISTINCT test. ** -** Space to hold the KeyInfo structure is obtain from malloc. The calling +** Space to hold the KeyInfo structure is obtained from malloc. The calling ** function is responsible for seeing that this structure is eventually ** freed. */ @@ -1541,7 +1541,7 @@ static void generateColumnNames( } /* -** Given a an expression list (which is really the list of expressions +** Given an expression list (which is really the list of expressions ** that form the result set of a SELECT statement) compute appropriate ** column names for a table that would hold the expression list. ** @@ -1614,7 +1614,7 @@ static int selectColumnsFromExprList( } /* Make sure the column name is unique. If the name is not unique, - ** append a integer to the name so that it becomes unique. + ** append an integer to the name so that it becomes unique. */ nName = sqlite3Strlen30(zName); for(j=cnt=0; j5 ** -** The code generated for this simpification gives the same result +** The code generated for this simplification gives the same result ** but only has to scan the data once. And because indices might ** exist on the table t1, a complete scan of the data might be ** avoided. @@ -3242,7 +3242,7 @@ static int flattenSubquery( pSubSrc = pSub->pSrc; assert( pSubSrc ); /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, - ** not arbitrary expresssions, we allowed some combining of LIMIT and OFFSET + ** not arbitrary expressions, we allowed some combining of LIMIT and OFFSET ** because they could be computed at compile-time. But when LIMIT and OFFSET ** became arbitrary expressions, we were forced to add restrictions (13) ** and (14). */ @@ -3628,7 +3628,7 @@ static u8 minMaxQuery(AggInfo *pAggInfo, ExprList **ppMinMax){ /* ** The select statement passed as the first argument is an aggregate query. -** The second argment is the associated aggregate-info object. This +** The second argument is the associated aggregate-info object. This ** function tests if the SELECT is of the form: ** ** SELECT count(*) FROM @@ -3958,10 +3958,10 @@ static void selectPopWith(Walker *pWalker, Select *p){ ** fill pTabList->a[].pSelect with a copy of the SELECT statement ** that implements the view. A copy is made of the view's SELECT ** statement so that we can freely modify or delete that statement -** without worrying about messing up the presistent representation +** without worrying about messing up the persistent representation ** of the view. ** -** (3) Add terms to the WHERE clause to accomodate the NATURAL keyword +** (3) Add terms to the WHERE clause to accommodate the NATURAL keyword ** on joins and the ON and USING clause of joins. ** ** (4) Scan the list of columns in the result set (pEList) looking diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d446d2ea98..56dede8ba8 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2094,7 +2094,7 @@ int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); ** turns off all busy handlers. ** ** ^(There can only be a single busy handler for a particular -** [database connection] any any given moment. If another busy handler +** [database connection] at any given moment. If another busy handler ** was defined (using [sqlite3_busy_handler()]) prior to calling ** this routine, that other busy handler is cleared.)^ ** @@ -4164,7 +4164,7 @@ SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), ** object results in undefined behavior. ** ** ^These routines work just like the corresponding [column access functions] -** except that these routines take a single [protected sqlite3_value] object +** except that these routines take a single [protected sqlite3_value] object ** pointer instead of a [sqlite3_stmt*] pointer and an integer column number. ** ** ^The sqlite3_value_text16() interface extracts a UTF-16 string @@ -6357,12 +6357,12 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** the current value is always zero.)^ ** ** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
    SQLITE_DBSTATUS_CACHE_USED
    -**
    This parameter returns the approximate number of of bytes of heap +**
    This parameter returns the approximate number of bytes of heap ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ** ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    -**
    This parameter returns the approximate number of of bytes of heap +**
    This parameter returns the approximate number of bytes of heap ** memory used to store the schema for all databases associated ** with the connection - main, temp, and any [ATTACH]-ed databases.)^ ** ^The full amount of memory used by the schemas is reported, even if the @@ -6371,7 +6371,7 @@ int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); ** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. ** ** [[SQLITE_DBSTATUS_STMT_USED]] ^(
    SQLITE_DBSTATUS_STMT_USED
    -**
    This parameter returns the approximate number of of bytes of heap +**
    This parameter returns the approximate number of bytes of heap ** and lookaside memory used by all prepared statements associated with ** the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index ecf93f62f6..b4baea2cc5 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -28,7 +28,7 @@ typedef struct sqlite3_api_routines sqlite3_api_routines; ** WARNING: In order to maintain backwards compatibility, add new ** interfaces to the end of this structure only. If you insert new ** interfaces in the middle of this structure, then older different -** versions of SQLite will not be able to load each others' shared +** versions of SQLite will not be able to load each other's shared ** libraries! */ struct sqlite3_api_routines { @@ -254,7 +254,7 @@ struct sqlite3_api_routines { /* ** The following macros redefine the API routines so that they are -** redirected throught the global sqlite3_api structure. +** redirected through the global sqlite3_api structure. ** ** This header file is also used by the loadext.c source file ** (part of the main SQLite library - not an extension) so that diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e56da4edd0..ceacdbb335 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -351,7 +351,7 @@ #endif /* -** Return true (non-zero) if the input is a integer that is too large +** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() ** macros to verify that we have tested SQLite for large-file support. */ @@ -639,7 +639,7 @@ extern const int sqlite3one; ** all alignment restrictions correct. ** ** Except, if SQLITE_4_BYTE_ALIGNED_MALLOC is defined, then the -** underlying malloc() implemention might return us 4-byte aligned +** underlying malloc() implementation might return us 4-byte aligned ** pointers. In that case, only verify 4-byte alignment. */ #ifdef SQLITE_4_BYTE_ALIGNED_MALLOC @@ -2883,8 +2883,8 @@ int sqlite3CantopenError(int); /* ** FTS4 is really an extension for FTS3. It is enabled using the -** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also all -** the SQLITE_ENABLE_FTS4 macro to serve as an alisse for SQLITE_ENABLE_FTS3. +** SQLITE_ENABLE_FTS3 macro. But to avoid confusion we also call +** the SQLITE_ENABLE_FTS4 macro to serve as an alias for SQLITE_ENABLE_FTS3. */ #if defined(SQLITE_ENABLE_FTS4) && !defined(SQLITE_ENABLE_FTS3) # define SQLITE_ENABLE_FTS3 diff --git a/src/table.c b/src/table.c index 26bbfb4f45..10b0d627f9 100644 --- a/src/table.c +++ b/src/table.c @@ -182,7 +182,7 @@ int sqlite3_get_table( ** This routine frees the space the sqlite3_get_table() malloced. */ void sqlite3_free_table( - char **azResult /* Result returned from from sqlite3_get_table() */ + char **azResult /* Result returned from sqlite3_get_table() */ ){ if( azResult ){ int i, n; diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 2b98b6aab4..945fd95982 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -760,7 +760,7 @@ static void tclSqlFunc(sqlite3_context *context, int argc, sqlite3_value**argv){ /* If there are arguments to the function, make a shallow copy of the ** script object, lappend the arguments, then evaluate the copy. ** - ** By "shallow" copy, we mean a only the outer list Tcl_Obj is duplicated. + ** By "shallow" copy, we mean only the outer list Tcl_Obj is duplicated. ** The new Tcl_Obj contains pointers to the original list elements. ** That way, when Tcl_EvalObjv() is run and shimmers the first element ** of the list to tclCmdNameType, that alternate representation will diff --git a/src/test1.c b/src/test1.c index 34faeaadf8..d050e683f4 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2605,7 +2605,7 @@ static int test_bind( ** SQLite selected to call. The TCL test script implements the ** "test_collate" proc. ** -** Note that this will only work with one intepreter at a time, as the +** Note that this will only work with one interpreter at a time, as the ** interp pointer to use when evaluating the TCL script is stored in ** pTestCollateInterp. */ @@ -3758,7 +3758,7 @@ static int test_prepare_v2( ** Usage: sqlite3_prepare_tkt3134 DB ** ** Generate a prepared statement for a zero-byte string as a test -** for ticket #3134. The string should be preceeded by a zero byte. +** for ticket #3134. The string should be preceded by a zero byte. */ static int test_prepare_tkt3134( void * clientData, diff --git a/src/test_intarray.c b/src/test_intarray.c index efcd21d404..7235fbceda 100644 --- a/src/test_intarray.c +++ b/src/test_intarray.c @@ -37,13 +37,13 @@ struct sqlite3_intarray { typedef struct intarray_vtab intarray_vtab; typedef struct intarray_cursor intarray_cursor; -/* A intarray table object */ +/* An intarray table object */ struct intarray_vtab { sqlite3_vtab base; /* Base class */ sqlite3_intarray *pContent; /* Content of the integer array */ }; -/* A intarray cursor object */ +/* An intarray cursor object */ struct intarray_cursor { sqlite3_vtab_cursor base; /* Base class */ int i; /* Current cursor position */ diff --git a/src/test_malloc.c b/src/test_malloc.c index b937cea7cf..bd0a3d1ffd 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -887,7 +887,7 @@ static int test_memdebug_log( ** ** Set the scratch memory buffer using SQLITE_CONFIG_SCRATCH. ** The buffer is static and is of limited size. N might be -** adjusted downward as needed to accomodate the requested size. +** adjusted downward as needed to accommodate the requested size. ** The revised value of N is returned. ** ** A negative SIZE causes the buffer pointer to be NULL. @@ -927,7 +927,7 @@ static int test_config_scratch( ** ** Set the page-cache memory buffer using SQLITE_CONFIG_PAGECACHE. ** The buffer is static and is of limited size. N might be -** adjusted downward as needed to accomodate the requested size. +** adjusted downward as needed to accommodate the requested size. ** The revised value of N is returned. ** ** A negative SIZE causes the buffer pointer to be NULL. diff --git a/src/test_schema.c b/src/test_schema.c index 00a9f4dd90..4ee18193b0 100644 --- a/src/test_schema.c +++ b/src/test_schema.c @@ -189,7 +189,7 @@ static int schemaNext(sqlite3_vtab_cursor *cur){ /* Set zSql to the SQL to pull the list of tables from the ** sqlite_master (or sqlite_temp_master) table of the database - ** identfied by the row pointed to by the SQL statement pCur->pDbList + ** identified by the row pointed to by the SQL statement pCur->pDbList ** (iterating through a "PRAGMA database_list;" statement). */ if( sqlite3_column_int(pCur->pDbList, 0)==1 ){ diff --git a/src/tokenize.c b/src/tokenize.c index 4017c3b816..3f1de221aa 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -77,7 +77,7 @@ const unsigned char ebcdicToAscii[] = { ** end result. ** ** Ticket #1066. the SQL standard does not allow '$' in the -** middle of identfiers. But many SQL implementations do. +** middle of identifiers. But many SQL implementations do. ** SQLite will allow '$' in identifiers for compatibility. ** But the feature is undocumented. */ diff --git a/src/trigger.c b/src/trigger.c index fc32a663bf..d2e7b5a1e6 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -127,7 +127,7 @@ void sqlite3BeginTrigger( ** ^^^^^^^^ ** ** To maintain backwards compatibility, ignore the database - ** name on pTableName if we are reparsing our of SQLITE_MASTER. + ** name on pTableName if we are reparsing out of SQLITE_MASTER. */ if( db->init.busy && iDb!=1 ){ sqlite3DbFree(db, pTableName->a[0].zDatabase); diff --git a/src/update.c b/src/update.c index e152e9057c..f781a60ccd 100644 --- a/src/update.c +++ b/src/update.c @@ -327,7 +327,7 @@ void sqlite3Update( } /* If we are trying to update a view, realize that view into - ** a ephemeral table. + ** an ephemeral table. */ #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ @@ -488,7 +488,7 @@ void sqlite3Update( } /* Populate the array of registers beginning at regNew with the new - ** row data. This array is used to check constaints, create the new + ** row data. This array is used to check constants, create the new ** table and index records, and as the values for any new.* references ** made by triggers. ** @@ -668,7 +668,7 @@ update_cleanup: return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise -** thely may interfere with compilation of other functions in this file +** they may interfere with compilation of other functions in this file ** (or in another file, if this file becomes part of the amalgamation). */ #ifdef isView #undef isView @@ -681,7 +681,7 @@ update_cleanup: /* ** Generate code for an UPDATE of a virtual table. ** -** The strategy is that we create an ephemerial table that contains +** The strategy is that we create an ephemeral table that contains ** for each row to be changed: ** ** (A) The original rowid of that row. @@ -689,7 +689,7 @@ update_cleanup: ** (C) The content of every column in the row. ** ** Then we loop over this ephemeral table and for each row in -** the ephermeral table call VUpdate. +** the ephemeral table call VUpdate. ** ** When finished, drop the ephemeral table. ** diff --git a/src/util.c b/src/util.c index 839a4a4636..9bb8d89157 100644 --- a/src/util.c +++ b/src/util.c @@ -204,7 +204,7 @@ void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ ** occur. ** ** 2002-Feb-14: This routine is extended to remove MS-Access style -** brackets from around identifers. For example: "[a-b-c]" becomes +** brackets from around identifiers. For example: "[a-b-c]" becomes ** "a-b-c". */ int sqlite3Dequote(char *z){ diff --git a/src/vacuum.c b/src/vacuum.c index 936a44a7fe..4d0c0976a1 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -87,7 +87,7 @@ static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ ** step (3) requires additional temporary disk space approximately equal ** to the size of the original database for the rollback journal. ** Hence, temporary disk space that is approximately 2x the size of the -** orginal database is required. Every page of the database is written +** original database is required. Every page of the database is written ** approximately 3 times: Once for step (2) and twice for step (3). ** Two writes per page are required in step (3) because the original ** database content must be written into the rollback journal prior to diff --git a/src/vdbe.c b/src/vdbe.c index 0f6e59bf15..cc9e317e4b 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2560,7 +2560,7 @@ case OP_MakeRecord: { ** ------------------------------------------------------------------------ ** ** Data(0) is taken from register P1. Data(1) comes from register P1+1 - ** and so froth. + ** and so forth. ** ** Each type field is a varint representing the serial type of the ** corresponding data element (see sqlite3VdbeSerialType()). The @@ -3547,7 +3547,7 @@ case OP_SeekGT: { /* jump, in3 */ if( pC->isTable ){ /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do - ** the seek, so covert it. */ + ** the seek, so convert it. */ pIn3 = &aMem[pOp->p3]; if( (pIn3->flags & (MEM_Int|MEM_Real))==0 ){ applyNumericAffinity(pIn3, 0); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index ef096bebb6..27b266986a 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -238,7 +238,7 @@ struct Mem { #endif /* -** Each auxilliary data pointer stored by a user defined function +** Each auxiliary data pointer stored by a user defined function ** implementation calling sqlite3_set_auxdata() is stored in an instance ** of this structure. All such structures associated with a single VM ** are stored in a linked list headed at Vdbe.pAuxData. All are destroyed @@ -253,7 +253,7 @@ struct AuxData { }; /* -** The "context" argument for a installable function. A pointer to an +** The "context" argument for an installable function. A pointer to an ** instance of this structure is the first argument to the routines used ** implement the SQL functions. ** diff --git a/src/vdbeapi.c b/src/vdbeapi.c index c4a2eebe08..e141ddfb94 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -212,7 +212,7 @@ int sqlite3_value_type(sqlite3_value* pVal){ ** The following routines are used by user-defined functions to specify ** the function result. ** -** The setStrOrError() funtion calls sqlite3VdbeMemSetStr() to store the +** The setStrOrError() function calls sqlite3VdbeMemSetStr() to store the ** result as a string or blob but if the string or blob is too large, it ** then sets the error code to SQLITE_TOOBIG */ @@ -645,7 +645,7 @@ void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ } /* -** Return the auxilary data pointer, if any, for the iArg'th argument to +** Return the auxiliary data pointer, if any, for the iArg'th argument to ** the user-function defined by pCtx. */ void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ @@ -660,7 +660,7 @@ void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ } /* -** Set the auxilary data pointer and delete function, for the iArg'th +** Set the auxiliary data pointer and delete function, for the iArg'th ** argument to the user-function defined by pCtx. Any previous value is ** deleted by calling the delete function specified when it was set. */ @@ -706,7 +706,7 @@ failed: #ifndef SQLITE_OMIT_DEPRECATED /* -** Return the number of times the Step function of a aggregate has been +** Return the number of times the Step function of an aggregate has been ** called. ** ** This function is deprecated. Do not use it for new code. It is @@ -976,7 +976,7 @@ const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ /* ** Return the name of the database from which a result column derives. ** NULL is returned if the result column is an expression or constant or -** anything else which is not an unabiguous reference to a database column. +** anything else which is not an unambiguous reference to a database column. */ const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ return columnName( @@ -992,7 +992,7 @@ const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ /* ** Return the name of the table from which a result column derives. ** NULL is returned if the result column is an expression or constant or -** anything else which is not an unabiguous reference to a database column. +** anything else which is not an unambiguous reference to a database column. */ const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ return columnName( @@ -1008,7 +1008,7 @@ const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ /* ** Return the name of the table column from which a result column derives. ** NULL is returned if the result column is an expression or constant or -** anything else which is not an unabiguous reference to a database column. +** anything else which is not an unambiguous reference to a database column. */ const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ return columnName( @@ -1288,7 +1288,7 @@ int sqlite3TransferBindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ ** Deprecated external interface. Internal/core SQLite code ** should call sqlite3TransferBindings. ** -** Is is misuse to call this routine with statements from different +** It is misuse to call this routine with statements from different ** database connections. But as this is a deprecated interface, we ** will not bother to check for that condition. ** diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 84d7cd3013..a653587319 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -895,7 +895,7 @@ void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){ ** routine, then a pointer to a dummy VdbeOp will be returned. That opcode ** is readable but not writable, though it is cast to a writable value. ** The return of a dummy opcode allows the call to continue functioning -** after a OOM fault without having to check to see if the return from +** after an OOM fault without having to check to see if the return from ** this routine is a valid pointer. But because the dummy.opcode is 0, ** dummy will never be written to. This is verified by code inspection and ** by running with Valgrind. @@ -1610,9 +1610,9 @@ void sqlite3VdbeRewind(Vdbe *p){ ** After the VDBE has be prepped, it can be executed by one or more ** calls to sqlite3VdbeExec(). ** -** This function may be called exact once on a each virtual machine. +** This function may be called exactly once on each virtual machine. ** After this routine is called the VM has been "packaged" and is ready -** to run. After this routine is called, futher calls to +** to run. After this routine is called, further calls to ** sqlite3VdbeAddOp() functions are prohibited. This routine disconnects ** the Vdbe from the Parse object that helped generate it so that the ** the Vdbe becomes an independent entity and the Parse object can be @@ -1990,7 +1990,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ /* The complex case - There is a multi-file write-transaction active. ** This requires a master journal file to ensure the transaction is - ** committed atomicly. + ** committed atomically. */ #ifndef SQLITE_OMIT_DISKIO else{ @@ -2638,7 +2638,7 @@ int sqlite3VdbeFinalize(Vdbe *p){ ** from left to right), or ** ** * the corresponding bit in argument mask is clear (where the first -** function parameter corrsponds to bit 0 etc.). +** function parameter corresponds to bit 0 etc.). */ void sqlite3VdbeDeleteAuxData(Vdbe *pVdbe, int iOp, int mask){ AuxData **pp = &pVdbe->pAuxData; @@ -2742,7 +2742,7 @@ static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ ** Something has moved cursor "p" out of place. Maybe the row it was ** pointed to was deleted out from under it. Or maybe the btree was ** rebalanced. Whatever the cause, try to restore "p" to the place it -** is suppose to be pointing. If the row was deleted out from under the +** is supposed to be pointing. If the row was deleted out from under the ** cursor, set the cursor to point to a NULL row. */ static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ @@ -3267,7 +3267,7 @@ static int vdbeRecordCompareDebug( assert( mem1.zMalloc==0 ); /* rc==0 here means that one of the keys ran out of fields and - ** all the fields up to that point were equal. Return the the default_rc + ** all the fields up to that point were equal. Return the default_rc ** value. */ rc = pPKey2->default_rc; @@ -3458,7 +3458,7 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero ** or positive integer if key1 is less than, equal to or ** greater than key2. The {nKey1, pKey1} key must be a blob -** created by th OP_MakeRecord opcode of the VDBE. The pPKey2 +** created by the OP_MakeRecord opcode of the VDBE. The pPKey2 ** key must be a parsed key such as obtained from ** sqlite3VdbeParseRecord. ** @@ -3648,7 +3648,7 @@ int sqlite3VdbeRecordCompare( assert( mem1.zMalloc==0 ); /* rc==0 here means that one or both of the keys ran out of fields and - ** all the fields up to that point were equal. Return the the default_rc + ** all the fields up to that point were equal. Return the default_rc ** value. */ assert( CORRUPT_DB || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) diff --git a/src/vdbemem.c b/src/vdbemem.c index 95e23c61a7..ea4def3f86 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -37,7 +37,7 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){ ** ** (1) Memory in Mem.zMalloc and managed by the Mem object ** (2) Memory to be freed using Mem.xDel - ** (3) An ephermal string or blob + ** (3) An ephemeral string or blob ** (4) A static string or blob */ if( (p->flags & (MEM_Str|MEM_Blob)) && p->z!=0 ){ @@ -240,7 +240,7 @@ int sqlite3VdbeMemNulTerminate(Mem *pMem){ ** used for converting values to text for returning to the user (i.e. via ** sqlite3_value_text()), or for ensuring that values to be used as btree ** keys are strings. In the former case a NULL pointer is returned the -** user and the later is an internal programming error. +** user and the latter is an internal programming error. */ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ int fg = pMem->flags; @@ -405,7 +405,7 @@ static i64 doubleToInt64(double r){ ** If pMem is an integer, then the value is exact. If pMem is ** a floating-point then the value returned is the integer part. ** If pMem is a string or blob, then we make an attempt to convert -** it into a integer and return that. If pMem represents an +** it into an integer and return that. If pMem represents an ** an SQL-NULL value, return 0. ** ** If pMem represents a string value, its encoding might be changed. @@ -697,7 +697,7 @@ int sqlite3VdbeMemTooBig(Mem *p){ #ifdef SQLITE_DEBUG /* -** This routine prepares a memory cell for modication by breaking +** This routine prepares a memory cell for modification by breaking ** its link to a shallow copy and by marking any current shallow ** copies of this cell as invalid. ** diff --git a/src/vdbesort.c b/src/vdbesort.c index 7318ea409e..d1ccf6089d 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -2059,7 +2059,7 @@ static void *vdbePmaReaderBgInit(void *pCtx){ /* ** Use a background thread to invoke vdbePmaReaderIncrMergeInit(INCRINIT_TASK) -** on the the PmaReader object passed as the first argument. +** on the PmaReader object passed as the first argument. ** ** This call will initialize the various fields of the pReadr->pIncr ** structure and, if it is a multi-threaded IncrMerger, launch a diff --git a/src/vdbetrace.c b/src/vdbetrace.c index 4a39e26521..362530a1d9 100644 --- a/src/vdbetrace.c +++ b/src/vdbetrace.c @@ -64,7 +64,7 @@ static int findNextHostParameter(const char *zSql, int *pnToken){ ** ALGORITHM: Scan the input string looking for host parameters in any of ** these forms: ?, ?N, $A, @A, :A. Take care to avoid text within ** string literals, quoted identifier names, and comments. For text forms, -** the host parameter index is found by scanning the perpared +** the host parameter index is found by scanning the prepared ** statement for the corresponding OP_Variable opcode. Once the host ** parameter index is known, locate the value in p->aVar[]. Then render ** the value as a literal in place of the host parameter name. diff --git a/src/wal.c b/src/wal.c index 70395f8bee..c0861d5be7 100644 --- a/src/wal.c +++ b/src/wal.c @@ -574,7 +574,7 @@ static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ ** The argument to this macro must be of type u32. On a little-endian ** architecture, it returns the u32 value that results from interpreting ** the 4 bytes as a big-endian value. On a big-endian architecture, it -** returns the value that would be produced by intepreting the 4 bytes +** returns the value that would be produced by interpreting the 4 bytes ** of the input value as a little-endian integer. */ #define BYTESWAP32(x) ( \ @@ -988,7 +988,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ assert( idx <= HASHTABLE_NSLOT/2 + 1 ); /* If this is the first entry to be added to this hash-table, zero the - ** entire hash table and aPgno[] array before proceding. + ** entire hash table and aPgno[] array before proceeding. */ if( idx==1 ){ int nByte = (int)((u8 *)&aHash[HASHTABLE_NSLOT] - (u8 *)&aPgno[1]); @@ -1646,7 +1646,7 @@ static int walPagesize(Wal *pWal){ ** database file. ** ** This routine uses and updates the nBackfill field of the wal-index header. -** This is the only routine tha will increase the value of nBackfill. +** This is the only routine that will increase the value of nBackfill. ** (A WAL reset or recovery will revert nBackfill to zero, but not increase ** its value.) ** @@ -1950,7 +1950,7 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ ** wal-index from the WAL before returning. ** ** Set *pChanged to 1 if the wal-index header value in pWal->hdr is -** changed by this opertion. If pWal->hdr is unchanged, set *pChanged +** changed by this operation. If pWal->hdr is unchanged, set *pChanged ** to 0. ** ** If the wal-index header is successfully read, return SQLITE_OK. @@ -2154,7 +2154,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** may have been appended to the log before READ_LOCK(0) was obtained. ** When holding READ_LOCK(0), the reader ignores the entire log file, ** which implies that the database file contains a trustworthy - ** snapshoT. Since holding READ_LOCK(0) prevents a checkpoint from + ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from ** happening, this is usually correct. ** ** However, if frames have been appended to the log (or if the log @@ -2822,7 +2822,7 @@ int sqlite3WalFrames( ** ** Padding and syncing only occur if this set of frames complete a ** transaction and if PRAGMA synchronous=FULL. If synchronous==NORMAL - ** or synchonous==OFF, then no padding or syncing are needed. + ** or synchronous==OFF, then no padding or syncing are needed. ** ** If SQLITE_IOCAP_POWERSAFE_OVERWRITE is defined, then padding is not ** needed and only the sync is done. If padding is needed, then the diff --git a/src/walker.c b/src/walker.c index 016ae77a92..e30bb60b5a 100644 --- a/src/walker.c +++ b/src/walker.c @@ -19,7 +19,7 @@ /* ** Walk an expression tree. Invoke the callback once for each node -** of the expression, while decending. (In other words, the callback +** of the expression, while descending. (In other words, the callback ** is invoked before visiting children.) ** ** The return value from the callback should be one of the WRC_* diff --git a/src/where.c b/src/where.c index 921baa29eb..5b990fc108 100644 --- a/src/where.c +++ b/src/where.c @@ -701,7 +701,7 @@ static int isLikeOrGlob( ** value of the variable means there is no need to invoke the LIKE ** function, then no OP_Variable will be added to the program. ** This causes problems for the sqlite3_bind_parameter_name() - ** API. To workaround them, add a dummy OP_Variable here. + ** API. To work around them, add a dummy OP_Variable here. */ int r1 = sqlite3GetTempReg(pParse); sqlite3ExprCodeTarget(pParse, pRight, r1); @@ -821,7 +821,7 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ ** appropriate for indexing exist. ** ** All examples A through E above satisfy case 2. But if a term -** also statisfies case 1 (such as B) we know that the optimizer will +** also satisfies case 1 (such as B) we know that the optimizer will ** always prefer case 1, so in that case we pretend that case 2 is not ** satisfied. ** @@ -979,7 +979,7 @@ static void exprAnalyzeOrTerm( } if( (chngToIN & getMask(&pWInfo->sMaskSet, pOrTerm->leftCursor))==0 ){ /* This term must be of the form t1.a==t2.b where t2 is in the - ** chngToIN set but t1 is not. This term will be either preceeded + ** chngToIN set but t1 is not. This term will be either preceded ** or follwed by an inverted copy (t2.b==t1.a). Skip this term ** and use its inversion. */ testcase( pOrTerm->wtFlags & TERM_COPIED ); @@ -1390,7 +1390,7 @@ static void exprAnalyze( } /* -** This function searches pList for a entry that matches the iCol-th column +** This function searches pList for an entry that matches the iCol-th column ** of index pIdx. ** ** If such an expression is found, its index in pList->a[] is returned. If @@ -2140,7 +2140,7 @@ static int whereRangeSkipScanEst( ** number of rows that the index scan is expected to visit without ** considering the range constraints. If nEq is 0, this is the number of ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) -** to account for the range contraints pLower and pUpper. +** to account for the range constraints pLower and pUpper. ** ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be ** used, a single range inequality reduces the search space by a factor of 4. @@ -3417,7 +3417,7 @@ static Bitmask codeOneLoopStart( ** B: ** ** Added 2014-05-26: If the table is a WITHOUT ROWID table, then - ** use an ephermeral index instead of a RowSet to record the primary + ** use an ephemeral index instead of a RowSet to record the primary ** keys of the rows we have already seen. ** */ @@ -3468,7 +3468,7 @@ static Bitmask codeOneLoopStart( } /* Initialize the rowset register to contain NULL. An SQL NULL is - ** equivalent to an empty rowset. Or, create an ephermeral index + ** equivalent to an empty rowset. Or, create an ephemeral index ** capable of holding primary keys in the case of a WITHOUT ROWID. ** ** Also initialize regReturn to contain the address of the instruction @@ -4723,7 +4723,7 @@ static int whereLoopAddBtree( ApplyCostMultiplier(pNew->rSetup, pTab->costMult); /* TUNING: Each index lookup yields 20 rows in the table. This ** is more than the usual guess of 10 rows, since we have no way - ** of knowning how selective the index will ultimately be. It would + ** of knowing how selective the index will ultimately be. It would ** not be unreasonable to make this value much larger. */ pNew->nOut = 43; assert( 43==sqlite3LogEst(20) ); pNew->rRun = sqlite3LogEstAdd(rLogSize,pNew->nOut); @@ -5153,7 +5153,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** strict. With GROUP BY and DISTINCT the only requirement is that ** equivalent rows appear immediately adjacent to one another. GROUP BY ** and DISTINCT do not require rows to appear in any particular order as long -** as equivelent rows are grouped together. Thus for GROUP BY and DISTINCT +** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT ** the pOrderBy terms can be matched in any order. With ORDER BY, the ** pOrderBy terms must be matched in strict left-to-right order. */ diff --git a/src/whereInt.h b/src/whereInt.h index 81f4a03667..f17906e63a 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -176,7 +176,7 @@ static int whereLoopResize(sqlite3*, WhereLoop*, int); ** 1. Then using those as a basis to compute the N best WherePath objects ** of length 2. And so forth until the length of WherePaths equals the ** number of nodes in the FROM clause. The best (lowest cost) WherePath -** at the end is the choosen query plan. +** at the end is the chosen query plan. */ struct WherePath { Bitmask maskLoop; /* Bitmask of all WhereLoop objects in this path */ From 2ee53412888324b00964a8766f4f78ccc80218a9 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 6 Sep 2014 16:49:40 +0000 Subject: [PATCH 306/710] Fixes to os_unix.c to support database (and other) files larger than 2GiB. FossilOrigin-Name: e7fae33c0754488336ce093189a83dfe1b818d89 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/os_unix.c | 23 ++++++++++++++++++++++- 3 files changed, 33 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8ae6336224..bb3b4e6579 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning. -D 2014-09-05T05:58:37.378 +C Fixes\sto\sos_unix.c\sto\ssupport\sdatabase\s(and\sother)\sfiles\slarger\sthan\s2GiB. +D 2014-09-06T16:49:40.555 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -208,7 +208,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 60d419395e32a8029fa380a80a3da2e9030f635e F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 8525ca79457c5b4673a5fda2774ee39fe155f40f +F src/os_unix.c 22225d7969b3e4d7aaef9804ea5af5d02d7abd6a F src/os_win.c 3c9f7df710cb6c757b04b78bf3d98f03830e67b9 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 3e732d2bbdd8d8d95fed0c5ae7e718d73153c4c5 @@ -1193,7 +1193,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9779c7a9eb1e2bd36e9286331a9314f064014d80 -R 05490459da8c4172c4abbda0933bfe68 -U mistachkin -Z b93a43f7e285c77b049b454cce5380d9 +P 733119067757814609a9cea6b975818607bee4e3 +R da7850c1d5c67f21c676f6a6ed44959c +T *branch * android-large-filles +T *sym-android-large-filles * +T -sym-trunk * +U dan +Z 5be16efe9db58eba68bedd24a6d0231c diff --git a/manifest.uuid b/manifest.uuid index fb28827b7e..2f4152264c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -733119067757814609a9cea6b975818607bee4e3 \ No newline at end of file +e7fae33c0754488336ce093189a83dfe1b818d89 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index f63afc6bc5..d80a706b93 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -299,6 +299,14 @@ static int randomnessPid = 0; # endif #endif +/* +** Explicitly call the 64-bit version of lseek() on Android. Otherwise, lseek() +** is the 32-bit version, even if _FILE_OFFSET_BITS=64 is defined. +*/ +#ifdef __ANDROID__ +# define lseek lseek64 +#endif + /* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). @@ -708,9 +716,22 @@ static int lockTrace(int fd, int op, struct flock *p){ /* ** Retry ftruncate() calls that fail due to EINTR +** +** All calls to ftruncate() within this file should be made through this wrapper. +** On the Android platform, bypassing the logic below could lead to a corrupt +** database. */ static int robust_ftruncate(int h, sqlite3_int64 sz){ int rc; +#ifdef __ANDROID__ + /* On Android, ftruncate() always uses 32-bit offsets, even if + ** _FILE_OFFSET_BITS=64 is defined. This means it is unsafe to attempt to + ** truncate a file to any size smaller than 2GiB. Silently ignore any + ** such attempts. */ + if( sz>(sqlite3_int64)0x7FFFFFFF ){ + rc = SQLITE_OK; + }else +#endif do{ rc = osFtruncate(h,sz); }while( rc<0 && errno==EINTR ); return rc; } @@ -3595,7 +3616,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; } - rc = robust_ftruncate(pFile->h, (off_t)nByte); + rc = robust_ftruncate(pFile->h, nByte); if( rc ){ pFile->lastErrno = errno; return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); From d45309796bac57233a063af72f0ac5a08bde8a04 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Sep 2014 14:47:53 +0000 Subject: [PATCH 307/710] Non-working preliminary implementation attempts on user authentication. FossilOrigin-Name: 8440f093bac19a41d44ee352744354eab897fe4e --- ext/userauth/user-auth.txt | 151 ++++++++++++++++++++++++++++ ext/userauth/userauth.c | 197 +++++++++++++++++++++++++++++++++++++ ext/userauth/userauth.h | 88 +++++++++++++++++ manifest | 28 +++--- manifest.uuid | 2 +- src/build.c | 18 ++++ src/main.c | 7 ++ src/pragma.c | 6 ++ src/prepare.c | 22 +++++ src/sqliteInt.h | 33 ++++++- 10 files changed, 539 insertions(+), 13 deletions(-) create mode 100644 ext/userauth/user-auth.txt create mode 100644 ext/userauth/userauth.c create mode 100644 ext/userauth/userauth.h diff --git a/ext/userauth/user-auth.txt b/ext/userauth/user-auth.txt new file mode 100644 index 0000000000..dd49dfcfb7 --- /dev/null +++ b/ext/userauth/user-auth.txt @@ -0,0 +1,151 @@ +Activate the user authentication logic by compiling SQLite with +the -DSQLITE_USER_AUTHENTICATION compile-time option. + +The following new APIs are available when user authentication is +activated: + + int sqlite3_user_authenticate( + sqlite3 *db, /* The database connection */ + const char *zUsername, /* Username */ + int nPW, /* Number of bytes in aPW[] */ + const void *aPW /* Password or credentials */ + ); + + int sqlite3_user_add( + sqlite3 *db, /* Database connection */ + const char *zUsername, /* Username to be added */ + int isAdmin, /* True to give new user admin privilege */ + int nPW, /* Number of bytes in aPW[] */ + const void *aPW /* Password or credentials */ + ); + + int sqlite3_user_change( + sqlite3 *db, /* Database connection */ + const char *zUsername, /* Username to change */ + int isAdmin, /* Modified admin privilege for the user */ + int nPW, /* Number of bytes in aPW[] */ + const void *aPW /* Modified password or credentials */ + ); + + int sqlite3_user_delete( + sqlite3 *db, /* Database connection */ + const char *zUsername /* Username to remove */ + ); + +The sqlite3_open(), sqlite3_open16(), and sqlite3_open_v2() interfaces +work as before: they open a new database connection. However, if the +database being opened requires authentication, then the database +connection will be unusable until after sqlite3_user_authenticate() +has been called successfully [1c]. The sqlite3_user_authenticate() call +will return SQLITE_OK if the authentication credentials are accepted +and SQLITE_ERROR if not. + +Calling sqlite3_user_authenticate() on a no-authentication-required +database connection is a harmless no-op. + +If the database is encrypted, then sqlite3_key_v2() must be called first, +with the correct decryption key, prior to invoking sqlite3_user_authenticate(). + +To recapitulate: When opening an existing unencrypted authentication- +required database, the call sequence is: + + sqlite3_open_v2() + sqlite3_user_authenticate(); + /* Database is now usable */ + +To open an existing, encrypted, authentication-required database, the +call sequence is: + + sqlite3_open_v2(); + sqlite3_key_v2(); + sqlite3_user_authenticate(); + /* Database is now usable */ + +When opening a no-authentication-required database, the database +connection is treated as if it was authenticated as an admin user. + +When ATTACH-ing new database files to a connection, each newly attached +database that is an authentication-required database is checked using +the same username and password as supplied to the main database. If that +check fails, then the ATTACH-ed database is unreadable [1g]. + +The sqlite3_user_add() interface can be used (by an admin user only) +to create a new user. When called on a no-authentication-required +database, this routine converts the database into an authentication- +required database [3], automatically makes the added user an +administrator [1b], and logs in the current connection as that user [1a]. +The sqlite3_user_add() interface only works for the "main" database, not +for any ATTACH-ed databases. Any call to sqlite3_user_add() by a +non-admin user results in an error. + +Hence, to create a new, unencrypted, authentication-required database, +the call sequence is: + + sqlite3_open_v2(); + sqlite3_user_add(); + +And to create a new, encrypted, authentication-required database, the call +sequence is: + + sqlite3_open_v2(); + sqlite3_key_v2(); + sqlite3_user_add(); + +The sqlite3_user_delete() interface can be used (by an admin user only) +to delete a user. The currently logged-in user cannot be deleted, +which guarantees that there is always an admin user and hence that +the database cannot be converted into a no-authentication-required +database [3]. + +The sqlite3_user_change() interface can be used to change a users +login credentials or admin privilege. Any user can change their own +login credentials [1b]. Only an admin user can change another users login +credentials or admin privilege setting. No user may change their own +admin privilege setting. + +The sqlite3_set_authorizer() callback is modified to take a 7th parameter +which is the username of the currently logged in user, or NULL for a +no-authentication-required database [1d]. + +----------------------------------------------------------------------------- +Implementation notes: + +An authentication-required database is identified by the presence of a +new table: + + CREATE TABLE sqlite_user( + uname TEXT PRIMARY KEY, + isAdmin BOOLEAN, + pw BLOB + ) WITHOUT ROWID; + +This table is inaccessible (unreadable and unwriteable) to non-admin users +and is read-only for admin users. However, if the same database file is +opened by a version of SQLite that omits the -DSQLITE_USER_AUTHENTICATION +compile-time option, then the sqlite_user table will be readable by +anybody and writeable by anybody if the "PRAGMA writable_schema=ON" +statement is run first. + +The sqlite_user.pw field is encoded by a built-in SQL function +"sqlite_crypt(X,Y)". The two arguments are both BLOBs. The first argument +is the plaintext password supplied to the sqlite3_user_authenticate() +interface. The second argument is the sqlite_user.pw value and is supplied +so that the function can extra the "salt" used by the password encoder. +the result of sqlite_crypt(X,Y) is another blob which is the value that +ends up being stored in sqlite_user.pw. To verify credentials X supplied +by the sqlite3_user_authenticate() routine, SQLite runs: + + sqlite_user.pw == sqlite_crypt(X, sqlite_user.pw) + +To compute an appropriate sqlite_user.pw value from a new or modified +password X, sqlite_crypt(X,NULL) is run. A new random salt is selected +when the second argument is NULL. + +The built-in version of of sqlite_crypt() uses a simple Ceasar-cypher +which prevents passwords from being revealed by search the raw database +for ASCII text, but is otherwise trivally broken. To truly secure the +passwords, the database should be encrypted using the SQLite Encryption +Extension or similar technology. Or, the application can use the +sqlite3_create_function() interface to provide an alternative +implementation of sqlite_crypt() that computes a stronger password hash, +perhaps using a cryptographic hash function like SHA1. diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c new file mode 100644 index 0000000000..305ae43a11 --- /dev/null +++ b/ext/userauth/userauth.c @@ -0,0 +1,197 @@ +/* +** 2014-09-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the bulk of the implementation of the +** user-authentication extension feature. Some parts of the user- +** authentication code are contained within the SQLite core (in the +** src/ subdirectory of the main source code tree) but those parts +** that could reasonable be separated out are moved into this file. +** +** To compile with the user-authentication feature, append this file to +** end of an SQLite amalgamation, then add the SQLITE_USER_AUTHENTICATION +** compile-time option. See the user-auth.txt file in the same source +** directory as this file for additional information. +*/ +#ifdef SQLITE_USER_AUTHENTICATION + +/* +** Prepare an SQL statement for use by the user authentication logic. +** Return a pointer to the prepared statement on success. Return a +** NULL pointer if there is an error of any kind. +*/ +static sqlite3_stmt *sqlite3UserAuthPrepare( + sqlite3 *db, + const char *zFormat, + ... +){ + sqlite3_stmt *pStmt; + char *zSql; + int rc; + va_list ap; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + if( zSql==0 ) return 0; + savedFlags = db->auth.authFlags; + db->auth.authFlags |= UAUTH_Ovrd; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + db->auth.authFlags = savedFlags; + sqlite3_free(zSql); + if( rc ){ + sqlite3_finalize(pStmt); + pStmt = 0; + } + return pStmt; +} + +/* +** Check to see if database zDb has a "sqlite_user" table and if it does +** whether that table can authenticate zUser with nPw,zPw. +*/ +static int sqlite3UserAuthCheckLogin( + sqlite3 *db, /* The database connection to check */ + const char *zDb, /* Name of specific database to check */ + const char *zUser, /* User name */ + int nPw, /* Size of password in bytes */ + const char *zPw, /* Password */ + int *pbOk /* OUT: write boolean result here */ +){ + sqlite3_stmt *pStmt; + char *zSql; + int rc; + int iResult; + + *pbOk = 0; + iResult = 0; + pStmt = sqlite3UserAuthPrepare(db, + "SELECT 1 FROM \"%w\".sqlite_master " + " WHERE name='sqlite_user' AND type='table'", zDb); + if( pStmt==0 ) return SQLITE_NOMEM; + rc = sqlite3_step(pStmt): + sqlite3_finalize(pStmt); + if( rc==SQLITE_DONE ){ + *pbOk = 1; + return SQLITE_OK; + } + if( rc!=SQLITE_OK ){ + return rc; + } + pStmt = sqlite3UserAuthPrepare(db, + "SELECT pw=sqlite_crypt(?1,pw), isAdmin FROM \"%w\".sqlite_user" + " WHERE uname=?2", zDb); + if( pStmt==0 ) return SQLITE_NOMEM; + sqlite3_bind_blob(pStmt, 1, zPw, nPw, SQLITE_STATIC); + sqlite3_bind_text(pStmt, 2, zUser, -1, SQLITE_STATIC); + rc = sqlite_step(pStmt); + if( rc==SQLITE_ROW && sqlite3_column_int(pStmt,0) ){ + *pbOk = sqlite3_column_int(pStmt, 1); + } + sqlite3_finalize(pStmt); + return rc; +} + +/* +** If a database contains the SQLITE_USER table, then the +** sqlite3_user_authenticate() interface must be invoked with an +** appropriate username and password prior to enable read and write +** access to the database. +** +** Return SQLITE_OK on success or SQLITE_ERROR if the username/password +** combination is incorrect or unknown. +** +** If the SQLITE_USER table is not present in the database file, then +** this interface is a harmless no-op returnning SQLITE_OK. +*/ +int sqlite3_user_authenticate( + sqlite3 *db, /* The database connection */ + const char *zUsername, /* Username */ + int nPW, /* Number of bytes in aPW[] */ + const void *aPW /* Password or credentials */ +){ + int bOk = 0; + int rc; + + rc = sqlite3UserAuthCheckLogin(db, zUsername, nPw, zPw, &bOk); + if( bOk ){ + db->auth.authFlags = bOk==2 ? UAUTH_Auth|UAUTH_Admin : UAUTH_Auth; + sqlite3_free(db->auth.zAuthUser); + db->auth.zAuthUser = sqlite3_malloc("%s", zUsername); + sqlite3_free(db->auth.zPw); + db->auth.zPw = sqlite3_malloc( nPw+1 ); + if( db->auth.zPw ){ + memcpy(db->auth.zPw,zPw,nPw); + db->auth.nPw = nPw; + rc = SQLITE_OK; + }else{ + rc = SQLITE_NOMEM; + } + }else{ + db->auth.authFlags = 0; + } + return rc; +} + +/* +** The sqlite3_user_add() interface can be used (by an admin user only) +** to create a new user. When called on a no-authentication-required +** database, this routine converts the database into an authentication- +** required database, automatically makes the added user an +** administrator, and logs in the current connection as that user. +** The sqlite3_user_add() interface only works for the "main" database, not +** for any ATTACH-ed databases. Any call to sqlite3_user_add() by a +** non-admin user results in an error. +*/ +int sqlite3_user_add( + sqlite3 *db, /* Database connection */ + const char *zUsername, /* Username to be added */ + int isAdmin, /* True to give new user admin privilege */ + int nPW, /* Number of bytes in aPW[] */ + const void *aPW /* Password or credentials */ +){ + if( !DbIsAdmin(db) ) return SQLITE_ERROR; + + return SQLITE_OK; +} + +/* +** The sqlite3_user_change() interface can be used to change a users +** login credentials or admin privilege. Any user can change their own +** login credentials. Only an admin user can change another users login +** credentials or admin privilege setting. No user may change their own +** admin privilege setting. +*/ +int sqlite3_user_change( + sqlite3 *db, /* Database connection */ + const char *zUsername, /* Username to change */ + int isAdmin, /* Modified admin privilege for the user */ + int nPW, /* Number of bytes in aPW[] */ + const void *aPW /* Modified password or credentials */ +){ + return SQLITE_OK; +} + +/* +** The sqlite3_user_delete() interface can be used (by an admin user only) +** to delete a user. The currently logged-in user cannot be deleted, +** which guarantees that there is always an admin user and hence that +** the database cannot be converted into a no-authentication-required +** database. +*/ +int sqlite3_user_delete( + sqlite3 *db, /* Database connection */ + const char *zUsername /* Username to remove */ +){ + return SQLITE_OK; +} + +#endif /* SQLITE_USER_AUTHENTICATION */ diff --git a/ext/userauth/userauth.h b/ext/userauth/userauth.h new file mode 100644 index 0000000000..279675f72c --- /dev/null +++ b/ext/userauth/userauth.h @@ -0,0 +1,88 @@ +/* +** 2014-09-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains the application interface definitions for the +** user-authentication extension feature. +** +** To compile with the user-authentication feature, append this file to +** end of an SQLite amalgamation header file ("sqlite3.h"), then add +** the SQLITE_USER_AUTHENTICATION compile-time option. See the +** user-auth.txt file in the same source directory as this file for +** additional information. +*/ +#ifdef SQLITE_USER_AUTHENTICATION + +/* +** If a database contains the SQLITE_USER table, then the +** sqlite3_user_authenticate() interface must be invoked with an +** appropriate username and password prior to enable read and write +** access to the database. +** +** Return SQLITE_OK on success or SQLITE_ERROR if the username/password +** combination is incorrect or unknown. +** +** If the SQLITE_USER table is not present in the database file, then +** this interface is a harmless no-op returnning SQLITE_OK. +*/ +int sqlite3_user_authenticate( + sqlite3 *db, /* The database connection */ + const char *zUsername, /* Username */ + int nPW, /* Number of bytes in aPW[] */ + const void *aPW /* Password or credentials */ +); + +/* +** The sqlite3_user_add() interface can be used (by an admin user only) +** to create a new user. When called on a no-authentication-required +** database, this routine converts the database into an authentication- +** required database, automatically makes the added user an +** administrator, and logs in the current connection as that user. +** The sqlite3_user_add() interface only works for the "main" database, not +** for any ATTACH-ed databases. Any call to sqlite3_user_add() by a +** non-admin user results in an error. +*/ +int sqlite3_user_add( + sqlite3 *db, /* Database connection */ + const char *zUsername, /* Username to be added */ + int isAdmin, /* True to give new user admin privilege */ + int nPW, /* Number of bytes in aPW[] */ + const void *aPW /* Password or credentials */ +); + +/* +** The sqlite3_user_change() interface can be used to change a users +** login credentials or admin privilege. Any user can change their own +** login credentials. Only an admin user can change another users login +** credentials or admin privilege setting. No user may change their own +** admin privilege setting. +*/ +int sqlite3_user_change( + sqlite3 *db, /* Database connection */ + const char *zUsername, /* Username to change */ + int isAdmin, /* Modified admin privilege for the user */ + int nPW, /* Number of bytes in aPW[] */ + const void *aPW /* Modified password or credentials */ +); + +/* +** The sqlite3_user_delete() interface can be used (by an admin user only) +** to delete a user. The currently logged-in user cannot be deleted, +** which guarantees that there is always an admin user and hence that +** the database cannot be converted into a no-authentication-required +** database. +*/ +int sqlite3_user_delete( + sqlite3 *db, /* Database connection */ + const char *zUsername /* Username to remove */ +); + +#endif /* SQLITE_USER_AUTHENTICATION */ diff --git a/manifest b/manifest index ccd849754b..0b16bdd037 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sto\sos_unix.c\sto\ssupport\sdatabase\s(and\sother)\sfiles\slarger\sthan\s2GiB\son\sAndroid. -D 2014-09-06T17:06:13.410 +C Non-working\spreliminary\simplementation\sattempts\son\suser\sauthentication. +D 2014-09-09T14:47:53.726 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -144,6 +144,9 @@ F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h 83349d519fe5f518b3ea025d18dd1fe51b1684bd F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 +F ext/userauth/user-auth.txt f471c5a363ab0682b109d85982ea857f9a144ccc +F ext/userauth/userauth.c cea064a9b66d7497c74cc504b3a01d7f390cbe6e +F ext/userauth/userauth.h efbfb68ff083749ad63b12dcb5877b936c3458d6 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -171,7 +174,7 @@ F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c b1c1cd1cc3ae2e433a23b9a6c9ab53805707d8cd F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc -F src/build.c 8cb237719c185eec7bd8449b2e747491ded11932 +F src/build.c 8b4c67c9fb638fb2e64b2bcd24677e71a5d61bfc F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a @@ -190,7 +193,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab -F src/main.c e48517e3da289d93ad86e8b7b4f68078df5e6e51 +F src/main.c 26299e9d9a72239f0652ac9a23e04ced3f536e70 F src/malloc.c 954de5f998c23237e04474a3f2159bf483bba65a F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -217,8 +220,8 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 2048affdb09a04478b5fc6e64cb1083078d369be F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa -F src/pragma.c 14bcdb504128a476cce5bbc086d5226c5e46c225 -F src/prepare.c 3842c1dfc0b053458e3adcf9f6efc48e03e3fe3d +F src/pragma.c e4f6421d7a1f23cd21bf281f2d8c56874879e1d0 +F src/prepare.c 9548c8efb80dbc14b0d54d58574c8a7414cde4db F src/printf.c e74925089a85e3c9f0e315595f41c139d3d118c2 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0d1621e45fffe4b4396477cf46e41a84b0145ffb @@ -228,7 +231,7 @@ F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50 F src/sqlite.h.in 64a77f2822f1325b12050972003184f99b655a0f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 1f40357fb9b12a80c5a3b2b109fd249b009213d4 -F src/sqliteInt.h 4d1b0488f097aa7be346176c4d5e3cc1e25d99da +F src/sqliteInt.h 883f905f17a78bb09061a8efa4f2708faa6ffacc F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 4e28a53e66bad8d014a510ef0205f5497c712b08 @@ -1193,7 +1196,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e62aab5e9290503869e1f4d5e0fefd2b4dee0a69 9dca7ce55797b3eb617859f6189c1a2ec6f66566 -R 4eacabb789e62917f709201f03eadd53 -U dan -Z f60ed4f30cad7188918c7dd18fc1b985 +P ad7063aa1a0db32cdbe71815545b2edca57d3bcc +R 8d9410cfb98bb3b10637c181b910724f +T *branch * user-auth +T *sym-user-auth * +T -sym-trunk * +U drh +Z 7af3a6ca660957a95167232843af8c9e diff --git a/manifest.uuid b/manifest.uuid index fc5888a5d9..c31b773fe5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ad7063aa1a0db32cdbe71815545b2edca57d3bcc \ No newline at end of file +8440f093bac19a41d44ee352744354eab897fe4e \ No newline at end of file diff --git a/src/build.c b/src/build.c index 6d54befbcc..c30645cdc6 100644 --- a/src/build.c +++ b/src/build.c @@ -271,6 +271,16 @@ void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ pParse->nested--; } +#if SQLITE_USER_AUTHENTICATION +/* +** Return TRUE if zTable is the name of the system table that stores the +** list of users and their access credentials. +*/ +int sqlite3UserAuthTable(const char *zTable){ + return sqlite3_stricmp(zTable, "sqlite_user")==0; +} +#endif + /* ** Locate the in-memory structure that describes a particular database ** table given the name of that table and (optionally) the name of the @@ -289,6 +299,11 @@ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ assert( zName!=0 ); /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); +#if SQLITE_USER_AUTHENTICATION + /* Only the admin user is allowed to know that the sqlite_user table + ** exists */ + if( DbIsAdmin(db)==0 && sqlite3UserAuthTable(zName)!=0 ) return 0; +#endif for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; @@ -2867,6 +2882,9 @@ Index *sqlite3CreateIndex( assert( pTab!=0 ); assert( pParse->nErr==0 ); if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 +#if SQLITE_USER_AUTHENTICATION + && sqlite3UserAuthTable(pTab->zName)==0 +#endif && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; diff --git a/src/main.c b/src/main.c index 977c786f38..f5114b7924 100644 --- a/src/main.c +++ b/src/main.c @@ -985,6 +985,10 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ sqlite3Error(db, SQLITE_OK); /* Deallocates any cached error strings. */ sqlite3ValueFree(db->pErr); sqlite3CloseExtensions(db); +#if SQLITE_USER_AUTHENTICATION + sqlite3_free(db->auth.zAuthUser); + sqlite3_free(db->auth.zAuthPW); +#endif db->magic = SQLITE_MAGIC_ERROR; @@ -2566,6 +2570,9 @@ static int openDatabase( db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); +#if SQLITE_USER_AUTHENTICATION + db->auth.authFlags = UAUTH_Auth|UAUTH_Admin; +#endif /* The default safety_level for the main database is 'full'; for the temp ** database it is 'NONE'. This matches the pager layer defaults. diff --git a/src/pragma.c b/src/pragma.c index 12446125fb..8ae79bda33 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1397,6 +1397,12 @@ void sqlite3Pragma( ** in auto-commit mode. */ mask &= ~(SQLITE_ForeignKeys); } +#if SQLITE_USER_AUTHENTICATION + if( !DbIsAdmin(db) ){ + /* Do not allow non-admin users to modify the schema arbitrarily */ + mask &= ~(SQLITE_WriteSchema); + } +#endif if( sqlite3GetBoolean(zRight, 0) ){ db->flags |= mask; diff --git a/src/prepare.c b/src/prepare.c index 5b92e88513..3703b35176 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -206,6 +206,9 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ if( ALWAYS(pTab) ){ pTab->tabFlags |= TF_Readonly; } +#if SQLITE_USER_AUTHENTICATION + db->auth.authFlags = UAUTH_Auth|UAUTH_Admin; +#endif /* Create a cursor to hold the database open */ @@ -361,6 +364,14 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ DbSetProperty(db, iDb, DB_SchemaLoaded); rc = SQLITE_OK; } +#if SQLITE_USER_AUTHENTICATION + if( rc==SQLITE_OK && iDb!=1 ){ + if( sqlite3FindTable(db, "sqlite_user", db->aDb[iDb].zName)!=0 ){ + db->auth.authFlags = UAUTH_AuthReqd; + } + } +#endif + /* Jump here for an error that occurs after successfully allocating ** curMain and calling sqlite3BtreeEnter(). For an error that occurs @@ -720,6 +731,17 @@ static int sqlite3LockAndPrepare( sqlite3_finalize(*ppStmt); rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail); } +#if SQLITE_USER_AUTHENTICATION + assert( rc==SQLITE_OK || *ppStmt==0 ); +printf("rc=%d init=%d auth=%d sql=[%.50s]\n", rc, db->init.busy, db->auth.authFlags, zSql); +fflush(stdout); + if( rc==SQLITE_OK && !DbIsAuth(db) && db->init.busy==0 ){ + sqlite3_finalize(*ppStmt); + *ppStmt = 0; + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "user not authenticated"); + rc = SQLITE_ERROR; + } +#endif sqlite3BtreeLeaveAll(db); sqlite3_mutex_leave(db->mutex); assert( rc==SQLITE_OK || *ppStmt==0 ); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ceacdbb335..ae43014c4c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -988,6 +988,35 @@ struct FuncDefHash { FuncDef *a[23]; /* Hash table for functions */ }; +#ifdef SQLITE_USER_AUTHENTICATION +/* +** Information held in the "sqlite3" database connection object and used +** to manage user authentication. +*/ +typedef struct sqlite3_userauth sqlite3_userauth; +struct sqlite3_userauth { + u8 authFlags; /* Status flags for user authentication */ + int nAuthPW; /* Size of the zAuthPW in bytes */ + char *zAuthPW; /* Password used to authenticate */ + char *zAuthUser; /* User name used to authenticate */ +}; + +/* Allowed values for sqlite3_userauth.authFlags */ +#define UAUTH_Ovrd 0x01 /* Do not enforce access restrictions */ +#define UAUTH_Auth 0x02 /* True if the user has authenticated */ +#define UAUTH_Admin 0x04 /* True if the user is an administrator */ +#define UAUTH_AuthReqd 0x08 /* True if main has an sqlite_user table */ + +/* Macros for accessing sqlite3.auth.authFlags */ +#define DbIsAuth(D) (((D)->auth.authFlags&UAUTH_Auth)!=0) +#define DbIsAdmin(D) (((D)->auth.authFlags&UAUTH_Admin)!=0) + +/* Functions used only by user authorization logic */ +int sqlite3UserAuthTable(const char*); + +#endif /* SQLITE_USER_AUTHENTICATION */ + + /* ** Each database connection is an instance of the following structure. */ @@ -1082,7 +1111,6 @@ struct sqlite3 { i64 nDeferredCons; /* Net deferred constraints this transaction. */ i64 nDeferredImmCons; /* Net deferred immediate constraints */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ - #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY /* The following variables are all protected by the STATIC_MASTER ** mutex, not by sqlite3.mutex. They are used by code in notify.c. @@ -1100,6 +1128,9 @@ struct sqlite3 { void (*xUnlockNotify)(void **, int); /* Unlock notify callback */ sqlite3 *pNextBlocked; /* Next in list of all blocked connections */ #endif +#ifdef SQLITE_USER_AUTHENTICATION + sqlite3_userauth auth; /* User authentication information */ +#endif }; /* From da4ca9d19cf92628f4989d95185f3b1fa1418e6d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Sep 2014 17:27:35 +0000 Subject: [PATCH 308/710] Add new APIs that take 64-bit length parameters: sqlite3_malloc64(), sqlite3_realloc64(), sqlite3_bind_blob64(), sqlite3_bind_texte64(), sqlite3_result_blob64(), and sqlite3_result_texte64(). Internal memory allocation routines also now use 64-bit unsigned length parameters for safety. Also add the sqlite3_msize() interface. Fix the sqlite3_get_table() to use sqlite3_realloc64() to avoid a integer overflow problem. FossilOrigin-Name: 94954850cf2e1ec0b7f590c7f46cdc54c72558ce --- manifest | 31 +++++++++++--------- manifest.uuid | 2 +- src/btree.c | 2 +- src/func.c | 2 +- src/malloc.c | 42 ++++++++++++++++++--------- src/pager.c | 2 +- src/sqlite.h.in | 76 ++++++++++++++++++++++++++++++++++++++----------- src/sqliteInt.h | 16 +++++------ src/table.c | 10 +++---- src/vdbeapi.c | 74 +++++++++++++++++++++++++++++++++++++++++++++++ 10 files changed, 196 insertions(+), 61 deletions(-) diff --git a/manifest b/manifest index ccd849754b..ae18c40673 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sto\sos_unix.c\sto\ssupport\sdatabase\s(and\sother)\sfiles\slarger\sthan\s2GiB\son\sAndroid. -D 2014-09-06T17:06:13.410 +C Add\snew\sAPIs\sthat\stake\s64-bit\slength\sparameters:\nsqlite3_malloc64(),\nsqlite3_realloc64(),\nsqlite3_bind_blob64(),\nsqlite3_bind_texte64(),\nsqlite3_result_blob64(),\nand\ssqlite3_result_texte64().\nInternal\smemory\sallocation\sroutines\salso\snow\suse\s64-bit\sunsigned\slength\nparameters\sfor\ssafety.\nAlso\sadd\sthe\ssqlite3_msize()\sinterface.\nFix\sthe\ssqlite3_get_table()\sto\suse\ssqlite3_realloc64()\sto\savoid\sa\ninteger\soverflow\sproblem. +D 2014-09-09T17:27:35.957 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c b1c1cd1cc3ae2e433a23b9a6c9ab53805707d8cd +F src/btree.c e4916b441bb036897cc69df275a2df3fea4d53b6 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc F src/build.c 8cb237719c185eec7bd8449b2e747491ded11932 @@ -180,7 +180,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c 441a7e24e2f7bea9475778fa8acce9e8a69ca8f0 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c 0517037766e18eff7dce298e6b3a8e6311df75ec +F src/func.c 2ae55b52aa95367ee11d1a8f658605093b13d250 F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -191,7 +191,7 @@ F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab F src/main.c e48517e3da289d93ad86e8b7b4f68078df5e6e51 -F src/malloc.c 954de5f998c23237e04474a3f2159bf483bba65a +F src/malloc.c cc015821ba267ad5c91dc8761d0498a3fc3ce6ce F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f @@ -211,7 +211,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c addd023b26c623fec4dedc110fc4370a65b4768c F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 31da9594ad4c3b5851bb6fe1a95c33835ab7ddce +F src/pager.c c6c809987f0c6a4e27634099d062d425527de173 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 2048affdb09a04478b5fc6e64cb1083078d369be @@ -225,13 +225,13 @@ F src/resolve.c 0d1621e45fffe4b4396477cf46e41a84b0145ffb F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c b4457526cee73c0b69fad42f799f619b1d5a8a8a F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50 -F src/sqlite.h.in 64a77f2822f1325b12050972003184f99b655a0f +F src/sqlite.h.in cbb079b1d89b45d53d44aab4dc291ce2bac0a4b1 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 1f40357fb9b12a80c5a3b2b109fd249b009213d4 -F src/sqliteInt.h 4d1b0488f097aa7be346176c4d5e3cc1e25d99da +F src/sqliteInt.h 0a9083f9d277bf8ca7e9327c01e01bd01f01a585 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 -F src/table.c 4e28a53e66bad8d014a510ef0205f5497c712b08 +F src/table.c 218ae2ba022881846741dfc8351aefdf129e0377 F src/tclsqlite.c 8d6d6833c0053f0b3b1aeb1c5c7a7eeff0ad4d3f F src/test1.c 22bfe1ce9f2f3746d682093a475ec0a33e0e55d8 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 @@ -288,7 +288,7 @@ F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 9a45dcbcc967fc0cb9248c75ba245d1d47de3e78 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h b4843c35c3ba533b69d4250f550b5bacf2fb013d -F src/vdbeapi.c e43a6b98f853d8064cc096e349ce47e63d4c72d2 +F src/vdbeapi.c fce69e5e9018ce3189da80eefe62ca67606723bc F src/vdbeaux.c 91fd1e0c54a765838dc61fcf79f31acce035ce38 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c dc36ea9fe26c25550c50085f388167086ef7d73a @@ -1193,7 +1193,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e62aab5e9290503869e1f4d5e0fefd2b4dee0a69 9dca7ce55797b3eb617859f6189c1a2ec6f66566 -R 4eacabb789e62917f709201f03eadd53 -U dan -Z f60ed4f30cad7188918c7dd18fc1b985 +P ad7063aa1a0db32cdbe71815545b2edca57d3bcc +R c9aabba8d6c73d7bad5e8196d2f3fcc0 +T *branch * 64-bit-lengths +T *sym-64-bit-lengths * +T -sym-trunk * +U drh +Z b65eab85daa713d0df4e10d4fe4ad4ae diff --git a/manifest.uuid b/manifest.uuid index fc5888a5d9..f3484cced1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ad7063aa1a0db32cdbe71815545b2edca57d3bcc \ No newline at end of file +94954850cf2e1ec0b7f590c7f46cdc54c72558ce \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 22880f8a54..fe34b4a836 100644 --- a/src/btree.c +++ b/src/btree.c @@ -606,7 +606,7 @@ static int saveCursorPosition(BtCursor *pCur){ ** data. */ if( 0==pCur->apPage[0]->intKey ){ - void *pKey = sqlite3Malloc( (int)pCur->nKey ); + void *pKey = sqlite3Malloc( pCur->nKey ); if( pKey ){ rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); if( rc==SQLITE_OK ){ diff --git a/src/func.c b/src/func.c index e338ab842b..eef16d3cab 100644 --- a/src/func.c +++ b/src/func.c @@ -390,7 +390,7 @@ static void *contextMalloc(sqlite3_context *context, i64 nByte){ sqlite3_result_error_toobig(context); z = 0; }else{ - z = sqlite3Malloc((int)nByte); + z = sqlite3Malloc(nByte); if( !z ){ sqlite3_result_error_nomem(context); } diff --git a/src/malloc.c b/src/malloc.c index b4b70350f4..daf646bc30 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -294,11 +294,9 @@ static int mallocWithAlarm(int n, void **pp){ ** Allocate memory. This routine is like sqlite3_malloc() except that it ** assumes the memory subsystem has already been initialized. */ -void *sqlite3Malloc(int n){ +void *sqlite3Malloc(u64 n){ void *p; - if( n<=0 /* IMP: R-65312-04917 */ - || n>=0x7fffff00 - ){ + if( n==0 || n>=0x7fffff00 ){ /* A memory allocation of a number of bytes which is near the maximum ** signed integer value might cause an integer overflow inside of the ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving @@ -310,7 +308,7 @@ void *sqlite3Malloc(int n){ mallocWithAlarm(n, &p); sqlite3_mutex_leave(mem0.mutex); }else{ - p = sqlite3GlobalConfig.m.xMalloc(n); + p = sqlite3GlobalConfig.m.xMalloc((int)n); } assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-04675-44850 */ return p; @@ -322,6 +320,12 @@ void *sqlite3Malloc(int n){ ** allocation. */ void *sqlite3_malloc(int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + return n<=0 ? 0 : sqlite3Malloc(n); +} +void *sqlite3_malloc64(sqlite3_uint64 n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -458,6 +462,9 @@ int sqlite3DbMallocSize(sqlite3 *db, void *p){ return sqlite3GlobalConfig.m.xSize(p); } } +sqlite3_uint64 sqlite3_msize(void *p){ + return (sqlite3_uint64)sqlite3GlobalConfig.m.xSize(p); +} /* ** Free memory previously obtained from sqlite3Malloc(). @@ -519,13 +526,13 @@ void sqlite3DbFree(sqlite3 *db, void *p){ /* ** Change the size of an existing memory allocation */ -void *sqlite3Realloc(void *pOld, int nBytes){ +void *sqlite3Realloc(void *pOld, u64 nBytes){ int nOld, nNew, nDiff; void *pNew; if( pOld==0 ){ return sqlite3Malloc(nBytes); /* IMP: R-28354-25769 */ } - if( nBytes<=0 ){ + if( nBytes==0 ){ sqlite3_free(pOld); /* IMP: R-31593-10574 */ return 0; } @@ -537,7 +544,7 @@ void *sqlite3Realloc(void *pOld, int nBytes){ /* IMPLEMENTATION-OF: R-46199-30249 SQLite guarantees that the second ** argument to xRealloc is always a value returned by a prior call to ** xRoundup. */ - nNew = sqlite3GlobalConfig.m.xRoundup(nBytes); + nNew = sqlite3GlobalConfig.m.xRoundup((int)nBytes); if( nOld==nNew ){ pNew = pOld; }else if( sqlite3GlobalConfig.bMemstat ){ @@ -572,6 +579,13 @@ void *sqlite3Realloc(void *pOld, int nBytes){ ** subsystem is initialized prior to invoking sqliteRealloc. */ void *sqlite3_realloc(void *pOld, int n){ +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return 0; +#endif + if( n<0 ) n = 0; + return sqlite3Realloc(pOld, n); +} +void *sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -582,7 +596,7 @@ void *sqlite3_realloc(void *pOld, int n){ /* ** Allocate and zero memory. */ -void *sqlite3MallocZero(int n){ +void *sqlite3MallocZero(u64 n){ void *p = sqlite3Malloc(n); if( p ){ memset(p, 0, n); @@ -594,7 +608,7 @@ void *sqlite3MallocZero(int n){ ** Allocate and zero memory. If the allocation fails, make ** the mallocFailed flag in the connection pointer. */ -void *sqlite3DbMallocZero(sqlite3 *db, int n){ +void *sqlite3DbMallocZero(sqlite3 *db, u64 n){ void *p = sqlite3DbMallocRaw(db, n); if( p ){ memset(p, 0, n); @@ -620,7 +634,7 @@ void *sqlite3DbMallocZero(sqlite3 *db, int n){ ** In other words, if a subsequent malloc (ex: "b") worked, it is assumed ** that all prior mallocs (ex: "a") worked too. */ -void *sqlite3DbMallocRaw(sqlite3 *db, int n){ +void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){ void *p; assert( db==0 || sqlite3_mutex_held(db->mutex) ); assert( db==0 || db->pnBytesFreed==0 ); @@ -664,7 +678,7 @@ void *sqlite3DbMallocRaw(sqlite3 *db, int n){ ** Resize the block of memory pointed to by p to n bytes. If the ** resize fails, set the mallocFailed flag in the connection object. */ -void *sqlite3DbRealloc(sqlite3 *db, void *p, int n){ +void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ void *pNew = 0; assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); @@ -701,7 +715,7 @@ void *sqlite3DbRealloc(sqlite3 *db, void *p, int n){ ** Attempt to reallocate p. If the reallocation fails, then free p ** and set the mallocFailed flag in the database connection. */ -void *sqlite3DbReallocOrFree(sqlite3 *db, void *p, int n){ +void *sqlite3DbReallocOrFree(sqlite3 *db, void *p, u64 n){ void *pNew; pNew = sqlite3DbRealloc(db, p, n); if( !pNew ){ @@ -731,7 +745,7 @@ char *sqlite3DbStrDup(sqlite3 *db, const char *z){ } return zNew; } -char *sqlite3DbStrNDup(sqlite3 *db, const char *z, int n){ +char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ char *zNew; if( z==0 ){ return 0; diff --git a/src/pager.c b/src/pager.c index 8059bee055..6ad6f73227 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2428,7 +2428,7 @@ static int pager_delmaster(Pager *pPager, const char *zMaster){ rc = sqlite3OsFileSize(pMaster, &nMasterJournal); if( rc!=SQLITE_OK ) goto delmaster_out; nMasterPtr = pVfs->mxPathname+1; - zMasterJournal = sqlite3Malloc((int)nMasterJournal + nMasterPtr + 1); + zMasterJournal = sqlite3Malloc(nMasterJournal + nMasterPtr + 1); if( !zMasterJournal ){ rc = SQLITE_NOMEM; goto delmaster_out; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 56dede8ba8..79dd4c34b9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2298,6 +2298,10 @@ char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** sqlite3_malloc() is zero or negative then sqlite3_malloc() returns ** a NULL pointer. ** +** ^The sqlite3_malloc64(N) routine works just like +** sqlite3_malloc(N) except that N is an unsigned 64-bit integer instead +** of a signed 32-bit integer. +** ** ^Calling sqlite3_free() with a pointer previously returned ** by sqlite3_malloc() or sqlite3_realloc() releases that memory so ** that it might be reused. ^The sqlite3_free() routine is @@ -2309,24 +2313,38 @@ char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** might result if sqlite3_free() is called with a non-NULL pointer that ** was not obtained from sqlite3_malloc() or sqlite3_realloc(). ** -** ^(The sqlite3_realloc() interface attempts to resize a -** prior memory allocation to be at least N bytes, where N is the -** second parameter. The memory allocation to be resized is the first -** parameter.)^ ^ If the first parameter to sqlite3_realloc() +** ^The sqlite3_realloc(X,N) interface attempts to resize a +** prior memory allocation X to be at least N bytes. +** ^If the X parameter to sqlite3_realloc(X,N) ** is a NULL pointer then its behavior is identical to calling -** sqlite3_malloc(N) where N is the second parameter to sqlite3_realloc(). -** ^If the second parameter to sqlite3_realloc() is zero or +** sqlite3_malloc(N). +** ^If the N parameter to sqlite3_realloc(X,N) is zero or ** negative then the behavior is exactly the same as calling -** sqlite3_free(P) where P is the first parameter to sqlite3_realloc(). -** ^sqlite3_realloc() returns a pointer to a memory allocation -** of at least N bytes in size or NULL if sufficient memory is unavailable. +** sqlite3_free(X). +** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation +** of at least N bytes in size or NULL if insufficient memory is available. ** ^If M is the size of the prior allocation, then min(N,M) bytes ** of the prior allocation are copied into the beginning of buffer returned -** by sqlite3_realloc() and the prior allocation is freed. -** ^If sqlite3_realloc() returns NULL, then the prior allocation -** is not freed. +** by sqlite3_realloc(X,N) and the prior allocation is freed. +** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the +** prior allocation is not freed. ** -** ^The memory returned by sqlite3_malloc() and sqlite3_realloc() +** ^The sqlite3_realloc64(X,N) interfaces works the same as +** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead +** of a 32-bit signed integer. +** +** ^If X is a memory allocation previously obtained from sqlite3_malloc(), +** sqlite3_malloc64(), sqlite3_realloc(), or sqlite3_realloc64(), then +** sqlite3_msize(X) returns the size of that memory allocation in bytes. +** ^The value returned by sqlite3_msize(X) might be larger than the number +** of bytes requested when X was allocated. ^If X is a NULL pointer then +** sqlite3_msize(X) returns zero. If X points to something that is not +** the beginning of memory allocation, or if it points to a formerly +** valid memory allocation that has now been freed, then the behavior +** of sqlite3_msize(X) is undefined and possibly harmful. +** +** ^The memory returned by sqlite3_malloc(), sqlite3_realloc(), +** sqlite3_malloc64(), and sqlite3_realloc64() ** is always aligned to at least an 8 byte boundary, or to a ** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time ** option is used. @@ -2354,8 +2372,11 @@ char *sqlite3_vsnprintf(int,char*,const char*, va_list); ** [sqlite3_free()] or [sqlite3_realloc()]. */ void *sqlite3_malloc(int); +void *sqlite3_malloc64(sqlite3_uint64); void *sqlite3_realloc(void*, int); +void *sqlite3_realloc64(void*, sqlite3_uint64); void sqlite3_free(void*); +sqlite3_uint64 sqlite3_msize(void*); /* ** CAPI3REF: Memory Allocator Statistics @@ -3364,7 +3385,8 @@ typedef struct sqlite3_context sqlite3_context; ** If the fourth parameter to sqlite3_bind_blob() is negative, then ** the behavior is undefined. ** If a non-negative fourth parameter is provided to sqlite3_bind_text() -** or sqlite3_bind_text16() then that parameter must be the byte offset +** or sqlite3_bind_text16() or sqlite3_bind_texte64() then +** that parameter must be the byte offset ** where the NUL terminator would occur assuming the string were NUL ** terminated. If any NUL characters occur at byte offsets less than ** the value of the fourth parameter then the resulting string value will @@ -3383,6 +3405,14 @@ typedef struct sqlite3_context sqlite3_context; ** SQLite makes its own private copy of the data immediately, before ** the sqlite3_bind_*() routine returns. ** +** ^The sixth argument to sqlite3_bind_texte64() must be one of +** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] +** to specify the encoding of the text in the third parameter. If +** the sixth argument to sqlite3_bind_texte64() is not how of the +** allowed values shown above, or if the text encoding is different +** from the encoding specified by the sixth parameter, then the behavior +** is undefined. +** ** ^The sqlite3_bind_zeroblob() routine binds a BLOB of length N that ** is filled with zeroes. ^A zeroblob uses a fixed amount of memory ** (just an integer to hold its size) while it is being processed. @@ -3403,6 +3433,9 @@ typedef struct sqlite3_context sqlite3_context; ** ** ^The sqlite3_bind_* routines return [SQLITE_OK] on success or an ** [error code] if anything goes wrong. +** ^[SQLITE_TOOBIG] might be returned if the size of a string or BLOB +** exceeds limits imposed by [sqlite3_limit]([SQLITE_LIMIT_LENGTH]) or +** [SQLITE_MAX_LENGTH]. ** ^[SQLITE_RANGE] is returned if the parameter ** index is out of range. ^[SQLITE_NOMEM] is returned if malloc() fails. ** @@ -3410,12 +3443,16 @@ typedef struct sqlite3_context sqlite3_context; ** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()]. */ int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); +int sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, + void(*)(void*)); int sqlite3_bind_double(sqlite3_stmt*, int, double); int sqlite3_bind_int(sqlite3_stmt*, int, int); int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); int sqlite3_bind_null(sqlite3_stmt*, int); -int sqlite3_bind_text(sqlite3_stmt*, int, const char*, int n, void(*)(void*)); +int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); +int sqlite3_bind_texte64(sqlite3_stmt*, int, const char*, sqlite3_uint64, + void(*)(void*), unsigned char encoding); int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); @@ -4407,10 +4444,14 @@ typedef void (*sqlite3_destructor_type)(void*); ** of the application-defined function to be NULL. ** ** ^The sqlite3_result_text(), sqlite3_result_text16(), -** sqlite3_result_text16le(), and sqlite3_result_text16be() interfaces +** sqlite3_result_text16le(), and sqlite3_result_text16be() ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. +** ^The sqlite3_result_texte64() interface sets the return value of an +** application-defined function to be a text string in an encoding +** specified by the fifth (and last) parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. ** ^If the 3rd parameter to the sqlite3_result_text* interfaces @@ -4454,6 +4495,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** the [sqlite3_context] pointer, the results are undefined. */ void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +void sqlite3_result_blob64(sqlite3_context*,const void*,sqlite3_uint64,void(*)(void*)); void sqlite3_result_double(sqlite3_context*, double); void sqlite3_result_error(sqlite3_context*, const char*, int); void sqlite3_result_error16(sqlite3_context*, const void*, int); @@ -4464,6 +4506,8 @@ void sqlite3_result_int(sqlite3_context*, int); void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); void sqlite3_result_null(sqlite3_context*); void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +void sqlite3_result_texte64(sqlite3_context*, const char*,sqlite3_uint64, + void(*)(void*), unsigned char encoding); void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ceacdbb335..8e3a6364c6 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2931,15 +2931,15 @@ int sqlite3Strlen30(const char*); int sqlite3MallocInit(void); void sqlite3MallocEnd(void); -void *sqlite3Malloc(int); -void *sqlite3MallocZero(int); -void *sqlite3DbMallocZero(sqlite3*, int); -void *sqlite3DbMallocRaw(sqlite3*, int); +void *sqlite3Malloc(u64); +void *sqlite3MallocZero(u64); +void *sqlite3DbMallocZero(sqlite3*, u64); +void *sqlite3DbMallocRaw(sqlite3*, u64); char *sqlite3DbStrDup(sqlite3*,const char*); -char *sqlite3DbStrNDup(sqlite3*,const char*, int); -void *sqlite3Realloc(void*, int); -void *sqlite3DbReallocOrFree(sqlite3 *, void *, int); -void *sqlite3DbRealloc(sqlite3 *, void *, int); +char *sqlite3DbStrNDup(sqlite3*,const char*, u64); +void *sqlite3Realloc(void*, u64); +void *sqlite3DbReallocOrFree(sqlite3 *, void *, u64); +void *sqlite3DbRealloc(sqlite3 *, void *, u64); void sqlite3DbFree(sqlite3*, void*); int sqlite3MallocSize(void*); int sqlite3DbMallocSize(sqlite3*, void*); diff --git a/src/table.c b/src/table.c index 10b0d627f9..12d0cf548e 100644 --- a/src/table.c +++ b/src/table.c @@ -29,10 +29,10 @@ typedef struct TabResult { char **azResult; /* Accumulated output */ char *zErrMsg; /* Error message text, if an error occurs */ - int nAlloc; /* Slots allocated for azResult[] */ - int nRow; /* Number of rows in the result */ - int nColumn; /* Number of columns in the result */ - int nData; /* Slots used in azResult[]. (nRow+1)*nColumn */ + u32 nAlloc; /* Slots allocated for azResult[] */ + u32 nRow; /* Number of rows in the result */ + u32 nColumn; /* Number of columns in the result */ + u32 nData; /* Slots used in azResult[]. (nRow+1)*nColumn */ int rc; /* Return code from sqlite3_exec() */ } TabResult; @@ -58,7 +58,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ if( p->nData + need > p->nAlloc ){ char **azNew; p->nAlloc = p->nAlloc*2 + need; - azNew = sqlite3_realloc( p->azResult, sizeof(char*)*p->nAlloc ); + azNew = sqlite3_realloc64( p->azResult, sizeof(char*)*p->nAlloc ); if( azNew==0 ) goto malloc_failed; p->azResult = azNew; } diff --git a/src/vdbeapi.c b/src/vdbeapi.c index e141ddfb94..087ea5c109 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -215,6 +215,9 @@ int sqlite3_value_type(sqlite3_value* pVal){ ** The setStrOrError() function calls sqlite3VdbeMemSetStr() to store the ** result as a string or blob but if the string or blob is too large, it ** then sets the error code to SQLITE_TOOBIG +** +** The invokeValueDestructor(P,X) routine invokes destructor function X() +** on value P is not going to be used and need to be destroyed. */ static void setResultStrOrError( sqlite3_context *pCtx, /* Function context */ @@ -227,6 +230,23 @@ static void setResultStrOrError( sqlite3_result_error_toobig(pCtx); } } +static int invokeValueDestructor( + const void *p, /* Value to destroy */ + void (*xDel)(void*), /* The destructor */ + sqlite3_context *pCtx /* Set a SQLITE_TOOBIG error if no NULL */ +){ + if( xDel==0 ){ + /* noop */ + }else if( xDel==SQLITE_TRANSIENT ){ + /* noop */ + }else if( xDel==SQLITE_DYNAMIC ){ + sqlite3_free((void*)p); + }else{ + xDel((void*)p); + } + if( pCtx ) sqlite3_result_error_toobig(pCtx); + return SQLITE_TOOBIG; +} void sqlite3_result_blob( sqlite3_context *pCtx, const void *z, @@ -237,6 +257,19 @@ void sqlite3_result_blob( assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, 0, xDel); } +void sqlite3_result_blob64( + sqlite3_context *pCtx, + const void *z, + sqlite3_uint64 n, + void (*xDel)(void *) +){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + if( n>0x7fffffff ){ + (void)invokeValueDestructor(z, xDel, pCtx); + }else{ + setResultStrOrError(pCtx, z, (int)n, 0, xDel); + } +} void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetDouble(pCtx->pOut, rVal); @@ -276,6 +309,20 @@ void sqlite3_result_text( assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); } +void sqlite3_result_texte64( + sqlite3_context *pCtx, + const char *z, + sqlite3_uint64 n, + void (*xDel)(void *), + unsigned char enc +){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + if( n>0x7fffffff ){ + (void)invokeValueDestructor(z, xDel, pCtx); + }else{ + setResultStrOrError(pCtx, z, (int)n, enc, xDel); + } +} #ifndef SQLITE_OMIT_UTF16 void sqlite3_result_text16( sqlite3_context *pCtx, @@ -1125,6 +1172,19 @@ int sqlite3_bind_blob( ){ return bindText(pStmt, i, zData, nData, xDel, 0); } +int sqlite3_bind_blob64( + sqlite3_stmt *pStmt, + int i, + const void *zData, + sqlite3_uint64 nData, + void (*xDel)(void*) +){ + if( nData>0x7fffffff ){ + return invokeValueDestructor(zData, xDel, 0); + }else{ + return bindText(pStmt, i, zData, nData, xDel, 0); + } +} int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; @@ -1166,6 +1226,20 @@ int sqlite3_bind_text( ){ return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8); } +int sqlite3_bind_texte64( + sqlite3_stmt *pStmt, + int i, + const char *zData, + sqlite3_uint64 nData, + void (*xDel)(void*), + unsigned char enc +){ + if( nData>0x7fffffff ){ + return invokeValueDestructor(zData, xDel, 0); + }else{ + return bindText(pStmt, i, zData, nData, xDel, enc); + } +} #ifndef SQLITE_OMIT_UTF16 int sqlite3_bind_text16( sqlite3_stmt *pStmt, From 0807cc2c29593d4bd1cbe4544bacff0509d1ed0e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Sep 2014 18:41:32 +0000 Subject: [PATCH 309/710] Add new interfaces to the loadable extension mechanism. FossilOrigin-Name: 18d80cbc590165913d82056aa69ddaeea07b76ec --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/loadext.c | 15 ++++++++++++++- src/sqlite3ext.h | 30 ++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index ae18c40673..c4063387f4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sAPIs\sthat\stake\s64-bit\slength\sparameters:\nsqlite3_malloc64(),\nsqlite3_realloc64(),\nsqlite3_bind_blob64(),\nsqlite3_bind_texte64(),\nsqlite3_result_blob64(),\nand\ssqlite3_result_texte64().\nInternal\smemory\sallocation\sroutines\salso\snow\suse\s64-bit\sunsigned\slength\nparameters\sfor\ssafety.\nAlso\sadd\sthe\ssqlite3_msize()\sinterface.\nFix\sthe\ssqlite3_get_table()\sto\suse\ssqlite3_realloc64()\sto\savoid\sa\ninteger\soverflow\sproblem. -D 2014-09-09T17:27:35.957 +C Add\snew\sinterfaces\sto\sthe\sloadable\sextension\smechanism. +D 2014-09-09T18:41:32.385 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -189,7 +189,7 @@ F src/insert.c 0b073fade178d9dbd990bbb32b4438e50b884a06 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b -F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab +F src/loadext.c 0cb3e0394c21e7fc513b9e3b013f9675f4c93774 F src/main.c e48517e3da289d93ad86e8b7b4f68078df5e6e51 F src/malloc.c cc015821ba267ad5c91dc8761d0498a3fc3ce6ce F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -227,7 +227,7 @@ F src/select.c b4457526cee73c0b69fad42f799f619b1d5a8a8a F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50 F src/sqlite.h.in cbb079b1d89b45d53d44aab4dc291ce2bac0a4b1 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad -F src/sqlite3ext.h 1f40357fb9b12a80c5a3b2b109fd249b009213d4 +F src/sqlite3ext.h b8ff57953a1c160e5aa6e0f03119e90ea41fbb4d F src/sqliteInt.h 0a9083f9d277bf8ca7e9327c01e01bd01f01a585 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 @@ -1193,10 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ad7063aa1a0db32cdbe71815545b2edca57d3bcc -R c9aabba8d6c73d7bad5e8196d2f3fcc0 -T *branch * 64-bit-lengths -T *sym-64-bit-lengths * -T -sym-trunk * +P 94954850cf2e1ec0b7f590c7f46cdc54c72558ce +R c1ce17f309170a7fa900bd5f2fa22c3a U drh -Z b65eab85daa713d0df4e10d4fe4ad4ae +Z 9deb67bcd7039b69f5843b47ed73f48c diff --git a/manifest.uuid b/manifest.uuid index f3484cced1..3f9cdc63c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -94954850cf2e1ec0b7f590c7f46cdc54c72558ce \ No newline at end of file +18d80cbc590165913d82056aa69ddaeea07b76ec \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 05045dedb3..be8262989a 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -390,7 +390,20 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_uri_int64, sqlite3_uri_parameter, sqlite3_vsnprintf, - sqlite3_wal_checkpoint_v2 + sqlite3_wal_checkpoint_v2, + /* Version 3.8.7 and later */ + sqlite3_auto_extension, + sqlite3_bind_blob64, + sqlite3_bind_texte64, + sqlite3_cancel_auto_extension, + sqlite3_load_extension, + sqlite3_malloc64, + sqlite3_msize, + sqlite3_realloc64, + sqlite3_reset_auto_extension, + sqlite3_result_blob64, + sqlite3_result_texte64, + sqlite3_strglob }; /* diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index b4baea2cc5..17d6197fb7 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -250,6 +250,23 @@ struct sqlite3_api_routines { const char *(*uri_parameter)(const char*,const char*); char *(*vsnprintf)(int,char*,const char*,va_list); int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); + /* Version 3.8.7 and later */ + int (*auto_extension)(void(*)(void)); + int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64, + void(*)(void*)); + int (*bind_texte64)(sqlite3_stmt*,int,const char*,sqlite3_uint64, + void(*)(void*),unsigned char); + int (*cancel_auto_extension)(void(*)(void)); + int (*load_extension)(sqlite3*,const char*,const char*,char**); + void *(*malloc64)(sqlite3_uint64); + sqlite3_uint64 (*msize)(void*); + void *(*realloc64)(void*,sqlite3_uint64); + void (*reset_auto_extension)(void); + void (*result_blob64)(sqlite3_context*,const void*,sqlite3_uint64, + void(*)(void*)); + void (*result_texte64)(sqlite3_context*,const char*,sqlite3_uint64, + void(*)(void*), unsigned char); + int (*strglob)(const char*,const char*); }; /* @@ -467,6 +484,19 @@ struct sqlite3_api_routines { #define sqlite3_uri_parameter sqlite3_api->uri_parameter #define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf #define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 +/* Version 3.8.7 and later */ +#define sqlite3_auto_extension sqlite3_api->auto_extension +#define sqlite3_bind_blob64 sqlite3_api->bind_blob64 +#define sqlite3_bind_texte64 sqlite3_api->bind_texte64 +#define sqlite3_cancel_auto_extension sqlite3_api->cancel_auto_extension +#define sqlite3_load_extension sqlite3_api->load_extension +#define sqlite3_malloc64 sqlite3_api->malloc64 +#define sqlite3_msize sqlite3_api->msize +#define sqlite3_realloc64 sqlite3_api->realloc64 +#define sqlite3_reset_auto_extension sqlite3_api->reset_auto_extension +#define sqlite3_result_blob64 sqlite3_api->result_blob64 +#define sqlite3_result_texte64 sqlite3_api->result_texte64 +#define sqlite3_strglob sqlite3_api->strglob #endif /* SQLITE_CORE */ #ifndef SQLITE_CORE From bbf483f8553fb37ac73f58e0ec9f7d1e469792f3 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Sep 2014 20:30:24 +0000 Subject: [PATCH 310/710] Change the name of the _texte64() interfaces to just _test64(), without the "e". FossilOrigin-Name: 6ab76c5fedfe568b0f0f34a600f9135bf6558148 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/func.c | 7 ++++--- src/loadext.c | 4 ++-- src/sqlite.h.in | 14 +++++++------- src/sqlite3ext.h | 8 ++++---- src/vdbeapi.c | 4 ++-- 7 files changed, 30 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index c4063387f4..637b73b365 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sinterfaces\sto\sthe\sloadable\sextension\smechanism. -D 2014-09-09T18:41:32.385 +C Change\sthe\sname\sof\sthe\s_texte64()\sinterfaces\sto\sjust\s_test64(),\swithout\sthe\s"e". +D 2014-09-09T20:30:24.037 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c 441a7e24e2f7bea9475778fa8acce9e8a69ca8f0 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c 2ae55b52aa95367ee11d1a8f658605093b13d250 +F src/func.c 63a0da710cdef3dd195bde045e55af34d775b851 F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -189,7 +189,7 @@ F src/insert.c 0b073fade178d9dbd990bbb32b4438e50b884a06 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b -F src/loadext.c 0cb3e0394c21e7fc513b9e3b013f9675f4c93774 +F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 F src/main.c e48517e3da289d93ad86e8b7b4f68078df5e6e51 F src/malloc.c cc015821ba267ad5c91dc8761d0498a3fc3ce6ce F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 @@ -225,9 +225,9 @@ F src/resolve.c 0d1621e45fffe4b4396477cf46e41a84b0145ffb F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c b4457526cee73c0b69fad42f799f619b1d5a8a8a F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50 -F src/sqlite.h.in cbb079b1d89b45d53d44aab4dc291ce2bac0a4b1 +F src/sqlite.h.in cabd2e9e3a8acb15c5a0f23317e423a17d111e7d F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad -F src/sqlite3ext.h b8ff57953a1c160e5aa6e0f03119e90ea41fbb4d +F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 0a9083f9d277bf8ca7e9327c01e01bd01f01a585 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 @@ -288,7 +288,7 @@ F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 9a45dcbcc967fc0cb9248c75ba245d1d47de3e78 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h b4843c35c3ba533b69d4250f550b5bacf2fb013d -F src/vdbeapi.c fce69e5e9018ce3189da80eefe62ca67606723bc +F src/vdbeapi.c cec65a12dc2fe9072d5108d9f75df57b0324883a F src/vdbeaux.c 91fd1e0c54a765838dc61fcf79f31acce035ce38 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c dc36ea9fe26c25550c50085f388167086ef7d73a @@ -1193,7 +1193,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 94954850cf2e1ec0b7f590c7f46cdc54c72558ce -R c1ce17f309170a7fa900bd5f2fa22c3a +P 18d80cbc590165913d82056aa69ddaeea07b76ec +R 3de8eb96dbf21697cafd6a57c5ce9a51 U drh -Z 9deb67bcd7039b69f5843b47ed73f48c +Z 5ae42b376f3f0dc69341930379e42b64 diff --git a/manifest.uuid b/manifest.uuid index 3f9cdc63c8..3cbc755153 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -18d80cbc590165913d82056aa69ddaeea07b76ec \ No newline at end of file +6ab76c5fedfe568b0f0f34a600f9135bf6558148 \ No newline at end of file diff --git a/src/func.c b/src/func.c index eef16d3cab..e421b2861d 100644 --- a/src/func.c +++ b/src/func.c @@ -325,13 +325,14 @@ static void substrFunc( for(z2=z; *z2 && p2; p2--){ SQLITE_SKIP_UTF8(z2); } - sqlite3_result_text(context, (char*)z, (int)(z2-z), SQLITE_TRANSIENT); + sqlite3_result_text64(context, (char*)z, z2-z, SQLITE_TRANSIENT, + SQLITE_UTF8); }else{ if( p1+p2>len ){ p2 = len-p1; if( p2<0 ) p2 = 0; } - sqlite3_result_blob(context, (char*)&z[p1], (int)p2, SQLITE_TRANSIENT); + sqlite3_result_blob64(context, (char*)&z[p1], (u64)p2, SQLITE_TRANSIENT); } } @@ -1041,7 +1042,7 @@ static void charFunc( *zOut++ = 0x80 + (u8)(c & 0x3F); } \ } - sqlite3_result_text(context, (char*)z, (int)(zOut-z), sqlite3_free); + sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); } /* diff --git a/src/loadext.c b/src/loadext.c index be8262989a..2a2afd8654 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -394,7 +394,7 @@ static const sqlite3_api_routines sqlite3Apis = { /* Version 3.8.7 and later */ sqlite3_auto_extension, sqlite3_bind_blob64, - sqlite3_bind_texte64, + sqlite3_bind_text64, sqlite3_cancel_auto_extension, sqlite3_load_extension, sqlite3_malloc64, @@ -402,7 +402,7 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_realloc64, sqlite3_reset_auto_extension, sqlite3_result_blob64, - sqlite3_result_texte64, + sqlite3_result_text64, sqlite3_strglob }; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 79dd4c34b9..f9249b05bc 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -3385,7 +3385,7 @@ typedef struct sqlite3_context sqlite3_context; ** If the fourth parameter to sqlite3_bind_blob() is negative, then ** the behavior is undefined. ** If a non-negative fourth parameter is provided to sqlite3_bind_text() -** or sqlite3_bind_text16() or sqlite3_bind_texte64() then +** or sqlite3_bind_text16() or sqlite3_bind_text64() then ** that parameter must be the byte offset ** where the NUL terminator would occur assuming the string were NUL ** terminated. If any NUL characters occur at byte offsets less than @@ -3405,10 +3405,10 @@ typedef struct sqlite3_context sqlite3_context; ** SQLite makes its own private copy of the data immediately, before ** the sqlite3_bind_*() routine returns. ** -** ^The sixth argument to sqlite3_bind_texte64() must be one of +** ^The sixth argument to sqlite3_bind_text64() must be one of ** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] ** to specify the encoding of the text in the third parameter. If -** the sixth argument to sqlite3_bind_texte64() is not how of the +** the sixth argument to sqlite3_bind_text64() is not how of the ** allowed values shown above, or if the text encoding is different ** from the encoding specified by the sixth parameter, then the behavior ** is undefined. @@ -3451,7 +3451,7 @@ int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); int sqlite3_bind_null(sqlite3_stmt*, int); int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); -int sqlite3_bind_texte64(sqlite3_stmt*, int, const char*, sqlite3_uint64, +int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, void(*)(void*), unsigned char encoding); int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); @@ -4448,7 +4448,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. -** ^The sqlite3_result_texte64() interface sets the return value of an +** ^The sqlite3_result_text64() interface sets the return value of an ** application-defined function to be a text string in an encoding ** specified by the fifth (and last) parameter, which must be one ** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. @@ -4506,8 +4506,8 @@ void sqlite3_result_int(sqlite3_context*, int); void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); void sqlite3_result_null(sqlite3_context*); void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); -void sqlite3_result_texte64(sqlite3_context*, const char*,sqlite3_uint64, - void(*)(void*), unsigned char encoding); +void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, + void(*)(void*), unsigned char encoding); void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); diff --git a/src/sqlite3ext.h b/src/sqlite3ext.h index 17d6197fb7..f9a066592d 100644 --- a/src/sqlite3ext.h +++ b/src/sqlite3ext.h @@ -254,7 +254,7 @@ struct sqlite3_api_routines { int (*auto_extension)(void(*)(void)); int (*bind_blob64)(sqlite3_stmt*,int,const void*,sqlite3_uint64, void(*)(void*)); - int (*bind_texte64)(sqlite3_stmt*,int,const char*,sqlite3_uint64, + int (*bind_text64)(sqlite3_stmt*,int,const char*,sqlite3_uint64, void(*)(void*),unsigned char); int (*cancel_auto_extension)(void(*)(void)); int (*load_extension)(sqlite3*,const char*,const char*,char**); @@ -264,7 +264,7 @@ struct sqlite3_api_routines { void (*reset_auto_extension)(void); void (*result_blob64)(sqlite3_context*,const void*,sqlite3_uint64, void(*)(void*)); - void (*result_texte64)(sqlite3_context*,const char*,sqlite3_uint64, + void (*result_text64)(sqlite3_context*,const char*,sqlite3_uint64, void(*)(void*), unsigned char); int (*strglob)(const char*,const char*); }; @@ -487,7 +487,7 @@ struct sqlite3_api_routines { /* Version 3.8.7 and later */ #define sqlite3_auto_extension sqlite3_api->auto_extension #define sqlite3_bind_blob64 sqlite3_api->bind_blob64 -#define sqlite3_bind_texte64 sqlite3_api->bind_texte64 +#define sqlite3_bind_text64 sqlite3_api->bind_text64 #define sqlite3_cancel_auto_extension sqlite3_api->cancel_auto_extension #define sqlite3_load_extension sqlite3_api->load_extension #define sqlite3_malloc64 sqlite3_api->malloc64 @@ -495,7 +495,7 @@ struct sqlite3_api_routines { #define sqlite3_realloc64 sqlite3_api->realloc64 #define sqlite3_reset_auto_extension sqlite3_api->reset_auto_extension #define sqlite3_result_blob64 sqlite3_api->result_blob64 -#define sqlite3_result_texte64 sqlite3_api->result_texte64 +#define sqlite3_result_text64 sqlite3_api->result_text64 #define sqlite3_strglob sqlite3_api->strglob #endif /* SQLITE_CORE */ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 087ea5c109..4dccb30c15 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -309,7 +309,7 @@ void sqlite3_result_text( assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); } -void sqlite3_result_texte64( +void sqlite3_result_text64( sqlite3_context *pCtx, const char *z, sqlite3_uint64 n, @@ -1226,7 +1226,7 @@ int sqlite3_bind_text( ){ return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8); } -int sqlite3_bind_texte64( +int sqlite3_bind_text64( sqlite3_stmt *pStmt, int i, const char *zData, From e933b83f029d0c749346391c86a47217f7e294cb Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 10 Sep 2014 17:34:28 +0000 Subject: [PATCH 311/710] Further ideas on user authentication. Not yet working code. FossilOrigin-Name: c8171ecd0d6f097c9e95d5f6643bae8d67f44750 --- ext/userauth/userauth.c | 105 ++++++++++++++++++++++++---------------- main.mk | 11 ++++- manifest | 27 +++++------ manifest.uuid | 2 +- src/build.c | 10 +++- src/main.c | 4 -- src/pragma.c | 2 +- src/prepare.c | 40 ++++++--------- src/sqliteInt.h | 17 +++---- 9 files changed, 117 insertions(+), 101 deletions(-) diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c index 305ae43a11..518f466f14 100644 --- a/ext/userauth/userauth.c +++ b/ext/userauth/userauth.c @@ -22,6 +22,7 @@ ** directory as this file for additional information. */ #ifdef SQLITE_USER_AUTHENTICATION +#include "sqliteInt.h" /* ** Prepare an SQL statement for use by the user authentication logic. @@ -42,10 +43,7 @@ static sqlite3_stmt *sqlite3UserAuthPrepare( zSql = sqlite3_vmprintf(zFormat, ap); va_end(ap); if( zSql==0 ) return 0; - savedFlags = db->auth.authFlags; - db->auth.authFlags |= UAUTH_Ovrd; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - db->auth.authFlags = savedFlags; sqlite3_free(zSql); if( rc ){ sqlite3_finalize(pStmt); @@ -56,49 +54,67 @@ static sqlite3_stmt *sqlite3UserAuthPrepare( /* ** Check to see if database zDb has a "sqlite_user" table and if it does -** whether that table can authenticate zUser with nPw,zPw. +** whether that table can authenticate zUser with nPw,zPw. Write one of +** the UAUTH_* user authorization level codes into *peAuth and return a +** result code. */ -static int sqlite3UserAuthCheckLogin( +static int userAuthCheckLogin( sqlite3 *db, /* The database connection to check */ const char *zDb, /* Name of specific database to check */ - const char *zUser, /* User name */ - int nPw, /* Size of password in bytes */ - const char *zPw, /* Password */ - int *pbOk /* OUT: write boolean result here */ + u8 *peAuth /* OUT: One of UAUTH_* constants */ ){ sqlite3_stmt *pStmt; - char *zSql; int rc; - int iResult; - *pbOk = 0; - iResult = 0; + *peAuth = UAUTH_Unknown; pStmt = sqlite3UserAuthPrepare(db, "SELECT 1 FROM \"%w\".sqlite_master " " WHERE name='sqlite_user' AND type='table'", zDb); - if( pStmt==0 ) return SQLITE_NOMEM; - rc = sqlite3_step(pStmt): + if( pStmt==0 ){ + return SQLITE_NOMEM; + } + rc = sqlite3_step(pStmt); sqlite3_finalize(pStmt); if( rc==SQLITE_DONE ){ - *pbOk = 1; + *peAuth = UAUTH_Admin; /* No sqlite_user table. Everybody is admin. */ return SQLITE_OK; } - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_ROW ){ return rc; } + if( db->auth.zAuthUser==0 ){ + *peAuth = UAUTH_Fail; + return SQLITE_OK; + } pStmt = sqlite3UserAuthPrepare(db, "SELECT pw=sqlite_crypt(?1,pw), isAdmin FROM \"%w\".sqlite_user" " WHERE uname=?2", zDb); if( pStmt==0 ) return SQLITE_NOMEM; - sqlite3_bind_blob(pStmt, 1, zPw, nPw, SQLITE_STATIC); - sqlite3_bind_text(pStmt, 2, zUser, -1, SQLITE_STATIC); - rc = sqlite_step(pStmt); + sqlite3_bind_blob(pStmt, 1, db->auth.zAuthPW, db->auth.nAuthPW,SQLITE_STATIC); + sqlite3_bind_text(pStmt, 2, db->auth.zAuthUser, -1, SQLITE_STATIC); + rc = sqlite3_step(pStmt); if( rc==SQLITE_ROW && sqlite3_column_int(pStmt,0) ){ - *pbOk = sqlite3_column_int(pStmt, 1); + *peAuth = sqlite3_column_int(pStmt, 1) + UAUTH_User; + }else{ + *peAuth = UAUTH_Fail; } sqlite3_finalize(pStmt); return rc; } +int sqlite3UserAuthCheckLogin( + sqlite3 *db, /* The database connection to check */ + const char *zDb, /* Name of specific database to check */ + u8 *peAuth /* OUT: One of UAUTH_* constants */ +){ + int rc; + u8 savedAuthLevel; + savedAuthLevel = db->auth.authLevel; + db->auth.authLevel = UAUTH_Admin; + rc = userAuthCheckLogin(db, zDb, peAuth); + db->auth.authLevel = savedAuthLevel; + return rc; +} + /* ** If a database contains the SQLITE_USER table, then the @@ -116,29 +132,29 @@ int sqlite3_user_authenticate( sqlite3 *db, /* The database connection */ const char *zUsername, /* Username */ int nPW, /* Number of bytes in aPW[] */ - const void *aPW /* Password or credentials */ + const char *zPW /* Password or credentials */ ){ - int bOk = 0; int rc; - - rc = sqlite3UserAuthCheckLogin(db, zUsername, nPw, zPw, &bOk); - if( bOk ){ - db->auth.authFlags = bOk==2 ? UAUTH_Auth|UAUTH_Admin : UAUTH_Auth; - sqlite3_free(db->auth.zAuthUser); - db->auth.zAuthUser = sqlite3_malloc("%s", zUsername); - sqlite3_free(db->auth.zPw); - db->auth.zPw = sqlite3_malloc( nPw+1 ); - if( db->auth.zPw ){ - memcpy(db->auth.zPw,zPw,nPw); - db->auth.nPw = nPw; - rc = SQLITE_OK; - }else{ - rc = SQLITE_NOMEM; - } - }else{ - db->auth.authFlags = 0; + u8 authLevel = UAUTH_Fail; + db->auth.authLevel = UAUTH_Unknown; + sqlite3_free(db->auth.zAuthUser); + sqlite3_free(db->auth.zAuthPW); + memset(&db->auth, 0, sizeof(db->auth)); + db->auth.zAuthUser = sqlite3_mprintf("%s", zUsername); + if( db->auth.zAuthUser==0 ) return SQLITE_NOMEM; + db->auth.zAuthPW = sqlite3_malloc( nPW+1 ); + if( db->auth.zAuthPW==0 ) return SQLITE_NOMEM; + memcpy(db->auth.zAuthPW,zPW,nPW); + db->auth.nAuthPW = nPW; + rc = sqlite3UserAuthCheckLogin(db, "main", &authLevel); + db->auth.authLevel = authLevel; + if( rc ){ + return rc; /* OOM error, I/O error, etc. */ } - return rc; + if( authLevelauth.authLevelauth.authLevelauth.zAuthUser, zUsername)!=0 + && db->auth.authLevelauth.authLevelauth.authLevelnDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ @@ -348,6 +350,12 @@ Table *sqlite3LocateTable( } pParse->checkSchema = 1; } +#if SQLITE_USER_AUTHENICATION + else if( pParse->db->auth.authLevelaDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); -#if SQLITE_USER_AUTHENTICATION - db->auth.authFlags = UAUTH_Auth|UAUTH_Admin; -#endif - /* The default safety_level for the main database is 'full'; for the temp ** database it is 'NONE'. This matches the pager layer defaults. */ diff --git a/src/pragma.c b/src/pragma.c index 8ae79bda33..cbd593f215 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1398,7 +1398,7 @@ void sqlite3Pragma( mask &= ~(SQLITE_ForeignKeys); } #if SQLITE_USER_AUTHENTICATION - if( !DbIsAdmin(db) ){ + if( db->auth.authLeveltabFlags |= TF_Readonly; } -#if SQLITE_USER_AUTHENTICATION - db->auth.authFlags = UAUTH_Auth|UAUTH_Admin; -#endif /* Create a cursor to hold the database open */ @@ -364,14 +361,6 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ DbSetProperty(db, iDb, DB_SchemaLoaded); rc = SQLITE_OK; } -#if SQLITE_USER_AUTHENTICATION - if( rc==SQLITE_OK && iDb!=1 ){ - if( sqlite3FindTable(db, "sqlite_user", db->aDb[iDb].zName)!=0 ){ - db->auth.authFlags = UAUTH_AuthReqd; - } - } -#endif - /* Jump here for an error that occurs after successfully allocating ** curMain and calling sqlite3BtreeEnter(). For an error that occurs @@ -420,8 +409,8 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ ** schema may contain references to objects in other databases. */ #ifndef SQLITE_OMIT_TEMPDB - if( rc==SQLITE_OK && ALWAYS(db->nDb>1) - && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ + assert( db->nDb>1 ); + if( rc==SQLITE_OK && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ rc = sqlite3InitOne(db, 1, pzErrMsg); if( rc ){ sqlite3ResetOneSchema(db, 1); @@ -725,23 +714,26 @@ static int sqlite3LockAndPrepare( return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); +#if SQLITE_USER_AUTHENTICATION + if( db->auth.authLevelauth.authLevel==UAUTH_Unknown ){ + u8 authLevel = UAUTH_Fail; + sqlite3UserAuthCheckLogin(db, "main", &authLevel); + db->auth.authLevel = authLevel; + } + if( db->auth.authLevelmutex); + return SQLITE_ERROR; + } + } +#endif sqlite3BtreeEnterAll(db); rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail); if( rc==SQLITE_SCHEMA ){ sqlite3_finalize(*ppStmt); rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail); } -#if SQLITE_USER_AUTHENTICATION - assert( rc==SQLITE_OK || *ppStmt==0 ); -printf("rc=%d init=%d auth=%d sql=[%.50s]\n", rc, db->init.busy, db->auth.authFlags, zSql); -fflush(stdout); - if( rc==SQLITE_OK && !DbIsAuth(db) && db->init.busy==0 ){ - sqlite3_finalize(*ppStmt); - *ppStmt = 0; - sqlite3ErrorWithMsg(db, SQLITE_ERROR, "user not authenticated"); - rc = SQLITE_ERROR; - } -#endif sqlite3BtreeLeaveAll(db); sqlite3_mutex_leave(db->mutex); assert( rc==SQLITE_OK || *ppStmt==0 ); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ae43014c4c..0849ee842b 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -995,24 +995,21 @@ struct FuncDefHash { */ typedef struct sqlite3_userauth sqlite3_userauth; struct sqlite3_userauth { - u8 authFlags; /* Status flags for user authentication */ + u8 authLevel; /* Current authentication level */ int nAuthPW; /* Size of the zAuthPW in bytes */ char *zAuthPW; /* Password used to authenticate */ char *zAuthUser; /* User name used to authenticate */ }; -/* Allowed values for sqlite3_userauth.authFlags */ -#define UAUTH_Ovrd 0x01 /* Do not enforce access restrictions */ -#define UAUTH_Auth 0x02 /* True if the user has authenticated */ -#define UAUTH_Admin 0x04 /* True if the user is an administrator */ -#define UAUTH_AuthReqd 0x08 /* True if main has an sqlite_user table */ - -/* Macros for accessing sqlite3.auth.authFlags */ -#define DbIsAuth(D) (((D)->auth.authFlags&UAUTH_Auth)!=0) -#define DbIsAdmin(D) (((D)->auth.authFlags&UAUTH_Admin)!=0) +/* Allowed values for sqlite3_userauth.authLevel */ +#define UAUTH_Unknown 0 /* Authentication not yet checked */ +#define UAUTH_Fail 1 /* User authentication failed */ +#define UAUTH_User 2 /* Authenticated as a normal user */ +#define UAUTH_Admin 3 /* Authenticated as an administrator */ /* Functions used only by user authorization logic */ int sqlite3UserAuthTable(const char*); +int sqlite3UserAuthCheckLogin(sqlite3*,const char*,u8*); #endif /* SQLITE_USER_AUTHENTICATION */ From f442e33e3a794b198071a9c5f76aa33eb74c3327 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 10 Sep 2014 19:01:14 +0000 Subject: [PATCH 312/710] Add the ".user" shell command and implement the sqlite3_user_add() routine. Incremental check-in. The code compiles but does not work. FossilOrigin-Name: a0455f9deb603bf91684158d911269622720fc1a --- .../{userauth.h => sqlite3userauth.h} | 6 +- ext/userauth/userauth.c | 106 ++++++++++++++---- main.mk | 6 +- manifest | 26 ++--- manifest.uuid | 2 +- src/func.c | 3 + src/prepare.c | 2 +- src/shell.c | 64 +++++++++++ src/sqlite.h.in | 1 + src/sqliteInt.h | 2 + 10 files changed, 178 insertions(+), 40 deletions(-) rename ext/userauth/{userauth.h => sqlite3userauth.h} (95%) diff --git a/ext/userauth/userauth.h b/ext/userauth/sqlite3userauth.h similarity index 95% rename from ext/userauth/userauth.h rename to ext/userauth/sqlite3userauth.h index 279675f72c..9f95e9fe7a 100644 --- a/ext/userauth/userauth.h +++ b/ext/userauth/sqlite3userauth.h @@ -37,7 +37,7 @@ int sqlite3_user_authenticate( sqlite3 *db, /* The database connection */ const char *zUsername, /* Username */ int nPW, /* Number of bytes in aPW[] */ - const void *aPW /* Password or credentials */ + const char *aPW /* Password or credentials */ ); /* @@ -55,7 +55,7 @@ int sqlite3_user_add( const char *zUsername, /* Username to be added */ int isAdmin, /* True to give new user admin privilege */ int nPW, /* Number of bytes in aPW[] */ - const void *aPW /* Password or credentials */ + const char *aPW /* Password or credentials */ ); /* @@ -70,7 +70,7 @@ int sqlite3_user_change( const char *zUsername, /* Username to change */ int isAdmin, /* Modified admin privilege for the user */ int nPW, /* Number of bytes in aPW[] */ - const void *aPW /* Modified password or credentials */ + const char *aPW /* Modified password or credentials */ ); /* diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c index 518f466f14..05d21bf10e 100644 --- a/ext/userauth/userauth.c +++ b/ext/userauth/userauth.c @@ -23,6 +23,7 @@ */ #ifdef SQLITE_USER_AUTHENTICATION #include "sqliteInt.h" +#include "sqlite3userauth.h" /* ** Prepare an SQL statement for use by the user authentication logic. @@ -52,6 +53,19 @@ static sqlite3_stmt *sqlite3UserAuthPrepare( return pStmt; } +/* +** Check to see if the sqlite_user table exists in database zDb. +*/ +static int userTableExists(sqlite3 *db, const char *zDb){ + int rc; + sqlite3_mutex_enter(db->mutex); + sqlite3BtreeEnterAll(db); + rc = sqlite3FindTable(db, "sqlite_user", zDb)!=0; + sqlite3BtreeLeaveAll(db); + sqlite3_mutex_leave(db->mutex); + return rc; +} + /* ** Check to see if database zDb has a "sqlite_user" table and if it does ** whether that table can authenticate zUser with nPw,zPw. Write one of @@ -67,20 +81,8 @@ static int userAuthCheckLogin( int rc; *peAuth = UAUTH_Unknown; - pStmt = sqlite3UserAuthPrepare(db, - "SELECT 1 FROM \"%w\".sqlite_master " - " WHERE name='sqlite_user' AND type='table'", zDb); - if( pStmt==0 ){ - return SQLITE_NOMEM; - } - rc = sqlite3_step(pStmt); - sqlite3_finalize(pStmt); - if( rc==SQLITE_DONE ){ + if( !userTableExists(db, "main") ){ *peAuth = UAUTH_Admin; /* No sqlite_user table. Everybody is admin. */ - return SQLITE_OK; - } - if( rc!=SQLITE_ROW ){ - return rc; } if( db->auth.zAuthUser==0 ){ *peAuth = UAUTH_Fail; @@ -115,6 +117,42 @@ int sqlite3UserAuthCheckLogin( return rc; } +/* +** Implementation of the sqlite_crypt(X,Y) function. +** +** If Y is NULL then generate a new hash for password X and return that +** hash. If Y is not null, then generate a hash for password X using the +** same salt as the previous hash Y and return the new hash. +*/ +void sqlite3CryptFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + const char *zIn; + int nIn, ii; + u8 *zOut; + char zSalt[8]; + zIn = sqlite3_value_blob(argv[0]); + nIn = sqlite3_value_bytes(argv[0]); + if( sqlite3_value_type(argv[1])==SQLITE_BLOB + && sqlite3_value_bytes(argv[1])==nIn+sizeof(zSalt) + ){ + memcpy(zSalt, sqlite3_value_blob(argv[1]), sizeof(zSalt)); + }else{ + sqlite3_randomness(sizeof(zSalt), zSalt); + } + zOut = sqlite3_malloc( nIn+sizeof(zSalt) ); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + memcpy(zOut, zSalt, sizeof(zSalt)); + for(ii=0; iiauth.nAuthPW = nPW; rc = sqlite3UserAuthCheckLogin(db, "main", &authLevel); db->auth.authLevel = authLevel; + sqlite3ExpirePreparedStatements(db); if( rc ){ return rc; /* OOM error, I/O error, etc. */ } @@ -172,9 +211,37 @@ int sqlite3_user_add( const char *zUsername, /* Username to be added */ int isAdmin, /* True to give new user admin privilege */ int nPW, /* Number of bytes in aPW[] */ - const void *aPW /* Password or credentials */ + const char *aPW /* Password or credentials */ ){ - if( db->auth.authLevelauth.authLevelauth.zAuthUser==0 ){ + assert( isAdmin!=0 ); + sqlite3_user_authenticate(db, zUsername, nPW, aPW); + } return SQLITE_OK; } @@ -190,11 +257,11 @@ int sqlite3_user_change( const char *zUsername, /* Username to change */ int isAdmin, /* Modified admin privilege for the user */ int nPW, /* Number of bytes in aPW[] */ - const void *aPW /* Modified password or credentials */ + const char *aPW /* Modified password or credentials */ ){ - if( db->auth.authLevelauth.authLevelauth.zAuthUser, zUsername)!=0 - && db->auth.authLevelauth.authLevelauth.authLevelauth.authLevelauth.zAuthUser, zUsername)==0 ) return SQLITE_AUTH; return SQLITE_OK; } diff --git a/main.mk b/main.mk index 2de8a230ff..4a7ac02710 100644 --- a/main.mk +++ b/main.mk @@ -46,7 +46,7 @@ # TCCX = $(TCC) $(OPTS) -I. -I$(TOP)/src -I$(TOP) TCCX += -I$(TOP)/ext/rtree -I$(TOP)/ext/icu -I$(TOP)/ext/fts3 -TCCX += -I$(TOP)/ext/async +TCCX += -I$(TOP)/ext/async -I$(TOP)/ext/userauth # Object files for the SQLite library. # @@ -216,7 +216,7 @@ SRC += \ $(TOP)/ext/rtree/rtree.c SRC += \ $(TOP)/ext/userauth/userauth.c \ - $(TOP)/ext/userauth/userauth.h + $(TOP)/ext/userauth/sqlite3userauth.h # Generated source code files # @@ -380,7 +380,7 @@ EXTHDR += \ EXTHDR += \ $(TOP)/ext/icu/sqliteicu.h EXTHDR += \ - $(TOP)/ext/userauth/userauth.h + $(TOP)/ext/userauth/sqlite3userauth.h # This is the default Makefile target. The objects listed here # are what get build when you type just "make" with no arguments. diff --git a/manifest b/manifest index fcb6f3314e..28f2b64a45 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\sideas\son\suser\sauthentication.\s\sNot\syet\sworking\scode. -D 2014-09-10T17:34:28.937 +C Add\sthe\s".user"\sshell\scommand\sand\simplement\sthe\ssqlite3_user_add()\nroutine.\s\sIncremental\scheck-in.\s\sThe\scode\scompiles\sbut\sdoes\snot\swork. +D 2014-09-10T19:01:14.206 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -144,13 +144,13 @@ F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h 83349d519fe5f518b3ea025d18dd1fe51b1684bd F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 +F ext/userauth/sqlite3userauth.h 6e15b0006e7b07b7b008c9f9297b3781a7514337 w ext/userauth/userauth.h F ext/userauth/user-auth.txt f471c5a363ab0682b109d85982ea857f9a144ccc -F ext/userauth/userauth.c 0d24bcd4a18b354797b9cc6f8e4ba152d385cebe -F ext/userauth/userauth.h efbfb68ff083749ad63b12dcb5877b936c3458d6 +F ext/userauth/userauth.c 5a3f8a7ac79eb1315c7e0313ff87d8c30e33d837 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk ac53fd5d61941c0ff1f05e710999b64ffd03f069 +F main.mk bbc8b6000ed143a1a8d31d3b4995c359a3188fa1 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -183,7 +183,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c 441a7e24e2f7bea9475778fa8acce9e8a69ca8f0 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c 0517037766e18eff7dce298e6b3a8e6311df75ec +F src/func.c 1b7ac915eb83255eba90906cc2e317b1f29ae5c9 F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -221,17 +221,17 @@ F src/pcache.c 2048affdb09a04478b5fc6e64cb1083078d369be F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa F src/pragma.c 3b7b1a5e90804006f44c65464c7032ee6a1d24e3 -F src/prepare.c 51ca716a2f73364d8f57c69c89423a0831d17572 +F src/prepare.c 8c2f992a3b3949ab0bf9d4862f7a271f0af0bd5b F src/printf.c e74925089a85e3c9f0e315595f41c139d3d118c2 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0d1621e45fffe4b4396477cf46e41a84b0145ffb F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c b4457526cee73c0b69fad42f799f619b1d5a8a8a -F src/shell.c 713cef4d73c05fc8e12f4960072329d767a05d50 -F src/sqlite.h.in 64a77f2822f1325b12050972003184f99b655a0f +F src/shell.c 4dac2ec625fb15a51b06ab998e7cec8c1e6a40eb +F src/sqlite.h.in 577876beef2264a0b031c0d744c81855983088f9 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 1f40357fb9b12a80c5a3b2b109fd249b009213d4 -F src/sqliteInt.h 10a1f056b6b40449a81cf5d708bc0d9fac053c53 +F src/sqliteInt.h fdc23ef0c5475888d0e532204a7451507ce17206 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 4e28a53e66bad8d014a510ef0205f5497c712b08 @@ -1196,7 +1196,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8440f093bac19a41d44ee352744354eab897fe4e -R 6f268f8f48a352ef36f94cb71204780d +P c8171ecd0d6f097c9e95d5f6643bae8d67f44750 +R f252935e505dbc9ddcbfc78d0487cc51 U drh -Z 6484e1728dcc8ed1c2a7dbcf5e6f2393 +Z 5adba3159d6bf335715850631d1526a9 diff --git a/manifest.uuid b/manifest.uuid index cff06d83f6..2ed68f3b13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8171ecd0d6f097c9e95d5f6643bae8d67f44750 \ No newline at end of file +a0455f9deb603bf91684158d911269622720fc1a \ No newline at end of file diff --git a/src/func.c b/src/func.c index e338ab842b..94ad1b62da 100644 --- a/src/func.c +++ b/src/func.c @@ -1695,6 +1695,9 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(sqlite_version, 0, 0, 0, versionFunc ), FUNCTION(sqlite_source_id, 0, 0, 0, sourceidFunc ), FUNCTION(sqlite_log, 2, 0, 0, errlogFunc ), +#if SQLITE_USER_AUTHENTICATION + FUNCTION(sqlite_crypt, 2, 0, 0, sqlite3CryptFunc ), +#endif #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS FUNCTION(sqlite_compileoption_used,1, 0, 0, compileoptionusedFunc ), FUNCTION(sqlite_compileoption_get, 1, 0, 0, compileoptiongetFunc ), diff --git a/src/prepare.c b/src/prepare.c index d3531f114b..e1739ba3f6 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -722,7 +722,7 @@ static int sqlite3LockAndPrepare( db->auth.authLevel = authLevel; } if( db->auth.authLevelmutex); return SQLITE_ERROR; } diff --git a/src/shell.c b/src/shell.c index afe01ef1a1..2312a7d321 100644 --- a/src/shell.c +++ b/src/shell.c @@ -33,6 +33,9 @@ #include #include #include "sqlite3.h" +#if SQLITE_USER_AUTHENTICATION +# include "sqlite3userauth.h" +#endif #include #include @@ -3435,6 +3438,67 @@ static int do_meta_command(char *zLine, ShellState *p){ #endif }else +#if SQLITE_USER_AUTHENTICATION + if( c=='u' && strncmp(azArg[0], "user", n)==0 ){ + if( nArg<2 ){ + fprintf(stderr, "Usage: .user SUBCOMMAND ...\n"); + rc = 1; + goto meta_command_exit; + } + if( strcmp(azArg[1],"login")==0 ){ + if( nArg!=4 ){ + fprintf(stderr, "Usage: .user login USER PASSWORD\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_authenticate(p->db, azArg[2], (int)strlen(azArg[3]), azArg[3]); + if( rc ){ + fprintf(stderr, "Authentication failed for user %s\n", azArg[2]); + rc = 1; + } + }else if( strcmp(azArg[1],"add")==0 ){ + if( nArg!=5 ){ + fprintf(stderr, "Usage: .user add USER ISADMIN PASSWORD\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_add(p->db, azArg[2], booleanValue(azArg[3]), + (int)strlen(azArg[4]), azArg[4]); + if( rc ){ + fprintf(stderr, "User-Add failed: %d\n", rc); + rc = 1; + } + }else if( strcmp(azArg[1],"edit")==0 ){ + if( nArg!=5 ){ + fprintf(stderr, "Usage: .user edit USER ISADMIN PASSWORD\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_change(p->db, azArg[2], booleanValue(azArg[3]), + (int)strlen(azArg[4]), azArg[4]); + if( rc ){ + fprintf(stderr, "User-Edit failed: %d\n", rc); + rc = 1; + } + }else if( strcmp(azArg[1],"delete")==0 ){ + if( nArg!=3 ){ + fprintf(stderr, "Usage: .user delete USER\n"); + rc = 1; + goto meta_command_exit; + } + rc = sqlite3_user_delete(p->db, azArg[2]); + if( rc ){ + fprintf(stderr, "User-Delete failed: %d\n", rc); + rc = 1; + } + }else{ + fprintf(stderr, "Usage: .user login|add|edit|delete ...\n"); + rc = 1; + goto meta_command_exit; + } + }else +#endif /* SQLITE_USER_AUTHENTICATION */ + if( c=='v' && strncmp(azArg[0], "version", n)==0 ){ fprintf(p->out, "SQLite %s %s\n" /*extra-version-info*/, sqlite3_libversion(), sqlite3_sourceid()); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 56dede8ba8..e947c3e19a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -492,6 +492,7 @@ int sqlite3_exec( #define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) +#define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) /* ** CAPI3REF: Flags For File Open Operations diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0849ee842b..9ef4e36208 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1010,6 +1010,8 @@ struct sqlite3_userauth { /* Functions used only by user authorization logic */ int sqlite3UserAuthTable(const char*); int sqlite3UserAuthCheckLogin(sqlite3*,const char*,u8*); +void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); + #endif /* SQLITE_USER_AUTHENTICATION */ From 09e60541aecde780c2d16d7d7e8284235247b375 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 10 Sep 2014 22:46:46 +0000 Subject: [PATCH 313/710] Complete the implementation of the various APIs. Fix several problems. This is another incremental check-in that does not completely work. FossilOrigin-Name: 4eaaa7fa87aa912d24f8b35440ab60310dc08310 --- ext/userauth/userauth.c | 66 ++++++++++++++++++++++++++++++++++------- manifest | 22 +++++++------- manifest.uuid | 2 +- src/ctime.c | 3 ++ src/legacy.c | 2 +- src/prepare.c | 1 + src/test_config.c | 6 ++++ 7 files changed, 78 insertions(+), 24 deletions(-) diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c index 05d21bf10e..5e00c5cfb2 100644 --- a/ext/userauth/userauth.c +++ b/ext/userauth/userauth.c @@ -39,12 +39,15 @@ static sqlite3_stmt *sqlite3UserAuthPrepare( char *zSql; int rc; va_list ap; + int savedFlags = db->flags; va_start(ap, zFormat); zSql = sqlite3_vmprintf(zFormat, ap); va_end(ap); if( zSql==0 ) return 0; + db->flags |= SQLITE_WriteSchema; rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + db->flags = savedFlags; sqlite3_free(zSql); if( rc ){ sqlite3_finalize(pStmt); @@ -60,6 +63,11 @@ static int userTableExists(sqlite3 *db, const char *zDb){ int rc; sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); + if( db->init.busy==0 ){ + char *zErr = 0; + sqlite3Init(db, &zErr); + sqlite3DbFree(db, zErr); + } rc = sqlite3FindTable(db, "sqlite_user", zDb)!=0; sqlite3BtreeLeaveAll(db); sqlite3_mutex_leave(db->mutex); @@ -83,6 +91,7 @@ static int userAuthCheckLogin( *peAuth = UAUTH_Unknown; if( !userTableExists(db, "main") ){ *peAuth = UAUTH_Admin; /* No sqlite_user table. Everybody is admin. */ + return SQLITE_OK; } if( db->auth.zAuthUser==0 ){ *peAuth = UAUTH_Fail; @@ -100,8 +109,7 @@ static int userAuthCheckLogin( }else{ *peAuth = UAUTH_Fail; } - sqlite3_finalize(pStmt); - return rc; + return sqlite3_finalize(pStmt); } int sqlite3UserAuthCheckLogin( sqlite3 *db, /* The database connection to check */ @@ -230,8 +238,8 @@ int sqlite3_user_add( if( rc ) return rc; } pStmt = sqlite3UserAuthPrepare(db, - "INSERT INTO sqlite_user(uname,isAdmin,sqlite_crypt(pw,NULL))" - " VALUES(%Q,%d,?1)", + "INSERT INTO sqlite_user(uname,isAdmin,pw)" + " VALUES(%Q,%d,sqlite_crypt(?1,NULL))", zUsername, isAdmin!=0); if( pStmt==0 ) return SQLITE_NOMEM; sqlite3_bind_blob(pStmt, 1, aPW, nPW, SQLITE_STATIC); @@ -259,10 +267,31 @@ int sqlite3_user_change( int nPW, /* Number of bytes in aPW[] */ const char *aPW /* Modified password or credentials */ ){ - if( db->auth.authLevelauth.zAuthUser, zUsername)!=0 - && db->auth.authLevelauth.authLevelauth.zAuthUser, zUsername)!=0 ){ + if( db->auth.authLevelauth.authLevel==UAUTH_Admin) ){ + /* Cannot change the isAdmin setting for self */ + return SQLITE_AUTH; + } + if( !userTableExists(db, "main") ){ + /* This routine is a no-op if the user to be modified does not exist */ + return SQLITE_OK; + } + pStmt = sqlite3UserAuthPrepare(db, + "UPDATE sqlite_user SET isAdmin=%d, pw=sqlite_crypt(?1,NULL)" + " WHERE uname=%Q", isAdmin, zUsername); + if( pStmt==0 ) return SQLITE_NOMEM; + sqlite3_bind_blob(pStmt, 1, aPW, nPW, SQLITE_STATIC); + sqlite3_step(pStmt); + return sqlite3_finalize(pStmt); } /* @@ -276,9 +305,24 @@ int sqlite3_user_delete( sqlite3 *db, /* Database connection */ const char *zUsername /* Username to remove */ ){ - if( db->auth.authLevelauth.zAuthUser, zUsername)==0 ) return SQLITE_AUTH; - return SQLITE_OK; + sqlite3_stmt *pStmt; + if( db->auth.authLevelauth.zAuthUser, zUsername)==0 ){ + /* Cannot delete self */ + return SQLITE_AUTH; + } + if( !userTableExists(db, "main") ){ + /* This routine is a no-op if the user to be deleted does not exist */ + return SQLITE_OK; + } + pStmt = sqlite3UserAuthPrepare(db, + "SELECT FROM sqlite_user WHERE uname=%Q", zUsername); + if( pStmt==0 ) return SQLITE_NOMEM; + sqlite3_step(pStmt); + return sqlite3_finalize(pStmt); } #endif /* SQLITE_USER_AUTHENTICATION */ diff --git a/manifest b/manifest index 28f2b64a45..01cd4221fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s".user"\sshell\scommand\sand\simplement\sthe\ssqlite3_user_add()\nroutine.\s\sIncremental\scheck-in.\s\sThe\scode\scompiles\sbut\sdoes\snot\swork. -D 2014-09-10T19:01:14.206 +C Complete\sthe\simplementation\sof\sthe\svarious\sAPIs.\s\sFix\sseveral\sproblems.\nThis\sis\sanother\sincremental\scheck-in\sthat\sdoes\snot\scompletely\swork. +D 2014-09-10T22:46:46.526 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -144,9 +144,9 @@ F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea F ext/rtree/sqlite3rtree.h 83349d519fe5f518b3ea025d18dd1fe51b1684bd F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 -F ext/userauth/sqlite3userauth.h 6e15b0006e7b07b7b008c9f9297b3781a7514337 w ext/userauth/userauth.h +F ext/userauth/sqlite3userauth.h 6e15b0006e7b07b7b008c9f9297b3781a7514337 F ext/userauth/user-auth.txt f471c5a363ab0682b109d85982ea857f9a144ccc -F ext/userauth/userauth.c 5a3f8a7ac79eb1315c7e0313ff87d8c30e33d837 +F ext/userauth/userauth.c e14ab212e1e2cd3f3a5d324f2c3e0b0c5a950c86 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -177,7 +177,7 @@ F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc F src/build.c 3a61555d469de2e0f5bcd1ac4d58a2a19ab093d5 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 -F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a +F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c 441a7e24e2f7bea9475778fa8acce9e8a69ca8f0 @@ -190,7 +190,7 @@ F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 0b073fade178d9dbd990bbb32b4438e50b884a06 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d -F src/legacy.c 87c92f4a08e2f70220e3b22a9c3b2482d36a134a +F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 31c2122b7dd05a179049bbf163fd4839f181cbab F src/main.c d15621461fb0c52675eba2b650492ed1beef69ab @@ -221,7 +221,7 @@ F src/pcache.c 2048affdb09a04478b5fc6e64cb1083078d369be F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa F src/pragma.c 3b7b1a5e90804006f44c65464c7032ee6a1d24e3 -F src/prepare.c 8c2f992a3b3949ab0bf9d4862f7a271f0af0bd5b +F src/prepare.c 10dd9833d7aa992baf84b8640224853576119d84 F src/printf.c e74925089a85e3c9f0e315595f41c139d3d118c2 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0d1621e45fffe4b4396477cf46e41a84b0145ffb @@ -249,7 +249,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c d5f00627c4f47515a57f905806558153cccd7253 +F src/test_config.c 6f721f0337b96d58e81ff69bba101113c8168c2b F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -1196,7 +1196,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c8171ecd0d6f097c9e95d5f6643bae8d67f44750 -R f252935e505dbc9ddcbfc78d0487cc51 +P a0455f9deb603bf91684158d911269622720fc1a +R 7c5d50077d463af5ab8f09588f919ad0 U drh -Z 5adba3159d6bf335715850631d1526a9 +Z 0eb3e0d5d27e81783efc050a8d458d7f diff --git a/manifest.uuid b/manifest.uuid index 2ed68f3b13..5efaf01425 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a0455f9deb603bf91684158d911269622720fc1a \ No newline at end of file +4eaaa7fa87aa912d24f8b35440ab60310dc08310 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index 286f66e061..6f7ac8fcba 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -368,6 +368,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_USE_ALLOCA "USE_ALLOCA", #endif +#ifdef SQLITE_USER_AUTHENTICATION + "USER_AUTHENTICATION", +#endif #ifdef SQLITE_WIN32_MALLOC "WIN32_MALLOC", #endif diff --git a/src/legacy.c b/src/legacy.c index b8cb90d707..a10006e558 100644 --- a/src/legacy.c +++ b/src/legacy.c @@ -125,7 +125,7 @@ exec_out: sqlite3DbFree(db, azCols); rc = sqlite3ApiExit(db, rc); - if( rc!=SQLITE_OK && ALWAYS(rc==sqlite3_errcode(db)) && pzErrMsg ){ + if( rc!=SQLITE_OK && pzErrMsg ){ int nErrMsg = 1 + sqlite3Strlen30(sqlite3_errmsg(db)); *pzErrMsg = sqlite3Malloc(nErrMsg); if( *pzErrMsg ){ diff --git a/src/prepare.c b/src/prepare.c index e1739ba3f6..c6752548ff 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -394,6 +394,7 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ int commit_internal = !(db->flags&SQLITE_InternChanges); assert( sqlite3_mutex_held(db->mutex) ); + assert( db->init.busy==0 ); rc = SQLITE_OK; db->init.busy = 1; for(i=0; rc==SQLITE_OK && inDb; i++){ diff --git a/src/test_config.c b/src/test_config.c index 78c65a8e52..074faf2116 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -603,6 +603,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "secure_delete", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_USER_AUTHENTICATION + Tcl_SetVar2(interp, "sqlite_options", "userauth", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "userauth", "0", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_MULTIPLEX_EXT_OVWR Tcl_SetVar2(interp, "sqlite_options", "multiplex_ext_overwrite", "1", TCL_GLOBAL_ONLY); #else From d39c40ff5e89601a242f004faa45735e80275fcb Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Sep 2014 00:27:53 +0000 Subject: [PATCH 314/710] Reorder parameters on the sqlite3_user_*() interfaces for consistency. Add the first TCL test cases. FossilOrigin-Name: 2f6d8f32eef526b5912f42ab467e3c7812480d8b --- ext/userauth/sqlite3userauth.h | 12 +-- ext/userauth/user-auth.txt | 27 ++++--- ext/userauth/userauth.c | 14 ++-- manifest | 21 +++--- manifest.uuid | 2 +- src/shell.c | 17 +++-- src/test1.c | 133 +++++++++++++++++++++++++++++++++ test/userauth01.test | 74 ++++++++++++++++++ 8 files changed, 257 insertions(+), 43 deletions(-) create mode 100644 test/userauth01.test diff --git a/ext/userauth/sqlite3userauth.h b/ext/userauth/sqlite3userauth.h index 9f95e9fe7a..619477cac9 100644 --- a/ext/userauth/sqlite3userauth.h +++ b/ext/userauth/sqlite3userauth.h @@ -36,8 +36,8 @@ int sqlite3_user_authenticate( sqlite3 *db, /* The database connection */ const char *zUsername, /* Username */ - int nPW, /* Number of bytes in aPW[] */ - const char *aPW /* Password or credentials */ + const char *aPW, /* Password or credentials */ + int nPW /* Number of bytes in aPW[] */ ); /* @@ -53,9 +53,9 @@ int sqlite3_user_authenticate( int sqlite3_user_add( sqlite3 *db, /* Database connection */ const char *zUsername, /* Username to be added */ - int isAdmin, /* True to give new user admin privilege */ + const char *aPW, /* Password or credentials */ int nPW, /* Number of bytes in aPW[] */ - const char *aPW /* Password or credentials */ + int isAdmin /* True to give new user admin privilege */ ); /* @@ -68,9 +68,9 @@ int sqlite3_user_add( int sqlite3_user_change( sqlite3 *db, /* Database connection */ const char *zUsername, /* Username to change */ - int isAdmin, /* Modified admin privilege for the user */ + const char *aPW, /* New password or credentials */ int nPW, /* Number of bytes in aPW[] */ - const char *aPW /* Modified password or credentials */ + int isAdmin /* Modified admin privilege for the user */ ); /* diff --git a/ext/userauth/user-auth.txt b/ext/userauth/user-auth.txt index dd49dfcfb7..85083d01c2 100644 --- a/ext/userauth/user-auth.txt +++ b/ext/userauth/user-auth.txt @@ -7,24 +7,24 @@ activated: int sqlite3_user_authenticate( sqlite3 *db, /* The database connection */ const char *zUsername, /* Username */ - int nPW, /* Number of bytes in aPW[] */ - const void *aPW /* Password or credentials */ + const char *aPW, /* Password or credentials */ + int nPW /* Number of bytes in aPW[] */ ); int sqlite3_user_add( sqlite3 *db, /* Database connection */ const char *zUsername, /* Username to be added */ - int isAdmin, /* True to give new user admin privilege */ + const char *aPW, /* Password or credentials */ int nPW, /* Number of bytes in aPW[] */ - const void *aPW /* Password or credentials */ + int isAdmin /* True to give new user admin privilege */ ); int sqlite3_user_change( sqlite3 *db, /* Database connection */ const char *zUsername, /* Username to change */ - int isAdmin, /* Modified admin privilege for the user */ + const void *aPW, /* Modified password or credentials */ int nPW, /* Number of bytes in aPW[] */ - const void *aPW /* Modified password or credentials */ + int isAdmin /* Modified admin privilege for the user */ ); int sqlite3_user_delete( @@ -71,12 +71,15 @@ check fails, then the ATTACH-ed database is unreadable [1g]. The sqlite3_user_add() interface can be used (by an admin user only) to create a new user. When called on a no-authentication-required -database, this routine converts the database into an authentication- -required database [3], automatically makes the added user an -administrator [1b], and logs in the current connection as that user [1a]. -The sqlite3_user_add() interface only works for the "main" database, not -for any ATTACH-ed databases. Any call to sqlite3_user_add() by a -non-admin user results in an error. +database and when A is true, the sqlite3_user_add(D,U,P,N,A) routine +converts the database into an authentication-required database and +logs the database connection D in using user U with password P,N. +To convert a no-authentication-required database into an authentication- +required database, the isAdmin parameter must be true. If +sqlite3_user_add(D,U,P,N,A) is called on a no-authentication-required +database and A is false, then the call fails with an SQLITE_AUTH error. + +Any call to sqlite3_user_add() by a non-admin user results in an error. Hence, to create a new, unencrypted, authentication-required database, the call sequence is: diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c index 5e00c5cfb2..343e49e6ff 100644 --- a/ext/userauth/userauth.c +++ b/ext/userauth/userauth.c @@ -177,8 +177,8 @@ void sqlite3CryptFunc( int sqlite3_user_authenticate( sqlite3 *db, /* The database connection */ const char *zUsername, /* Username */ - int nPW, /* Number of bytes in aPW[] */ - const char *zPW /* Password or credentials */ + const char *zPW, /* Password or credentials */ + int nPW /* Number of bytes in aPW[] */ ){ int rc; u8 authLevel = UAUTH_Fail; @@ -217,9 +217,9 @@ int sqlite3_user_authenticate( int sqlite3_user_add( sqlite3 *db, /* Database connection */ const char *zUsername, /* Username to be added */ - int isAdmin, /* True to give new user admin privilege */ + const char *aPW, /* Password or credentials */ int nPW, /* Number of bytes in aPW[] */ - const char *aPW /* Password or credentials */ + int isAdmin /* True to give new user admin privilege */ ){ sqlite3_stmt *pStmt; int rc; @@ -248,7 +248,7 @@ int sqlite3_user_add( if( rc ) return rc; if( db->auth.zAuthUser==0 ){ assert( isAdmin!=0 ); - sqlite3_user_authenticate(db, zUsername, nPW, aPW); + sqlite3_user_authenticate(db, zUsername, aPW, nPW); } return SQLITE_OK; } @@ -263,9 +263,9 @@ int sqlite3_user_add( int sqlite3_user_change( sqlite3 *db, /* Database connection */ const char *zUsername, /* Username to change */ - int isAdmin, /* Modified admin privilege for the user */ + const char *aPW, /* Modified password or credentials */ int nPW, /* Number of bytes in aPW[] */ - const char *aPW /* Modified password or credentials */ + int isAdmin /* Modified admin privilege for the user */ ){ sqlite3_stmt *pStmt; if( db->auth.authLeveldb, azArg[2], (int)strlen(azArg[3]), azArg[3]); + rc = sqlite3_user_authenticate(p->db, azArg[2], azArg[3], + (int)strlen(azArg[3])); if( rc ){ fprintf(stderr, "Authentication failed for user %s\n", azArg[2]); rc = 1; } }else if( strcmp(azArg[1],"add")==0 ){ if( nArg!=5 ){ - fprintf(stderr, "Usage: .user add USER ISADMIN PASSWORD\n"); + fprintf(stderr, "Usage: .user add USER PASSWORD ISADMIN\n"); rc = 1; goto meta_command_exit; } - rc = sqlite3_user_add(p->db, azArg[2], booleanValue(azArg[3]), - (int)strlen(azArg[4]), azArg[4]); + rc = sqlite3_user_add(p->db, azArg[2], + azArg[3], (int)strlen(azArg[3]), + booleanValue(azArg[4])); if( rc ){ fprintf(stderr, "User-Add failed: %d\n", rc); rc = 1; } }else if( strcmp(azArg[1],"edit")==0 ){ if( nArg!=5 ){ - fprintf(stderr, "Usage: .user edit USER ISADMIN PASSWORD\n"); + fprintf(stderr, "Usage: .user edit USER PASSWORD ISADMIN\n"); rc = 1; goto meta_command_exit; } - rc = sqlite3_user_change(p->db, azArg[2], booleanValue(azArg[3]), - (int)strlen(azArg[4]), azArg[4]); + rc = sqlite3_user_change(p->db, azArg[2], + azArg[3], (int)strlen(azArg[3]), + booleanValue(azArg[4])); if( rc ){ fprintf(stderr, "User-Edit failed: %d\n", rc); rc = 1; diff --git a/src/test1.c b/src/test1.c index d050e683f4..62b575989d 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6497,6 +6497,132 @@ static int sorter_test_sort4_helper( } +#ifdef SQLITE_USER_AUTHENTICATION +#include "sqlite3userauth.h" +/* +** tclcmd: sqlite3_user_authenticate DB USERNAME PASSWORD +*/ +static int test_user_authenticate( + ClientData clientData, /* Unused */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + char *zUser = 0; + char *zPasswd = 0; + int nPasswd = 0; + sqlite3 *db; + int rc; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB USERNAME PASSWORD"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + zUser = Tcl_GetString(objv[2]); + zPasswd = Tcl_GetStringFromObj(objv[3], &nPasswd); + rc = sqlite3_user_authenticate(db, zUser, zPasswd, nPasswd); + Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC); + return TCL_OK; +} +#endif /* SQLITE_USER_AUTHENTICATION */ + +#ifdef SQLITE_USER_AUTHENTICATION +/* +** tclcmd: sqlite3_user_add DB USERNAME PASSWORD ISADMIN +*/ +static int test_user_add( + ClientData clientData, /* Unused */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + char *zUser = 0; + char *zPasswd = 0; + int nPasswd = 0; + int isAdmin = 0; + sqlite3 *db; + int rc; + + if( objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB USERNAME PASSWORD ISADMIN"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + zUser = Tcl_GetString(objv[2]); + zPasswd = Tcl_GetStringFromObj(objv[3], &nPasswd); + Tcl_GetBooleanFromObj(interp, objv[4], &isAdmin); + rc = sqlite3_user_add(db, zUser, zPasswd, nPasswd, isAdmin); + Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC); + return TCL_OK; +} +#endif /* SQLITE_USER_AUTHENTICATION */ + +#ifdef SQLITE_USER_AUTHENTICATION +/* +** tclcmd: sqlite3_user_change DB USERNAME PASSWORD ISADMIN +*/ +static int test_user_change( + ClientData clientData, /* Unused */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + char *zUser = 0; + char *zPasswd = 0; + int nPasswd = 0; + int isAdmin = 0; + sqlite3 *db; + int rc; + + if( objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB USERNAME PASSWORD ISADMIN"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + zUser = Tcl_GetString(objv[2]); + zPasswd = Tcl_GetStringFromObj(objv[3], &nPasswd); + Tcl_GetBooleanFromObj(interp, objv[4], &isAdmin); + rc = sqlite3_user_change(db, zUser, zPasswd, nPasswd, isAdmin); + Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC); + return TCL_OK; +} +#endif /* SQLITE_USER_AUTHENTICATION */ + +#ifdef SQLITE_USER_AUTHENTICATION +/* +** tclcmd: sqlite3_user_delete DB USERNAME +*/ +static int test_user_delete( + ClientData clientData, /* Unused */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + char *zUser = 0; + sqlite3 *db; + int rc; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB USERNAME"); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ){ + return TCL_ERROR; + } + zUser = Tcl_GetString(objv[2]); + rc = sqlite3_user_delete(db, zUser); + Tcl_SetResult(interp, (char *)t1ErrorName(rc), TCL_STATIC); + return TCL_OK; +} +#endif /* SQLITE_USER_AUTHENTICATION */ + /* ** Register commands with the TCL interpreter. */ @@ -6734,6 +6860,13 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "load_static_extension", tclLoadStaticExtensionCmd }, { "sorter_test_fakeheap", sorter_test_fakeheap }, { "sorter_test_sort4_helper", sorter_test_sort4_helper }, +#ifdef SQLITE_USER_AUTHENTICATION + { "sqlite3_user_authenticate", test_user_authenticate, 0 }, + { "sqlite3_user_add", test_user_add, 0 }, + { "sqlite3_user_change", test_user_change, 0 }, + { "sqlite3_user_delete", test_user_delete, 0 }, +#endif + }; static int bitmask_size = sizeof(Bitmask)*8; int i; diff --git a/test/userauth01.test b/test/userauth01.test new file mode 100644 index 0000000000..60e617e3ba --- /dev/null +++ b/test/userauth01.test @@ -0,0 +1,74 @@ +# 2014-09-10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests of the SQLITE_USER_AUTHENTICATION extension. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix userauth01 + +ifcapable !userauth { + finish_test + return +} + +# Create a no-authentication-required database +# +do_execsql_test userauth01-1.0 { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1),(2.5),('three'),(x'4444'),(NULL); + SELECT quote(x) FROM t1 ORDER BY x; + SELECT name FROM sqlite_master; +} {NULL 1 2.5 'three' X'4444' t1} + +# Calling sqlite3_user_authenticate() on a no-authentication-required +# database connection is a harmless no-op. +# +do_test userauth01-1.1 { + sqlite3_user_authenticate db alice pw-4-alice + execsql { + SELECT quote(x) FROM t1 ORDER BY x; + SELECT name FROM sqlite_master; + } +} {NULL 1 2.5 'three' X'4444' t1} + +# If sqlite3_user_add(D,U,P,N,A) is called on a no-authentication-required +# database and A is false, then the call fails with an SQLITE_AUTH error. +# +do_test userauth01-1.2 { + sqlite3_user_add db bob pw-4-bob 0 +} {SQLITE_AUTH} +do_test userauth01-1.3 { + execsql { + SELECT quote(x) FROM t1 ORDER BY x; + SELECT name FROM sqlite_master; + } +} {NULL 1 2.5 'three' X'4444' t1} + +# When called on a no-authentication-required +# database and when A is true, the sqlite3_user_add(D,U,P,N,A) routine +# converts the database into an authentication-required database and +# logs the database connection D in using user U with password P,N. +# +do_test userauth01-1.4 { + sqlite3_user_add db alice pw-4-alice 1 +} {SQLITE_OK} +do_test userauth01-1.5 { + execsql { + SELECT quote(x) FROM t1 ORDER BY x; + SELECT uname, isadmin FROM sqlite_user ORDER BY uname; + SELECT name FROM sqlite_master ORDER BY name; + } +} {NULL 1 2.5 'three' X'4444' alice 1 sqlite_user t1} + + +finish_test From 32c6a48b5ec977fc849833f9c340b85893103074 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Sep 2014 13:44:52 +0000 Subject: [PATCH 315/710] Add support for the extra parameter on the sqlite3_set_authorizer() callback and support for failing an ATTACH with an authentication-required database using bad credentials. The extension is now feature complete, but much testing and bug-fixing remains. FossilOrigin-Name: 596e728b0eb19a34c888e33d4d37978ca2bf1e00 --- ext/userauth/user-auth.txt | 43 +++--- manifest | 40 +++--- manifest.uuid | 2 +- src/attach.c | 9 ++ src/auth.c | 14 +- src/build.c | 2 +- src/prepare.c | 2 +- src/sqliteInt.h | 15 +- src/tclsqlite.c | 11 +- test/auth.test | 274 ++++++++++++++++++------------------- test/auth2.test | 4 +- test/auth3.test | 2 +- test/fkey2.test | 2 +- test/fts4aa.test | 2 +- test/savepoint.test | 4 +- test/vtab3.test | 2 +- test/without_rowid3.test | 2 +- 17 files changed, 238 insertions(+), 192 deletions(-) diff --git a/ext/userauth/user-auth.txt b/ext/userauth/user-auth.txt index 85083d01c2..8b64617c36 100644 --- a/ext/userauth/user-auth.txt +++ b/ext/userauth/user-auth.txt @@ -1,5 +1,12 @@ -Activate the user authentication logic by compiling SQLite with -the -DSQLITE_USER_AUTHENTICATION compile-time option. +Activate the user authentication logic by including the +ext/userauth/userauth.c source code file in the build and +adding the -DSQLITE_USER_AUTHENTICATION compile-time option. +The ext/userauth/sqlite3userauth.h header file is available to +applications to define the interface. + +When using the SQLite amalgamation, it is sufficient to append +the ext/userauth/userauth.c source file onto the end of the +amalgamation. The following new APIs are available when user authentication is activated: @@ -31,12 +38,16 @@ activated: sqlite3 *db, /* Database connection */ const char *zUsername /* Username to remove */ ); - + +With this extension, a database can be marked as requiring authentication. +By default a database does not require authentication. + The sqlite3_open(), sqlite3_open16(), and sqlite3_open_v2() interfaces work as before: they open a new database connection. However, if the -database being opened requires authentication, then the database -connection will be unusable until after sqlite3_user_authenticate() -has been called successfully [1c]. The sqlite3_user_authenticate() call +database being opened requires authentication, then attempts to prepare +SQL statements (using sqlite3_prepare_v2(), for example) will fail +with an SQLITE_AUTH error until after sqlite3_user_authenticate() +has been called successfully. The sqlite3_user_authenticate() call will return SQLITE_OK if the authentication credentials are accepted and SQLITE_ERROR if not. @@ -67,13 +78,13 @@ connection is treated as if it was authenticated as an admin user. When ATTACH-ing new database files to a connection, each newly attached database that is an authentication-required database is checked using the same username and password as supplied to the main database. If that -check fails, then the ATTACH-ed database is unreadable [1g]. +check fails, then the ATTACH command fails with an SQLITE_AUTH error. The sqlite3_user_add() interface can be used (by an admin user only) to create a new user. When called on a no-authentication-required database and when A is true, the sqlite3_user_add(D,U,P,N,A) routine converts the database into an authentication-required database and -logs the database connection D in using user U with password P,N. +logs in the database connection D as user U with password P,N. To convert a no-authentication-required database into an authentication- required database, the isAdmin parameter must be true. If sqlite3_user_add(D,U,P,N,A) is called on a no-authentication-required @@ -98,17 +109,17 @@ The sqlite3_user_delete() interface can be used (by an admin user only) to delete a user. The currently logged-in user cannot be deleted, which guarantees that there is always an admin user and hence that the database cannot be converted into a no-authentication-required -database [3]. +database. The sqlite3_user_change() interface can be used to change a users login credentials or admin privilege. Any user can change their own -login credentials [1b]. Only an admin user can change another users login +password. Only an admin user can change another users login credentials or admin privilege setting. No user may change their own admin privilege setting. The sqlite3_set_authorizer() callback is modified to take a 7th parameter which is the username of the currently logged in user, or NULL for a -no-authentication-required database [1d]. +no-authentication-required database. ----------------------------------------------------------------------------- Implementation notes: @@ -133,8 +144,8 @@ The sqlite_user.pw field is encoded by a built-in SQL function "sqlite_crypt(X,Y)". The two arguments are both BLOBs. The first argument is the plaintext password supplied to the sqlite3_user_authenticate() interface. The second argument is the sqlite_user.pw value and is supplied -so that the function can extra the "salt" used by the password encoder. -the result of sqlite_crypt(X,Y) is another blob which is the value that +so that the function can extract the "salt" used by the password encoder. +The result of sqlite_crypt(X,Y) is another blob which is the value that ends up being stored in sqlite_user.pw. To verify credentials X supplied by the sqlite3_user_authenticate() routine, SQLite runs: @@ -145,9 +156,9 @@ password X, sqlite_crypt(X,NULL) is run. A new random salt is selected when the second argument is NULL. The built-in version of of sqlite_crypt() uses a simple Ceasar-cypher -which prevents passwords from being revealed by search the raw database -for ASCII text, but is otherwise trivally broken. To truly secure the -passwords, the database should be encrypted using the SQLite Encryption +which prevents passwords from being revealed by searching the raw database +for ASCII text, but is otherwise trivally broken. For better password +security, the database should be encrypted using the SQLite Encryption Extension or similar technology. Or, the application can use the sqlite3_create_function() interface to provide an alternative implementation of sqlite_crypt() that computes a stronger password hash, diff --git a/manifest b/manifest index ad8259c2fc..12328063d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reorder\sparameters\son\sthe\ssqlite3_user_*()\sinterfaces\sfor\sconsistency.\nAdd\sthe\sfirst\sTCL\stest\scases. -D 2014-09-11T00:27:53.371 +C Add\ssupport\sfor\sthe\sextra\sparameter\son\sthe\ssqlite3_set_authorizer()\scallback\nand\ssupport\sfor\sfailing\san\sATTACH\swith\san\sauthentication-required\sdatabase\nusing\sbad\scredentials.\s\sThe\sextension\sis\snow\sfeature\scomplete,\sbut\smuch\ntesting\sand\sbug-fixing\sremains. +D 2014-09-11T13:44:52.150 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -145,7 +145,7 @@ F ext/rtree/sqlite3rtree.h 83349d519fe5f518b3ea025d18dd1fe51b1684bd F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 -F ext/userauth/user-auth.txt a0340e6219f0b70fde57502c8f6b1c5fdb23023e +F ext/userauth/user-auth.txt 527aaec593ae34dcaf543324623c8351a5d33d3f F ext/userauth/userauth.c a66cd3abcc3b2c10b3999ab49f900d561e8ddd33 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 @@ -166,15 +166,15 @@ F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c 79383a54fee3b7f1fb03dd4c8c8115583f506de5 -F src/attach.c 3801129015ef59d76bf23c95ef9b0069d18a0c52 -F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 +F src/attach.c cc9b30041dfcd24be0a47986c87c384515c54449 +F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c b1c1cd1cc3ae2e433a23b9a6c9ab53805707d8cd F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc -F src/build.c 3a61555d469de2e0f5bcd1ac4d58a2a19ab093d5 +F src/build.c 4c7aac1ddda782c6f1cad84aeabec6e8d0be7495 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958 @@ -221,7 +221,7 @@ F src/pcache.c 2048affdb09a04478b5fc6e64cb1083078d369be F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa F src/pragma.c 3b7b1a5e90804006f44c65464c7032ee6a1d24e3 -F src/prepare.c 10dd9833d7aa992baf84b8640224853576119d84 +F src/prepare.c f82c009a763e739c6bdf02a270ccfda9e54f783c F src/printf.c e74925089a85e3c9f0e315595f41c139d3d118c2 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0d1621e45fffe4b4396477cf46e41a84b0145ffb @@ -231,11 +231,11 @@ F src/shell.c 7d26b6526fb9daab994265446b751fb86fd9d675 F src/sqlite.h.in 577876beef2264a0b031c0d744c81855983088f9 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 1f40357fb9b12a80c5a3b2b109fd249b009213d4 -F src/sqliteInt.h fdc23ef0c5475888d0e532204a7451507ce17206 +F src/sqliteInt.h cb44c24e5c5d52f33bb9e78bfcc9c703b1d178c4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 4e28a53e66bad8d014a510ef0205f5497c712b08 -F src/tclsqlite.c 8d6d6833c0053f0b3b1aeb1c5c7a7eeff0ad4d3f +F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 F src/test1.c 523cd70ded28db71af9a30ec184cbe0957de9575 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c @@ -336,9 +336,9 @@ F test/attach2.test 0ec5defa340363de6cd50fd595046465e9aaba2d F test/attach3.test 359eb65d00102cdfcef6fa4e81dc1648f8f80b27 F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 -F test/auth.test 5bdf154eb28c0e4bbc0473f335858c0d96171768 -F test/auth2.test c3b415b76c033bedb81292118fb7c01f5f10cbcd -F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 +F test/auth.test 855233ef26eb3601b6886567ea4e326c72959360 +F test/auth2.test 264c6af53cad9aba5218c68bbe18036e39007bfa +F test/auth3.test 5cfa94ed90c6617c42b7ba4b133fd79678b251c7 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61 F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de @@ -479,7 +479,7 @@ F test/fallocate.test 3e979af17dfa7e5e9dda5eba1a696c04fa9d47f7 F test/filectrl.test 14fa712e42c4cb791e09dfd58a6a03efb47ef13a F test/filefmt.test cb34663f126cbc2d358af552dcaf5c72769b0146 F test/fkey1.test e1d1fa84cde579185ea01358436839703e415a5b -F test/fkey2.test 32ca728bcb854feed72d1406ea375fe423eebff2 +F test/fkey2.test 1db212cda86b0d3ce72714001f7b6381c321341c F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d F test/fkey5.test 8a1fde4e7721ae00b05b3178888833726ca2df8d @@ -577,7 +577,7 @@ F test/fts3sort.test ed34c716a11cc2009a35210e84ad5f9c102362ca F test/fts3tok1.test c551043de056b0b1582a54e878991f57bad074bc F test/fts3tok_err.test 52273cd193b9036282f7bacb43da78c6be87418d F test/fts3varint.test 752c08ed5d32c5d7dc211b056f4ed68a76b7e36e -F test/fts4aa.test 0c3152322c7f0b548cc942ad763eaba0da87ccca +F test/fts4aa.test 10aac8e9d62c7357590acfabe3fad01e9a9ce1cb F test/fts4check.test 74d77f6cdb768ac49df5afda575cef14ae3d239a F test/fts4content.test 2e7252557d6d24afa101d9ba1de710d6140e6d06 F test/fts4docid.test e33c383cfbdff0284685604d256f347a18fdbf01 @@ -785,7 +785,7 @@ F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test b78b30afb9537a73788ca1233a23a32190a3bb1f F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 -F test/savepoint.test 6c53f76dffe5df0dd87646efe3e7aa159c36e07b +F test/savepoint.test 51d3900dc071a7c2ad4248578a5925631b476313 F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 @@ -1068,7 +1068,7 @@ F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test f311691d696a5cc27e3c1b875cec1b0866b4ccd9 F test/vtab1.test b631d147b198cfd7903ab5fed028eb2a3d321dc6 F test/vtab2.test 7bcffc050da5c68f4f312e49e443063e2d391c0d -F test/vtab3.test baad99fd27217f5d6db10660522e0b7192446de1 +F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e F test/vtab4.test 942f8b8280b3ea8a41dae20e7822d065ca1cb275 F test/vtab5.test 889f444970393c73f1e077e2bdc5d845e157a391 F test/vtab6.test 5f5380c425e52993560ab4763db4f826d2ba7b09 @@ -1140,7 +1140,7 @@ F test/with2.test ee227a663586aa09771cafd4fa269c5217eaf775 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 F test/without_rowid1.test 7862e605753c8d25329f665fa09072e842183151 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 -F test/without_rowid3.test eac3d5c8a1924725b58503a368f2cbd24fd6c8a0 +F test/without_rowid3.test 1081aabf60a1e1123b7f9a8f6ae19954351843b0 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a F test/without_rowid5.test b4a639a367f04d382d20e8f44fc1be4f2d57d107 F test/wordcount.c 9915e06cb33d8ca8109b8700791afe80d305afda @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4eaaa7fa87aa912d24f8b35440ab60310dc08310 -R b62ab47ef5a1f118b5c6bf446676036b +P 2f6d8f32eef526b5912f42ab467e3c7812480d8b +R 8b68610f4a9d73b92fb9d68a3359baf2 U drh -Z 388f9ca2f5e0d91d315989cc617a17a1 +Z 6a1c7f3384e27f9ad12e8ea277382ab2 diff --git a/manifest.uuid b/manifest.uuid index 2679d57ef9..1b3da62f36 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f6d8f32eef526b5912f42ab467e3c7812480d8b \ No newline at end of file +596e728b0eb19a34c888e33d4d37978ca2bf1e00 \ No newline at end of file diff --git a/src/attach.c b/src/attach.c index 89050fd9dc..5763e631ad 100644 --- a/src/attach.c +++ b/src/attach.c @@ -207,6 +207,15 @@ static void attachFunc( rc = sqlite3Init(db, &zErrDyn); sqlite3BtreeLeaveAll(db); } +#ifdef SQLITE_USER_AUTHENTICATION + if( rc==SQLITE_OK ){ + u8 newAuth = 0; + rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth); + if( newAuthauth.authLevel ){ + rc = SQLITE_AUTH; + } + } +#endif if( rc ){ int iDb = db->nDb - 1; assert( iDb>=2 ); diff --git a/src/auth.c b/src/auth.c index d38bb836a7..1680c9a7c2 100644 --- a/src/auth.c +++ b/src/auth.c @@ -73,7 +73,7 @@ int sqlite3_set_authorizer( void *pArg ){ sqlite3_mutex_enter(db->mutex); - db->xAuth = xAuth; + db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; sqlite3ExpirePreparedStatements(db); sqlite3_mutex_leave(db->mutex); @@ -108,7 +108,11 @@ int sqlite3AuthReadCol( char *zDb = db->aDb[iDb].zName; /* Name of attached database */ int rc; /* Auth callback return code */ - rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext); + rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext +#ifdef SQLITE_USER_AUTHENTICATION + ,db->auth.zAuthUser +#endif + ); if( rc==SQLITE_DENY ){ if( db->nDb>2 || iDb!=0 ){ sqlite3ErrorMsg(pParse, "access to %s.%s.%s is prohibited",zDb,zTab,zCol); @@ -208,7 +212,11 @@ int sqlite3AuthCheck( if( db->xAuth==0 ){ return SQLITE_OK; } - rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext); + rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext +#ifdef SQLITE_USER_AUTHENTICATION + ,db->auth.zAuthUser +#endif + ); if( rc==SQLITE_DENY ){ sqlite3ErrorMsg(pParse, "not authorized"); pParse->rc = SQLITE_AUTH; diff --git a/src/build.c b/src/build.c index f745164e54..c94c5a1310 100644 --- a/src/build.c +++ b/src/build.c @@ -2075,7 +2075,7 @@ int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ int nErr = 0; /* Number of errors encountered */ int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + sqlite3_xauth xAuth; /* Saved xAuth pointer */ assert( pTable ); diff --git a/src/prepare.c b/src/prepare.c index c6752548ff..c57fda0026 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -328,7 +328,7 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ db->aDb[iDb].zName, zMasterName); #ifndef SQLITE_OMIT_AUTHORIZATION { - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); + sqlite3_xauth xAuth; xAuth = db->xAuth; db->xAuth = 0; #endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9ef4e36208..a9f2001457 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1012,9 +1012,19 @@ int sqlite3UserAuthTable(const char*); int sqlite3UserAuthCheckLogin(sqlite3*,const char*,u8*); void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); - #endif /* SQLITE_USER_AUTHENTICATION */ +/* +** typedef for the authorization callback function. +*/ +#ifdef SQLITE_USER_AUTHENTICATION + typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, + const char*, const char*); +#else + typedef int (*sqlite3_xauth)(void*,int,const char*,const char*,const char*, + const char*); +#endif + /* ** Each database connection is an instance of the following structure. @@ -1083,8 +1093,7 @@ struct sqlite3 { } u1; Lookaside lookaside; /* Lookaside malloc configuration */ #ifndef SQLITE_OMIT_AUTHORIZATION - int (*xAuth)(void*,int,const char*,const char*,const char*,const char*); - /* Access authorization function */ + sqlite3_xauth xAuth; /* Access authorization function */ void *pAuthArg; /* 1st argument to the access auth function */ #endif #ifndef SQLITE_OMIT_PROGRESS_CALLBACK diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 945fd95982..756d0daa5a 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -872,6 +872,9 @@ static int auth_callback( const char *zArg2, const char *zArg3, const char *zArg4 +#ifdef SQLITE_USER_AUTHENTICATION + ,const char *zArg5 +#endif ){ const char *zCode; Tcl_DString str; @@ -924,6 +927,9 @@ static int auth_callback( Tcl_DStringAppendElement(&str, zArg2 ? zArg2 : ""); Tcl_DStringAppendElement(&str, zArg3 ? zArg3 : ""); Tcl_DStringAppendElement(&str, zArg4 ? zArg4 : ""); +#ifdef SQLITE_USER_AUTHENTICATION + Tcl_DStringAppendElement(&str, zArg5 ? zArg5 : ""); +#endif rc = Tcl_GlobalEval(pDb->interp, Tcl_DStringValue(&str)); Tcl_DStringFree(&str); zReply = rc==TCL_OK ? Tcl_GetStringResult(pDb->interp) : "SQLITE_DENY"; @@ -1700,8 +1706,11 @@ static int DbObjCmd(void *cd, Tcl_Interp *interp, int objc,Tcl_Obj *const*objv){ pDb->zAuth = 0; } if( pDb->zAuth ){ + typedef int (*sqlite3_auth_cb)( + void*,int,const char*,const char*, + const char*,const char*); pDb->interp = interp; - sqlite3_set_authorizer(pDb->db, auth_callback, pDb); + sqlite3_set_authorizer(pDb->db,(sqlite3_auth_cb)auth_callback,pDb); }else{ sqlite3_set_authorizer(pDb->db, 0, 0); } diff --git a/test/auth.test b/test/auth.test index 43e53ef2e3..f3c2fa79e8 100644 --- a/test/auth.test +++ b/test/auth.test @@ -36,7 +36,7 @@ proc_real proc {name arguments script} { do_test auth-1.1.1 { db close set ::DB [sqlite3 db test.db] - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -61,7 +61,7 @@ do_test auth-1.2 { execsql {SELECT name FROM sqlite_master} } {} do_test auth-1.3.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -82,7 +82,7 @@ do_test auth-1.4 { ifcapable tempdb { do_test auth-1.5 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -94,7 +94,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {} do_test auth-1.7.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -112,7 +112,7 @@ ifcapable tempdb { } do_test auth-1.9 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -124,7 +124,7 @@ do_test auth-1.10 { execsql {SELECT name FROM sqlite_master} } {} do_test auth-1.11 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -139,7 +139,7 @@ do_test auth-1.12 { ifcapable tempdb { do_test auth-1.13 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -151,7 +151,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {} do_test auth-1.15 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -165,7 +165,7 @@ ifcapable tempdb { } {} do_test auth-1.17 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -181,7 +181,7 @@ ifcapable tempdb { do_test auth-1.19.1 { set ::authargs {} - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -198,7 +198,7 @@ do_test auth-1.20 { } {t2} do_test auth-1.21.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -214,7 +214,7 @@ do_test auth-1.22 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.23.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -232,7 +232,7 @@ do_test auth-1.24 { ifcapable tempdb { do_test auth-1.25 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -245,7 +245,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.27 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -260,7 +260,7 @@ ifcapable tempdb { } do_test auth-1.29 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="t2"} { return SQLITE_DENY } @@ -272,7 +272,7 @@ do_test auth-1.30 { execsql {SELECT * FROM t2} } {} do_test auth-1.31 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="t2"} { return SQLITE_IGNORE } @@ -284,7 +284,7 @@ do_test auth-1.32 { execsql {SELECT * FROM t2} } {} do_test auth-1.33 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="t1"} { return SQLITE_IGNORE } @@ -297,7 +297,7 @@ do_test auth-1.34 { } {1 2 3} do_test auth-1.35.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="b"} { return SQLITE_DENY } @@ -313,7 +313,7 @@ ifcapable attach { execsql {DETACH DATABASE two} } do_test auth-1.36 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="b"} { return SQLITE_IGNORE } @@ -322,7 +322,7 @@ do_test auth-1.36 { catchsql {SELECT * FROM t2} } {0 {1 {} 3}} do_test auth-1.37 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="b"} { return SQLITE_IGNORE } @@ -331,7 +331,7 @@ do_test auth-1.37 { catchsql {SELECT * FROM t2 WHERE b=2} } {0 {}} do_test auth-1.38 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="a"} { return SQLITE_IGNORE } @@ -340,7 +340,7 @@ do_test auth-1.38 { catchsql {SELECT * FROM t2 WHERE b=2} } {0 {{} 2 3}} do_test auth-1.39 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="b"} { return SQLITE_IGNORE } @@ -349,7 +349,7 @@ do_test auth-1.39 { catchsql {SELECT * FROM t2 WHERE b IS NULL} } {0 {1 {} 3}} do_test auth-1.40 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="b"} { return SQLITE_DENY } @@ -359,7 +359,7 @@ do_test auth-1.40 { } {1 {access to t2.b is prohibited}} do_test auth-1.41 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_UPDATE" && $arg1=="t2" && $arg2=="b"} { return SQLITE_DENY } @@ -371,7 +371,7 @@ do_test auth-1.42 { execsql {SELECT * FROM t2} } {11 2 3} do_test auth-1.43 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_UPDATE" && $arg1=="t2" && $arg2=="b"} { return SQLITE_DENY } @@ -383,7 +383,7 @@ do_test auth-1.44 { execsql {SELECT * FROM t2} } {11 2 3} do_test auth-1.45 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_UPDATE" && $arg1=="t2" && $arg2=="b"} { return SQLITE_IGNORE } @@ -396,7 +396,7 @@ do_test auth-1.46 { } {11 2 33} do_test auth-1.47 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="t2"} { return SQLITE_DENY } @@ -408,7 +408,7 @@ do_test auth-1.48 { execsql {SELECT * FROM t2} } {11 2 33} do_test auth-1.49 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="t2"} { return SQLITE_IGNORE } @@ -424,7 +424,7 @@ do_test auth-1.50.2 { } {} do_test auth-1.51 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_SELECT"} { return SQLITE_DENY } @@ -433,7 +433,7 @@ do_test auth-1.51 { catchsql {SELECT * FROM t2} } {1 {not authorized}} do_test auth-1.52 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_SELECT"} { return SQLITE_IGNORE } @@ -442,7 +442,7 @@ do_test auth-1.52 { catchsql {SELECT * FROM t2} } {0 {}} do_test auth-1.53 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_SELECT"} { return SQLITE_OK } @@ -462,7 +462,7 @@ do_test auth-1.55 { } {11 2 33 7 8 9} do_test auth-1.63 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -474,7 +474,7 @@ do_test auth-1.64 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.65 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="t2"} { return SQLITE_DENY } @@ -488,7 +488,7 @@ do_test auth-1.66 { ifcapable tempdb { do_test auth-1.67 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -500,7 +500,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.69 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="t1"} { return SQLITE_DENY } @@ -514,7 +514,7 @@ ifcapable tempdb { } do_test auth-1.71 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -526,7 +526,7 @@ do_test auth-1.72 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.73 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="t2"} { return SQLITE_IGNORE } @@ -540,7 +540,7 @@ do_test auth-1.74 { ifcapable tempdb { do_test auth-1.75 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -552,7 +552,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.77 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="t1"} { return SQLITE_IGNORE } @@ -569,7 +569,7 @@ ifcapable tempdb { # Omit these if the library was compiled with views omitted. ifcapable view { do_test auth-1.79 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -585,7 +585,7 @@ do_test auth-1.81 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.82 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -603,7 +603,7 @@ do_test auth-1.84 { ifcapable tempdb { do_test auth-1.85 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -619,7 +619,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.88 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -637,7 +637,7 @@ ifcapable tempdb { } do_test auth-1.91 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -649,7 +649,7 @@ do_test auth-1.92 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.93 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -663,7 +663,7 @@ do_test auth-1.94 { ifcapable tempdb { do_test auth-1.95 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -675,7 +675,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.97 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -689,7 +689,7 @@ ifcapable tempdb { } do_test auth-1.99 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -704,7 +704,7 @@ do_test auth-1.100 { execsql {SELECT name FROM sqlite_master} } {t2 v2} do_test auth-1.101 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -720,7 +720,7 @@ do_test auth-1.103 { execsql {SELECT name FROM sqlite_master} } {t2 v2} do_test auth-1.104 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -732,7 +732,7 @@ do_test auth-1.105 { execsql {SELECT name FROM sqlite_master} } {t2 v2} do_test auth-1.106 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -748,7 +748,7 @@ do_test auth-1.108 { execsql {SELECT name FROM sqlite_master} } {t2 v2} do_test auth-1.109 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -767,7 +767,7 @@ do_test auth-1.111 { ifcapable tempdb { do_test auth-1.112 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -782,7 +782,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 v1} do_test auth-1.114 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -798,7 +798,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 v1} do_test auth-1.117 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -810,7 +810,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 v1} do_test auth-1.119 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -826,7 +826,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 v1} do_test auth-1.122 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_VIEW"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -849,7 +849,7 @@ ifcapable tempdb { # ifcapable trigger&&tempdb { do_test auth-1.125 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -869,7 +869,7 @@ do_test auth-1.127 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.128 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -885,7 +885,7 @@ do_test auth-1.129 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.130 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -905,7 +905,7 @@ do_test auth-1.132 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.133 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -921,7 +921,7 @@ do_test auth-1.134 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.135 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -944,7 +944,7 @@ do_test auth-1.136.2 { } } {r2} do_test auth-1.136.3 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { lappend ::authargs $code $arg1 $arg2 $arg3 $arg4 return SQLITE_OK } @@ -963,7 +963,7 @@ do_test auth-1.137 { execsql {SELECT name FROM sqlite_master} } {t2 tx r2} do_test auth-1.138 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -983,7 +983,7 @@ do_test auth-1.140 { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.141 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -999,7 +999,7 @@ do_test auth-1.142 { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.143 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1019,7 +1019,7 @@ do_test auth-1.145 { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.146 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -1035,7 +1035,7 @@ do_test auth-1.147 { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.148 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1056,7 +1056,7 @@ do_test auth-1.150 { } {t1 r1} do_test auth-1.151 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -1068,7 +1068,7 @@ do_test auth-1.152 { execsql {SELECT name FROM sqlite_master} } {t2 tx r2} do_test auth-1.153 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1084,7 +1084,7 @@ do_test auth-1.155 { execsql {SELECT name FROM sqlite_master} } {t2 tx r2} do_test auth-1.156 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -1096,7 +1096,7 @@ do_test auth-1.157 { execsql {SELECT name FROM sqlite_master} } {t2 tx r2} do_test auth-1.158 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1112,7 +1112,7 @@ do_test auth-1.160 { execsql {SELECT name FROM sqlite_master} } {t2 tx r2} do_test auth-1.161 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1133,7 +1133,7 @@ do_test auth-1.163 { } {t2} do_test auth-1.164 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -1145,7 +1145,7 @@ do_test auth-1.165 { execsql {SELECT name FROM sqlite_temp_master} } {t1 r1} do_test auth-1.166 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1161,7 +1161,7 @@ do_test auth-1.168 { execsql {SELECT name FROM sqlite_temp_master} } {t1 r1} do_test auth-1.169 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -1173,7 +1173,7 @@ do_test auth-1.170 { execsql {SELECT name FROM sqlite_temp_master} } {t1 r1} do_test auth-1.171 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1189,7 +1189,7 @@ do_test auth-1.173 { execsql {SELECT name FROM sqlite_temp_master} } {t1 r1} do_test auth-1.174 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_TRIGGER"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1207,7 +1207,7 @@ do_test auth-1.176 { } ;# ifcapable trigger do_test auth-1.177 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1223,7 +1223,7 @@ do_test auth-1.179 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.180 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -1235,7 +1235,7 @@ do_test auth-1.181 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.182 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1251,7 +1251,7 @@ do_test auth-1.184 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.185 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -1263,7 +1263,7 @@ do_test auth-1.186 { execsql {SELECT name FROM sqlite_master} } {t2} do_test auth-1.187 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1281,7 +1281,7 @@ do_test auth-1.189 { ifcapable tempdb { do_test auth-1.190 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1297,7 +1297,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.193 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -1309,7 +1309,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.195 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1325,7 +1325,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.198 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_INSERT" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -1337,7 +1337,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1} do_test auth-1.200 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_CREATE_TEMP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1355,7 +1355,7 @@ ifcapable tempdb { } do_test auth-1.203 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_DENY } @@ -1367,7 +1367,7 @@ do_test auth-1.204 { execsql {SELECT name FROM sqlite_master} } {t2 i2} do_test auth-1.205 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1383,7 +1383,7 @@ do_test auth-1.207 { execsql {SELECT name FROM sqlite_master} } {t2 i2} do_test auth-1.208 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_master"} { return SQLITE_IGNORE } @@ -1395,7 +1395,7 @@ do_test auth-1.209 { execsql {SELECT name FROM sqlite_master} } {t2 i2} do_test auth-1.210 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1411,7 +1411,7 @@ do_test auth-1.212 { execsql {SELECT name FROM sqlite_master} } {t2 i2} do_test auth-1.213 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1429,7 +1429,7 @@ do_test auth-1.215 { ifcapable tempdb { do_test auth-1.216 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_DENY } @@ -1441,7 +1441,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 i1} do_test auth-1.218 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1457,7 +1457,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 i1} do_test auth-1.221 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE" && $arg1=="sqlite_temp_master"} { return SQLITE_IGNORE } @@ -1469,7 +1469,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 i1} do_test auth-1.223 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1485,7 +1485,7 @@ ifcapable tempdb { execsql {SELECT name FROM sqlite_temp_master} } {t1 i1} do_test auth-1.226 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DROP_TEMP_INDEX"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1503,7 +1503,7 @@ ifcapable tempdb { } do_test auth-1.229 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_PRAGMA"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1519,7 +1519,7 @@ do_test auth-1.231 { execsql2 {SELECT a FROM t2} } {a 11 a 7} do_test auth-1.232 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_PRAGMA"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1535,7 +1535,7 @@ do_test auth-1.234 { execsql2 {SELECT a FROM t2} } {a 11 a 7} do_test auth-1.235 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_PRAGMA"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1548,7 +1548,7 @@ do_test auth-1.236 { execsql2 {SELECT a FROM t2} } {t2.a 11 t2.a 7} do_test auth-1.237 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_PRAGMA"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1565,7 +1565,7 @@ do_test auth-1.239 { } {a 11 a 7} do_test auth-1.240 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_TRANSACTION"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1578,7 +1578,7 @@ do_test auth-1.241 { set ::authargs } {BEGIN {} {} {}} do_test auth-1.242 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_TRANSACTION" && $arg1!="BEGIN"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1618,7 +1618,7 @@ do_test auth-1.250 { ifcapable attach { do_test auth-1.251 { db authorizer ::auth - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ATTACH"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] } @@ -1644,7 +1644,7 @@ ifcapable attach { } {{} {} {} {}} do_test auth-1.253 { catchsql {DETACH DATABASE test1} - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ATTACH"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1660,7 +1660,7 @@ ifcapable attach { } {} do_test auth-1.255 { catchsql {DETACH DATABASE test1} - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ATTACH"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1675,7 +1675,7 @@ ifcapable attach { lindex [execsql {PRAGMA database_list}] 7 } {} do_test auth-1.257 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DETACH"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1692,7 +1692,7 @@ ifcapable attach { } {} do_test auth-1.259 { execsql {ATTACH DATABASE ':memory:' AS test1} - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DETACH"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1710,7 +1710,7 @@ ifcapable attach { } {test1} } ;# ifcapable schema_pragmas do_test auth-1.261 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DETACH"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1735,7 +1735,7 @@ ifcapable attach { ifcapable altertable { do_test auth-1.263 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1753,7 +1753,7 @@ ifcapable attach { set authargs } {temp t1 {} {}} do_test auth-1.266 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1771,7 +1771,7 @@ ifcapable attach { set authargs } {temp t1x {} {}} do_test auth-1.269 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1804,7 +1804,7 @@ db authorizer {} catchsql {ALTER TABLE t1x RENAME TO t1} db authorizer ::auth do_test auth-1.272 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -1822,7 +1822,7 @@ do_test auth-1.274 { set authargs } {main t2 {} {}} do_test auth-1.275 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -1840,7 +1840,7 @@ do_test auth-1.277 { set authargs } {main t2x {} {}} do_test auth-1.278 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -1867,7 +1867,7 @@ ifcapable reindex { proc auth {code args} { if {$code=="SQLITE_REINDEX"} { - set ::authargs [concat $::authargs $args] + set ::authargs [concat $::authargs [lrange $args 0 3]] } return SQLITE_OK } @@ -1950,7 +1950,7 @@ ifcapable tempdb { } {t3_idx2 {} temp {} t3_idx1 {} temp {} sqlite_autoindex_t3_1 {} temp {}} proc auth {code args} { if {$code=="SQLITE_REINDEX"} { - set ::authargs [concat $::authargs $args] + set ::authargs [concat $::authargs [lrange $args 0 3]] return SQLITE_DENY } return SQLITE_OK @@ -1973,7 +1973,7 @@ ifcapable tempdb { ifcapable analyze { proc auth {code args} { if {$code=="SQLITE_ANALYZE"} { - set ::authargs [concat $::authargs $args] + set ::authargs [concat $::authargs [lrange $args 0 3]] } return SQLITE_OK } @@ -2020,7 +2020,7 @@ ifcapable analyze { ifcapable {altertable} { do_test auth-1.300 { execsql {CREATE TABLE t5(x)} - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_OK @@ -2039,7 +2039,7 @@ ifcapable {altertable} { set authargs } {main t5 {} {}} do_test auth-1.303 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_IGNORE @@ -2058,7 +2058,7 @@ ifcapable {altertable} { set authargs } {main t5 {} {}} do_test auth-1.306 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_ALTER_TABLE"} { set ::authargs [list $arg1 $arg2 $arg3 $arg4] return SQLITE_DENY @@ -2082,7 +2082,7 @@ ifcapable {altertable} { ifcapable {cte} { do_test auth-1.310 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_RECURSIVE"} { return SQLITE_DENY } @@ -2117,7 +2117,7 @@ ifcapable {cte} { } ;# ifcapable cte do_test auth-2.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t3" && $arg2=="x"} { return SQLITE_DENY } @@ -2137,7 +2137,7 @@ do_test auth-2.3 { catchsql {SELECT OID,y,z FROM t3} } {1 {access to t3.x is prohibited}} do_test auth-2.4 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t3" && $arg2=="x"} { return SQLITE_IGNORE } @@ -2150,7 +2150,7 @@ do_test auth-2.5 { catchsql {SELECT rowid,y,z FROM t3} } {0 {{} 55 66}} do_test auth-2.6 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t3" && $arg2=="ROWID"} { return SQLITE_IGNORE } @@ -2162,7 +2162,7 @@ do_test auth-2.7 { catchsql {SELECT ROWID,y,z FROM t3} } {0 {44 55 66}} do_test auth-2.8 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="ROWID"} { return SQLITE_IGNORE } @@ -2181,7 +2181,7 @@ do_test auth-2.9.1 { # db cache flush - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="ROWID"} { return bogus } @@ -2193,7 +2193,7 @@ do_test auth-2.9.2 { db errorcode } {1} do_test auth-2.10 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_SELECT"} { return bogus } @@ -2202,7 +2202,7 @@ do_test auth-2.10 { catchsql {SELECT ROWID,b,c FROM t2} } {1 {authorizer malfunction}} do_test auth-2.11.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg2=="a"} { return SQLITE_IGNORE } @@ -2211,7 +2211,7 @@ do_test auth-2.11.1 { catchsql {SELECT * FROM t2, t3} } {0 {{} 2 33 44 55 66 {} 8 9 44 55 66}} do_test auth-2.11.2 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg2=="x"} { return SQLITE_IGNORE } @@ -2224,7 +2224,7 @@ do_test auth-2.11.2 { # ifcapable trigger { do_test auth-3.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { return SQLITE_OK } execsql { @@ -2237,7 +2237,7 @@ ifcapable trigger { } } {11 12 2 2 33 33 7 8 8 8 9 9} do_test auth-3.2 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_READ" && $arg1=="t2" && $arg2=="c"} { return SQLITE_IGNORE } @@ -2255,7 +2255,7 @@ ifcapable trigger { # ifcapable trigger { do_test auth-4.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { lappend ::authargs $code $arg1 $arg2 $arg3 $arg4 return SQLITE_OK } @@ -2340,7 +2340,7 @@ do_test auth-4.5 { # clause. # do_test auth-5.1 { - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { return SQLITE_OK } execsql { @@ -2393,7 +2393,7 @@ ifcapable trigger { } {} set ::authargs [list] proc auth {args} { - eval lappend ::authargs $args + eval lappend ::authargs [lrange $args 0 4] return SQLITE_OK } do_test auth-5.3.2 { @@ -2419,7 +2419,7 @@ do_test auth-6.1 { } {} set ::authargs [list] proc auth {args} { - eval lappend ::authargs $args + eval lappend ::authargs [lrange $args 0 4] return SQLITE_OK } do_test auth-6.2 { diff --git a/test/auth2.test b/test/auth2.test index 65e0591249..a9d64d08af 100644 --- a/test/auth2.test +++ b/test/auth2.test @@ -31,7 +31,7 @@ do_test auth2-1.1 { INSERT INTO t1 VALUES(1,2,3); } set ::flist {} - proc auth {code arg1 arg2 arg3 arg4} { + proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_FUNCTION"} { lappend ::flist $arg2 if {$arg2=="max"} { @@ -80,7 +80,7 @@ sqlite3 db test.db sqlite3 db2 test.db proc auth {args} { global authargs - append authargs $args\n + append authargs [lrange $args 0 4]\n return SQLITE_OK } db auth auth diff --git a/test/auth3.test b/test/auth3.test index 21e2b3b65d..eef10b398f 100644 --- a/test/auth3.test +++ b/test/auth3.test @@ -30,7 +30,7 @@ if {[catch {db auth {}} msg]} { db cache size 0 db authorizer ::auth -proc auth {code arg1 arg2 arg3 arg4} { +proc auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_DELETE"} { return $::authcode } diff --git a/test/fkey2.test b/test/fkey2.test index 4c8daa0b02..53b90dc91c 100644 --- a/test/fkey2.test +++ b/test/fkey2.test @@ -1554,7 +1554,7 @@ ifcapable auth { } } {} - proc auth {args} {eval lappend ::authargs $args ; return SQLITE_OK} + proc auth {args} {eval lappend ::authargs [lrange $args 0 4]; return SQLITE_OK} db auth auth # An insert on the parent table must read the child key of any deferred diff --git a/test/fts4aa.test b/test/fts4aa.test index 88550c99f7..e6c7f9336e 100644 --- a/test/fts4aa.test +++ b/test/fts4aa.test @@ -170,7 +170,7 @@ foreach {q r} [array get fts4aa_res] { # Should get the same search results when an authorizer prevents # all PRAGMA statements. # -proc no_pragma_auth {code arg1 arg2 arg3 arg4} { +proc no_pragma_auth {code arg1 arg2 arg3 arg4 args} { if {$code=="SQLITE_PRAGMA"} {return SQLITE_DENY} return SQLITE_OK; } diff --git a/test/savepoint.test b/test/savepoint.test index 9f4571abef..9362c8fe19 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -561,7 +561,7 @@ do_test savepoint-8-2 { # ifcapable auth { proc auth {args} { - eval lappend ::authdata $args + eval lappend ::authdata [lrange $args 0 4] return SQLITE_OK } db auth auth @@ -583,7 +583,7 @@ ifcapable auth { } {SQLITE_SAVEPOINT RELEASE sp1 {} {}} proc auth {args} { - eval lappend ::authdata $args + eval lappend ::authdata [lrange $args 0 4] return SQLITE_DENY } db auth auth diff --git a/test/vtab3.test b/test/vtab3.test index ebf8369d5d..e8c6982a57 100644 --- a/test/vtab3.test +++ b/test/vtab3.test @@ -25,7 +25,7 @@ set ::auth_fail 0 set ::auth_log [list] set ::auth_filter [list SQLITE_READ SQLITE_UPDATE SQLITE_SELECT SQLITE_PRAGMA] -proc auth {code arg1 arg2 arg3 arg4} { +proc auth {code arg1 arg2 arg3 arg4 args} { if {[lsearch $::auth_filter $code]>-1} { return SQLITE_OK } diff --git a/test/without_rowid3.test b/test/without_rowid3.test index c4c2d6f483..a0dc76d3f9 100644 --- a/test/without_rowid3.test +++ b/test/without_rowid3.test @@ -1621,7 +1621,7 @@ ifcapable auth { } } {} - proc auth {args} {eval lappend ::authargs $args ; return SQLITE_OK} + proc auth {args} {eval lappend ::authargs [lrange $args 0 4]; return SQLITE_OK} db auth auth # An insert on the parent table must read the child key of any deferred From b2445d5ee8b58a4e9e006d697e14dae3f4c43cad Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Sep 2014 14:01:41 +0000 Subject: [PATCH 316/710] Move user authentication blocking from sqlite3_prepare() over to the table lock generator, thus allowing SQL statements (like "PRAGMA locking_mode") that do not touch database content to run prior to authentication. FossilOrigin-Name: 70121e7cf868b7d6d19bf98794ddc3809a901456 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/build.c | 18 ++++++++++++++++++ src/pragma.c | 2 +- src/prepare.c | 14 -------------- 5 files changed, 28 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 12328063d0..2e3e9eb42d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sthe\sextra\sparameter\son\sthe\ssqlite3_set_authorizer()\scallback\nand\ssupport\sfor\sfailing\san\sATTACH\swith\san\sauthentication-required\sdatabase\nusing\sbad\scredentials.\s\sThe\sextension\sis\snow\sfeature\scomplete,\sbut\smuch\ntesting\sand\sbug-fixing\sremains. -D 2014-09-11T13:44:52.150 +C Move\suser\sauthentication\sblocking\sfrom\ssqlite3_prepare()\sover\sto\sthe\ntable\slock\sgenerator,\sthus\sallowing\sSQL\sstatements\s(like\s\n"PRAGMA\slocking_mode")\sthat\sdo\snot\stouch\sdatabase\scontent\sto\srun\nprior\sto\sauthentication. +D 2014-09-11T14:01:41.319 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,7 +174,7 @@ F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c b1c1cd1cc3ae2e433a23b9a6c9ab53805707d8cd F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc -F src/build.c 4c7aac1ddda782c6f1cad84aeabec6e8d0be7495 +F src/build.c 8b02494e4dc9c4a6c9aff1cac8b40c426733f025 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958 @@ -220,8 +220,8 @@ F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 2048affdb09a04478b5fc6e64cb1083078d369be F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa -F src/pragma.c 3b7b1a5e90804006f44c65464c7032ee6a1d24e3 -F src/prepare.c f82c009a763e739c6bdf02a270ccfda9e54f783c +F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f +F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c e74925089a85e3c9f0e315595f41c139d3d118c2 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0d1621e45fffe4b4396477cf46e41a84b0145ffb @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2f6d8f32eef526b5912f42ab467e3c7812480d8b -R 8b68610f4a9d73b92fb9d68a3359baf2 +P 596e728b0eb19a34c888e33d4d37978ca2bf1e00 +R c54f70fc39612d9ea503f6c456a7ab1f U drh -Z 6a1c7f3384e27f9ad12e8ea277382ab2 +Z e2993baf94376fc5d680228ffe933715 diff --git a/manifest.uuid b/manifest.uuid index 1b3da62f36..a5702df9c7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -596e728b0eb19a34c888e33d4d37978ca2bf1e00 \ No newline at end of file +70121e7cf868b7d6d19bf98794ddc3809a901456 \ No newline at end of file diff --git a/src/build.c b/src/build.c index c94c5a1310..d95cf849e6 100644 --- a/src/build.c +++ b/src/build.c @@ -156,6 +156,24 @@ void sqlite3FinishCoding(Parse *pParse){ while( sqlite3VdbeDeletePriorOpcode(v, OP_Close) ){} sqlite3VdbeAddOp0(v, OP_Halt); +#if SQLITE_USER_AUTHENTICATION + if( pParse->nTableLock>0 && db->init.busy==0 ){ + if( db->auth.authLevelauth.authLevel==UAUTH_Unknown ){ + u8 authLevel = UAUTH_Fail; + sqlite3UserAuthCheckLogin(db, "main", &authLevel); + db->auth.authLevel = authLevel; + if( authLevelflags &= ~SQLITE_WriteSchema; + } + if( db->auth.authLevelrc = SQLITE_AUTH_USER; + sqlite3ErrorMsg(pParse, "user not authenticated"); + return; + } + } + } +#endif + /* The cookie mask contains one bit for each database file open. ** (Bit 0 is for main, bit 1 is for temp, and so forth.) Bits are ** set for each database that is used. Generate code to start a diff --git a/src/pragma.c b/src/pragma.c index cbd593f215..543f265ba9 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1398,7 +1398,7 @@ void sqlite3Pragma( mask &= ~(SQLITE_ForeignKeys); } #if SQLITE_USER_AUTHENTICATION - if( db->auth.authLevelauth.authLevel==UAUTH_User ){ /* Do not allow non-admin users to modify the schema arbitrarily */ mask &= ~(SQLITE_WriteSchema); } diff --git a/src/prepare.c b/src/prepare.c index c57fda0026..a05e619f3e 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -715,20 +715,6 @@ static int sqlite3LockAndPrepare( return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); -#if SQLITE_USER_AUTHENTICATION - if( db->auth.authLevelauth.authLevel==UAUTH_Unknown ){ - u8 authLevel = UAUTH_Fail; - sqlite3UserAuthCheckLogin(db, "main", &authLevel); - db->auth.authLevel = authLevel; - } - if( db->auth.authLevelmutex); - return SQLITE_ERROR; - } - } -#endif sqlite3BtreeEnterAll(db); rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail); if( rc==SQLITE_SCHEMA ){ From 570f187f78872b349cea66822595cc23304dc378 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Sep 2014 14:40:27 +0000 Subject: [PATCH 317/710] Fix the sqlite3_user_change() interface so that it does allow a non-admin user to change their own password. FossilOrigin-Name: 52d440c7e1b07fc03f14ed5fa4cc4c89a75cd430 --- ext/userauth/user-auth.txt | 23 ++++++++++----------- ext/userauth/userauth.c | 30 ++++++++++++++++++--------- manifest | 16 +++++++-------- manifest.uuid | 2 +- test/userauth01.test | 42 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 82 insertions(+), 31 deletions(-) diff --git a/ext/userauth/user-auth.txt b/ext/userauth/user-auth.txt index 8b64617c36..ba4eabc137 100644 --- a/ext/userauth/user-auth.txt +++ b/ext/userauth/user-auth.txt @@ -44,12 +44,11 @@ By default a database does not require authentication. The sqlite3_open(), sqlite3_open16(), and sqlite3_open_v2() interfaces work as before: they open a new database connection. However, if the -database being opened requires authentication, then attempts to prepare -SQL statements (using sqlite3_prepare_v2(), for example) will fail -with an SQLITE_AUTH error until after sqlite3_user_authenticate() -has been called successfully. The sqlite3_user_authenticate() call -will return SQLITE_OK if the authentication credentials are accepted -and SQLITE_ERROR if not. +database being opened requires authentication, then attempts to read +or write from the database will fail with an SQLITE_AUTH error until +after sqlite3_user_authenticate() has been called successfully. The +sqlite3_user_authenticate() call will return SQLITE_OK if the +authentication credentials are accepted and SQLITE_ERROR if not. Calling sqlite3_user_authenticate() on a no-authentication-required database connection is a harmless no-op. @@ -133,12 +132,12 @@ new table: pw BLOB ) WITHOUT ROWID; -This table is inaccessible (unreadable and unwriteable) to non-admin users -and is read-only for admin users. However, if the same database file is -opened by a version of SQLite that omits the -DSQLITE_USER_AUTHENTICATION -compile-time option, then the sqlite_user table will be readable by -anybody and writeable by anybody if the "PRAGMA writable_schema=ON" -statement is run first. +The sqlite_user table is inaccessible (unreadable and unwriteable) to +non-admin users and is read-only for admin users. However, if the same +database file is opened by a version of SQLite that omits +the -DSQLITE_USER_AUTHENTICATION compile-time option, then the sqlite_user +table will be readable by anybody and writeable by anybody if +the "PRAGMA writable_schema=ON" statement is run first. The sqlite_user.pw field is encoded by a built-in SQL function "sqlite_crypt(X,Y)". The two arguments are both BLOBs. The first argument diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c index 343e49e6ff..19e9f6f762 100644 --- a/ext/userauth/userauth.c +++ b/ext/userauth/userauth.c @@ -268,7 +268,11 @@ int sqlite3_user_change( int isAdmin /* Modified admin privilege for the user */ ){ sqlite3_stmt *pStmt; - if( db->auth.authLevelauth.authLevel; + if( authLevelauth.authLevel==UAUTH_Admin) ){ + }else if( isAdmin!=(authLevel==UAUTH_Admin) ){ /* Cannot change the isAdmin setting for self */ return SQLITE_AUTH; } + db->auth.authLevel = UAUTH_Admin; if( !userTableExists(db, "main") ){ /* This routine is a no-op if the user to be modified does not exist */ - return SQLITE_OK; + }else{ + pStmt = sqlite3UserAuthPrepare(db, + "UPDATE sqlite_user SET isAdmin=%d, pw=sqlite_crypt(?1,NULL)" + " WHERE uname=%Q", isAdmin, zUsername); + if( pStmt==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_bind_blob(pStmt, 1, aPW, nPW, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_finalize(pStmt); + } } - pStmt = sqlite3UserAuthPrepare(db, - "UPDATE sqlite_user SET isAdmin=%d, pw=sqlite_crypt(?1,NULL)" - " WHERE uname=%Q", isAdmin, zUsername); - if( pStmt==0 ) return SQLITE_NOMEM; - sqlite3_bind_blob(pStmt, 1, aPW, nPW, SQLITE_STATIC); - sqlite3_step(pStmt); - return sqlite3_finalize(pStmt); + db->auth.authLevel = authLevel; + return rc; } /* diff --git a/manifest b/manifest index 2e3e9eb42d..b1a2451fb1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Move\suser\sauthentication\sblocking\sfrom\ssqlite3_prepare()\sover\sto\sthe\ntable\slock\sgenerator,\sthus\sallowing\sSQL\sstatements\s(like\s\n"PRAGMA\slocking_mode")\sthat\sdo\snot\stouch\sdatabase\scontent\sto\srun\nprior\sto\sauthentication. -D 2014-09-11T14:01:41.319 +C Fix\sthe\ssqlite3_user_change()\sinterface\sso\sthat\sit\sdoes\sallow\sa\nnon-admin\suser\sto\schange\stheir\sown\spassword. +D 2014-09-11T14:40:27.156 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -145,8 +145,8 @@ F ext/rtree/sqlite3rtree.h 83349d519fe5f518b3ea025d18dd1fe51b1684bd F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 -F ext/userauth/user-auth.txt 527aaec593ae34dcaf543324623c8351a5d33d3f -F ext/userauth/userauth.c a66cd3abcc3b2c10b3999ab49f900d561e8ddd33 +F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 +F ext/userauth/userauth.c bcd1aedb0b810b1a1125945e637af54ce3d299f1 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1057,7 +1057,7 @@ F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b -F test/userauth01.test 695ead5a47e8827dea283abca69c121b679176af +F test/userauth01.test 3be4d454af151aa8d59804f2fe4e593f367014c3 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test af432e6e3bfc0ea20a80cb86a03c7d9876d38324 @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 596e728b0eb19a34c888e33d4d37978ca2bf1e00 -R c54f70fc39612d9ea503f6c456a7ab1f +P 70121e7cf868b7d6d19bf98794ddc3809a901456 +R fc5d3c327123005770c7e41594123732 U drh -Z e2993baf94376fc5d680228ffe933715 +Z 21dad64c9f9cfa4863908b784242b9b1 diff --git a/manifest.uuid b/manifest.uuid index a5702df9c7..5b03dbec59 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -70121e7cf868b7d6d19bf98794ddc3809a901456 \ No newline at end of file +52d440c7e1b07fc03f14ed5fa4cc4c89a75cd430 \ No newline at end of file diff --git a/test/userauth01.test b/test/userauth01.test index 60e617e3ba..9f0eb5d85a 100644 --- a/test/userauth01.test +++ b/test/userauth01.test @@ -70,5 +70,47 @@ do_test userauth01-1.5 { } } {NULL 1 2.5 'three' X'4444' alice 1 sqlite_user t1} +# The sqlite3_user_add() interface can be used (by an admin user only) +# to create a new user. +# +do_test userauth01-1.6 { + sqlite3_user_add db bob pw-4-bob 0 + sqlite3_user_add db cindy pw-4-cindy 0 + sqlite3_user_add db david pw-4-david 0 + execsql { + SELECT uname, isadmin FROM sqlite_user ORDER BY uname; + } +} {alice 1 bob 0 cindy 0 david 0} + +# The sqlite_user table is inaccessible (unreadable and unwriteable) to +# non-admin users and is read-only for admin users. However, if the same +# +do_test userauth01-1.7 { + sqlite3 db2 test.db + sqlite3_user_authenticate db2 cindy pw-4-cindy + db2 eval { + SELECT quote(x) FROM t1 ORDER BY x; + SELECT name FROM sqlite_master ORDER BY name; + } +} {NULL 1 2.5 'three' X'4444' sqlite_user t1} +do_test userauth01-1.8 { + catchsql { + SELECT uname, isadmin FROM sqlite_user ORDER BY uname; + } db2 +} {1 {no such table: sqlite_user}} + +# Any user can change their own password. +# +do_test userauth01-1.9 { + sqlite3_user_change db2 cindy xyzzy-cindy 0 +} {SQLITE_OK} +do_test userauth01-1.10 { + sqlite3_user_authenticate db2 cindy pw-4-cindy +} {SQLITE_AUTH} +do_test userauth01-1.11 { + sqlite3_user_authenticate db2 cindy xyzzy-cindy +} {SQLITE_OK} + + finish_test From 9d5b0df132ad3abec0bc0aa2913814e845909b05 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Sep 2014 14:56:45 +0000 Subject: [PATCH 318/710] Get the sqlite3_user_delete() interface working. FossilOrigin-Name: 974a9c65583f7ab438d5673dc00c347ab8322855 --- ext/userauth/userauth.c | 2 +- manifest | 14 +++---- manifest.uuid | 2 +- test/userauth01.test | 89 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 98 insertions(+), 9 deletions(-) diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c index 19e9f6f762..21d33ce80f 100644 --- a/ext/userauth/userauth.c +++ b/ext/userauth/userauth.c @@ -329,7 +329,7 @@ int sqlite3_user_delete( return SQLITE_OK; } pStmt = sqlite3UserAuthPrepare(db, - "SELECT FROM sqlite_user WHERE uname=%Q", zUsername); + "DELETE FROM sqlite_user WHERE uname=%Q", zUsername); if( pStmt==0 ) return SQLITE_NOMEM; sqlite3_step(pStmt); return sqlite3_finalize(pStmt); diff --git a/manifest b/manifest index b1a2451fb1..f5797939b4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssqlite3_user_change()\sinterface\sso\sthat\sit\sdoes\sallow\sa\nnon-admin\suser\sto\schange\stheir\sown\spassword. -D 2014-09-11T14:40:27.156 +C Get\sthe\ssqlite3_user_delete()\sinterface\sworking. +D 2014-09-11T14:56:45.837 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -146,7 +146,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 -F ext/userauth/userauth.c bcd1aedb0b810b1a1125945e637af54ce3d299f1 +F ext/userauth/userauth.c 7942172fe537a6eedb797535b7558e726e00f728 F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1057,7 +1057,7 @@ F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b -F test/userauth01.test 3be4d454af151aa8d59804f2fe4e593f367014c3 +F test/userauth01.test 77f6762fd30e09a70c08ae1734fc63cc1e2f214a F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test af432e6e3bfc0ea20a80cb86a03c7d9876d38324 @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 70121e7cf868b7d6d19bf98794ddc3809a901456 -R fc5d3c327123005770c7e41594123732 +P 52d440c7e1b07fc03f14ed5fa4cc4c89a75cd430 +R f24750ae4706249df9137bdaab459848 U drh -Z 21dad64c9f9cfa4863908b784242b9b1 +Z 7444bd078e3d35d0ed86c3e87803a00a diff --git a/manifest.uuid b/manifest.uuid index 5b03dbec59..d912814cf2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -52d440c7e1b07fc03f14ed5fa4cc4c89a75cd430 \ No newline at end of file +974a9c65583f7ab438d5673dc00c347ab8322855 \ No newline at end of file diff --git a/test/userauth01.test b/test/userauth01.test index 9f0eb5d85a..a914ea07ca 100644 --- a/test/userauth01.test +++ b/test/userauth01.test @@ -110,7 +110,96 @@ do_test userauth01-1.10 { do_test userauth01-1.11 { sqlite3_user_authenticate db2 cindy xyzzy-cindy } {SQLITE_OK} +do_test userauth01-1.12 { + sqlite3_user_change db alice xyzzy-alice 1 +} {SQLITE_OK} +do_test userauth01-1.13 { + sqlite3_user_authenticate db alice pw-4-alice +} {SQLITE_AUTH} +do_test userauth01-1.14 { + sqlite3_user_authenticate db alice xyzzy-alice +} {SQLITE_OK} +# No user may change their own admin privilege setting. +# +do_test userauth01-1.15 { + sqlite3_user_change db alice xyzzy-alice 0 +} {SQLITE_AUTH} +do_test userauth01-1.16 { + db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname} +} {alice 1 bob 0 cindy 0 david 0} +do_test userauth01-1.17 { + sqlite3_user_change db2 cindy xyzzy-cindy 1 +} {SQLITE_AUTH} +do_test userauth01-1.18 { + db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname} +} {alice 1 bob 0 cindy 0 david 0} + +# The sqlite3_user_change() interface can be used to change a users +# login credentials or admin privilege. +# +do_test userauth01-1.20 { + sqlite3_user_change db david xyzzy-david 1 +} {SQLITE_OK} +do_test userauth01-1.21 { + db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname} +} {alice 1 bob 0 cindy 0 david 1} +do_test userauth01-1.22 { + sqlite3_user_authenticate db2 david xyzzy-david +} {SQLITE_OK} +do_test userauth01-1.23 { + db2 eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname} +} {alice 1 bob 0 cindy 0 david 1} +do_test userauth01-1.24 { + sqlite3_user_change db david pw-4-david 0 +} {SQLITE_OK} +do_test userauth01-1.25 { + sqlite3_user_authenticate db2 david pw-4-david +} {SQLITE_OK} +do_test userauth01-1.26 { + db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname} +} {alice 1 bob 0 cindy 0 david 0} +do_test userauth01-1.27 { + catchsql {SELECT uname, isadmin FROM sqlite_user ORDER BY uname} db2 +} {1 {no such table: sqlite_user}} + +# Only an admin user can change another users login +# credentials or admin privilege setting. +# +do_test userauth01-1.30 { + sqlite3_user_change db2 bob xyzzy-bob 1 +} {SQLITE_AUTH} +do_test userauth01-1.31 { + db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname} +} {alice 1 bob 0 cindy 0 david 0} + +# The sqlite3_user_delete() interface can be used (by an admin user only) +# to delete a user. +# +do_test userauth01-1.40 { + sqlite3_user_delete db bob +} {SQLITE_OK} +do_test userauth01-1.41 { + db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname} +} {alice 1 cindy 0 david 0} +do_test userauth01-1.42 { + sqlite3_user_delete db2 cindy +} {SQLITE_AUTH} +do_test userauth01-1.43 { + sqlite3_user_delete db2 alice +} {SQLITE_AUTH} +do_test userauth01-1.44 { + db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname} +} {alice 1 cindy 0 david 0} + +# The currently logged-in user cannot be deleted +# +do_test userauth01-1.50 { + sqlite3_user_delete db alice +} {SQLITE_AUTH} +do_test userauth01-1.51 { + db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname} +} {alice 1 cindy 0 david 0} finish_test From a000ca681aa71c7efb4b9ab91f8ae6ca2bf289d3 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Sep 2014 15:25:02 +0000 Subject: [PATCH 319/710] All interfaces working and tested. FossilOrigin-Name: 96ea5c0b3cd1dec81d490f2f958ebd2e47a24921 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/attach.c | 2 +- test/userauth01.test | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f5797939b4..f6d0875475 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\ssqlite3_user_delete()\sinterface\sworking. -D 2014-09-11T14:56:45.837 +C All\sinterfaces\sworking\sand\stested. +D 2014-09-11T15:25:02.114 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c 79383a54fee3b7f1fb03dd4c8c8115583f506de5 -F src/attach.c cc9b30041dfcd24be0a47986c87c384515c54449 +F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb @@ -1057,7 +1057,7 @@ F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b -F test/userauth01.test 77f6762fd30e09a70c08ae1734fc63cc1e2f214a +F test/userauth01.test de260ba56ca288e36f10fc86cdd6e30be0c96edb F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae F test/vacuum.test ce91c39f7f91a4273bf620efad21086b5aa6ef1d F test/vacuum2.test af432e6e3bfc0ea20a80cb86a03c7d9876d38324 @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 52d440c7e1b07fc03f14ed5fa4cc4c89a75cd430 -R f24750ae4706249df9137bdaab459848 +P 974a9c65583f7ab438d5673dc00c347ab8322855 +R 1b82e61677dc0f31739c2e6eefc2d964 U drh -Z 7444bd078e3d35d0ed86c3e87803a00a +Z fb0835cb3153cdbefaf30b784736228d diff --git a/manifest.uuid b/manifest.uuid index d912814cf2..adf124ebaf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -974a9c65583f7ab438d5673dc00c347ab8322855 \ No newline at end of file +96ea5c0b3cd1dec81d490f2f958ebd2e47a24921 \ No newline at end of file diff --git a/src/attach.c b/src/attach.c index 5763e631ad..cf52bb24b1 100644 --- a/src/attach.c +++ b/src/attach.c @@ -212,7 +212,7 @@ static void attachFunc( u8 newAuth = 0; rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth); if( newAuthauth.authLevel ){ - rc = SQLITE_AUTH; + rc = SQLITE_AUTH_USER; } } #endif diff --git a/test/userauth01.test b/test/userauth01.test index a914ea07ca..a4621dc72f 100644 --- a/test/userauth01.test +++ b/test/userauth01.test @@ -201,5 +201,41 @@ do_test userauth01-1.51 { db eval {SELECT uname, isadmin FROM sqlite_user ORDER BY uname} } {alice 1 cindy 0 david 0} +# When ATTACH-ing new database files to a connection, each newly attached +# database that is an authentication-required database is checked using +# the same username and password as supplied to the main database. If that +# check fails, then the ATTACH command fails with an SQLITE_AUTH error. +# +do_test userauth01-1.60 { + forcedelete test3.db + sqlite3 db3 test3.db + db3 eval { + CREATE TABLE t3(a,b,c); INSERT INTO t3 VALUES(1,2,3); + SELECT * FROM t3; + } +} {1 2 3} +do_test userauth01-1.61 { + sqlite3_user_add db3 alice xyzzy-alice 1 +} {SQLITE_OK} +do_test userauth01-1.62 { + db eval { + ATTACH 'test3.db' AS aux; + SELECT * FROM t1, t3 ORDER BY x LIMIT 1; + DETACH aux; + } +} {{} 1 2 3} +do_test userauth01-1.63 { + sqlite3_user_change db alice pw-4-alice 1 + sqlite3_user_authenticate db alice pw-4-alice + catchsql { + ATTACH 'test3.db' AS aux; + } +} {1 {unable to open database: test3.db}} +do_test userauth01-1.64 { + sqlite3_extended_errcode db +} {SQLITE_AUTH} +do_test userauth01-1.65 { + db eval {PRAGMA database_list} +} {~/test3.db/} finish_test From 7883ecfcd40c8bcf60cb69a36459f6f4f6242824 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Sep 2014 16:19:31 +0000 Subject: [PATCH 320/710] Enhance the sqlite3_user_add() interface to initialize the user authentication logic. Add test cases for the extra argument on the end of the authorizer callback. FossilOrigin-Name: 842c6da8f1a62bd13a1b4089a98b0835a46a2285 --- ext/userauth/userauth.c | 16 ++++++++++++++++ manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/build.c | 15 ++++----------- src/shell.c | 1 + src/sqliteInt.h | 1 + test/userauth01.test | 22 +++++++++++++++++++--- 7 files changed, 52 insertions(+), 25 deletions(-) diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c index 21d33ce80f..d368df8f9d 100644 --- a/ext/userauth/userauth.c +++ b/ext/userauth/userauth.c @@ -118,6 +118,8 @@ int sqlite3UserAuthCheckLogin( ){ int rc; u8 savedAuthLevel; + assert( zDb!=0 ); + assert( peAuth!=0 ); savedAuthLevel = db->auth.authLevel; db->auth.authLevel = UAUTH_Admin; rc = userAuthCheckLogin(db, zDb, peAuth); @@ -125,6 +127,19 @@ int sqlite3UserAuthCheckLogin( return rc; } +/* +** If the current authLevel is UAUTH_Unknown, the take actions to figure +** out what authLevel should be +*/ +void sqlite3UserAuthInit(sqlite3 *db){ + if( db->auth.authLevel==UAUTH_Unknown ){ + u8 authLevel = UAUTH_Fail; + sqlite3UserAuthCheckLogin(db, "main", &authLevel); + db->auth.authLevel = authLevel; + if( authLevelflags &= ~SQLITE_WriteSchema; + } +} + /* ** Implementation of the sqlite_crypt(X,Y) function. ** @@ -223,6 +238,7 @@ int sqlite3_user_add( ){ sqlite3_stmt *pStmt; int rc; + sqlite3UserAuthInit(db); if( db->auth.authLevelnTableLock>0 && db->init.busy==0 ){ + sqlite3UserAuthInit(db); if( db->auth.authLevelauth.authLevel==UAUTH_Unknown ){ - u8 authLevel = UAUTH_Fail; - sqlite3UserAuthCheckLogin(db, "main", &authLevel); - db->auth.authLevel = authLevel; - if( authLevelflags &= ~SQLITE_WriteSchema; - } - if( db->auth.authLevelrc = SQLITE_AUTH_USER; - sqlite3ErrorMsg(pParse, "user not authenticated"); - return; - } + pParse->rc = SQLITE_AUTH_USER; + sqlite3ErrorMsg(pParse, "user not authenticated"); + return; } } #endif diff --git a/src/shell.c b/src/shell.c index b8ab2dbe9f..ec83b13910 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3445,6 +3445,7 @@ static int do_meta_command(char *zLine, ShellState *p){ rc = 1; goto meta_command_exit; } + open_db(p, 0); if( strcmp(azArg[1],"login")==0 ){ if( nArg!=4 ){ fprintf(stderr, "Usage: .user login USER PASSWORD\n"); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a9f2001457..805c925f79 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1010,6 +1010,7 @@ struct sqlite3_userauth { /* Functions used only by user authorization logic */ int sqlite3UserAuthTable(const char*); int sqlite3UserAuthCheckLogin(sqlite3*,const char*,u8*); +void sqlite3UserAuthInit(sqlite3*); void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); #endif /* SQLITE_USER_AUTHENTICATION */ diff --git a/test/userauth01.test b/test/userauth01.test index a4621dc72f..644937b192 100644 --- a/test/userauth01.test +++ b/test/userauth01.test @@ -209,14 +209,14 @@ do_test userauth01-1.51 { do_test userauth01-1.60 { forcedelete test3.db sqlite3 db3 test3.db + sqlite3_user_add db3 alice xyzzy-alice 1 +} {SQLITE_OK} +do_test userauth01-1.61 { db3 eval { CREATE TABLE t3(a,b,c); INSERT INTO t3 VALUES(1,2,3); SELECT * FROM t3; } } {1 2 3} -do_test userauth01-1.61 { - sqlite3_user_add db3 alice xyzzy-alice 1 -} {SQLITE_OK} do_test userauth01-1.62 { db eval { ATTACH 'test3.db' AS aux; @@ -238,4 +238,20 @@ do_test userauth01-1.65 { db eval {PRAGMA database_list} } {~/test3.db/} +# The sqlite3_set_authorizer() callback is modified to take a 7th parameter +# which is the username of the currently logged in user, or NULL for a +# no-authentication-required database. +# +proc auth {args} { + lappend ::authargs $args + return SQLITE_OK +} +do_test authuser01-2.1 { + unset -nocomplain ::authargs + db auth auth + db eval {SELECT x FROM t1} + set ::authargs +} {/SQLITE_SELECT {} {} {} {} alice/} + + finish_test From 3a3a03f29e0072dd0ded8d644307f5b1a31633f5 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Sep 2014 16:36:43 +0000 Subject: [PATCH 321/710] Suppress the potential schema error that occurs when a non-user-auth SQLite library tries to parse the sqlite_user table definition in a user-auth database. FossilOrigin-Name: cda33c1ef35416a155af602c0b4e9d42ccf8633f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/build.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index bbf33a5fb6..3fc0ea1b3f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\ssqlite3_user_add()\sinterface\sto\sinitialize\sthe\suser\nauthentication\slogic.\s\sAdd\stest\scases\sfor\sthe\sextra\sargument\son\sthe\nend\sof\sthe\sauthorizer\scallback. -D 2014-09-11T16:19:31.719 +C Suppress\sthe\spotential\sschema\serror\sthat\soccurs\swhen\sa\snon-user-auth\nSQLite\slibrary\stries\sto\sparse\sthe\ssqlite_user\stable\sdefinition\sin\sa\nuser-auth\sdatabase. +D 2014-09-11T16:36:43.549 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,7 +174,7 @@ F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c b1c1cd1cc3ae2e433a23b9a6c9ab53805707d8cd F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc -F src/build.c 555826ae03c3bc589a7b09b279c2e5ba989a4178 +F src/build.c 047d7e1d2d89fa55134fa1d6b669c9c2983c0d11 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958 @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 96ea5c0b3cd1dec81d490f2f958ebd2e47a24921 -R 279311d0c4a57cb81a913da29cc5dd35 +P 842c6da8f1a62bd13a1b4089a98b0835a46a2285 +R 7de809a255ed1cf8a77cea399ddd7268 U drh -Z cf3239448d4f92059057a29736710c2c +Z 0b3d163c7ef70d6122b5132c30860c28 diff --git a/manifest.uuid b/manifest.uuid index 78c06de5d3..8aae504135 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -842c6da8f1a62bd13a1b4089a98b0835a46a2285 \ No newline at end of file +cda33c1ef35416a155af602c0b4e9d42ccf8633f \ No newline at end of file diff --git a/src/build.c b/src/build.c index 84ffb0a38c..791f6f2033 100644 --- a/src/build.c +++ b/src/build.c @@ -2901,6 +2901,7 @@ Index *sqlite3CreateIndex( assert( pTab!=0 ); assert( pParse->nErr==0 ); if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 + && db->init.busy==0 #if SQLITE_USER_AUTHENTICATION && sqlite3UserAuthTable(pTab->zName)==0 #endif From c891c6c7adf9be3296fba6fef24012893a9bb8c2 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Sep 2014 17:14:54 +0000 Subject: [PATCH 322/710] Clean up some #includes in the extension API implementation. FossilOrigin-Name: b149ef5c639e6bcff7bd1c7866353e7f7f468070 --- ext/userauth/userauth.c | 5 +++-- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/ext/userauth/userauth.c b/ext/userauth/userauth.c index d368df8f9d..6ce99053d3 100644 --- a/ext/userauth/userauth.c +++ b/ext/userauth/userauth.c @@ -22,8 +22,9 @@ ** directory as this file for additional information. */ #ifdef SQLITE_USER_AUTHENTICATION -#include "sqliteInt.h" -#include "sqlite3userauth.h" +#ifndef _SQLITEINT_H_ +# include "sqliteInt.h" +#endif /* ** Prepare an SQL statement for use by the user authentication logic. diff --git a/manifest b/manifest index 3fc0ea1b3f..10ecd71450 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Suppress\sthe\spotential\sschema\serror\sthat\soccurs\swhen\sa\snon-user-auth\nSQLite\slibrary\stries\sto\sparse\sthe\ssqlite_user\stable\sdefinition\sin\sa\nuser-auth\sdatabase. -D 2014-09-11T16:36:43.549 +C Clean\sup\ssome\s#includes\sin\sthe\sextension\sAPI\simplementation. +D 2014-09-11T17:14:54.772 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -146,7 +146,7 @@ F ext/rtree/tkt3363.test 142ab96eded44a3615ec79fba98c7bde7d0f96de F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024 F ext/userauth/sqlite3userauth.h 19cb6f0e31316d0ee4afdfb7a85ef9da3333a220 F ext/userauth/user-auth.txt e6641021a9210364665fe625d067617d03f27b04 -F ext/userauth/userauth.c 02a52c3c345a8dede3a1018c08840b74230acc51 +F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 842c6da8f1a62bd13a1b4089a98b0835a46a2285 -R 7de809a255ed1cf8a77cea399ddd7268 +P cda33c1ef35416a155af602c0b4e9d42ccf8633f +R 6795c3f7d8a461e29887bf50932299a0 U drh -Z 0b3d163c7ef70d6122b5132c30860c28 +Z 25539858b9c10651bb38a3461b6abb42 diff --git a/manifest.uuid b/manifest.uuid index 8aae504135..9d753fddf9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cda33c1ef35416a155af602c0b4e9d42ccf8633f \ No newline at end of file +b149ef5c639e6bcff7bd1c7866353e7f7f468070 \ No newline at end of file From fc59a954cbce32be71da37d7ca1f447396ba39e0 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Sep 2014 23:34:55 +0000 Subject: [PATCH 323/710] Fix an issue with sqlite3_bind_text64() with the SQLITE_UTF16 encoding parameter. Remove some unreachable code from the text64() and blob64() implementation. FossilOrigin-Name: 34292b084ef48cd6e9ca5704f6b072a29733b4c2 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/vdbeapi.c | 8 ++++++-- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 24f2ea483a..c83ebe16ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sAPIs\sthat\stake\s64-bit\slength\sparameters:\ssqlite3_malloc64(),\nsqlite3_realloc64(),\ssqlite3_bind_blob64(),\ssqlite3_bind_text64(),\nsqlite3_result_blob64(),\sand\ssqlite3_result_text64().\nAdd\sthe\ssqlite3_msize()\sinterface.\nInternal\smemory\sallocation\sroutines\snow\suse\s64-bit\sunsigned\nlength\sparameters\sfor\ssafety.\nFix\sthe\ssqlite3_get_table()\sto\suse\ssqlite3_realloc64()\sto\savoid\na\sinteger\soverflow\sproblem. -D 2014-09-11T18:44:04.913 +C Fix\san\sissue\swith\ssqlite3_bind_text64()\swith\sthe\sSQLITE_UTF16\sencoding\nparameter.\s\sRemove\ssome\sunreachable\scode\sfrom\sthe\stext64()\sand\sblob64()\nimplementation. +D 2014-09-11T23:34:55.607 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 9a45dcbcc967fc0cb9248c75ba245d1d47de3e78 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h b4843c35c3ba533b69d4250f550b5bacf2fb013d -F src/vdbeapi.c cec65a12dc2fe9072d5108d9f75df57b0324883a +F src/vdbeapi.c 06b712d4772b318b69cd37a416deb1ff0426aa8c F src/vdbeaux.c 91fd1e0c54a765838dc61fcf79f31acce035ce38 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c dc36ea9fe26c25550c50085f388167086ef7d73a @@ -1197,8 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 65884d4f81a4705b0356b6cb8ec4909945ff5c19 6ab76c5fedfe568b0f0f34a600f9135bf6558148 -R a34346a4fba04f6707f06567bf21377a -T +closed 6ab76c5fedfe568b0f0f34a600f9135bf6558148 +P 7e4978c003867d1b532b69221013dda75ca61953 +R 5b504a805873df7c692d762cd653895b U drh -Z ecfd648f0ae419a16a536c86444de6f4 +Z 9235587180e257c32aca7830c18cbf2e diff --git a/manifest.uuid b/manifest.uuid index 0feb233a2e..1b709cc082 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7e4978c003867d1b532b69221013dda75ca61953 \ No newline at end of file +34292b084ef48cd6e9ca5704f6b072a29733b4c2 \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 4dccb30c15..b64f33c8c6 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -235,12 +235,11 @@ static int invokeValueDestructor( void (*xDel)(void*), /* The destructor */ sqlite3_context *pCtx /* Set a SQLITE_TOOBIG error if no NULL */ ){ + assert( xDel!=SQLITE_DYNAMIC ); if( xDel==0 ){ /* noop */ }else if( xDel==SQLITE_TRANSIENT ){ /* noop */ - }else if( xDel==SQLITE_DYNAMIC ){ - sqlite3_free((void*)p); }else{ xDel((void*)p); } @@ -264,6 +263,7 @@ void sqlite3_result_blob64( void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + assert( xDel!=SQLITE_DYNAMIC ); if( n>0x7fffffff ){ (void)invokeValueDestructor(z, xDel, pCtx); }else{ @@ -317,6 +317,7 @@ void sqlite3_result_text64( unsigned char enc ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + assert( xDel!=SQLITE_DYNAMIC ); if( n>0x7fffffff ){ (void)invokeValueDestructor(z, xDel, pCtx); }else{ @@ -1179,6 +1180,7 @@ int sqlite3_bind_blob64( sqlite3_uint64 nData, void (*xDel)(void*) ){ + assert( xDel!=SQLITE_DYNAMIC ); if( nData>0x7fffffff ){ return invokeValueDestructor(zData, xDel, 0); }else{ @@ -1234,9 +1236,11 @@ int sqlite3_bind_text64( void (*xDel)(void*), unsigned char enc ){ + assert( xDel!=SQLITE_DYNAMIC ); if( nData>0x7fffffff ){ return invokeValueDestructor(zData, xDel, 0); }else{ + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; return bindText(pStmt, i, zData, nData, xDel, enc); } } From fb046e7653a1dee6e34b8b3b2ab71ebf0582de2c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 12 Sep 2014 04:28:33 +0000 Subject: [PATCH 324/710] Fix a problem with parser memory allocation on 32-bit systems. FossilOrigin-Name: 2f69a1fa6adc9377149ae7faa586a5d30b6a631b --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/lempar.c | 4 ++-- src/sqliteInt.h | 2 +- src/tokenize.c | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index c83ebe16ff..4c19df8ec5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sissue\swith\ssqlite3_bind_text64()\swith\sthe\sSQLITE_UTF16\sencoding\nparameter.\s\sRemove\ssome\sunreachable\scode\sfrom\sthe\stext64()\sand\sblob64()\nimplementation. -D 2014-09-11T23:34:55.607 +C Fix\sa\sproblem\swith\sparser\smemory\sallocation\son\s32-bit\ssystems. +D 2014-09-12T04:28:33.110 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -191,7 +191,7 @@ F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 F src/insert.c 0b073fade178d9dbd990bbb32b4438e50b884a06 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e -F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b +F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 F src/main.c d15621461fb0c52675eba2b650492ed1beef69ab F src/malloc.c cc015821ba267ad5c91dc8761d0498a3fc3ce6ce @@ -231,7 +231,7 @@ F src/shell.c c00220cdd7f2027780bc25b78376c16dc24e4b7d F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 0f6bd5c2bc864b1261898a73673bed1030cfbfe2 +F src/sqliteInt.h e9030816d5ee6fe2063553b40359096f994664ee F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 218ae2ba022881846741dfc8351aefdf129e0377 @@ -282,7 +282,7 @@ F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 22dded4283dc4b25422f6444cdcb8d6b1ea0b5ff -F src/tokenize.c 722872c816887fd66931333c59570ebd9622a95f +F src/tokenize.c 3df63041994f55afeb168b463ec836e8f1c50e7c F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7e4978c003867d1b532b69221013dda75ca61953 -R 5b504a805873df7c692d762cd653895b +P 34292b084ef48cd6e9ca5704f6b072a29733b4c2 +R 9210ace0d122525851cce20a27ee9393 U drh -Z 9235587180e257c32aca7830c18cbf2e +Z 8a9f5697995e1302a3aaac24500097fb diff --git a/manifest.uuid b/manifest.uuid index 1b709cc082..2828da8f81 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -34292b084ef48cd6e9ca5704f6b072a29733b4c2 \ No newline at end of file +2f69a1fa6adc9377149ae7faa586a5d30b6a631b \ No newline at end of file diff --git a/src/lempar.c b/src/lempar.c index 2afaa6cea6..ba0837c0ab 100644 --- a/src/lempar.c +++ b/src/lempar.c @@ -271,9 +271,9 @@ static void yyGrowStack(yyParser *p){ ** A pointer to a parser. This pointer is used in subsequent calls ** to Parse and ParseFree. */ -void *ParseAlloc(void *(*mallocProc)(size_t)){ +void *ParseAlloc(void *(*mallocProc)(u64)){ yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (size_t)sizeof(yyParser) ); + pParser = (yyParser*)(*mallocProc)( (u64)sizeof(yyParser) ); if( pParser ){ pParser->yyidx = -1; #ifdef YYTRACKMAXSTACKDEPTH diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e1faccb211..a254796abd 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3520,7 +3520,7 @@ int sqlite3Stat4Column(sqlite3*, const void*, int, int, sqlite3_value**); /* ** The interface to the LEMON-generated parser */ -void *sqlite3ParserAlloc(void*(*)(size_t)); +void *sqlite3ParserAlloc(void*(*)(u64)); void sqlite3ParserFree(void*, void(*)(void*)); void sqlite3Parser(void*, int, Token, Parse*); #ifdef YYTRACKMAXSTACKDEPTH diff --git a/src/tokenize.c b/src/tokenize.c index 3f1de221aa..8a7894514c 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -398,7 +398,7 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ pParse->zTail = zSql; i = 0; assert( pzErrMsg!=0 ); - pEngine = sqlite3ParserAlloc((void*(*)(size_t))sqlite3Malloc); + pEngine = sqlite3ParserAlloc(sqlite3Malloc); if( pEngine==0 ){ db->mallocFailed = 1; return SQLITE_NOMEM; From 236241aeb030ad3b23cea8a1f4d133e02f7dc60a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 12 Sep 2014 17:41:30 +0000 Subject: [PATCH 325/710] Simplify the way the column cache is managed around OP_Move instructions. FossilOrigin-Name: 320556233e19cdd9d590a09655c3465754700d39 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 9 +-------- src/select.c | 4 ++-- 4 files changed, 11 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 4c19df8ec5..8501d7bcd0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\swith\sparser\smemory\sallocation\son\s32-bit\ssystems. -D 2014-09-12T04:28:33.110 +C Simplify\sthe\sway\sthe\scolumn\scache\sis\smanaged\saround\sOP_Move\sinstructions. +D 2014-09-12T17:41:30.440 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f -F src/expr.c 441a7e24e2f7bea9475778fa8acce9e8a69ca8f0 +F src/expr.c 19392d98e089640c3336e65b4254cc337efef7d1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c d4d218704b13bc1814b7d76874e405743c903773 @@ -226,7 +226,7 @@ F src/printf.c e74925089a85e3c9f0e315595f41c139d3d118c2 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0d1621e45fffe4b4396477cf46e41a84b0145ffb F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c b4457526cee73c0b69fad42f799f619b1d5a8a8a +F src/select.c bd8ed75592642f79871fd331d869c37fc6a14846 F src/shell.c c00220cdd7f2027780bc25b78376c16dc24e4b7d F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 34292b084ef48cd6e9ca5704f6b072a29733b4c2 -R 9210ace0d122525851cce20a27ee9393 +P 2f69a1fa6adc9377149ae7faa586a5d30b6a631b +R 2196198b201612a018f7c15a4c2f1424 U drh -Z 8a9f5697995e1302a3aaac24500097fb +Z a609740927532bd8f27cb3bc904eb4ff diff --git a/manifest.uuid b/manifest.uuid index 2828da8f81..2b28a6b8f4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f69a1fa6adc9377149ae7faa586a5d30b6a631b \ No newline at end of file +320556233e19cdd9d590a09655c3465754700d39 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index cd0983c579..c6d8b9e5f3 100644 --- a/src/expr.c +++ b/src/expr.c @@ -2432,16 +2432,9 @@ void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, int iCount){ ** over to iTo..iTo+nReg-1. Keep the column cache up-to-date. */ void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ - int i; - struct yColCache *p; assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); - for(i=0, p=pParse->aColCache; iiReg; - if( x>=iFrom && xiReg += iTo-iFrom; - } - } + sqlite3ExprCacheRemove(pParse, iFrom, nReg); } #if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) diff --git a/src/select.c b/src/select.c index 31a70bff1b..7ecbf30697 100644 --- a/src/select.c +++ b/src/select.c @@ -488,7 +488,7 @@ static void pushOntoSorter( sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } if( nPrefixReg==0 ){ - sqlite3VdbeAddOp3(v, OP_Move, regData, regBase+nExpr+bSeq, nData); + sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); } sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); @@ -524,7 +524,7 @@ static void pushOntoSorter( sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); sqlite3VdbeJumpHere(v, addrFirst); - sqlite3VdbeAddOp3(v, OP_Move, regBase, regPrevKey, pSort->nOBSat); + sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); sqlite3VdbeJumpHere(v, addrJmp); } if( pSort->sortFlags & SORTFLAG_UseSorter ){ From 36ce91913c7923e06f1b6cdb8385812fd0fe9998 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 12 Sep 2014 20:30:59 +0000 Subject: [PATCH 326/710] Small performance improvement to the dirty list handling in the pager. FossilOrigin-Name: b332a84d5154f70f3197537df4af243eaebbb011 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pcache.c | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 8501d7bcd0..81daedebde 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\sway\sthe\scolumn\scache\sis\smanaged\saround\sOP_Move\sinstructions. -D 2014-09-12T17:41:30.440 +C Small\sperformance\simprovement\sto\sthe\sdirty\slist\shandling\sin\sthe\spager. +D 2014-09-12T20:30:59.762 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c c6c809987f0c6a4e27634099d062d425527de173 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 -F src/pcache.c 2048affdb09a04478b5fc6e64cb1083078d369be +F src/pcache.c b42c513da255c33e99dc0e23d16c3caf30dc9175 F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2f69a1fa6adc9377149ae7faa586a5d30b6a631b -R 2196198b201612a018f7c15a4c2f1424 +P 320556233e19cdd9d590a09655c3465754700d39 +R 576d19acba6677524711080ef6f39102 U drh -Z a609740927532bd8f27cb3bc904eb4ff +Z 60cbf2357ae42b4a38f44133d16be246 diff --git a/manifest.uuid b/manifest.uuid index 2b28a6b8f4..7821ed52ba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -320556233e19cdd9d590a09655c3465754700d39 \ No newline at end of file +b332a84d5154f70f3197537df4af243eaebbb011 \ No newline at end of file diff --git a/src/pcache.c b/src/pcache.c index 801df2b02a..e975e43889 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -116,14 +116,14 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ if( pPage->pDirtyNext ){ assert( pPage->pDirtyNext->pDirtyPrev==0 ); pPage->pDirtyNext->pDirtyPrev = pPage; - }else if( p->bPurgeable ){ - assert( p->eCreate==2 ); - p->eCreate = 1; + }else{ + p->pDirtyTail = pPage; + if( p->bPurgeable ){ + assert( p->eCreate==2 ); + p->eCreate = 1; + } } p->pDirty = pPage; - if( !p->pDirtyTail ){ - p->pDirtyTail = pPage; - } if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ p->pSynced = pPage; } @@ -399,7 +399,7 @@ void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ p->pCache->nRef--; if( (p->flags&PGHDR_DIRTY)==0 ){ pcacheUnpin(p); - }else{ + }else if( p->pDirtyPrev!=0 ){ /* Move the page to the head of the dirty list. */ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); } From 6092d2bcfebf699c970146308f910cb4f9423ace Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 15 Sep 2014 11:14:50 +0000 Subject: [PATCH 327/710] Adjust comments to show that subquery flattening restriction (10) was removed from the code back in 2005. This is a comment change only. FossilOrigin-Name: 4ff0eb96bc364baed2d8005c69291ca9240b99dd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 81daedebde..160c3f8d2f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\simprovement\sto\sthe\sdirty\slist\shandling\sin\sthe\spager. -D 2014-09-12T20:30:59.762 +C Adjust\scomments\sto\sshow\sthat\ssubquery\sflattening\srestriction\s(10)\swas\nremoved\sfrom\sthe\scode\sback\sin\s2005.\s\sThis\sis\sa\scomment\schange\sonly. +D 2014-09-15T11:14:50.871 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -226,7 +226,7 @@ F src/printf.c e74925089a85e3c9f0e315595f41c139d3d118c2 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 0d1621e45fffe4b4396477cf46e41a84b0145ffb F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c bd8ed75592642f79871fd331d869c37fc6a14846 +F src/select.c 70312b18bf56f741556ed494c398afa0ca3c0e1a F src/shell.c c00220cdd7f2027780bc25b78376c16dc24e4b7d F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 320556233e19cdd9d590a09655c3465754700d39 -R 576d19acba6677524711080ef6f39102 +P b332a84d5154f70f3197537df4af243eaebbb011 +R 4fde0e674c64eada90bf3beabe45d5b4 U drh -Z 60cbf2357ae42b4a38f44133d16be246 +Z 7e796116236157f7f97effb3ae76fb8a diff --git a/manifest.uuid b/manifest.uuid index 7821ed52ba..30c79b290d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b332a84d5154f70f3197537df4af243eaebbb011 \ No newline at end of file +4ff0eb96bc364baed2d8005c69291ca9240b99dd \ No newline at end of file diff --git a/src/select.c b/src/select.c index 7ecbf30697..fe6052d6e8 100644 --- a/src/select.c +++ b/src/select.c @@ -3131,8 +3131,10 @@ static void substSelect( ** (9) The subquery does not use LIMIT or the outer query does not use ** aggregates. ** -** (10) The subquery does not use aggregates or the outer query does not -** use LIMIT. +** (**) Restriction (10) was removed from the code on 2005-02-05 but we +** accidently carried the comment forward until 2014-09-15. Original +** text: "The subquery does not use aggregates or the outer query does not +** use LIMIT." ** ** (11) The subquery and the outer query do not both have ORDER BY clauses. ** From 9588ad95c1a9cc7cd5add559b43536d48a4f2e4e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 15 Sep 2014 14:46:02 +0000 Subject: [PATCH 328/710] Do not flatten aggregate subqueries that contain min() or max() functions so that if the min()/max() are discarded by the outer query, they still function and cause non-aggregate expression to be evaluated on the minimal or maximal row. FossilOrigin-Name: 0bdf1a086b3946722f4d4b328e25917f61c14713 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/func.c | 10 ++++++---- src/resolve.c | 24 ++++++++++++++---------- src/select.c | 15 +++++++++++++-- src/sqliteInt.h | 25 +++++++++++++++++-------- test/aggnested.test | 8 +++++++- test/minmax4.test | 9 ++++++--- 8 files changed, 75 insertions(+), 40 deletions(-) diff --git a/manifest b/manifest index 160c3f8d2f..f13c9ab026 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\scomments\sto\sshow\sthat\ssubquery\sflattening\srestriction\s(10)\swas\nremoved\sfrom\sthe\scode\sback\sin\s2005.\s\sThis\sis\sa\scomment\schange\sonly. -D 2014-09-15T11:14:50.871 +C Do\snot\sflatten\saggregate\ssubqueries\sthat\scontain\smin()\sor\smax()\sfunctions\nso\sthat\sif\sthe\smin()/max()\sare\sdiscarded\sby\sthe\souter\squery,\sthey\sstill\nfunction\sand\scause\snon-aggregate\sexpression\sto\sbe\sevaluated\son\sthe\sminimal\nor\smaximal\srow. +D 2014-09-15T14:46:02.082 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -183,7 +183,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c 19392d98e089640c3336e65b4254cc337efef7d1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c d4d218704b13bc1814b7d76874e405743c903773 +F src/func.c 5d498933f6168dff80941c873805fe04dc2b766d F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -224,14 +224,14 @@ F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c e74925089a85e3c9f0e315595f41c139d3d118c2 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c 0d1621e45fffe4b4396477cf46e41a84b0145ffb +F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 70312b18bf56f741556ed494c398afa0ca3c0e1a +F src/select.c 0cd6706fd52ae5db229e9041094db6ec27195335 F src/shell.c c00220cdd7f2027780bc25b78376c16dc24e4b7d F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h e9030816d5ee6fe2063553b40359096f994664ee +F src/sqliteInt.h 0803e900eb1882f7dd88e86ddcddd2d1b27c8d86 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 218ae2ba022881846741dfc8351aefdf129e0377 @@ -305,7 +305,7 @@ F src/where.c 839b5e1db2507e221ad1c308f148a8519ed750be F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 -F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 +F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 6ff7b43c2b4b905c74dc4a813d201d0fa64c5783 F test/alter.test 547dc2d292644301ac9a7dda22b319b74f9c08d2 @@ -712,7 +712,7 @@ F test/memsubsys2.test 3a1c1a9de48e5726faa85108b02459fae8cb9ee9 F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 -F test/minmax4.test 536a3360470633a177e42fbc19660d146b51daef +F test/minmax4.test 936941484ebdceb8adec7c86b6cd9b6e5e897c1f F test/misc1.test 1201a037c24f982cc0e956cdaa34fcaf6439c417 F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b332a84d5154f70f3197537df4af243eaebbb011 -R 4fde0e674c64eada90bf3beabe45d5b4 +P 4ff0eb96bc364baed2d8005c69291ca9240b99dd +R d83c0c27bf862a8b9f30cb2ccec5fde3 U drh -Z 7e796116236157f7f97effb3ae76fb8a +Z 5049f8c8dea35bd8f91ea699f6fd9960 diff --git a/manifest.uuid b/manifest.uuid index 30c79b290d..69d001e682 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4ff0eb96bc364baed2d8005c69291ca9240b99dd \ No newline at end of file +0bdf1a086b3946722f4d4b328e25917f61c14713 \ No newline at end of file diff --git a/src/func.c b/src/func.c index f7e50f3374..0a8a9dda38 100644 --- a/src/func.c +++ b/src/func.c @@ -1663,10 +1663,12 @@ void sqlite3RegisterGlobalFunctions(void){ FUNCTION(trim, 2, 3, 0, trimFunc ), FUNCTION(min, -1, 0, 1, minmaxFunc ), FUNCTION(min, 0, 0, 1, 0 ), - AGGREGATE(min, 1, 0, 1, minmaxStep, minMaxFinalize ), + AGGREGATE2(min, 1, 0, 1, minmaxStep, minMaxFinalize, + SQLITE_FUNC_MINMAX ), FUNCTION(max, -1, 1, 1, minmaxFunc ), FUNCTION(max, 0, 1, 1, 0 ), - AGGREGATE(max, 1, 1, 1, minmaxStep, minMaxFinalize ), + AGGREGATE2(max, 1, 1, 1, minmaxStep, minMaxFinalize, + SQLITE_FUNC_MINMAX ), FUNCTION2(typeof, 1, 0, 0, typeofFunc, SQLITE_FUNC_TYPEOF), FUNCTION2(length, 1, 0, 0, lengthFunc, SQLITE_FUNC_LENGTH), FUNCTION(instr, 2, 0, 0, instrFunc ), @@ -1719,8 +1721,8 @@ void sqlite3RegisterGlobalFunctions(void){ AGGREGATE(sum, 1, 0, 0, sumStep, sumFinalize ), AGGREGATE(total, 1, 0, 0, sumStep, totalFinalize ), AGGREGATE(avg, 1, 0, 0, sumStep, avgFinalize ), - /* AGGREGATE(count, 0, 0, 0, countStep, countFinalize ), */ - {0,SQLITE_UTF8|SQLITE_FUNC_COUNT,0,0,0,countStep,countFinalize,"count",0,0}, + AGGREGATE2(count, 0, 0, 0, countStep, countFinalize, + SQLITE_FUNC_COUNT ), AGGREGATE(count, 1, 0, 0, countStep, countFinalize ), AGGREGATE(group_concat, 1, 0, 0, groupConcatStep, groupConcatFinalize), AGGREGATE(group_concat, 2, 0, 0, groupConcatStep, groupConcatFinalize), diff --git a/src/resolve.c b/src/resolve.c index a77fd9d367..d6a865caef 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -719,9 +719,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pExpr->iTable = pDef->zName[0]=='u' ? 62 : 938; } } - } #ifndef SQLITE_OMIT_AUTHORIZATION - if( pDef ){ auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0); if( auth!=SQLITE_OK ){ if( auth==SQLITE_DENY ){ @@ -732,9 +730,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pExpr->op = TK_NULL; return WRC_Prune; } +#endif if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ) ExprSetProperty(pExpr,EP_Constant); } -#endif if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); pNC->nErr++; @@ -757,7 +755,13 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pExpr->op2++; pNC2 = pNC2->pNext; } - if( pNC2 ) pNC2->ncFlags |= NC_HasAgg; + assert( pDef!=0 ); + if( pNC2 ){ + assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); + testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); + pNC2->ncFlags |= NC_HasAgg | (pDef->funcFlags & SQLITE_FUNC_MINMAX); + + } pNC->ncFlags |= NC_AllowAgg; } /* FIX ME: Compute pExpr->affinity based on the expected return @@ -1222,7 +1226,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ assert( (p->selFlags & SF_Aggregate)==0 ); pGroupBy = p->pGroupBy; if( pGroupBy || (sNC.ncFlags & NC_HasAgg)!=0 ){ - p->selFlags |= SF_Aggregate; + assert( NC_MinMaxAgg==SF_MinMaxAgg ); + p->selFlags |= SF_Aggregate | (sNC.ncFlags&NC_MinMaxAgg); }else{ sNC.ncFlags &= ~NC_AllowAgg; } @@ -1350,7 +1355,7 @@ int sqlite3ResolveExprNames( NameContext *pNC, /* Namespace to resolve expressions in. */ Expr *pExpr /* The expression to be analyzed. */ ){ - u8 savedHasAgg; + u16 savedHasAgg; Walker w; if( pExpr==0 ) return 0; @@ -1363,8 +1368,8 @@ int sqlite3ResolveExprNames( pParse->nHeight += pExpr->nHeight; } #endif - savedHasAgg = pNC->ncFlags & NC_HasAgg; - pNC->ncFlags &= ~NC_HasAgg; + savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg); memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; @@ -1379,9 +1384,8 @@ int sqlite3ResolveExprNames( } if( pNC->ncFlags & NC_HasAgg ){ ExprSetProperty(pExpr, EP_Agg); - }else if( savedHasAgg ){ - pNC->ncFlags |= NC_HasAgg; } + pNC->ncFlags |= savedHasAgg; return ExprHasProperty(pExpr, EP_Error); } diff --git a/src/select.c b/src/select.c index fe6052d6e8..d3ffaf451a 100644 --- a/src/select.c +++ b/src/select.c @@ -3197,6 +3197,11 @@ static void substSelect( ** parent to a compound query confuses the code that handles ** recursive queries in multiSelect(). ** +** (24) The subquery is not an aggregate that uses the built-in min() or +** or max() functions. (Without this restriction, a query like: +** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily +** return the value X for which Y was maximal.) +** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query @@ -3269,8 +3274,14 @@ static int flattenSubquery( if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ return 0; /* Restriction (21) */ } - if( pSub->selFlags & SF_Recursive ) return 0; /* Restriction (22) */ - if( (p->selFlags & SF_Recursive) && pSub->pPrior ) return 0; /* (23) */ + testcase( pSub->selFlags & SF_Recursive ); + testcase( pSub->selFlags & SF_MinMaxAgg ); + if( pSub->selFlags & (SF_Recursive|SF_MinMaxAgg) ){ + return 0; /* Restrictions (22) and (24) */ + } + if( (p->selFlags & SF_Recursive) && pSub->pPrior ){ + return 0; /* Restriction (23) */ + } /* OBSOLETE COMMENT 1: ** Restriction 3: If the subquery is a join, make sure the subquery is diff --git a/src/sqliteInt.h b/src/sqliteInt.h index a254796abd..fd3731d817 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1285,6 +1285,7 @@ struct FuncDestructor { #define SQLITE_FUNC_COALESCE 0x200 /* Built-in coalesce() or ifnull() */ #define SQLITE_FUNC_UNLIKELY 0x400 /* Built-in unlikely() function */ #define SQLITE_FUNC_CONSTANT 0x800 /* Constant inputs give a constant output */ +#define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -1332,6 +1333,9 @@ struct FuncDestructor { #define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \ SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} +#define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \ + {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ + SQLITE_INT_TO_PTR(arg), 0, 0, xStep,xFinal,#zName,0,0} /* ** All current savepoints are stored in a linked list starting at @@ -2254,17 +2258,22 @@ struct NameContext { NameContext *pNext; /* Next outer name context. NULL for outermost */ int nRef; /* Number of names resolved by this context */ int nErr; /* Number of errors encountered while resolving names */ - u8 ncFlags; /* Zero or more NC_* flags defined below */ + u16 ncFlags; /* Zero or more NC_* flags defined below */ }; /* ** Allowed values for the NameContext, ncFlags field. +** +** Note: NC_MinMaxAgg must have the same value as SF_MinMaxAgg and +** SQLITE_FUNC_MINMAX. +** */ -#define NC_AllowAgg 0x01 /* Aggregate functions are allowed here */ -#define NC_HasAgg 0x02 /* One or more aggregate functions seen */ -#define NC_IsCheck 0x04 /* True if resolving names in a CHECK constraint */ -#define NC_InAggFunc 0x08 /* True if analyzing arguments to an agg func */ -#define NC_PartIdx 0x10 /* True if resolving a partial index WHERE */ +#define NC_AllowAgg 0x0001 /* Aggregate functions are allowed here */ +#define NC_HasAgg 0x0002 /* One or more aggregate functions seen */ +#define NC_IsCheck 0x0004 /* True if resolving names in a CHECK constraint */ +#define NC_InAggFunc 0x0008 /* True if analyzing arguments to an agg func */ +#define NC_PartIdx 0x0010 /* True if resolving a partial index WHERE */ +#define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ /* ** An instance of the following structure contains all information @@ -2315,13 +2324,13 @@ struct Select { #define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ #define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ #define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ - /* 0x0040 NOT USED */ +#define SF_Compound 0x0040 /* Part of a compound query */ #define SF_Values 0x0080 /* Synthesized from VALUES clause */ /* 0x0100 NOT USED */ #define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ #define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ #define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ -#define SF_Compound 0x1000 /* Part of a compound query */ +#define SF_MinMaxAgg 0x1000 /* Aggregate containing min() or max() */ /* diff --git a/test/aggnested.test b/test/aggnested.test index 6e2fd6554b..a87c751eda 100644 --- a/test/aggnested.test +++ b/test/aggnested.test @@ -156,8 +156,14 @@ do_test aggnested-3.2 { (SELECT value1 as xyz, max(x1) AS pqr FROM t1 GROUP BY id1); + SELECT + (SELECT sum(value2<>xyz) FROM t2) + FROM + (SELECT value1 as xyz, max(x1) AS pqr + FROM t1 + GROUP BY id1); } -} {0} +} {1 0} do_test aggnested-3.3 { db eval { DROP TABLE IF EXISTS t1; diff --git a/test/minmax4.test b/test/minmax4.test index 0d8305b5ff..8063538bfd 100644 --- a/test/minmax4.test +++ b/test/minmax4.test @@ -56,14 +56,16 @@ do_test minmax4-1.5 { do_test minmax4-1.6 { db eval { SELECT p, min(q) FROM t1; + SELECT p FROM (SELECT p, min(q) FROM t1); } -} {1 2} +} {1 2 1} do_test minmax4-1.7 { db eval { INSERT INTO t1 VALUES(5,0); SELECT p, max(q) FROM t1; + SELECT p FROM (SELECT max(q), p FROM t1); } -} {3 4} +} {3 4 3} do_test minmax4-1.8 { db eval { SELECT p, min(q) FROM t1; @@ -73,8 +75,9 @@ do_test minmax4-1.9 { db eval { INSERT INTO t1 VALUES(6,1); SELECT p, max(q) FROM t1; + SELECT p FROM (SELECT max(q), p FROM t1); } -} {3 4} +} {3 4 3} do_test minmax4-1.10 { db eval { SELECT p, min(q) FROM t1; From 907214c8e83a3d1f35b6cce5768016089193b3c2 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 15 Sep 2014 14:59:12 +0000 Subject: [PATCH 329/710] Remove the EXPENSIVE_ASSERTS in pcache.c having to do with the pSynced field of the Pcache object, as they are incorrect, as revealed by recent pcache enhancements. FossilOrigin-Name: 69a64560777f85b47349b4b2aab01dc99298592e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pcache.c | 20 -------------------- 3 files changed, 7 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index f13c9ab026..7e35806fec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sflatten\saggregate\ssubqueries\sthat\scontain\smin()\sor\smax()\sfunctions\nso\sthat\sif\sthe\smin()/max()\sare\sdiscarded\sby\sthe\souter\squery,\sthey\sstill\nfunction\sand\scause\snon-aggregate\sexpression\sto\sbe\sevaluated\son\sthe\sminimal\nor\smaximal\srow. -D 2014-09-15T14:46:02.082 +C Remove\sthe\sEXPENSIVE_ASSERTS\sin\spcache.c\shaving\sto\sdo\swith\sthe\spSynced\sfield\nof\sthe\sPcache\sobject,\sas\sthey\sare\sincorrect,\sas\srevealed\sby\srecent\spcache\nenhancements. +D 2014-09-15T14:59:12.274 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c c6c809987f0c6a4e27634099d062d425527de173 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 -F src/pcache.c b42c513da255c33e99dc0e23d16c3caf30dc9175 +F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f @@ -1197,7 +1197,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4ff0eb96bc364baed2d8005c69291ca9240b99dd -R d83c0c27bf862a8b9f30cb2ccec5fde3 +P 0bdf1a086b3946722f4d4b328e25917f61c14713 +R 6604c3404d8f943a6eb10e6ee4c1398f U drh -Z 5049f8c8dea35bd8f91ea699f6fd9960 +Z 9f457bfb74de1e07b081e001cb58874a diff --git a/manifest.uuid b/manifest.uuid index 69d001e682..0cb3407f29 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0bdf1a086b3946722f4d4b328e25917f61c14713 \ No newline at end of file +69a64560777f85b47349b4b2aab01dc99298592e \ No newline at end of file diff --git a/src/pcache.c b/src/pcache.c index e975e43889..191a9d00f4 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -45,23 +45,6 @@ struct PCache { /********************************** Linked List Management ********************/ -#if !defined(NDEBUG) && defined(SQLITE_ENABLE_EXPENSIVE_ASSERT) -/* -** Check that the pCache->pSynced variable is set correctly. If it -** is not, either fail an assert or return zero. Otherwise, return -** non-zero. This is only used in debugging builds, as follows: -** -** expensive_assert( pcacheCheckSynced(pCache) ); -*/ -static int pcacheCheckSynced(PCache *pCache){ - PgHdr *p; - for(p=pCache->pDirtyTail; p!=pCache->pSynced; p=p->pDirtyPrev){ - assert( p->nRef || (p->flags&PGHDR_NEED_SYNC) ); - } - return (p==0 || p->nRef || (p->flags&PGHDR_NEED_SYNC)==0); -} -#endif /* !NDEBUG && SQLITE_ENABLE_EXPENSIVE_ASSERT */ - /* Allowed values for second argument to pcacheManageDirtyList() */ #define PCACHE_DIRTYLIST_REMOVE 1 /* Remove pPage from dirty list */ #define PCACHE_DIRTYLIST_ADD 2 /* Add pPage to the dirty list */ @@ -107,7 +90,6 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ } pPage->pDirtyNext = 0; pPage->pDirtyPrev = 0; - expensive_assert( pcacheCheckSynced(p) ); } if( addRemove & PCACHE_DIRTYLIST_ADD ){ assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); @@ -127,7 +109,6 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) ){ p->pSynced = pPage; } - expensive_assert( pcacheCheckSynced(p) ); } } @@ -304,7 +285,6 @@ int sqlite3PcacheFetchStress( ** cleared), but if that is not possible settle for any other ** unreferenced dirty page. */ - expensive_assert( pcacheCheckSynced(pCache) ); for(pPg=pCache->pSynced; pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); pPg=pPg->pDirtyPrev From ed7bcba7980a59d400b592e7306d72e1599678fd Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 15 Sep 2014 16:50:34 +0000 Subject: [PATCH 330/710] Avoid attempting to call the xFetch() method of an sqlite3_io_methods object with a version number less than 3. FossilOrigin-Name: dedaa6fb3d2e6e697d4a48649af5f42d9a11c333 --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/vdbesort.c | 9 ++++++--- test/sort5.test | 45 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 test/sort5.test diff --git a/manifest b/manifest index 7e35806fec..bf269d15d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sEXPENSIVE_ASSERTS\sin\spcache.c\shaving\sto\sdo\swith\sthe\spSynced\sfield\nof\sthe\sPcache\sobject,\sas\sthey\sare\sincorrect,\sas\srevealed\sby\srecent\spcache\nenhancements. -D 2014-09-15T14:59:12.274 +C Avoid\sattempting\sto\scall\sthe\sxFetch()\smethod\sof\san\ssqlite3_io_methods\sobject\swith\sa\sversion\snumber\sless\sthan\s3. +D 2014-09-15T16:50:34.423 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -295,7 +295,7 @@ F src/vdbeapi.c 06b712d4772b318b69cd37a416deb1ff0426aa8c F src/vdbeaux.c 91fd1e0c54a765838dc61fcf79f31acce035ce38 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c dc36ea9fe26c25550c50085f388167086ef7d73a -F src/vdbesort.c ab39574ec6e0c6213bd2a5c09cca9f9f8ba98450 +F src/vdbesort.c a7a40ceca6325b853040ffcc363dcd49a45f201b F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 @@ -845,6 +845,7 @@ F test/sort.test 15e1d3014abc3f6d4357ed81b93b82117aefd235 F test/sort2.test 269f4f50c6e468cc32b302ae7ff0add8338ec6de F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 F test/sort4.test 6c37d85f7cd28d50cce222fcab84ccd771e105cb +F test/sort5.test a448240a42b49239edc00f85d6d7ac7a1b261e1f F test/sortfault.test b8e35177f97438b930ee87c9419ca2599e8073e1 F test/speed1.test f2974a91d79f58507ada01864c0e323093065452 F test/speed1p.explain d841e650a04728b39e6740296b852dccdca9b2cb @@ -1197,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0bdf1a086b3946722f4d4b328e25917f61c14713 -R 6604c3404d8f943a6eb10e6ee4c1398f -U drh -Z 9f457bfb74de1e07b081e001cb58874a +P 69a64560777f85b47349b4b2aab01dc99298592e +R 41452df15ff1b435df3a846f31103df4 +U dan +Z 356b29f991ba0eb562f8fca4ef2b2ea4 diff --git a/manifest.uuid b/manifest.uuid index 0cb3407f29..13bc35293d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69a64560777f85b47349b4b2aab01dc99298592e \ No newline at end of file +dedaa6fb3d2e6e697d4a48649af5f42d9a11c333 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index d1ccf6089d..b9a62bf569 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -602,8 +602,11 @@ static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ int rc = SQLITE_OK; if( pFile->iEof<=(i64)(pTask->pSorter->db->nMaxSorterMmap) ){ - rc = sqlite3OsFetch(pFile->pFd, 0, (int)pFile->iEof, (void**)pp); - testcase( rc!=SQLITE_OK ); + sqlite3_file *pFd = pFile->pFd; + if( pFd->pMethods->iVersion>=3 ){ + rc = sqlite3OsFetch(pFd, 0, (int)pFile->iEof, (void**)pp); + testcase( rc!=SQLITE_OK ); + } } return rc; } @@ -1123,7 +1126,7 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ if( nByte<=(i64)(db->nMaxSorterMmap) ){ int rc = sqlite3OsTruncate(pFd, nByte); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && pFd->pMethods->iVersion>=3 ){ void *p = 0; sqlite3OsFetch(pFd, 0, (int)nByte, &p); sqlite3OsUnfetch(pFd, 0, p); diff --git a/test/sort5.test b/test/sort5.test new file mode 100644 index 0000000000..a02b8f920f --- /dev/null +++ b/test/sort5.test @@ -0,0 +1,45 @@ +# 2014 September 15. +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix sort5 + + +#------------------------------------------------------------------------- +# Verify that sorting works with a version 1 sqlite3_io_methods structure. +# +testvfs tvfs -iversion 1 -default true +reset_db +do_execsql_test 1.0 { + PRAGMA mmap_size = 10000000; + PRAGMA cache_size = 10; + CREATE TABLE t1(a, b); +} {0} + +do_test 1.1 { + execsql BEGIN + for {set i 0} {$i < 2000} {incr i} { + execsql { INSERT INTO t1 VALUES($i, randomblob(2000)) } + } + execsql COMMIT +} {} + +do_execsql_test 1.2 { + CREATE INDEX i1 ON t1(b); +} + +db close +tvfs delete +finish_test + From b6dea49f3d30bd348676eb5c6795f6f1ea5c173c Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 15 Sep 2014 16:53:23 +0000 Subject: [PATCH 331/710] Fix tool/showwal.c so that it handles WAL files that contain 64KiB pages. FossilOrigin-Name: 4060efb646c873c4abde7ab9ddf330489a44f274 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/showwal.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index bf269d15d0..44c6f5bfa6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sattempting\sto\scall\sthe\sxFetch()\smethod\sof\san\ssqlite3_io_methods\sobject\swith\sa\sversion\snumber\sless\sthan\s3. -D 2014-09-15T16:50:34.423 +C Fix\stool/showwal.c\sso\sthat\sit\shandles\sWAL\sfiles\sthat\scontain\s64KiB\spages. +D 2014-09-15T16:53:23.593 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1179,7 +1179,7 @@ F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/showdb.c bd073a78bce714a0e42d92ea474b3eb8cb53be5d F tool/showjournal.c 053eb1cc774710c6890b7dd6293300cc297b16a5 F tool/showstat4.c c39279d6bd37cb999b634f0064f6f86ad7af008f -F tool/showwal.c 3209120269cdf9380f091459e47b776b4f81dfd3 +F tool/showwal.c 85cb36d4fe3e93e2fbd63e786e0d1ce42d0c4fad F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b F tool/spaceanal.tcl 8e50b217c56a6a086a1b47eac9d09c5cd65b996f @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 69a64560777f85b47349b4b2aab01dc99298592e -R 41452df15ff1b435df3a846f31103df4 +P dedaa6fb3d2e6e697d4a48649af5f42d9a11c333 +R 9c897c2b35082efdf1b1cdba3d9a6e48 U dan -Z 356b29f991ba0eb562f8fca4ef2b2ea4 +Z fbf2bdf01c7200ebedc4b699448a8435 diff --git a/manifest.uuid b/manifest.uuid index 13bc35293d..f4f07abe9e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dedaa6fb3d2e6e697d4a48649af5f42d9a11c333 \ No newline at end of file +4060efb646c873c4abde7ab9ddf330489a44f274 \ No newline at end of file diff --git a/tool/showwal.c b/tool/showwal.c index 6dc1de173f..35810c66a9 100644 --- a/tool/showwal.c +++ b/tool/showwal.c @@ -510,7 +510,7 @@ static void decode_btree_page( int main(int argc, char **argv){ struct stat sbuf; - unsigned char zPgSz[2]; + unsigned char zPgSz[4]; if( argc<2 ){ fprintf(stderr,"Usage: %s FILENAME ?PAGE? ...\n", argv[0]); exit(1); @@ -522,9 +522,9 @@ int main(int argc, char **argv){ } zPgSz[0] = 0; zPgSz[1] = 0; - lseek(fd, 10, SEEK_SET); - read(fd, zPgSz, 2); - pagesize = zPgSz[0]*256 + zPgSz[1]; + lseek(fd, 8, SEEK_SET); + read(fd, zPgSz, 4); + pagesize = zPgSz[1]*65536 + zPgSz[2]*256 + zPgSz[3]; if( pagesize==0 ) pagesize = 1024; printf("Pagesize: %d\n", pagesize); fstat(fd, &sbuf); From 982ff72f0f90e9bc5bc314482e6fba1cc4ca2579 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Sep 2014 03:24:43 +0000 Subject: [PATCH 332/710] Performance improvement to the sqlite3MemCompare() routine by factoring out sqlite3BlobCompare(). FossilOrigin-Name: 20ed2321b09ba076e50f9fc2f42c135b25746d72 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 19 +++++++++++++------ 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 44c6f5bfa6..8a0b3a189a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stool/showwal.c\sso\sthat\sit\shandles\sWAL\sfiles\sthat\scontain\s64KiB\spages. -D 2014-09-15T16:53:23.593 +C Performance\simprovement\sto\sthe\ssqlite3MemCompare()\sroutine\sby\sfactoring\sout\nsqlite3BlobCompare(). +D 2014-09-16T03:24:43.248 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vdbe.c 9a45dcbcc967fc0cb9248c75ba245d1d47de3e78 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h b4843c35c3ba533b69d4250f550b5bacf2fb013d F src/vdbeapi.c 06b712d4772b318b69cd37a416deb1ff0426aa8c -F src/vdbeaux.c 91fd1e0c54a765838dc61fcf79f31acce035ce38 +F src/vdbeaux.c cde99fa6659f5f9000d2d84bb5c4cc85d9e0a200 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c dc36ea9fe26c25550c50085f388167086ef7d73a F src/vdbesort.c a7a40ceca6325b853040ffcc363dcd49a45f201b @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dedaa6fb3d2e6e697d4a48649af5f42d9a11c333 -R 9c897c2b35082efdf1b1cdba3d9a6e48 -U dan -Z fbf2bdf01c7200ebedc4b699448a8435 +P 4060efb646c873c4abde7ab9ddf330489a44f274 +R 2d14ce03cde1c84220e0226983629fd3 +U drh +Z 10220c87d8978c341785bd01c3f5069d diff --git a/manifest.uuid b/manifest.uuid index f4f07abe9e..bbfe9273dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4060efb646c873c4abde7ab9ddf330489a44f274 \ No newline at end of file +20ed2321b09ba076e50f9fc2f42c135b25746d72 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index a653587319..86f36aba87 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3319,6 +3319,18 @@ static int vdbeCompareMemString( } } +/* +** Compare two blobs. Return negative, zero, or positive if the first +** is less than, equal to, or greater than the second, respectively. +** If one blob is a prefix of the other, then the shorter is the lessor. +*/ +static SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem *pB2){ + int c = memcmp(pB1->z, pB2->z, pB1->n>pB2->n ? pB2->n : pB1->n); + if( c ) return c; + return pB1->n - pB2->n; +} + + /* ** Compare the values contained by the two memory cells, returning ** negative, zero or positive if pMem1 is less than, equal to, or greater @@ -3329,7 +3341,6 @@ static int vdbeCompareMemString( ** Two NULL values are considered equal by this function. */ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ - int rc; int f1, f2; int combined_flags; @@ -3404,11 +3415,7 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ } /* Both values must be blobs. Compare using memcmp(). */ - rc = memcmp(pMem1->z, pMem2->z, (pMem1->n>pMem2->n)?pMem2->n:pMem1->n); - if( rc==0 ){ - rc = pMem1->n - pMem2->n; - } - return rc; + return sqlite3BlobCompare(pMem1, pMem2); } From 7f4b19f170ff8010cd88c248aa9bf7b909d1dc75 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Sep 2014 13:30:05 +0000 Subject: [PATCH 333/710] Changes to sqlite3VdbeRecordUnpack() to make it slightly smaller and faster. FossilOrigin-Name: 8fb90da77ce0e662c1ef1ae0d854e5164494b7af --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8a0b3a189a..6eff938f17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovement\sto\sthe\ssqlite3MemCompare()\sroutine\sby\sfactoring\sout\nsqlite3BlobCompare(). -D 2014-09-16T03:24:43.248 +C Changes\sto\ssqlite3VdbeRecordUnpack()\sto\smake\sit\sslightly\ssmaller\sand\sfaster. +D 2014-09-16T13:30:05.468 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vdbe.c 9a45dcbcc967fc0cb9248c75ba245d1d47de3e78 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 F src/vdbeInt.h b4843c35c3ba533b69d4250f550b5bacf2fb013d F src/vdbeapi.c 06b712d4772b318b69cd37a416deb1ff0426aa8c -F src/vdbeaux.c cde99fa6659f5f9000d2d84bb5c4cc85d9e0a200 +F src/vdbeaux.c 2f284f8b052acc5b484682fbd3de35c990c2ca1c F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c dc36ea9fe26c25550c50085f388167086ef7d73a F src/vdbesort.c a7a40ceca6325b853040ffcc363dcd49a45f201b @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4060efb646c873c4abde7ab9ddf330489a44f274 -R 2d14ce03cde1c84220e0226983629fd3 +P 20ed2321b09ba076e50f9fc2f42c135b25746d72 +R 052add0d7794460f3fc0b9bdb3e7879a U drh -Z 10220c87d8978c341785bd01c3f5069d +Z bf1aaf6028da77cf4241e813f52ee696 diff --git a/manifest.uuid b/manifest.uuid index bbfe9273dd..d8ad59bb30 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -20ed2321b09ba076e50f9fc2f42c135b25746d72 \ No newline at end of file +8fb90da77ce0e662c1ef1ae0d854e5164494b7af \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 86f36aba87..3360d919b2 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3161,7 +3161,7 @@ void sqlite3VdbeRecordUnpack( idx = getVarint32(aKey, szHdr); d = szHdr; u = 0; - while( idxnField && d<=nKey ){ + while( idxzMalloc = 0; d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); pMem++; - u++; + if( (++u)>=p->nField ) break; } assert( u<=pKeyInfo->nField + 1 ); p->nField = u; From 035e563bf62e749036825d85fdd885a4110810ce Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Sep 2014 14:16:31 +0000 Subject: [PATCH 334/710] Reorder the elements of the Mem object for a small size reduction and performance improvement. FossilOrigin-Name: 0be3019ed794c10de67dfd645ceea7d45815bc4b --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/func.c | 1 + src/vdbeInt.h | 13 +++++++------ src/vdbeapi.c | 16 +++++++++++++--- src/vdbeaux.c | 5 +++-- src/vdbemem.c | 2 ++ 7 files changed, 37 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 6eff938f17..30bef2c31f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\ssqlite3VdbeRecordUnpack()\sto\smake\sit\sslightly\ssmaller\sand\sfaster. -D 2014-09-16T13:30:05.468 +C Reorder\sthe\selements\sof\sthe\sMem\sobject\sfor\sa\ssmall\ssize\sreduction\sand\nperformance\simprovement. +D 2014-09-16T14:16:31.846 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -183,7 +183,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c 19392d98e089640c3336e65b4254cc337efef7d1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c 5d498933f6168dff80941c873805fe04dc2b766d +F src/func.c 1629ccdd8ef3f19d7accc9d9287190489469ff81 F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -290,11 +290,11 @@ F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 9a45dcbcc967fc0cb9248c75ba245d1d47de3e78 F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 -F src/vdbeInt.h b4843c35c3ba533b69d4250f550b5bacf2fb013d -F src/vdbeapi.c 06b712d4772b318b69cd37a416deb1ff0426aa8c -F src/vdbeaux.c 2f284f8b052acc5b484682fbd3de35c990c2ca1c +F src/vdbeInt.h 0de8705e38b5f28808e37cebb9ec6df995ac3304 +F src/vdbeapi.c 4d2aa56efa1b4a010012466bf8e97dbf179081a6 +F src/vdbeaux.c 4d607ce804cc4ee129df5745f7b73ed472f808b6 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c dc36ea9fe26c25550c50085f388167086ef7d73a +F src/vdbemem.c 4d1e1398be24f85805196c20a80699be0699a9ca F src/vdbesort.c a7a40ceca6325b853040ffcc363dcd49a45f201b F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 20ed2321b09ba076e50f9fc2f42c135b25746d72 -R 052add0d7794460f3fc0b9bdb3e7879a +P 8fb90da77ce0e662c1ef1ae0d854e5164494b7af +R 66ed6d98b57d603d2a5f34403ecc91fe U drh -Z bf1aaf6028da77cf4241e813f52ee696 +Z d53857782fade1fcfcd0d6fed6dee986 diff --git a/manifest.uuid b/manifest.uuid index d8ad59bb30..d1ee51fd68 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8fb90da77ce0e662c1ef1ae0d854e5164494b7af \ No newline at end of file +0be3019ed794c10de67dfd645ceea7d45815bc4b \ No newline at end of file diff --git a/src/func.c b/src/func.c index 0a8a9dda38..e1961118fd 100644 --- a/src/func.c +++ b/src/func.c @@ -1492,6 +1492,7 @@ static void minmaxStep( sqlite3SkipAccumulatorLoad(context); } }else{ + pBest->db = sqlite3_context_db_handle(context); sqlite3VdbeMemCopy(pBest, pArg); } } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 27b266986a..5379d6b667 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -161,9 +161,6 @@ struct VdbeFrame { ** integer etc.) of the same value. */ struct Mem { - sqlite3 *db; /* The associated database connection */ - char *z; /* String or BLOB value */ - double r; /* Real value */ union { i64 i; /* Integer value used when MEM_Int is set in flags */ int nZero; /* Used when bit MEM_Zero is set in flags */ @@ -171,15 +168,19 @@ struct Mem { RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ } u; - int n; /* Number of characters in string value, excluding '\0' */ u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ + int n; /* Number of characters in string value, excluding '\0' */ + double r; /* Real value */ + char *z; /* String or BLOB value */ + char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ + /* ShallowCopy only needs to copy the information above */ + sqlite3 *db; /* The associated database connection */ + void (*xDel)(void*);/* If not null, call this function to delete Mem.z */ #ifdef SQLITE_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ #endif - void (*xDel)(void *); /* If not null, call this function to delete Mem.z */ - char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ }; /* One or more of the following flags are set to indicate the validOK diff --git a/src/vdbeapi.c b/src/vdbeapi.c index b64f33c8c6..14d6b8412e 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -803,11 +803,21 @@ static const Mem *columnNullValue(void){ #if defined(SQLITE_DEBUG) && defined(__GNUC__) __attribute__((aligned(8))) #endif - = {0, "", (double)0, {0}, 0, MEM_Null, 0, + = { + .flags = MEM_Null, + .enc = 0, + .n = 0, + .r = (double)0, + .u = {0}, + .z = 0, + .zMalloc = 0, + .db = 0, + .xDel = 0, #ifdef SQLITE_DEBUG - 0, 0, /* pScopyFrom, pFiller */ + .pScopyFrom = 0, + .pFiller = 0, #endif - 0, 0 }; + }; return &nullMem; } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 3360d919b2..65792b485e 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3303,8 +3303,9 @@ static int vdbeCompareMemString( int n1, n2; Mem c1; Mem c2; - memset(&c1, 0, sizeof(c1)); - memset(&c2, 0, sizeof(c2)); + c1.db = c2.db = pMem1->db; + c1.flags = c2.flags = 0; + c1.zMalloc = c2.zMalloc = 0; sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); diff --git a/src/vdbemem.c b/src/vdbemem.c index ea4def3f86..7785bc0de5 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -730,6 +730,7 @@ void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ */ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( (pFrom->flags & MEM_RowSet)==0 ); + assert( pTo->db==pFrom->db ); VdbeMemReleaseExtern(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->xDel = 0; @@ -747,6 +748,7 @@ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; + assert( pTo->db==pFrom->db ); assert( (pFrom->flags & MEM_RowSet)==0 ); VdbeMemReleaseExtern(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); From 75179ded976666932155d84f7ed805a8f6e167ca Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Sep 2014 14:37:35 +0000 Subject: [PATCH 335/710] Reduce the number of arguments to RecordCompare functions from 4 to 3, resulting in a small performance increase. FossilOrigin-Name: 8239c35aedd583af79505378bb7dbb78346a3f45 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/btree.c | 6 +++--- src/vdbe.h | 4 ++-- src/vdbeaux.c | 30 +++++++++++++++--------------- src/vdbesort.c | 4 ++-- src/where.c | 8 ++++---- 7 files changed, 37 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index 30bef2c31f..154cd1a029 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reorder\sthe\selements\sof\sthe\sMem\sobject\sfor\sa\ssmall\ssize\sreduction\sand\nperformance\simprovement. -D 2014-09-16T14:16:31.846 +C Reduce\sthe\snumber\sof\sarguments\sto\sRecordCompare\sfunctions\sfrom\s4\sto\s3,\nresulting\sin\sa\ssmall\sperformance\sincrease. +D 2014-09-16T14:37:35.076 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -171,7 +171,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c e4916b441bb036897cc69df275a2df3fea4d53b6 +F src/btree.c 6aa61c0e3d20d1d1acc8fb33d8f0ebd675305d3c F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc F src/build.c 047d7e1d2d89fa55134fa1d6b669c9c2983c0d11 @@ -289,19 +289,19 @@ F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 9a45dcbcc967fc0cb9248c75ba245d1d47de3e78 -F src/vdbe.h c63fad052c9e7388d551e556e119c0bcf6bebdf8 +F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h 0de8705e38b5f28808e37cebb9ec6df995ac3304 F src/vdbeapi.c 4d2aa56efa1b4a010012466bf8e97dbf179081a6 -F src/vdbeaux.c 4d607ce804cc4ee129df5745f7b73ed472f808b6 +F src/vdbeaux.c d06769a7c1f8db9c04fe1457d357203a06684662 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 4d1e1398be24f85805196c20a80699be0699a9ca -F src/vdbesort.c a7a40ceca6325b853040ffcc363dcd49a45f201b +F src/vdbesort.c 09efa5e5098d1a159cd21f588eb118e4fe87cfde F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 839b5e1db2507e221ad1c308f148a8519ed750be +F src/where.c dc276288039fb45ce23c80e4535980f5a152d8ec F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8fb90da77ce0e662c1ef1ae0d854e5164494b7af -R 66ed6d98b57d603d2a5f34403ecc91fe +P 0be3019ed794c10de67dfd645ceea7d45815bc4b +R 4ac54fa45280398cdbf86b6ba8db7793 U drh -Z d53857782fade1fcfcd0d6fed6dee986 +Z fba86109c421937fd36d2c7c73cc5fa1 diff --git a/manifest.uuid b/manifest.uuid index d1ee51fd68..d03654a7d8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0be3019ed794c10de67dfd645ceea7d45815bc4b \ No newline at end of file +8239c35aedd583af79505378bb7dbb78346a3f45 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index fe34b4a836..522e945ac2 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4746,14 +4746,14 @@ int sqlite3BtreeMovetoUnpacked( ** single byte varint and the record fits entirely on the main ** b-tree page. */ testcase( pCell+nCell+1==pPage->aDataEnd ); - c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey, 0); + c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); }else if( !(pCell[1] & 0x80) && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal ){ /* The record-size field is a 2 byte varint and the record ** fits entirely on the main b-tree page. */ testcase( pCell+nCell+2==pPage->aDataEnd ); - c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey, 0); + c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); }else{ /* The record flows over onto one or more overflow pages. In ** this case the whole cell needs to be parsed, a buffer allocated @@ -4774,7 +4774,7 @@ int sqlite3BtreeMovetoUnpacked( sqlite3_free(pCellKey); goto moveto_finish; } - c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); + c = xRecordCompare(nCell, pCellKey, pIdxKey); sqlite3_free(pCellKey); } assert( diff --git a/src/vdbe.h b/src/vdbe.h index ef91010d80..f975f95543 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -212,10 +212,10 @@ void sqlite3VdbeSetVarmask(Vdbe*, int); int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); -int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*,int); +int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); -typedef int (*RecordCompare)(int,const void*,UnpackedRecord*,int); +typedef int (*RecordCompare)(int,const void*,UnpackedRecord*); RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); #ifndef SQLITE_OMIT_TRIGGER diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 65792b485e..5a0b330060 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3482,7 +3482,7 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** pPKey2->errCode is set to SQLITE_NOMEM and, if it is not NULL, the ** malloc-failed flag set on database handle (pPKey2->pKeyInfo->db). */ -int sqlite3VdbeRecordCompare( +static int vdbeRecordCompareWithSkip( int nKey1, const void *pKey1, /* Left key */ UnpackedRecord *pPKey2, /* Right key */ int bSkip /* If true, skip the first field */ @@ -3664,6 +3664,13 @@ int sqlite3VdbeRecordCompare( ); return pPKey2->default_rc; } +int sqlite3VdbeRecordCompare( + int nKey1, const void *pKey1, /* Left key */ + UnpackedRecord *pPKey2 /* Right key */ +){ + return vdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 0); +} + /* ** This function is an optimized version of sqlite3VdbeRecordCompare() @@ -3676,8 +3683,7 @@ int sqlite3VdbeRecordCompare( */ static int vdbeRecordCompareInt( int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2, /* Right key */ - int bSkip /* Ignored */ + UnpackedRecord *pPKey2 /* Right key */ ){ const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F]; int serial_type = ((const u8*)pKey1)[1]; @@ -3686,9 +3692,7 @@ static int vdbeRecordCompareInt( u64 x; i64 v = pPKey2->aMem[0].u.i; i64 lhs; - UNUSED_PARAMETER(bSkip); - assert( bSkip==0 ); assert( (*(u8*)pKey1)<=0x3F || CORRUPT_DB ); switch( serial_type ){ case 1: { /* 1-byte signed integer */ @@ -3738,10 +3742,10 @@ static int vdbeRecordCompareInt( ** (as gcc is clever enough to combine the two like cases). Other ** compilers might be similar. */ case 0: case 7: - return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 0); + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2); default: - return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 0); + return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2); } if( v>lhs ){ @@ -3751,7 +3755,7 @@ static int vdbeRecordCompareInt( }else if( pPKey2->nField>1 ){ /* The first fields of the two keys are equal. Compare the trailing ** fields. */ - res = sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 1); + res = vdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ /* The first fields of the two keys are equal and there are no trailing ** fields. Return pPKey2->default_rc in this case. */ @@ -3770,17 +3774,13 @@ static int vdbeRecordCompareInt( */ static int vdbeRecordCompareString( int nKey1, const void *pKey1, /* Left key */ - UnpackedRecord *pPKey2, /* Right key */ - int bSkip + UnpackedRecord *pPKey2 /* Right key */ ){ const u8 *aKey1 = (const u8*)pKey1; int serial_type; int res; - UNUSED_PARAMETER(bSkip); - assert( bSkip==0 ); getVarint32(&aKey1[1], serial_type); - if( serial_type<12 ){ res = pPKey2->r1; /* (pKey1/nKey1) is a number or a null */ }else if( !(serial_type & 0x01) ){ @@ -3802,7 +3802,7 @@ static int vdbeRecordCompareString( res = nStr - pPKey2->aMem[0].n; if( res==0 ){ if( pPKey2->nField>1 ){ - res = sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2, 1); + res = vdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ res = pPKey2->default_rc; } @@ -3980,7 +3980,7 @@ int sqlite3VdbeIdxKeyCompare( if( rc ){ return rc; } - *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked, 0); + *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked); sqlite3VdbeMemRelease(&m); return SQLITE_OK; } diff --git a/src/vdbesort.c b/src/vdbesort.c index b9a62bf569..2d7bc8a7a4 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -761,7 +761,7 @@ static int vdbeSorterCompare( if( pKey2 ){ sqlite3VdbeRecordUnpack(pTask->pSorter->pKeyInfo, nKey2, pKey2, r2); } - return sqlite3VdbeRecordCompare(nKey1, pKey1, r2, 0); + return sqlite3VdbeRecordCompare(nKey1, pKey1, r2); } /* @@ -2517,6 +2517,6 @@ int sqlite3VdbeSorterCompare( } } - *pRes = sqlite3VdbeRecordCompare(pVal->n, pVal->z, r2, 0); + *pRes = sqlite3VdbeRecordCompare(pVal->n, pVal->z, r2); return SQLITE_OK; } diff --git a/src/where.c b/src/where.c index 5b990fc108..c7cb7bb67c 100644 --- a/src/where.c +++ b/src/where.c @@ -1913,7 +1913,7 @@ static void whereKeyStats( assert( pRec->nField>0 && iColnSampleCol ); do{ iTest = (iMin+i)/2; - res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec, 0); + res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec); if( res<0 ){ iMin = iTest+1; }else{ @@ -1928,16 +1928,16 @@ static void whereKeyStats( if( res==0 ){ /* If (res==0) is true, then sample $i must be equal to pRec */ assert( inSample ); - assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec, 0) + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) || pParse->db->mallocFailed ); }else{ /* Otherwise, pRec must be smaller than sample $i and larger than ** sample ($i-1). */ assert( i==pIdx->nSample - || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec, 0)>0 + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 || pParse->db->mallocFailed ); assert( i==0 - || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec, 0)<0 + || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 || pParse->db->mallocFailed ); } #endif /* ifdef SQLITE_DEBUG */ From 897a2d50f222721bc71c9835705618dff1a7801a Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Sep 2014 14:55:08 +0000 Subject: [PATCH 336/710] Remove an unused parameter from sqlite3VdbeIdxRowid(). This is cosmetic only as the C-compiler optimizers were already omitting this parameter on amalgamation builds. FossilOrigin-Name: a10a6bba4963761b490b7243b388dcc920c4cfed --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- src/vdbeInt.h | 2 +- src/vdbeaux.c | 4 +--- 5 files changed, 12 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 154cd1a029..8c762d224f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\snumber\sof\sarguments\sto\sRecordCompare\sfunctions\sfrom\s4\sto\s3,\nresulting\sin\sa\ssmall\sperformance\sincrease. -D 2014-09-16T14:37:35.076 +C Remove\san\sunused\sparameter\sfrom\ssqlite3VdbeIdxRowid().\s\sThis\sis\scosmetic\nonly\sas\sthe\sC-compiler\soptimizers\swere\salready\somitting\sthis\sparameter\son\namalgamation\sbuilds. +D 2014-09-16T14:55:08.312 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,11 +288,11 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 9a45dcbcc967fc0cb9248c75ba245d1d47de3e78 +F src/vdbe.c ae07ee84667c190d31946ae31c2fc63a3aa81a32 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h 0de8705e38b5f28808e37cebb9ec6df995ac3304 +F src/vdbeInt.h dc1743de339f5556cc6687219cf8727ad0d35f72 F src/vdbeapi.c 4d2aa56efa1b4a010012466bf8e97dbf179081a6 -F src/vdbeaux.c d06769a7c1f8db9c04fe1457d357203a06684662 +F src/vdbeaux.c 79ce140ee79ecc7638eac070b48f1d24bbf9653c F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 4d1e1398be24f85805196c20a80699be0699a9ca F src/vdbesort.c 09efa5e5098d1a159cd21f588eb118e4fe87cfde @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0be3019ed794c10de67dfd645ceea7d45815bc4b -R 4ac54fa45280398cdbf86b6ba8db7793 +P 8239c35aedd583af79505378bb7dbb78346a3f45 +R 8e31ee1c1cfa56f2cb381e2e177d1a9b U drh -Z fba86109c421937fd36d2c7c73cc5fa1 +Z 717af3cbede201cebaf3af532573dee2 diff --git a/manifest.uuid b/manifest.uuid index d03654a7d8..1d2c86f0a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8239c35aedd583af79505378bb7dbb78346a3f45 \ No newline at end of file +a10a6bba4963761b490b7243b388dcc920c4cfed \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index cc9e317e4b..b1764a12f4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4769,7 +4769,7 @@ case OP_IdxRowid: { /* out2-prerelease */ assert( pC->isTable==0 ); if( !pC->nullRow ){ rowid = 0; /* Not needed. Only used to silence a warning. */ - rc = sqlite3VdbeIdxRowid(db, pCrsr, &rowid); + rc = sqlite3VdbeIdxRowid(pCrsr, &rowid); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 5379d6b667..14ae09130c 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -398,7 +398,7 @@ void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); -int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); +int sqlite3VdbeIdxRowid(BtCursor *, i64 *); int sqlite3VdbeExec(Vdbe*); int sqlite3VdbeList(Vdbe*); int sqlite3VdbeHalt(Vdbe*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 5a0b330060..b7e3da7584 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3876,7 +3876,7 @@ RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ ** pCur might be pointing to text obtained from a corrupt database file. ** So the content cannot be trusted. Do appropriate checks on the content. */ -int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ +int sqlite3VdbeIdxRowid(BtCursor *pCur, i64 *rowid){ i64 nCellKey = 0; int rc; u32 szHdr; /* Size of the header */ @@ -3884,8 +3884,6 @@ int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ u32 lenRowid; /* Size of the rowid */ Mem m, v; - UNUSED_PARAMETER(db); - /* Get the size of the index entry. Only indices entries of less ** than 2GiB are support - anything large must be database corruption. ** Any corruption is detected in sqlite3BtreeParseCellPtr(), though, so From 2a2a696cd754b117dc6b9dfa8e9bcdc46d9f1d72 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Sep 2014 18:22:44 +0000 Subject: [PATCH 337/710] Simplification of the OP_Column logic for the case of rows with overflow. FossilOrigin-Name: f73678038d8fc399c1ca55230ae45962007c909c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 38 ++++++++++++-------------------------- src/vdbemem.c | 7 +++++-- 4 files changed, 25 insertions(+), 36 deletions(-) diff --git a/manifest b/manifest index 8c762d224f..100a334778 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\sparameter\sfrom\ssqlite3VdbeIdxRowid().\s\sThis\sis\scosmetic\nonly\sas\sthe\sC-compiler\soptimizers\swere\salready\somitting\sthis\sparameter\son\namalgamation\sbuilds. -D 2014-09-16T14:55:08.312 +C Simplification\sof\sthe\sOP_Column\slogic\sfor\sthe\scase\sof\srows\swith\soverflow. +D 2014-09-16T18:22:44.852 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,13 +288,13 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c ae07ee84667c190d31946ae31c2fc63a3aa81a32 +F src/vdbe.c ff1b0b4f41355ba207bdc691b1017e7642f42c6b F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h dc1743de339f5556cc6687219cf8727ad0d35f72 F src/vdbeapi.c 4d2aa56efa1b4a010012466bf8e97dbf179081a6 F src/vdbeaux.c 79ce140ee79ecc7638eac070b48f1d24bbf9653c F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 4d1e1398be24f85805196c20a80699be0699a9ca +F src/vdbemem.c 8abc122ce5359a120196e0825dca9a08a787aff6 F src/vdbesort.c 09efa5e5098d1a159cd21f588eb118e4fe87cfde F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8239c35aedd583af79505378bb7dbb78346a3f45 -R 8e31ee1c1cfa56f2cb381e2e177d1a9b +P a10a6bba4963761b490b7243b388dcc920c4cfed +R e7cf963c2745f47a3657ad38e9817c1f U drh -Z 717af3cbede201cebaf3af532573dee2 +Z 153a841014a3e255f6703a490d77aa20 diff --git a/manifest.uuid b/manifest.uuid index 1d2c86f0a0..9162c28d1d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a10a6bba4963761b490b7243b388dcc920c4cfed \ No newline at end of file +f73678038d8fc399c1ca55230ae45962007c909c \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index b1764a12f4..20301edc56 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2449,37 +2449,23 @@ case OP_Column: { && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)) || (len = sqlite3VdbeSerialTypeLen(t))==0 ){ - /* Content is irrelevant for the typeof() function and for - ** the length(X) function if X is a blob. So we might as well use - ** bogus content rather than reading content from disk. NULL works - ** for text and blob and whatever is in the payloadSize64 variable - ** will work for everything else. Content is also irrelevant if - ** the content length is 0. */ - zData = t<=13 ? (u8*)&payloadSize64 : 0; - sMem.zMalloc = 0; + /* Content is irrelevant for + ** 1. the typeof() function, + ** 2. the length(X) function if X is a blob, and + ** 3. if the content length is zero. + ** So we might as well use bogus content rather than reading + ** content from disk. NULL will work for the value for strings + ** and blobs and whatever is in the payloadSize64 variable + ** will work for everything else. */ + sqlite3VdbeSerialGet(t<=13 ? (u8*)&payloadSize64 : 0, t, pDest); }else{ - memset(&sMem, 0, sizeof(sMem)); - sqlite3VdbeMemMove(&sMem, pDest); rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable, - &sMem); + pDest); if( rc!=SQLITE_OK ){ goto op_column_error; } - zData = (u8*)sMem.z; - } - sqlite3VdbeSerialGet(zData, t, pDest); - /* If we dynamically allocated space to hold the data (in the - ** sqlite3VdbeMemFromBtree() call above) then transfer control of that - ** dynamically allocated space over to the pDest structure. - ** This prevents a memory copy. */ - if( sMem.zMalloc ){ - assert( sMem.z==sMem.zMalloc ); - assert( VdbeMemDynamic(pDest)==0 ); - assert( (pDest->flags & (MEM_Blob|MEM_Str))==0 || pDest->z==sMem.z ); - pDest->flags &= ~(MEM_Ephem|MEM_Static); - pDest->flags |= MEM_Term; - pDest->z = sMem.z; - pDest->zMalloc = sMem.zMalloc; + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + pDest->flags &= ~MEM_Ephem; } } pDest->enc = encoding; diff --git a/src/vdbemem.c b/src/vdbemem.c index 7785bc0de5..432cd4e898 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -885,8 +885,11 @@ int sqlite3VdbeMemSetStr( ** key is true to get the key or false to get data. The result is written ** into the pMem element. ** -** The pMem structure is assumed to be uninitialized. Any prior content -** is overwritten without being freed. +** The pMem object must have been initialized. This routine will use +** pMem->zMalloc to hold the content from the btree, if possible. New +** pMem->zMalloc space will be allocated if necessary. The calling routine +** is responsible for making sure that the pMem object is eventually +** destroyed. ** ** If this routine fails for any reason (malloc returns NULL or unable ** to read from the disk) then the pMem is left in an inconsistent state. From 8740a600162808c84040356282fa2f5af95588ff Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Sep 2014 20:05:21 +0000 Subject: [PATCH 338/710] Make sure registers are cleared properly prior to being used to store the result of an OP_Column operator. FossilOrigin-Name: 78fb8838d80b229418c347c63579989432e1af7d --- manifest | 16 +++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- src/vdbeaux.c | 1 + src/vdbemem.c | 56 ++++++++++++++++++++++++++++++--------------------- 5 files changed, 44 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 100a334778..20bc2b4129 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplification\sof\sthe\sOP_Column\slogic\sfor\sthe\scase\sof\srows\swith\soverflow. -D 2014-09-16T18:22:44.852 +C Make\ssure\sregisters\sare\scleared\sproperly\sprior\sto\sbeing\sused\sto\sstore\nthe\sresult\sof\san\sOP_Column\soperator. +D 2014-09-16T20:05:21.909 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,13 +288,13 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c ff1b0b4f41355ba207bdc691b1017e7642f42c6b +F src/vdbe.c 6a45baf86fcc6c294d57e0aef8c9f2c54f07ff18 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h dc1743de339f5556cc6687219cf8727ad0d35f72 F src/vdbeapi.c 4d2aa56efa1b4a010012466bf8e97dbf179081a6 -F src/vdbeaux.c 79ce140ee79ecc7638eac070b48f1d24bbf9653c +F src/vdbeaux.c 211ad29d51e01c44a0db1ab69b74c11c8de1cccf F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 8abc122ce5359a120196e0825dca9a08a787aff6 +F src/vdbemem.c 18556fc614426886e380def839bdcf9cadbb752a F src/vdbesort.c 09efa5e5098d1a159cd21f588eb118e4fe87cfde F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a10a6bba4963761b490b7243b388dcc920c4cfed -R e7cf963c2745f47a3657ad38e9817c1f +P f73678038d8fc399c1ca55230ae45962007c909c +R 39f3c2bf48f680cf4e99aef8c3698c28 U drh -Z 153a841014a3e255f6703a490d77aa20 +Z e586af33404759487ed3f6653fe697fa diff --git a/manifest.uuid b/manifest.uuid index 9162c28d1d..0d00de3592 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f73678038d8fc399c1ca55230ae45962007c909c \ No newline at end of file +78fb8838d80b229418c347c63579989432e1af7d \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 20301edc56..4bdfbfbdeb 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2437,10 +2437,10 @@ case OP_Column: { assert( p2nHdrParsed ); assert( rc==SQLITE_OK ); assert( sqlite3VdbeCheckMemInvariants(pDest) ); + VdbeMemReleaseExtern(pDest); if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ - VdbeMemReleaseExtern(pDest); sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], aType[p2], pDest); }else{ /* This branch happens only when content is on overflow pages */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index b7e3da7584..10495e76e4 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3306,6 +3306,7 @@ static int vdbeCompareMemString( c1.db = c2.db = pMem1->db; c1.flags = c2.flags = 0; c1.zMalloc = c2.zMalloc = 0; + c1.xDel = c2.xDel = 0; sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); diff --git a/src/vdbemem.c b/src/vdbemem.c index 432cd4e898..c75046ab87 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -310,9 +310,13 @@ int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ } /* -** If the memory cell contains a string value that must be freed by -** invoking an external callback, free it now. Calling this function -** does not free any Mem.zMalloc buffer. +** If the memory cell contains a value that must be freed by +** invoking an external callback, then free it now. +** +** This routine does NOT do any of the following: +** (1) Set the Mem.flags field to a rational value. +** (2) Free memory held by Mem.zMalloc +** The caller is expected to take care of setting Mem.flags appropriately. ** ** The VdbeMemReleaseExtern() macro invokes this routine if only if there ** is work for this routine to do. @@ -340,8 +344,8 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ ** by p->xDel and memory in p->zMalloc. ** ** This is a helper routine invoked by sqlite3VdbeMemRelease() in -** the uncommon case when there really is memory in p that is -** need of freeing. +** the uncommon case when there really is memory in p that needs +** to be freeing. */ static SQLITE_NOINLINE void vdbeMemRelease(Mem *p){ if( VdbeMemDynamic(p) ){ @@ -355,9 +359,11 @@ static SQLITE_NOINLINE void vdbeMemRelease(Mem *p){ } /* -** Release any memory held by the Mem. This may leave the Mem in an -** inconsistent state, for example with (Mem.z==0) and -** (Mem.flags==MEM_Str). +** Release any memory held by the Mem. This may leave the Mem.flags in an +** inconsistent state, for example with (Mem.z==0) and (Mem.flags==MEM_Str). +** +** This routine releases both the Mem.xDel space and the Mem.zMalloc space. +** Use sqlite3VdbeMemReleaseExternal() to release just the Mem.xDel space. */ void sqlite3VdbeMemRelease(Mem *p){ assert( sqlite3VdbeCheckMemInvariants(p) ); @@ -733,7 +739,7 @@ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( pTo->db==pFrom->db ); VdbeMemReleaseExtern(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); - pTo->xDel = 0; + assert( pTo->xDel==0 ); if( (pFrom->flags&MEM_Static)==0 ){ pTo->flags &= ~(MEM_Dyn|MEM_Static|MEM_Ephem); assert( srcType==MEM_Ephem || srcType==MEM_Static ); @@ -753,7 +759,7 @@ int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ VdbeMemReleaseExtern(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; - pTo->xDel = 0; + assert( pTo->xDel==0 ); if( pTo->flags&(MEM_Str|MEM_Blob) ){ if( 0==(pFrom->flags&MEM_Static) ){ @@ -906,6 +912,7 @@ int sqlite3VdbeMemFromBtree( int rc = SQLITE_OK; /* Return code */ assert( sqlite3BtreeCursorIsValid(pCur) ); + assert( pMem->xDel==0 ); /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() ** that both the BtShared and database handle mutexes are held. */ @@ -922,19 +929,22 @@ int sqlite3VdbeMemFromBtree( pMem->z = &zData[offset]; pMem->flags = MEM_Blob|MEM_Ephem; pMem->n = (int)amt; - }else if( SQLITE_OK==(rc = sqlite3VdbeMemGrow(pMem, amt+2, 0)) ){ - if( key ){ - rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); - }else{ - rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); - } - if( rc==SQLITE_OK ){ - pMem->z[amt] = 0; - pMem->z[amt+1] = 0; - pMem->flags = MEM_Blob|MEM_Term; - pMem->n = (int)amt; - }else{ - sqlite3VdbeMemRelease(pMem); + }else{ + pMem->flags = MEM_Null; + if( SQLITE_OK==(rc = sqlite3VdbeMemGrow(pMem, amt+2, 0)) ){ + if( key ){ + rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); + }else{ + rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); + } + if( rc==SQLITE_OK ){ + pMem->z[amt] = 0; + pMem->z[amt+1] = 0; + pMem->flags = MEM_Blob|MEM_Term; + pMem->n = (int)amt; + }else{ + sqlite3VdbeMemRelease(pMem); + } } } From 6b478bcd9ec466846dba80446636a2cbd992b221 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Sep 2014 21:54:11 +0000 Subject: [PATCH 339/710] Continuing cleanup of memory register memory allocation handling. FossilOrigin-Name: 2598aedc5dd2bac67e2e518a31f2803e469c2ba6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/utf.c | 4 ++-- src/vdbemem.c | 17 ++++++++--------- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 20bc2b4129..0c48965589 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sregisters\sare\scleared\sproperly\sprior\sto\sbeing\sused\sto\sstore\nthe\sresult\sof\san\sOP_Column\soperator. -D 2014-09-16T20:05:21.909 +C Continuing\scleanup\sof\smemory\sregister\smemory\sallocation\shandling. +D 2014-09-16T21:54:11.409 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/threads.c 22dded4283dc4b25422f6444cdcb8d6b1ea0b5ff F src/tokenize.c 3df63041994f55afeb168b463ec836e8f1c50e7c F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 -F src/utf.c 77abb5e6d27f3d236e50f7c8fff1d00e15262359 +F src/utf.c 8f634b93d41c089029dd503161a7d3e685d59a9c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 6a45baf86fcc6c294d57e0aef8c9f2c54f07ff18 @@ -294,7 +294,7 @@ F src/vdbeInt.h dc1743de339f5556cc6687219cf8727ad0d35f72 F src/vdbeapi.c 4d2aa56efa1b4a010012466bf8e97dbf179081a6 F src/vdbeaux.c 211ad29d51e01c44a0db1ab69b74c11c8de1cccf F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 18556fc614426886e380def839bdcf9cadbb752a +F src/vdbemem.c 9615c9aba37ec42bd6ea705d3d72379c77720b00 F src/vdbesort.c 09efa5e5098d1a159cd21f588eb118e4fe87cfde F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f73678038d8fc399c1ca55230ae45962007c909c -R 39f3c2bf48f680cf4e99aef8c3698c28 +P 78fb8838d80b229418c347c63579989432e1af7d +R 386107f24552e12c4df7dfd8a184a53a U drh -Z e586af33404759487ed3f6653fe697fa +Z 90927a6d3c50834bddf5d6b0b977d8d6 diff --git a/manifest.uuid b/manifest.uuid index 0d00de3592..1ed225fb04 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -78fb8838d80b229418c347c63579989432e1af7d \ No newline at end of file +2598aedc5dd2bac67e2e518a31f2803e469c2ba6 \ No newline at end of file diff --git a/src/utf.c b/src/utf.c index 557f3a95e4..549983f6f1 100644 --- a/src/utf.c +++ b/src/utf.c @@ -314,10 +314,10 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ *z = 0; assert( (pMem->n+(desiredEnc==SQLITE_UTF8?1:2))<=len ); + c = pMem->flags; sqlite3VdbeMemRelease(pMem); - pMem->flags &= ~(MEM_Static|MEM_Dyn|MEM_Ephem); + pMem->flags = MEM_Str|MEM_Term|(c&MEM_AffMask); pMem->enc = desiredEnc; - pMem->flags |= (MEM_Term); pMem->z = (char*)zOut; pMem->zMalloc = pMem->z; diff --git a/src/vdbemem.c b/src/vdbemem.c index c75046ab87..27a2301a9d 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -335,8 +335,11 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ }else if( p->flags&MEM_RowSet ){ sqlite3RowSetClear(p->u.pRowSet); }else if( p->flags&MEM_Frame ){ - sqlite3VdbeMemSetNull(p); + VdbeFrame *pFrame = p->u.pFrame; + pFrame->pParent = pFrame->v->pDelFrame; + pFrame->v->pDelFrame = pFrame; } + p->flags = MEM_Null; } /* @@ -589,15 +592,11 @@ void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ ** Delete any previous value and set the value stored in *pMem to NULL. */ void sqlite3VdbeMemSetNull(Mem *pMem){ - if( pMem->flags & MEM_Frame ){ - VdbeFrame *pFrame = pMem->u.pFrame; - pFrame->pParent = pFrame->v->pDelFrame; - pFrame->v->pDelFrame = pFrame; + if( VdbeMemDynamic(pMem) ){ + sqlite3VdbeMemReleaseExternal(pMem); + }else{ + pMem->flags = MEM_Null; } - if( pMem->flags & MEM_RowSet ){ - sqlite3RowSetClear(pMem->u.pRowSet); - } - MemSetTypeFlag(pMem, MEM_Null); } void sqlite3ValueSetNull(sqlite3_value *p){ sqlite3VdbeMemSetNull((Mem*)p); From 0725cabe3a807b1fe8065ef3689dcfb8b4242265 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 17 Sep 2014 14:52:46 +0000 Subject: [PATCH 340/710] Improved interface to the Mem object handling. Small size reduction and performance increase. FossilOrigin-Name: 4e437844322cc20eef92928b53fa6b37eded586e --- manifest | 18 +++++------ manifest.uuid | 2 +- src/vdbe.c | 22 +++++++------- src/vdbeInt.h | 3 -- src/vdbeapi.c | 3 +- src/vdbemem.c | 83 +++++++++++++++++++++++++-------------------------- 6 files changed, 63 insertions(+), 68 deletions(-) diff --git a/manifest b/manifest index 0c48965589..56db1ec750 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Continuing\scleanup\sof\smemory\sregister\smemory\sallocation\shandling. -D 2014-09-16T21:54:11.409 +C Improved\sinterface\sto\sthe\sMem\sobject\shandling.\s\sSmall\ssize\sreduction\sand\nperformance\sincrease. +D 2014-09-17T14:52:46.727 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,13 +288,13 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c 8f634b93d41c089029dd503161a7d3e685d59a9c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 6a45baf86fcc6c294d57e0aef8c9f2c54f07ff18 +F src/vdbe.c d39487782c0c6a2448bd1b351eba6a2dce101343 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h dc1743de339f5556cc6687219cf8727ad0d35f72 -F src/vdbeapi.c 4d2aa56efa1b4a010012466bf8e97dbf179081a6 +F src/vdbeInt.h 9d398055c873980b61ce0f7bf82140a8e4e2ccbc +F src/vdbeapi.c c6e63f913bcb12977731a8b12e7e5c5762981527 F src/vdbeaux.c 211ad29d51e01c44a0db1ab69b74c11c8de1cccf F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 9615c9aba37ec42bd6ea705d3d72379c77720b00 +F src/vdbemem.c cf552a404f0e73a48bd266699aa27f0df26b096b F src/vdbesort.c 09efa5e5098d1a159cd21f588eb118e4fe87cfde F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 78fb8838d80b229418c347c63579989432e1af7d -R 386107f24552e12c4df7dfd8a184a53a +P 2598aedc5dd2bac67e2e518a31f2803e469c2ba6 +R f63e98cc1bcae44946c79c20cdee7610 U drh -Z 90927a6d3c50834bddf5d6b0b977d8d6 +Z 1559702e8c6b5218f324b4f43d7dc2c7 diff --git a/manifest.uuid b/manifest.uuid index 1ed225fb04..52a893ba83 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2598aedc5dd2bac67e2e518a31f2803e469c2ba6 \ No newline at end of file +4e437844322cc20eef92928b53fa6b37eded586e \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 4bdfbfbdeb..edccc1c601 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -640,7 +640,7 @@ int sqlite3VdbeExec( assert( pOp->p2<=(p->nMem-p->nCursor) ); pOut = &aMem[pOp->p2]; memAboutToChange(p, pOut); - VdbeMemReleaseExtern(pOut); + if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut); pOut->flags = MEM_Int; } @@ -1079,7 +1079,7 @@ case OP_Null: { /* out2-prerelease */ while( cnt>0 ){ pOut++; memAboutToChange(p, pOut); - VdbeMemReleaseExtern(pOut); + sqlite3VdbeMemSetNull(pOut); pOut->flags = nullFlag; cnt--; } @@ -2107,10 +2107,10 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ case OP_Not: { /* same as TK_NOT, in1, out2 */ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; - if( pIn1->flags & MEM_Null ){ - sqlite3VdbeMemSetNull(pOut); - }else{ - sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeIntValue(pIn1)); + sqlite3VdbeMemSetNull(pOut); + if( (pIn1->flags & MEM_Null)==0 ){ + pOut->flags = MEM_Int; + pOut->u.i = !sqlite3VdbeIntValue(pIn1); } break; } @@ -2125,10 +2125,10 @@ case OP_Not: { /* same as TK_NOT, in1, out2 */ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; - if( pIn1->flags & MEM_Null ){ - sqlite3VdbeMemSetNull(pOut); - }else{ - sqlite3VdbeMemSetInt64(pOut, ~sqlite3VdbeIntValue(pIn1)); + sqlite3VdbeMemSetNull(pOut); + if( (pIn1->flags & MEM_Null)==0 ){ + pOut->flags = MEM_Int; + pOut->u.i = ~sqlite3VdbeIntValue(pIn1); } break; } @@ -2437,7 +2437,7 @@ case OP_Column: { assert( p2nHdrParsed ); assert( rc==SQLITE_OK ); assert( sqlite3VdbeCheckMemInvariants(pDest) ); - VdbeMemReleaseExtern(pDest); + if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest); if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 14ae09130c..556dfa16c7 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -429,11 +429,8 @@ int sqlite3VdbeMemNumerify(Mem*); void sqlite3VdbeMemCast(Mem*,u8,u8); int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,int,Mem*); void sqlite3VdbeMemRelease(Mem *p); -void sqlite3VdbeMemReleaseExternal(Mem *p); #define VdbeMemDynamic(X) \ (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) -#define VdbeMemReleaseExtern(X) \ - if( VdbeMemDynamic(X) ) sqlite3VdbeMemReleaseExternal(X); int sqlite3VdbeMemFinalize(Mem*, FuncDef*); const char *sqlite3OpcodeName(int); int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 14d6b8412e..dbfabebbf0 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -662,8 +662,7 @@ static SQLITE_NOINLINE void *createAggContext(sqlite3_context *p, int nByte){ Mem *pMem = p->pMem; assert( (pMem->flags & MEM_Agg)==0 ); if( nByte<=0 ){ - sqlite3VdbeMemReleaseExternal(pMem); - pMem->flags = MEM_Null; + sqlite3VdbeMemSetNull(pMem); pMem->z = 0; }else{ sqlite3VdbeMemGrow(pMem, nByte, 0); diff --git a/src/vdbemem.c b/src/vdbemem.c index 27a2301a9d..e1b4e4949f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -48,7 +48,6 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){ ((p->flags&MEM_Static)!=0 ? 1 : 0) == 1 ); } - return 1; } #endif @@ -121,9 +120,8 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); } if( pMem->zMalloc==0 ){ - VdbeMemReleaseExtern(pMem); + sqlite3VdbeMemSetNull(pMem); pMem->z = 0; - pMem->flags = MEM_Null; return SQLITE_NOMEM; } } @@ -311,23 +309,22 @@ int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ /* ** If the memory cell contains a value that must be freed by -** invoking an external callback, then free it now. +** invoking the external callback in Mem.xDel, then this routine +** will free that value. It also sets Mem.flags to MEM_Null. ** -** This routine does NOT do any of the following: -** (1) Set the Mem.flags field to a rational value. -** (2) Free memory held by Mem.zMalloc -** The caller is expected to take care of setting Mem.flags appropriately. -** -** The VdbeMemReleaseExtern() macro invokes this routine if only if there -** is work for this routine to do. +** This is a helper routine for sqlite3VdbeMemSetNull() and +** for sqlite3VdbeMemRelease(). Use those other routines as the +** entry point for releasing Mem resources. */ -void sqlite3VdbeMemReleaseExternal(Mem *p){ +static SQLITE_NOINLINE void vdbeMemClearExternAndSetNull(Mem *p){ assert( p->db==0 || sqlite3_mutex_held(p->db->mutex) ); + assert( VdbeMemDynamic(p) ); if( p->flags&MEM_Agg ){ sqlite3VdbeMemFinalize(p, p->u.pDef); assert( (p->flags & MEM_Agg)==0 ); - sqlite3VdbeMemRelease(p); - }else if( p->flags&MEM_Dyn ){ + testcase( p->flags & MEM_Dyn ); + } + if( p->flags&MEM_Dyn ){ assert( (p->flags&MEM_RowSet)==0 ); assert( p->xDel!=SQLITE_DYNAMIC && p->xDel!=0 ); p->xDel((void *)p->z); @@ -347,12 +344,12 @@ void sqlite3VdbeMemReleaseExternal(Mem *p){ ** by p->xDel and memory in p->zMalloc. ** ** This is a helper routine invoked by sqlite3VdbeMemRelease() in -** the uncommon case when there really is memory in p that needs -** to be freeing. +** the unusual case where there really is memory in p that needs +** to be freed. */ -static SQLITE_NOINLINE void vdbeMemRelease(Mem *p){ +static SQLITE_NOINLINE void vdbeMemClear(Mem *p){ if( VdbeMemDynamic(p) ){ - sqlite3VdbeMemReleaseExternal(p); + vdbeMemClearExternAndSetNull(p); } if( p->zMalloc ){ sqlite3DbFree(p->db, p->zMalloc); @@ -362,18 +359,19 @@ static SQLITE_NOINLINE void vdbeMemRelease(Mem *p){ } /* -** Release any memory held by the Mem. This may leave the Mem.flags in an -** inconsistent state, for example with (Mem.z==0) and (Mem.flags==MEM_Str). +** Release any memory resources held by the Mem. Both the memory that is +** free by Mem.xDel and the Mem.zMalloc allocation are freed. ** -** This routine releases both the Mem.xDel space and the Mem.zMalloc space. -** Use sqlite3VdbeMemReleaseExternal() to release just the Mem.xDel space. +** Use this routine prior to clean up prior to abandoning a Mem, or to +** reset a Mem back to its minimum memory utilization. +** +** Use sqlite3VdbeMemSetNull() to release just the Mem.xDel space +** prior to inserting new content into the Mem. */ void sqlite3VdbeMemRelease(Mem *p){ assert( sqlite3VdbeCheckMemInvariants(p) ); if( VdbeMemDynamic(p) || p->zMalloc ){ - vdbeMemRelease(p); - }else{ - p->z = 0; + vdbeMemClear(p); } assert( p->xDel==0 ); } @@ -590,10 +588,19 @@ void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ /* ** Delete any previous value and set the value stored in *pMem to NULL. +** +** This routine calls the Mem.xDel destructor to dispose of values that +** require the destructor. But it preserves the Mem.zMalloc memory allocation. +** To free all resources, use sqlite3VdbeMemRelease(), which both calls this +** routine to invoke the destructor and deallocates Mem.zMalloc. +** +** Use this routine to reset the Mem prior to insert a new value. +** +** Use sqlite3VdbeMemRelease() to complete erase the Mem prior to abandoning it. */ void sqlite3VdbeMemSetNull(Mem *pMem){ if( VdbeMemDynamic(pMem) ){ - sqlite3VdbeMemReleaseExternal(pMem); + vdbeMemClearExternAndSetNull(pMem); }else{ pMem->flags = MEM_Null; } @@ -613,14 +620,7 @@ void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ if( n<0 ) n = 0; pMem->u.nZero = n; pMem->enc = SQLITE_UTF8; - -#ifdef SQLITE_OMIT_INCRBLOB - sqlite3VdbeMemGrow(pMem, n, 0); - if( pMem->z ){ - pMem->n = n; - memset(pMem->z, 0, n); - } -#endif + pMem->z = 0; } /* @@ -629,7 +629,7 @@ void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ ** a 64-bit integer. */ static SQLITE_NOINLINE void vdbeReleaseAndSetInt64(Mem *pMem, i64 val){ - sqlite3VdbeMemReleaseExternal(pMem); + sqlite3VdbeMemSetNull(pMem); pMem->u.i = val; pMem->flags = MEM_Int; } @@ -653,10 +653,8 @@ void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ ** manifest type REAL. */ void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ - if( sqlite3IsNaN(val) ){ - sqlite3VdbeMemSetNull(pMem); - }else{ - sqlite3VdbeMemRelease(pMem); + sqlite3VdbeMemSetNull(pMem); + if( !sqlite3IsNaN(val) ){ pMem->r = val; pMem->flags = MEM_Real; } @@ -736,7 +734,7 @@ void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( (pFrom->flags & MEM_RowSet)==0 ); assert( pTo->db==pFrom->db ); - VdbeMemReleaseExtern(pTo); + if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); assert( pTo->xDel==0 ); if( (pFrom->flags&MEM_Static)==0 ){ @@ -755,7 +753,7 @@ int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ assert( pTo->db==pFrom->db ); assert( (pFrom->flags & MEM_RowSet)==0 ); - VdbeMemReleaseExtern(pTo); + if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; assert( pTo->xDel==0 ); @@ -832,7 +830,8 @@ int sqlite3VdbeMemSetStr( if( nByte<0 ){ assert( enc!=0 ); if( enc==SQLITE_UTF8 ){ - for(nByte=0; nByte<=iLimit && z[nByte]; nByte++){} + nByte = sqlite3Strlen30(z); + if( nByte>iLimit ) nByte = iLimit+1; }else{ for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} } From d3b74200a871af8ac8e4b36a84eafe658aa0aa12 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 17 Sep 2014 16:41:15 +0000 Subject: [PATCH 341/710] In the Mem object, stop requiring that Mem.xDel be NULL when the MEM_Dyn bit is clear. Also reduce the amount of initialization of Mem objects. All for a small size reduction and performance increase. FossilOrigin-Name: fdddb477c89dabb9f7bf2d5ccb32534868df3a03 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/vdbe.c | 12 +++--------- src/vdbeInt.h | 7 ++++--- src/vdbeaux.c | 9 ++++----- src/vdbemem.c | 30 ++++++++++++++++-------------- 6 files changed, 37 insertions(+), 41 deletions(-) diff --git a/manifest b/manifest index 56db1ec750..369f641908 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sinterface\sto\sthe\sMem\sobject\shandling.\s\sSmall\ssize\sreduction\sand\nperformance\sincrease. -D 2014-09-17T14:52:46.727 +C In\sthe\sMem\sobject,\sstop\srequiring\sthat\sMem.xDel\sbe\sNULL\swhen\sthe\sMEM_Dyn\nbit\sis\sclear.\s\sAlso\sreduce\sthe\samount\sof\sinitialization\sof\sMem\sobjects.\nAll\sfor\sa\ssmall\ssize\sreduction\sand\sperformance\sincrease. +D 2014-09-17T16:41:15.094 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,13 +288,13 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c 8f634b93d41c089029dd503161a7d3e685d59a9c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c d39487782c0c6a2448bd1b351eba6a2dce101343 +F src/vdbe.c d3c548ad4ea6a5549d2d0b502070e0523023ff97 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h 9d398055c873980b61ce0f7bf82140a8e4e2ccbc +F src/vdbeInt.h f90b0de6153f50de630a5a113537efb47083812f F src/vdbeapi.c c6e63f913bcb12977731a8b12e7e5c5762981527 -F src/vdbeaux.c 211ad29d51e01c44a0db1ab69b74c11c8de1cccf +F src/vdbeaux.c 9ac63bc59d2783df77e591e4c4fa8c1153a07eab F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c cf552a404f0e73a48bd266699aa27f0df26b096b +F src/vdbemem.c 8b5e1083fed2da94e315858a7edf5604a5b91804 F src/vdbesort.c 09efa5e5098d1a159cd21f588eb118e4fe87cfde F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2598aedc5dd2bac67e2e518a31f2803e469c2ba6 -R f63e98cc1bcae44946c79c20cdee7610 +P 4e437844322cc20eef92928b53fa6b37eded586e +R 8244691fd19236cfb5e445e4012669fa U drh -Z 1559702e8c6b5218f324b4f43d7dc2c7 +Z ef23a1637c5e3046b6cead80453b022e diff --git a/manifest.uuid b/manifest.uuid index 52a893ba83..2cf0d22ed7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e437844322cc20eef92928b53fa6b37eded586e \ No newline at end of file +fdddb477c89dabb9f7bf2d5ccb32534868df3a03 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index edccc1c601..5ab14c5cbf 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1174,7 +1174,6 @@ case OP_Move: { } #endif pIn1->flags = MEM_Undefined; - pIn1->xDel = 0; pIn1->zMalloc = zMalloc; REGISTER_TRACE(p2++, pOut); pIn1++; @@ -2645,7 +2644,6 @@ case OP_MakeRecord: { assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pOut->n = (int)nByte; pOut->flags = MEM_Blob; - pOut->xDel = 0; if( nZero ){ pOut->u.nZero = nZero; pOut->flags |= MEM_Zero; @@ -4755,7 +4753,7 @@ case OP_IdxRowid: { /* out2-prerelease */ assert( pC->isTable==0 ); if( !pC->nullRow ){ rowid = 0; /* Not needed. Only used to silence a warning. */ - rc = sqlite3VdbeIdxRowid(pCrsr, &rowid); + rc = sqlite3VdbeIdxRowid(db, pCrsr, &rowid); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } @@ -4839,7 +4837,7 @@ case OP_IdxGE: { /* jump */ { int i; for(i=0; iopcode&1)==(OP_IdxLT&1) ){ assert( pOp->opcode==OP_IdxLE || pOp->opcode==OP_IdxLT ); @@ -5609,11 +5607,7 @@ case OP_AggStep: { assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); ctx.pMem = pMem = &aMem[pOp->p3]; pMem->n++; - t.flags = MEM_Null; - t.z = 0; - t.zMalloc = 0; - t.xDel = 0; - t.db = db; + sqlite3VdbeMemInit(&t, db, MEM_Null); ctx.pOut = &t; ctx.isError = 0; ctx.pColl = 0; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 556dfa16c7..9c7378a26c 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -176,7 +176,7 @@ struct Mem { char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ /* ShallowCopy only needs to copy the information above */ sqlite3 *db; /* The associated database connection */ - void (*xDel)(void*);/* If not null, call this function to delete Mem.z */ + void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */ #ifdef SQLITE_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ @@ -397,8 +397,8 @@ u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); -int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); -int sqlite3VdbeIdxRowid(BtCursor *, i64 *); +int sqlite3VdbeIdxKeyCompare(sqlite3*,VdbeCursor*,UnpackedRecord*,int*); +int sqlite3VdbeIdxRowid(sqlite3*, BtCursor*, i64*); int sqlite3VdbeExec(Vdbe*); int sqlite3VdbeList(Vdbe*); int sqlite3VdbeHalt(Vdbe*); @@ -415,6 +415,7 @@ void sqlite3VdbeMemSetInt64(Mem*, i64); #else void sqlite3VdbeMemSetDouble(Mem*, double); #endif +void sqlite3VdbeMemInit(Mem*,sqlite3*,u16); void sqlite3VdbeMemSetNull(Mem*); void sqlite3VdbeMemSetZeroBlob(Mem*,int); void sqlite3VdbeMemSetRowSet(Mem*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 10495e76e4..8466bfb30a 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3085,7 +3085,6 @@ u32 sqlite3VdbeSerialGet( static const u16 aFlag[] = { MEM_Blob|MEM_Ephem, MEM_Str|MEM_Ephem }; pMem->z = (char *)buf; pMem->n = (serial_type-12)/2; - pMem->xDel = 0; pMem->flags = aFlag[serial_type&1]; return pMem->n; } @@ -3306,7 +3305,6 @@ static int vdbeCompareMemString( c1.db = c2.db = pMem1->db; c1.flags = c2.flags = 0; c1.zMalloc = c2.zMalloc = 0; - c1.xDel = c2.xDel = 0; sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); @@ -3877,7 +3875,7 @@ RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ ** pCur might be pointing to text obtained from a corrupt database file. ** So the content cannot be trusted. Do appropriate checks on the content. */ -int sqlite3VdbeIdxRowid(BtCursor *pCur, i64 *rowid){ +int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ i64 nCellKey = 0; int rc; u32 szHdr; /* Size of the header */ @@ -3896,7 +3894,7 @@ int sqlite3VdbeIdxRowid(BtCursor *pCur, i64 *rowid){ assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey ); /* Read in the complete content of the index entry */ - memset(&m, 0, sizeof(m)); + sqlite3VdbeMemInit(&m, db, 0); rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, 1, &m); if( rc ){ return rc; @@ -3956,6 +3954,7 @@ idx_rowid_corruption: ** of the keys prior to the final rowid, not the entire key. */ int sqlite3VdbeIdxKeyCompare( + sqlite3 *db, /* Database connection */ VdbeCursor *pC, /* The cursor to compare against */ UnpackedRecord *pUnpacked, /* Unpacked version of key */ int *res /* Write the comparison result here */ @@ -3974,7 +3973,7 @@ int sqlite3VdbeIdxKeyCompare( *res = 0; return SQLITE_CORRUPT_BKPT; } - memset(&m, 0, sizeof(m)); + sqlite3VdbeMemInit(&m, db, 0); rc = sqlite3VdbeMemFromBtree(pC->pCursor, 0, (u32)nCellKey, 1, &m); if( rc ){ return rc; diff --git a/src/vdbemem.c b/src/vdbemem.c index e1b4e4949f..359abb891f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -26,11 +26,10 @@ ** this: assert( sqlite3VdbeCheckMemInvariants(pMem) ); */ int sqlite3VdbeCheckMemInvariants(Mem *p){ - /* The MEM_Dyn bit is set if and only if Mem.xDel is a non-NULL destructor - ** function for Mem.z + /* If MEM_Dyn is set then Mem.xDel!=0. + ** Mem.xDel is might not be initialized if MEM_Dyn is clear. */ assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 ); - assert( (p->flags & MEM_Dyn)!=0 || p->xDel==0 ); /* If p holds a string or blob, the Mem.z must point to exactly ** one of the following: @@ -136,7 +135,6 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ pMem->z = pMem->zMalloc; pMem->flags &= ~(MEM_Dyn|MEM_Ephem|MEM_Static); - pMem->xDel = 0; return SQLITE_OK; } @@ -299,7 +297,7 @@ int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ ctx.pMem = pMem; ctx.pFunc = pFunc; pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ - assert( 0==(pMem->flags&MEM_Dyn) && !pMem->xDel ); + assert( (pMem->flags & MEM_Dyn)==0 ); sqlite3DbFree(pMem->db, pMem->zMalloc); memcpy(pMem, &t, sizeof(t)); rc = ctx.isError; @@ -328,7 +326,6 @@ static SQLITE_NOINLINE void vdbeMemClearExternAndSetNull(Mem *p){ assert( (p->flags&MEM_RowSet)==0 ); assert( p->xDel!=SQLITE_DYNAMIC && p->xDel!=0 ); p->xDel((void *)p->z); - p->xDel = 0; }else if( p->flags&MEM_RowSet ){ sqlite3RowSetClear(p->u.pRowSet); }else if( p->flags&MEM_Frame ){ @@ -373,7 +370,6 @@ void sqlite3VdbeMemRelease(Mem *p){ if( VdbeMemDynamic(p) || p->zMalloc ){ vdbeMemClear(p); } - assert( p->xDel==0 ); } /* @@ -585,6 +581,18 @@ void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ } } +/* +** Initialize bulk memory to be a consistent Mem object. +** +** The minimum amount of initialization feasible is performed. +*/ +void sqlite3VdbeMemInit(Mem *pMem, sqlite3 *db, u16 flags){ + assert( (flags & ~MEM_TypeMask)==0 ); + pMem->flags = flags; + pMem->db = db; + pMem->zMalloc = 0; +} + /* ** Delete any previous value and set the value stored in *pMem to NULL. @@ -736,7 +744,6 @@ void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ assert( pTo->db==pFrom->db ); if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); - assert( pTo->xDel==0 ); if( (pFrom->flags&MEM_Static)==0 ){ pTo->flags &= ~(MEM_Dyn|MEM_Static|MEM_Ephem); assert( srcType==MEM_Ephem || srcType==MEM_Static ); @@ -756,8 +763,6 @@ int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; - assert( pTo->xDel==0 ); - if( pTo->flags&(MEM_Str|MEM_Blob) ){ if( 0==(pFrom->flags&MEM_Static) ){ pTo->flags |= MEM_Ephem; @@ -782,7 +787,6 @@ void sqlite3VdbeMemMove(Mem *pTo, Mem *pFrom){ sqlite3VdbeMemRelease(pTo); memcpy(pTo, pFrom, sizeof(Mem)); pFrom->flags = MEM_Null; - pFrom->xDel = 0; pFrom->zMalloc = 0; } @@ -857,7 +861,6 @@ int sqlite3VdbeMemSetStr( }else if( xDel==SQLITE_DYNAMIC ){ sqlite3VdbeMemRelease(pMem); pMem->zMalloc = pMem->z = (char *)z; - pMem->xDel = 0; }else{ sqlite3VdbeMemRelease(pMem); pMem->z = (char *)z; @@ -910,7 +913,7 @@ int sqlite3VdbeMemFromBtree( int rc = SQLITE_OK; /* Return code */ assert( sqlite3BtreeCursorIsValid(pCur) ); - assert( pMem->xDel==0 ); + assert( !VdbeMemDynamic(pMem) ); /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() ** that both the BtShared and database handle mutexes are held. */ @@ -923,7 +926,6 @@ int sqlite3VdbeMemFromBtree( assert( zData!=0 ); if( offset+amt<=available ){ - sqlite3VdbeMemRelease(pMem); pMem->z = &zData[offset]; pMem->flags = MEM_Blob|MEM_Ephem; pMem->n = (int)amt; From ca5506bdc482bbfde2fe3dd6a93f079e9801ce60 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 17 Sep 2014 23:37:38 +0000 Subject: [PATCH 342/710] Performance improvement and slight size reduction to the comparison operators in the VDBE. FossilOrigin-Name: 14052a7d088bed8196d90a3361ce717a5193bdc8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 14 ++++++++++---- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 369f641908..12094ebf72 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sMem\sobject,\sstop\srequiring\sthat\sMem.xDel\sbe\sNULL\swhen\sthe\sMEM_Dyn\nbit\sis\sclear.\s\sAlso\sreduce\sthe\samount\sof\sinitialization\sof\sMem\sobjects.\nAll\sfor\sa\ssmall\ssize\sreduction\sand\sperformance\sincrease. -D 2014-09-17T16:41:15.094 +C Performance\simprovement\sand\sslight\ssize\sreduction\sto\sthe\scomparison\soperators\nin\sthe\sVDBE. +D 2014-09-17T23:37:39.000 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,7 +288,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c 8f634b93d41c089029dd503161a7d3e685d59a9c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c d3c548ad4ea6a5549d2d0b502070e0523023ff97 +F src/vdbe.c 78606777e4ce5dba147ab75e71c0127b0d8d4c3d F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f90b0de6153f50de630a5a113537efb47083812f F src/vdbeapi.c c6e63f913bcb12977731a8b12e7e5c5762981527 @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4e437844322cc20eef92928b53fa6b37eded586e -R 8244691fd19236cfb5e445e4012669fa +P fdddb477c89dabb9f7bf2d5ccb32534868df3a03 +R 27b98108123b0758f30cbbc4e2693ca1 U drh -Z ef23a1637c5e3046b6cead80453b022e +Z 7dd9c2f8b2084fea7b27f362515d2b5a diff --git a/manifest.uuid b/manifest.uuid index 2cf0d22ed7..2a5acf5d23 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fdddb477c89dabb9f7bf2d5ccb32534868df3a03 \ No newline at end of file +14052a7d088bed8196d90a3361ce717a5193bdc8 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 5ab14c5cbf..f964a73878 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1911,8 +1911,14 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ } assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); - ExpandBlob(pIn1); - ExpandBlob(pIn3); + if( pIn1->flags & MEM_Zero ){ + sqlite3VdbeMemExpandBlob(pIn1); + flags1 &= ~MEM_Zero; + } + if( pIn3->flags & MEM_Zero ){ + sqlite3VdbeMemExpandBlob(pIn3); + flags3 &= ~MEM_Zero; + } res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); } switch( pOp->opcode ){ @@ -1937,8 +1943,8 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ } } /* Undo any changes made by applyAffinity() to the input registers. */ - pIn1->flags = (pIn1->flags&~MEM_TypeMask) | (flags1&MEM_TypeMask); - pIn3->flags = (pIn3->flags&~MEM_TypeMask) | (flags3&MEM_TypeMask); + pIn1->flags = flags1; + pIn3->flags = flags3; break; } From 3329a63ac5ca50861d926248b159106cd9fd96a5 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Sep 2014 01:21:43 +0000 Subject: [PATCH 343/710] Fix compiler warnings and change the nullMem structure initializer into a format that MSVC can understand. FossilOrigin-Name: 163bfae8583b2d3002a3a43d6bf8a66fefd73acb --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/malloc.c | 8 ++++---- src/table.c | 2 +- src/vdbeapi.c | 26 +++++++++++++------------- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 12094ebf72..a6cdd2c40c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovement\sand\sslight\ssize\sreduction\sto\sthe\scomparison\soperators\nin\sthe\sVDBE. -D 2014-09-17T23:37:39.000 +C Fix\scompiler\swarnings\sand\schange\sthe\snullMem\sstructure\sinitializer\sinto\sa\nformat\sthat\sMSVC\scan\sunderstand. +D 2014-09-18T01:21:43.977 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 F src/main.c d15621461fb0c52675eba2b650492ed1beef69ab -F src/malloc.c cc015821ba267ad5c91dc8761d0498a3fc3ce6ce +F src/malloc.c c6dc1e154eb7d4a33eb737bde2c4103d00007597 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f @@ -234,7 +234,7 @@ F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 0803e900eb1882f7dd88e86ddcddd2d1b27c8d86 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 -F src/table.c 218ae2ba022881846741dfc8351aefdf129e0377 +F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 F src/test1.c 523cd70ded28db71af9a30ec184cbe0957de9575 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 @@ -291,7 +291,7 @@ F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 78606777e4ce5dba147ab75e71c0127b0d8d4c3d F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f90b0de6153f50de630a5a113537efb47083812f -F src/vdbeapi.c c6e63f913bcb12977731a8b12e7e5c5762981527 +F src/vdbeapi.c c02242df5e9e8d1001e0086f405953833f9c426b F src/vdbeaux.c 9ac63bc59d2783df77e591e4c4fa8c1153a07eab F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 8b5e1083fed2da94e315858a7edf5604a5b91804 @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fdddb477c89dabb9f7bf2d5ccb32534868df3a03 -R 27b98108123b0758f30cbbc4e2693ca1 +P 14052a7d088bed8196d90a3361ce717a5193bdc8 +R fdb25cd80d01110b7b302f5203339ac7 U drh -Z 7dd9c2f8b2084fea7b27f362515d2b5a +Z 3225a18fbab1063469f56e7e3c1f239a diff --git a/manifest.uuid b/manifest.uuid index 2a5acf5d23..6cec555758 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -14052a7d088bed8196d90a3361ce717a5193bdc8 \ No newline at end of file +163bfae8583b2d3002a3a43d6bf8a66fefd73acb \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index daf646bc30..a4acd2f0ce 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -305,7 +305,7 @@ void *sqlite3Malloc(u64 n){ p = 0; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); - mallocWithAlarm(n, &p); + mallocWithAlarm((int)n, &p); sqlite3_mutex_leave(mem0.mutex); }else{ p = sqlite3GlobalConfig.m.xMalloc((int)n); @@ -549,7 +549,7 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){ pNew = pOld; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, nBytes); + sqlite3StatusSet(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); nDiff = nNew - nOld; if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= mem0.alarmThreshold-nDiff ){ @@ -559,7 +559,7 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){ assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); if( pNew==0 && mem0.alarmCallback ){ - sqlite3MallocAlarm(nBytes); + sqlite3MallocAlarm((int)nBytes); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } if( pNew ){ @@ -699,7 +699,7 @@ void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - pNew = sqlite3_realloc(p, n); + pNew = sqlite3_realloc64(p, n); if( !pNew ){ sqlite3MemdebugSetType(p, MEMTYPE_DB|MEMTYPE_HEAP); db->mallocFailed = 1; diff --git a/src/table.c b/src/table.c index 12d0cf548e..c435b2bc02 100644 --- a/src/table.c +++ b/src/table.c @@ -73,7 +73,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ if( z==0 ) goto malloc_failed; p->azResult[p->nData++] = z; } - }else if( p->nColumn!=nCol ){ + }else if( (int)p->nColumn!=nCol ){ sqlite3_free(p->zErrMsg); p->zErrMsg = sqlite3_mprintf( "sqlite3_get_table() called with two or more incompatible queries" diff --git a/src/vdbeapi.c b/src/vdbeapi.c index dbfabebbf0..ef1167a52d 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -803,18 +803,18 @@ static const Mem *columnNullValue(void){ __attribute__((aligned(8))) #endif = { - .flags = MEM_Null, - .enc = 0, - .n = 0, - .r = (double)0, - .u = {0}, - .z = 0, - .zMalloc = 0, - .db = 0, - .xDel = 0, + /* .u = */ {0}, + /* .flags = */ MEM_Null, + /* .enc = */ 0, + /* .n = */ 0, + /* .r = */ (double)0, + /* .z = */ 0, + /* .zMalloc = */ 0, + /* .db = */ 0, + /* .xDel = */ 0, #ifdef SQLITE_DEBUG - .pScopyFrom = 0, - .pFiller = 0, + /* .pScopyFrom = */ 0, + /* .pFiller = */ 0, #endif }; return &nullMem; @@ -1193,7 +1193,7 @@ int sqlite3_bind_blob64( if( nData>0x7fffffff ){ return invokeValueDestructor(zData, xDel, 0); }else{ - return bindText(pStmt, i, zData, nData, xDel, 0); + return bindText(pStmt, i, zData, (int)nData, xDel, 0); } } int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ @@ -1250,7 +1250,7 @@ int sqlite3_bind_text64( return invokeValueDestructor(zData, xDel, 0); }else{ if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; - return bindText(pStmt, i, zData, nData, xDel, enc); + return bindText(pStmt, i, zData, (int)nData, xDel, enc); } } #ifndef SQLITE_OMIT_UTF16 From 760b15984bc16e1e90eb89bc79daa25ba7dd9f40 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Sep 2014 01:50:09 +0000 Subject: [PATCH 344/710] Make sure of the strchrnul() library function on platforms where it is available. FossilOrigin-Name: ef1aa10b7f54912cba71cd0a98c5055d501de54f --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/printf.c | 23 +++++++++++++++++++++-- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d8b46a196d..f1b8ecd79c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\smicro-optimizations\sinto\strunk\safter\sfixing\sthe\sbuild\son\sMSVC.\nPerformance\snow\sshows\s7.58%\sfaster\sthan\sthe\s3.8.6\srelease\son\sx64\swith\ngcc\s4.8.1\sand\s-Os. -D 2014-09-18T01:29:11.287 +C Make\ssure\sof\sthe\sstrchrnul()\slibrary\sfunction\son\splatforms\swhere\sit\sis\navailable. +D 2014-09-18T01:50:09.699 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c e74925089a85e3c9f0e315595f41c139d3d118c2 +F src/printf.c 19e3e81addf593195369ec8d487ed063ad3170bb F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -1198,8 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8fb90da77ce0e662c1ef1ae0d854e5164494b7af 163bfae8583b2d3002a3a43d6bf8a66fefd73acb -R fdb25cd80d01110b7b302f5203339ac7 -T +closed 163bfae8583b2d3002a3a43d6bf8a66fefd73acb +P 1de558bcb13edc4e9a42a0b05e4b0ed6b14286a4 +R 5de5c16595c1a6599607c0d453169480 U drh -Z a578f94c122b57df374e40cc66ef0667 +Z 45b315c664f0110e7c4a00564360b273 diff --git a/manifest.uuid b/manifest.uuid index 5bbe565c7b..c029a6ebc7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1de558bcb13edc4e9a42a0b05e4b0ed6b14286a4 \ No newline at end of file +ef1aa10b7f54912cba71cd0a98c5055d501de54f \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 8e71ad8bc6..03e39085b9 100644 --- a/src/printf.c +++ b/src/printf.c @@ -14,6 +14,21 @@ */ #include "sqliteInt.h" +/* +** If the strchrnul() library function is available, then set +** HAVE_STRCHRNUL. If that routine is not available, this module +** will supply its own. The built-in version is slower than +** the glibc version so the glibc version is definitely preferred. +*/ +#if !defined(HAVE_STRCHRNUL) +# if defined(__linux__) && defined(_GNU_SOURCE) +# define HAVE_STRCHRNUL 1 +# else +# define HAVE_STRCHRNUL 0 +# endif +#endif + + /* ** Conversion types fall into various categories as defined by the ** following enumeration. @@ -224,9 +239,13 @@ void sqlite3VXPrintf( for(; (c=(*fmt))!=0; ++fmt){ if( c!='%' ){ bufpt = (char *)fmt; - while( (c=(*++fmt))!='%' && c!=0 ){}; +#if HAVE_STRCHRNUL + fmt = strchrnul(fmt, '%'); +#else + do{ fmt++; }while( *fmt && *fmt != '%' ); +#endif sqlite3StrAccumAppend(pAccum, bufpt, (int)(fmt - bufpt)); - if( c==0 ) break; + if( *fmt==0 ) break; } if( (c=(*++fmt))==0 ){ sqlite3StrAccumAppend(pAccum, "%", 1); From 20f3df046abb2da2c2030a89bbc23e3ad8b36f21 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Sep 2014 02:20:54 +0000 Subject: [PATCH 345/710] Fix harmless warnings on 32-bit MSVC builds. FossilOrigin-Name: 5192f964b2a85459553f1cea741b9791606ccc4e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/malloc.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f1b8ecd79c..2699c2c1a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sof\sthe\sstrchrnul()\slibrary\sfunction\son\splatforms\swhere\sit\sis\navailable. -D 2014-09-18T01:50:09.699 +C Fix\sharmless\swarnings\son\s32-bit\sMSVC\sbuilds. +D 2014-09-18T02:20:54.689 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 F src/main.c d15621461fb0c52675eba2b650492ed1beef69ab -F src/malloc.c c6dc1e154eb7d4a33eb737bde2c4103d00007597 +F src/malloc.c 4c1d511157defd7b1d023062cf05a1dc17b8f79b F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1de558bcb13edc4e9a42a0b05e4b0ed6b14286a4 -R 5de5c16595c1a6599607c0d453169480 +P ef1aa10b7f54912cba71cd0a98c5055d501de54f +R c34bfcc3b603f9401507c43eeaad2e7b U drh -Z 45b315c664f0110e7c4a00564360b273 +Z e8711818b1e0ba1f2fd0d1d927b89d2c diff --git a/manifest.uuid b/manifest.uuid index c029a6ebc7..290fa02af4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef1aa10b7f54912cba71cd0a98c5055d501de54f \ No newline at end of file +5192f964b2a85459553f1cea741b9791606ccc4e \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index a4acd2f0ce..e0d5b5ff9d 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -599,7 +599,7 @@ void *sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ void *sqlite3MallocZero(u64 n){ void *p = sqlite3Malloc(n); if( p ){ - memset(p, 0, n); + memset(p, 0, (size_t)n); } return p; } @@ -611,7 +611,7 @@ void *sqlite3MallocZero(u64 n){ void *sqlite3DbMallocZero(sqlite3 *db, u64 n){ void *p = sqlite3DbMallocRaw(db, n); if( p ){ - memset(p, 0, n); + memset(p, 0, (size_t)n); } return p; } @@ -753,7 +753,7 @@ char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ assert( (n&0x7fffffff)==n ); zNew = sqlite3DbMallocRaw(db, n+1); if( zNew ){ - memcpy(zNew, z, n); + memcpy(zNew, z, (size_t)n); zNew[n] = 0; } return zNew; From 9675d5dabc180230557a53faac66263a2f52477d Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 18 Sep 2014 09:59:28 +0000 Subject: [PATCH 346/710] Modify an assert() within sqlite3PagerWalFramesize(), a function only ever used by zipvfs, to account for recent zipvfs changes. FossilOrigin-Name: 3bd7c1b2faa2d4cc95b255633204006849bfd5e0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2699c2c1a8..c14922f59d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\swarnings\son\s32-bit\sMSVC\sbuilds. -D 2014-09-18T02:20:54.689 +C Modify\san\sassert()\swithin\ssqlite3PagerWalFramesize(),\sa\sfunction\sonly\sever\sused\sby\szipvfs,\sto\saccount\sfor\srecent\szipvfs\schanges. +D 2014-09-18T09:59:28.052 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c addd023b26c623fec4dedc110fc4370a65b4768c F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c c6c809987f0c6a4e27634099d062d425527de173 +F src/pager.c caab007743821d96752597c9cfd7351654697b06 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef1aa10b7f54912cba71cd0a98c5055d501de54f -R c34bfcc3b603f9401507c43eeaad2e7b -U drh -Z e8711818b1e0ba1f2fd0d1d927b89d2c +P 5192f964b2a85459553f1cea741b9791606ccc4e +R e8be2bb63b11cad540f46beed0dd6c95 +U dan +Z 6d111ffcef80c669aed6101b87cfb783 diff --git a/manifest.uuid b/manifest.uuid index 290fa02af4..273ff9d0ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5192f964b2a85459553f1cea741b9791606ccc4e \ No newline at end of file +3bd7c1b2faa2d4cc95b255633204006849bfd5e0 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 6ad6f73227..79bfe15f10 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7230,7 +7230,7 @@ int sqlite3PagerCloseWal(Pager *pPager){ ** is empty, return 0. */ int sqlite3PagerWalFramesize(Pager *pPager){ - assert( pPager->eState==PAGER_READER ); + assert( pPager->eState>=PAGER_READER ); return sqlite3WalFramesize(pPager->pWal); } #endif From 7ea31ccb015703f08ba29b328f79182c6ee2b557 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Sep 2014 14:36:00 +0000 Subject: [PATCH 347/710] Since numeric affinity is the most common case, check it first. Interchange the NONE and TEXT affinity codes for easier checking of no affinity. FossilOrigin-Name: 4ef4c9a7c8510203bce0941dda2f76ded8da1de2 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/build.c | 12 ++++++------ src/sqliteInt.h | 6 +++--- src/vdbe.c | 22 +++++++++++----------- 5 files changed, 30 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index c14922f59d..00c2f35ee9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\san\sassert()\swithin\ssqlite3PagerWalFramesize(),\sa\sfunction\sonly\sever\sused\sby\szipvfs,\sto\saccount\sfor\srecent\szipvfs\schanges. -D 2014-09-18T09:59:28.052 +C Since\snumeric\saffinity\sis\sthe\smost\scommon\scase,\scheck\sit\sfirst.\s\sInterchange\nthe\sNONE\sand\sTEXT\saffinity\scodes\sfor\seasier\schecking\sof\sno\saffinity. +D 2014-09-18T14:36:00.055 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,7 +174,7 @@ F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c 6aa61c0e3d20d1d1acc8fb33d8f0ebd675305d3c F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc -F src/build.c 047d7e1d2d89fa55134fa1d6b669c9c2983c0d11 +F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958 @@ -231,7 +231,7 @@ F src/shell.c c00220cdd7f2027780bc25b78376c16dc24e4b7d F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 0803e900eb1882f7dd88e86ddcddd2d1b27c8d86 +F src/sqliteInt.h 9bb8f655b076e1b9ed7cfe0b8c181e758d937369 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -288,7 +288,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c 8f634b93d41c089029dd503161a7d3e685d59a9c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 78606777e4ce5dba147ab75e71c0127b0d8d4c3d +F src/vdbe.c 2caa3b1e32f3fddc60fecb2cfd66afe2c2eb96b1 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f90b0de6153f50de630a5a113537efb47083812f F src/vdbeapi.c c02242df5e9e8d1001e0086f405953833f9c426b @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5192f964b2a85459553f1cea741b9791606ccc4e -R e8be2bb63b11cad540f46beed0dd6c95 -U dan -Z 6d111ffcef80c669aed6101b87cfb783 +P 3bd7c1b2faa2d4cc95b255633204006849bfd5e0 +R c1df8072077a2642e4ce7426502f62dc +U drh +Z 04ecbaaf95d3fdde0c1a120923b44258 diff --git a/manifest.uuid b/manifest.uuid index 273ff9d0ee..2c7c5a518e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3bd7c1b2faa2d4cc95b255633204006849bfd5e0 \ No newline at end of file +4ef4c9a7c8510203bce0941dda2f76ded8da1de2 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 791f6f2033..0c02a56fe7 100644 --- a/src/build.c +++ b/src/build.c @@ -1177,7 +1177,7 @@ char sqlite3AffinityType(const char *zIn, u8 *pszEst){ ** estimate is scaled so that the size of an integer is 1. */ if( pszEst ){ *pszEst = 1; /* default size is approx 4 bytes */ - if( aff<=SQLITE_AFF_NONE ){ + if( affaCol, i=0; inCol; i++, pCol++){ static const char * const azType[] = { - /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NONE */ "", + /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NUMERIC */ " NUM", /* SQLITE_AFF_INTEGER */ " INT", /* SQLITE_AFF_REAL */ " REAL" @@ -1561,15 +1561,15 @@ static char *createTableStmt(sqlite3 *db, Table *p){ k += sqlite3Strlen30(&zStmt[k]); zSep = zSep2; identPut(zStmt, &k, pCol->zName); - assert( pCol->affinity-SQLITE_AFF_TEXT >= 0 ); - assert( pCol->affinity-SQLITE_AFF_TEXT < ArraySize(azType) ); - testcase( pCol->affinity==SQLITE_AFF_TEXT ); + assert( pCol->affinity-SQLITE_AFF_NONE >= 0 ); + assert( pCol->affinity-SQLITE_AFF_NONE < ArraySize(azType) ); testcase( pCol->affinity==SQLITE_AFF_NONE ); + testcase( pCol->affinity==SQLITE_AFF_TEXT ); testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); testcase( pCol->affinity==SQLITE_AFF_INTEGER ); testcase( pCol->affinity==SQLITE_AFF_REAL ); - zType = azType[pCol->affinity - SQLITE_AFF_TEXT]; + zType = azType[pCol->affinity - SQLITE_AFF_NONE]; len = sqlite3Strlen30(zType); assert( pCol->affinity==SQLITE_AFF_NONE || pCol->affinity==sqlite3AffinityType(zType, 0) ); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index fd3731d817..ea8ad4573f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1427,10 +1427,10 @@ struct CollSeq { ** used as the P4 operand, they will be more readable. ** ** Note also that the numeric types are grouped together so that testing -** for a numeric type is a single comparison. +** for a numeric type is a single comparison. And the NONE type is first. */ -#define SQLITE_AFF_TEXT 'a' -#define SQLITE_AFF_NONE 'b' +#define SQLITE_AFF_NONE 'a' +#define SQLITE_AFF_TEXT 'b' #define SQLITE_AFF_NUMERIC 'c' #define SQLITE_AFF_INTEGER 'd' #define SQLITE_AFF_REAL 'e' diff --git a/src/vdbe.c b/src/vdbe.c index f964a73878..1ff33b3e49 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -277,15 +277,7 @@ static void applyAffinity( char affinity, /* The affinity to be applied */ u8 enc /* Use this text encoding */ ){ - if( affinity==SQLITE_AFF_TEXT ){ - /* Only attempt the conversion to TEXT if there is an integer or real - ** representation (blob and NULL do not get converted) but no string - ** representation. - */ - if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){ - sqlite3VdbeMemStringify(pRec, enc, 1); - } - }else if( affinity!=SQLITE_AFF_NONE ){ + if( affinity>=SQLITE_AFF_NUMERIC ){ assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL || affinity==SQLITE_AFF_NUMERIC ); if( (pRec->flags & MEM_Int)==0 ){ @@ -295,6 +287,14 @@ static void applyAffinity( sqlite3VdbeIntegerAffinity(pRec); } } + }else if( affinity==SQLITE_AFF_TEXT ){ + /* Only attempt the conversion to TEXT if there is an integer or real + ** representation (blob and NULL do not get converted) but no string + ** representation. + */ + if( 0==(pRec->flags&MEM_Str) && (pRec->flags&(MEM_Real|MEM_Int)) ){ + sqlite3VdbeMemStringify(pRec, enc, 1); + } } } @@ -1754,7 +1754,7 @@ case OP_RealAffinity: { /* in1 */ ** A NULL value is not changed by this routine. It remains NULL. */ case OP_Cast: { /* in1 */ - assert( pOp->p2>=SQLITE_AFF_TEXT && pOp->p2<=SQLITE_AFF_REAL ); + assert( pOp->p2>=SQLITE_AFF_NONE && pOp->p2<=SQLITE_AFF_REAL ); testcase( pOp->p2==SQLITE_AFF_TEXT ); testcase( pOp->p2==SQLITE_AFF_NONE ); testcase( pOp->p2==SQLITE_AFF_NUMERIC ); @@ -1904,7 +1904,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ }else{ /* Neither operand is NULL. Do a comparison. */ affinity = pOp->p5 & SQLITE_AFF_MASK; - if( affinity ){ + if( affinity>=SQLITE_AFF_TEXT ){ applyAffinity(pIn1, affinity, encoding); applyAffinity(pIn3, affinity, encoding); if( db->mallocFailed ) goto no_mem; From 24a096297ef66348e059b07328f1beb466b5e7c9 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Sep 2014 16:28:59 +0000 Subject: [PATCH 348/710] Performance improvement for affinity transformations on comparison operators. FossilOrigin-Name: d7afdcbac24350b73a30c06c45cf0f2122820e4f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 20 +++++++++++++++----- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 00c2f35ee9..8cf00918c0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Since\snumeric\saffinity\sis\sthe\smost\scommon\scase,\scheck\sit\sfirst.\s\sInterchange\nthe\sNONE\sand\sTEXT\saffinity\scodes\sfor\seasier\schecking\sof\sno\saffinity. -D 2014-09-18T14:36:00.055 +C Performance\simprovement\sfor\saffinity\stransformations\son\scomparison\soperators. +D 2014-09-18T16:28:59.796 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,7 +288,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c 8f634b93d41c089029dd503161a7d3e685d59a9c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 2caa3b1e32f3fddc60fecb2cfd66afe2c2eb96b1 +F src/vdbe.c b00ffadc43a588b02ca2b60b9128338a6f4efcba F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f90b0de6153f50de630a5a113537efb47083812f F src/vdbeapi.c c02242df5e9e8d1001e0086f405953833f9c426b @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3bd7c1b2faa2d4cc95b255633204006849bfd5e0 -R c1df8072077a2642e4ce7426502f62dc +P 4ef4c9a7c8510203bce0941dda2f76ded8da1de2 +R b808ae1c5ded91d91c2a612e5497b640 U drh -Z 04ecbaaf95d3fdde0c1a120923b44258 +Z 8ec5fac053656715ec4b7b6a3b299333 diff --git a/manifest.uuid b/manifest.uuid index 2c7c5a518e..8fa5f719ed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4ef4c9a7c8510203bce0941dda2f76ded8da1de2 \ No newline at end of file +d7afdcbac24350b73a30c06c45cf0f2122820e4f \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 1ff33b3e49..4bfa5f0e7e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1904,12 +1904,21 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ }else{ /* Neither operand is NULL. Do a comparison. */ affinity = pOp->p5 & SQLITE_AFF_MASK; - if( affinity>=SQLITE_AFF_TEXT ){ - applyAffinity(pIn1, affinity, encoding); - applyAffinity(pIn3, affinity, encoding); - if( db->mallocFailed ) goto no_mem; + if( affinity>=SQLITE_AFF_NUMERIC ){ + if( (pIn1->flags & (MEM_Int|MEM_Real))==0 && (pIn1->flags&MEM_Str)!=0 ){ + applyNumericAffinity(pIn1,0); + } + if( (pIn3->flags & (MEM_Int|MEM_Real))==0 && (pIn3->flags&MEM_Str)!=0 ){ + applyNumericAffinity(pIn3,0); + } + }else if( affinity==SQLITE_AFF_TEXT ){ + if( (pIn1->flags & MEM_Str)==0 && (pIn1->flags & (MEM_Int|MEM_Real))!=0 ){ + sqlite3VdbeMemStringify(pIn1, encoding, 1); + } + if( (pIn3->flags & MEM_Str)==0 && (pIn3->flags & (MEM_Int|MEM_Real))!=0 ){ + sqlite3VdbeMemStringify(pIn3, encoding, 1); + } } - assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); if( pIn1->flags & MEM_Zero ){ sqlite3VdbeMemExpandBlob(pIn1); @@ -1919,6 +1928,7 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ sqlite3VdbeMemExpandBlob(pIn3); flags3 &= ~MEM_Zero; } + if( db->mallocFailed ) goto no_mem; res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); } switch( pOp->opcode ){ From 74eaba4de25d955314df279e5ca27aa24f2698ea Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Sep 2014 17:52:15 +0000 Subject: [PATCH 349/710] Merge the Mem.r value into the MemValue union as Mem.u.r. Hence, a Mem can now store an integer or a real but not both at the same time. Strings are still stored in a separate element Mem.z, for now. FossilOrigin-Name: 4c8c89d7e62aecfe2eb735f7bb114aed6b452847 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/vdbe.c | 19 ++++++++++--------- src/vdbeInt.h | 6 +++--- src/vdbeapi.c | 3 +-- src/vdbeaux.c | 24 ++++++++++++------------ src/vdbemem.c | 36 +++++++++++++++++++----------------- src/vdbetrace.c | 2 +- 8 files changed, 58 insertions(+), 56 deletions(-) diff --git a/manifest b/manifest index 8cf00918c0..4bfcb72630 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\simprovement\sfor\saffinity\stransformations\son\scomparison\soperators. -D 2014-09-18T16:28:59.796 +C Merge\sthe\sMem.r\svalue\sinto\sthe\sMemValue\sunion\sas\sMem.u.r.\s\sHence,\sa\sMem\scan\nnow\sstore\san\sinteger\sor\sa\sreal\sbut\snot\sboth\sat\sthe\ssame\stime.\s\sStrings\sare\nstill\sstored\sin\sa\sseparate\selement\sMem.z,\sfor\snow. +D 2014-09-18T17:52:15.374 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,15 +288,15 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c 8f634b93d41c089029dd503161a7d3e685d59a9c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c b00ffadc43a588b02ca2b60b9128338a6f4efcba +F src/vdbe.c 17f285ff89d73b6af5bd1fc90c0943341f4003d5 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h f90b0de6153f50de630a5a113537efb47083812f -F src/vdbeapi.c c02242df5e9e8d1001e0086f405953833f9c426b -F src/vdbeaux.c 9ac63bc59d2783df77e591e4c4fa8c1153a07eab +F src/vdbeInt.h 45a0b4c5e4b38a2ff8af05102d45e7fa2e6c4439 +F src/vdbeapi.c 88929e02676fdbd5f436fcfd63fa0d371756a7ce +F src/vdbeaux.c ac3188f182f25eac58923b1a3c0840c69949ed28 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 8b5e1083fed2da94e315858a7edf5604a5b91804 +F src/vdbemem.c 1907e24ab431bd465f5709c30ec28a9119502f9a F src/vdbesort.c 09efa5e5098d1a159cd21f588eb118e4fe87cfde -F src/vdbetrace.c 16d39c1ef7d1f4a3a7464bea3b7b4bdd7849c415 +F src/vdbetrace.c 4f29b04edb0cec3d5fcd9b566d9f0e75c8984362 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4ef4c9a7c8510203bce0941dda2f76ded8da1de2 -R b808ae1c5ded91d91c2a612e5497b640 +P d7afdcbac24350b73a30c06c45cf0f2122820e4f +R 320866568edfdafc16dd0b942dd87d16 U drh -Z 8ec5fac053656715ec4b7b6a3b299333 +Z 7641aa17b0b2d51a4b828447b0d0dc99 diff --git a/manifest.uuid b/manifest.uuid index 8fa5f719ed..55f0527798 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d7afdcbac24350b73a30c06c45cf0f2122820e4f \ No newline at end of file +4c8c89d7e62aecfe2eb735f7bb114aed6b452847 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 4bfa5f0e7e..996cf8b9b8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -243,12 +243,13 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ i64 iValue; u8 enc = pRec->enc; if( (pRec->flags&MEM_Str)==0 ) return; + if( (pRec->flags&(MEM_Int|MEM_Real))!=0 ) return; if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ pRec->u.i = iValue; pRec->flags |= MEM_Int; }else{ - pRec->r = rValue; + pRec->u.r = rValue; pRec->flags |= MEM_Real; if( bTryForInt ) sqlite3VdbeIntegerAffinity(pRec); } @@ -329,13 +330,13 @@ void sqlite3ValueApplyAffinity( /* ** pMem currently only holds a string type (or maybe a BLOB that we can ** interpret as a string if we want to). Compute its corresponding -** numeric type, if has one. Set the pMem->r and pMem->u.i fields +** numeric type, if has one. Set the pMem->u.r and pMem->u.i fields ** accordingly. */ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ assert( (pMem->flags & (MEM_Int|MEM_Real))==0 ); assert( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ); - if( sqlite3AtoF(pMem->z, &pMem->r, pMem->n, pMem->enc)==0 ){ + if( sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc)==0 ){ return 0; } if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){ @@ -349,7 +350,7 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ ** none. ** ** Unlike applyNumericAffinity(), this routine does not modify pMem->flags. -** But it does set pMem->r and pMem->u.i appropriately. +** But it does set pMem->u.r and pMem->u.i appropriately. */ static u16 numericType(Mem *pMem){ if( pMem->flags & (MEM_Int|MEM_Real) ){ @@ -459,7 +460,7 @@ static void memTracePrint(Mem *p){ printf(" i:%lld", p->u.i); #ifndef SQLITE_OMIT_FLOATING_POINT }else if( p->flags & MEM_Real ){ - printf(" r:%g", p->r); + printf(" r:%g", p->u.r); #endif }else if( p->flags & MEM_RowSet ){ printf(" (rowset)"); @@ -1002,7 +1003,7 @@ case OP_Int64: { /* out2-prerelease */ case OP_Real: { /* same as TK_FLOAT, out2-prerelease */ pOut->flags = MEM_Real; assert( !sqlite3IsNaN(*pOp->p4.pReal) ); - pOut->r = *pOp->p4.pReal; + pOut->u.r = *pOp->p4.pReal; break; } #endif @@ -1479,7 +1480,7 @@ fp_math: if( sqlite3IsNaN(rB) ){ goto arithmetic_result_is_null; } - pOut->r = rB; + pOut->u.r = rB; MemSetTypeFlag(pOut, MEM_Real); if( ((type1|type2)&MEM_Real)==0 && !bIntint ){ sqlite3VdbeIntegerAffinity(pOut); @@ -3572,7 +3573,7 @@ case OP_SeekGT: { /* jump, in3 */ ** (x > 4.9) -> (x >= 5) ** (x <= 4.9) -> (x < 5) */ - if( pIn3->r<(double)iKey ){ + if( pIn3->u.r<(double)iKey ){ assert( OP_SeekGE==(OP_SeekGT-1) ); assert( OP_SeekLT==(OP_SeekLE-1) ); assert( (OP_SeekLE & 0x0001)==(OP_SeekGT & 0x0001) ); @@ -3581,7 +3582,7 @@ case OP_SeekGT: { /* jump, in3 */ /* If the approximation iKey is smaller than the actual real search ** term, substitute <= for < and > for >=. */ - else if( pIn3->r>(double)iKey ){ + else if( pIn3->u.r>(double)iKey ){ assert( OP_SeekLE==(OP_SeekLT+1) ); assert( OP_SeekGT==(OP_SeekGE+1) ); assert( (OP_SeekLT & 0x0001)==(OP_SeekGE & 0x0001) ); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 9c7378a26c..fb05e9aed2 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -161,7 +161,8 @@ struct VdbeFrame { ** integer etc.) of the same value. */ struct Mem { - union { + union MemValue { + double r; /* Real value used when MEM_Realis set in flags */ i64 i; /* Integer value used when MEM_Int is set in flags */ int nZero; /* Used when bit MEM_Zero is set in flags */ FuncDef *pDef; /* Used only when flags==MEM_Agg */ @@ -171,10 +172,9 @@ struct Mem { u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ int n; /* Number of characters in string value, excluding '\0' */ - double r; /* Real value */ char *z; /* String or BLOB value */ - char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ /* ShallowCopy only needs to copy the information above */ + char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ sqlite3 *db; /* The associated database connection */ void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */ #ifdef SQLITE_DEBUG diff --git a/src/vdbeapi.c b/src/vdbeapi.c index ef1167a52d..aad64aa64b 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -807,7 +807,6 @@ static const Mem *columnNullValue(void){ /* .flags = */ MEM_Null, /* .enc = */ 0, /* .n = */ 0, - /* .r = */ (double)0, /* .z = */ 0, /* .zMalloc = */ 0, /* .db = */ 0, @@ -1272,7 +1271,7 @@ int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ break; } case SQLITE_FLOAT: { - rc = sqlite3_bind_double(pStmt, i, pValue->r); + rc = sqlite3_bind_double(pStmt, i, pValue->u.r); break; } case SQLITE_BLOB: { diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 8466bfb30a..ad57f4ccd3 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1076,7 +1076,7 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ }else if( pMem->flags & MEM_Int ){ sqlite3_snprintf(nTemp, zTemp, "%lld", pMem->u.i); }else if( pMem->flags & MEM_Real ){ - sqlite3_snprintf(nTemp, zTemp, "%.16g", pMem->r); + sqlite3_snprintf(nTemp, zTemp, "%.16g", pMem->u.r); }else if( pMem->flags & MEM_Null ){ sqlite3_snprintf(nTemp, zTemp, "NULL"); }else{ @@ -2949,8 +2949,8 @@ u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ u64 v; u32 i; if( serial_type==7 ){ - assert( sizeof(v)==sizeof(pMem->r) ); - memcpy(&v, &pMem->r, sizeof(v)); + assert( sizeof(v)==sizeof(pMem->u.r) ); + memcpy(&v, &pMem->u.r, sizeof(v)); swapMixedEndianFloat(v); }else{ v = pMem->u.i; @@ -3020,10 +3020,10 @@ static u32 SQLITE_NOINLINE serialGet( swapMixedEndianFloat(t2); assert( sizeof(r1)==sizeof(t2) && memcmp(&r1, &t2, sizeof(r1))==0 ); #endif - assert( sizeof(x)==8 && sizeof(pMem->r)==8 ); + assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); swapMixedEndianFloat(x); - memcpy(&pMem->r, &x, sizeof(x)); - pMem->flags = sqlite3IsNaN(pMem->r) ? MEM_Null : MEM_Real; + memcpy(&pMem->u.r, &x, sizeof(x)); + pMem->flags = sqlite3IsNaN(pMem->u.r) ? MEM_Null : MEM_Real; } return 8; } @@ -3368,14 +3368,14 @@ int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const CollSeq *pColl){ return 0; } if( (f1&MEM_Real)!=0 ){ - r1 = pMem1->r; + r1 = pMem1->u.r; }else if( (f1&MEM_Int)!=0 ){ r1 = (double)pMem1->u.i; }else{ return 1; } if( (f2&MEM_Real)!=0 ){ - r2 = pMem2->r; + r2 = pMem2->u.r; }else if( (f2&MEM_Int)!=0 ){ r2 = (double)pMem2->u.i; }else{ @@ -3536,9 +3536,9 @@ static int vdbeRecordCompareWithSkip( }else if( serial_type==7 ){ double rhs = (double)pRhs->u.i; sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); - if( mem1.rrhs ){ + }else if( mem1.u.r>rhs ){ rc = +1; } }else{ @@ -3560,11 +3560,11 @@ static int vdbeRecordCompareWithSkip( }else if( serial_type==0 ){ rc = -1; }else{ - double rhs = pRhs->r; + double rhs = pRhs->u.r; double lhs; sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); if( serial_type==7 ){ - lhs = mem1.r; + lhs = mem1.u.r; }else{ lhs = (double)mem1.u.i; } diff --git a/src/vdbemem.c b/src/vdbemem.c index 359abb891f..780bc5286b 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -31,6 +31,9 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){ */ assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 ); + /* Cannot be both MEM_Int and MEM_Real at the same time */ + assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) ); + /* If p holds a string or blob, the Mem.z must point to exactly ** one of the following: ** @@ -264,7 +267,7 @@ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); }else{ assert( fg & MEM_Real ); - sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->r); + sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->u.r); } pMem->n = sqlite3Strlen30(pMem->z); pMem->enc = SQLITE_UTF8; @@ -421,7 +424,7 @@ i64 sqlite3VdbeIntValue(Mem *pMem){ if( flags & MEM_Int ){ return pMem->u.i; }else if( flags & MEM_Real ){ - return doubleToInt64(pMem->r); + return doubleToInt64(pMem->u.r); }else if( flags & (MEM_Str|MEM_Blob) ){ i64 value = 0; assert( pMem->z || pMem->n==0 ); @@ -442,7 +445,7 @@ double sqlite3VdbeRealValue(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); if( pMem->flags & MEM_Real ){ - return pMem->r; + return pMem->u.r; }else if( pMem->flags & MEM_Int ){ return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ @@ -461,12 +464,13 @@ double sqlite3VdbeRealValue(Mem *pMem){ ** MEM_Int if we can. */ void sqlite3VdbeIntegerAffinity(Mem *pMem){ + i64 ix; assert( pMem->flags & MEM_Real ); assert( (pMem->flags & MEM_RowSet)==0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - pMem->u.i = doubleToInt64(pMem->r); + ix = doubleToInt64(pMem->u.r); /* Only mark the value as an integer if ** @@ -478,11 +482,9 @@ void sqlite3VdbeIntegerAffinity(Mem *pMem){ ** the second condition under the assumption that addition overflow causes ** values to wrap around. */ - if( pMem->r==(double)pMem->u.i - && pMem->u.i>SMALLEST_INT64 - && pMem->u.iflags |= MEM_Int; + if( pMem->u.r==ix && ix>SMALLEST_INT64 && ixu.i = ix; + MemSetTypeFlag(pMem, MEM_Int); } } @@ -507,7 +509,7 @@ int sqlite3VdbeMemRealify(Mem *pMem){ assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - pMem->r = sqlite3VdbeRealValue(pMem); + pMem->u.r = sqlite3VdbeRealValue(pMem); MemSetTypeFlag(pMem, MEM_Real); return SQLITE_OK; } @@ -527,7 +529,7 @@ int sqlite3VdbeMemNumerify(Mem *pMem){ if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){ MemSetTypeFlag(pMem, MEM_Int); }else{ - pMem->r = sqlite3VdbeRealValue(pMem); + pMem->u.r = sqlite3VdbeRealValue(pMem); MemSetTypeFlag(pMem, MEM_Real); sqlite3VdbeIntegerAffinity(pMem); } @@ -663,7 +665,7 @@ void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ sqlite3VdbeMemSetNull(pMem); if( !sqlite3IsNaN(val) ){ - pMem->r = val; + pMem->u.r = val; pMem->flags = MEM_Real; } } @@ -1168,14 +1170,14 @@ static int valueFromExpr( && pVal!=0 ){ sqlite3VdbeMemNumerify(pVal); - if( pVal->u.i==SMALLEST_INT64 ){ - pVal->flags &= ~MEM_Int; - pVal->flags |= MEM_Real; - pVal->r = (double)SMALLEST_INT64; + if( pVal->flags & MEM_Real ){ + pVal->u.r = -pVal->u.r; + }else if( pVal->u.i==SMALLEST_INT64 ){ + pVal->u.r = -(double)SMALLEST_INT64; + MemSetTypeFlag(pVal, MEM_Real); }else{ pVal->u.i = -pVal->u.i; } - pVal->r = -pVal->r; sqlite3ValueApplyAffinity(pVal, affinity, enc); } }else if( op==TK_NULL ){ diff --git a/src/vdbetrace.c b/src/vdbetrace.c index 362530a1d9..d27693450e 100644 --- a/src/vdbetrace.c +++ b/src/vdbetrace.c @@ -127,7 +127,7 @@ char *sqlite3VdbeExpandSql( }else if( pVar->flags & MEM_Int ){ sqlite3XPrintf(&out, 0, "%lld", pVar->u.i); }else if( pVar->flags & MEM_Real ){ - sqlite3XPrintf(&out, 0, "%!.15g", pVar->r); + sqlite3XPrintf(&out, 0, "%!.15g", pVar->u.r); }else if( pVar->flags & MEM_Str ){ int nOut; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 From 26c79a060b2e7fdf6d05585a3b44d0498df3fe1c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 18 Sep 2014 18:55:47 +0000 Subject: [PATCH 350/710] Correct typos in comments. No changes to code. FossilOrigin-Name: 55879932116d373c95a5f32ec44b53a9c3f4db24 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeInt.h | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4bfcb72630..f3e0c0137b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sMem.r\svalue\sinto\sthe\sMemValue\sunion\sas\sMem.u.r.\s\sHence,\sa\sMem\scan\nnow\sstore\san\sinteger\sor\sa\sreal\sbut\snot\sboth\sat\sthe\ssame\stime.\s\sStrings\sare\nstill\sstored\sin\sa\sseparate\selement\sMem.z,\sfor\snow. -D 2014-09-18T17:52:15.374 +C Correct\stypos\sin\scomments.\s\sNo\schanges\sto\scode. +D 2014-09-18T18:55:47.619 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 17f285ff89d73b6af5bd1fc90c0943341f4003d5 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h 45a0b4c5e4b38a2ff8af05102d45e7fa2e6c4439 +F src/vdbeInt.h 1c31448d9d4f074e93863ebe69164ff29cc4f7f8 F src/vdbeapi.c 88929e02676fdbd5f436fcfd63fa0d371756a7ce F src/vdbeaux.c ac3188f182f25eac58923b1a3c0840c69949ed28 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d7afdcbac24350b73a30c06c45cf0f2122820e4f -R 320866568edfdafc16dd0b942dd87d16 -U drh -Z 7641aa17b0b2d51a4b828447b0d0dc99 +P 4c8c89d7e62aecfe2eb735f7bb114aed6b452847 +R a232caa53ffd8c1fd2d0c9fb51cf1463 +U mistachkin +Z b2beb30c7e105753dffdcae3bc62d871 diff --git a/manifest.uuid b/manifest.uuid index 55f0527798..cd35e4dca6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4c8c89d7e62aecfe2eb735f7bb114aed6b452847 \ No newline at end of file +55879932116d373c95a5f32ec44b53a9c3f4db24 \ No newline at end of file diff --git a/src/vdbeInt.h b/src/vdbeInt.h index fb05e9aed2..8e4405437b 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -162,7 +162,7 @@ struct VdbeFrame { */ struct Mem { union MemValue { - double r; /* Real value used when MEM_Realis set in flags */ + double r; /* Real value used when MEM_Real is set in flags */ i64 i; /* Integer value used when MEM_Int is set in flags */ int nZero; /* Used when bit MEM_Zero is set in flags */ FuncDef *pDef; /* Used only when flags==MEM_Agg */ @@ -268,13 +268,13 @@ struct AuxData { */ struct sqlite3_context { Mem *pOut; /* The return value is stored here */ - FuncDef *pFunc; /* Pointer to function information. MUST BE FIRST */ + FuncDef *pFunc; /* Pointer to function information */ Mem *pMem; /* Memory cell used to store aggregate context */ CollSeq *pColl; /* Collating sequence */ Vdbe *pVdbe; /* The VM that owns this context */ int iOp; /* Instruction number of OP_Function */ int isError; /* Error code returned by the function. */ - u8 skipFlag; /* Skip skip accumulator loading if true */ + u8 skipFlag; /* Skip accumulator loading if true */ u8 fErrorOrAux; /* isError!=0 or pVdbe->pAuxData modified */ }; From 17bcb102993a155dec69a58e2f079a880e1967b7 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 18 Sep 2014 21:25:33 +0000 Subject: [PATCH 351/710] Add the Mem.szMalloc element to the Mem object and use it to keep track of the size of the Mem.zMalloc allocation. FossilOrigin-Name: 9c09ac353df6041808cace41880f4729ee73f5e1 --- manifest | 28 ++++++++++++++-------------- manifest.uuid | 2 +- src/malloc.c | 19 +++++++++++-------- src/test_func.c | 4 ++-- src/utf.c | 1 + src/vdbe.c | 11 +++-------- src/vdbeInt.h | 4 +++- src/vdbeapi.c | 2 ++ src/vdbeaux.c | 29 ++++++++++++++--------------- src/vdbemem.c | 41 ++++++++++++++++++++++++++--------------- 10 files changed, 77 insertions(+), 64 deletions(-) diff --git a/manifest b/manifest index f3e0c0137b..1ced1cbf05 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correct\stypos\sin\scomments.\s\sNo\schanges\sto\scode. -D 2014-09-18T18:55:47.619 +C Add\sthe\sMem.szMalloc\selement\sto\sthe\sMem\sobject\sand\suse\sit\sto\skeep\strack\sof\nthe\ssize\sof\sthe\sMem.zMalloc\sallocation. +D 2014-09-18T21:25:33.845 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 F src/main.c d15621461fb0c52675eba2b650492ed1beef69ab -F src/malloc.c 4c1d511157defd7b1d023062cf05a1dc17b8f79b +F src/malloc.c 5bb99ee1e08ad58e457063cf79ce521db0e24195 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f @@ -253,7 +253,7 @@ F src/test_config.c 6f721f0337b96d58e81ff69bba101113c8168c2b F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f -F src/test_func.c d3013ce36f19ac72a99c73864930fd1fa41832f8 +F src/test_func.c 14e543ae4d905ee31dc322b2f8d31bfac1769d45 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 66b33120ffe9cd853b5a905ec850d51151337b32 F src/test_intarray.c 6c610a21ab8edde85a3a2c7f2b069244ecf4d834 @@ -285,16 +285,16 @@ F src/threads.c 22dded4283dc4b25422f6444cdcb8d6b1ea0b5ff F src/tokenize.c 3df63041994f55afeb168b463ec836e8f1c50e7c F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 -F src/utf.c 8f634b93d41c089029dd503161a7d3e685d59a9c +F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 17f285ff89d73b6af5bd1fc90c0943341f4003d5 +F src/vdbe.c fb490f5b1b2ee2f33f60c7dc678c0d0b70f2e0cb F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h 1c31448d9d4f074e93863ebe69164ff29cc4f7f8 -F src/vdbeapi.c 88929e02676fdbd5f436fcfd63fa0d371756a7ce -F src/vdbeaux.c ac3188f182f25eac58923b1a3c0840c69949ed28 +F src/vdbeInt.h 1ac536d1fa1260d72f81003ff5b283e8f3c40442 +F src/vdbeapi.c e088ed70b6cc42ed68985ab064397ebd452286d6 +F src/vdbeaux.c b3230032238df611aefee5907ea792786362a55d F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 1907e24ab431bd465f5709c30ec28a9119502f9a +F src/vdbemem.c 3aea3831a981378368ca058cd8fc700b1982772d F src/vdbesort.c 09efa5e5098d1a159cd21f588eb118e4fe87cfde F src/vdbetrace.c 4f29b04edb0cec3d5fcd9b566d9f0e75c8984362 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4c8c89d7e62aecfe2eb735f7bb114aed6b452847 -R a232caa53ffd8c1fd2d0c9fb51cf1463 -U mistachkin -Z b2beb30c7e105753dffdcae3bc62d871 +P 55879932116d373c95a5f32ec44b53a9c3f4db24 +R 508057a57cef866ce62608d1eb2d9e8e +U drh +Z 7d8fd1db974daf115bf085d0714b6de9 diff --git a/manifest.uuid b/manifest.uuid index cd35e4dca6..77cd05df09 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55879932116d373c95a5f32ec44b53a9c3f4db24 \ No newline at end of file +9c09ac353df6041808cace41880f4729ee73f5e1 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index e0d5b5ff9d..8ba5fa0a84 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -451,15 +451,18 @@ int sqlite3MallocSize(void *p){ return sqlite3GlobalConfig.m.xSize(p); } int sqlite3DbMallocSize(sqlite3 *db, void *p){ - assert( db!=0 ); - assert( sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db, p) ){ - return db->lookaside.sz; + if( db==0 ){ + return sqlite3MallocSize(p); }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); - assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); - return sqlite3GlobalConfig.m.xSize(p); + assert( sqlite3_mutex_held(db->mutex) ); + if( isLookaside(db, p) ){ + return db->lookaside.sz; + }else{ + assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); + return sqlite3GlobalConfig.m.xSize(p); + } } } sqlite3_uint64 sqlite3_msize(void *p){ diff --git a/src/test_func.c b/src/test_func.c index 9cf2f8002c..c7850631d7 100644 --- a/src/test_func.c +++ b/src/test_func.c @@ -504,7 +504,7 @@ static void test_extract( sqlite3_result_value(context, &mem); } - sqlite3DbFree(db, mem.zMalloc); + if( mem.szMalloc ) sqlite3DbFree(db, mem.zMalloc); } } @@ -591,7 +591,7 @@ static void test_decode( Tcl_ListObjAppendElement(0, pRet, pVal); - if( mem.zMalloc ){ + if( mem.szMalloc ){ sqlite3DbFree(db, mem.zMalloc); } } diff --git a/src/utf.c b/src/utf.c index 549983f6f1..25f4dadf0c 100644 --- a/src/utf.c +++ b/src/utf.c @@ -320,6 +320,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desiredEnc){ pMem->enc = desiredEnc; pMem->z = (char*)zOut; pMem->zMalloc = pMem->z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->z); translate_out: #if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG) diff --git a/src/vdbe.c b/src/vdbe.c index 996cf8b9b8..4a63f30af8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1026,9 +1026,9 @@ case OP_String8: { /* same as TK_STRING, out2-prerelease */ rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); if( rc==SQLITE_TOOBIG ) goto too_big; if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; - assert( pOut->zMalloc==pOut->z ); + assert( pOut->szMalloc>0 && pOut->zMalloc==pOut->z ); assert( VdbeMemDynamic(pOut)==0 ); - pOut->zMalloc = 0; + pOut->szMalloc = 0; pOut->flags |= MEM_Static; if( pOp->p4type==P4_DYNAMIC ){ sqlite3DbFree(db, pOp->p4.z); @@ -1148,7 +1148,6 @@ case OP_Variable: { /* out2-prerelease */ ** for P3 to be less than 1. */ case OP_Move: { - char *zMalloc; /* Holding variable for allocated memory */ int n; /* Number of registers left to copy */ int p1; /* Register to copy from */ int p2; /* Register to copy to */ @@ -1166,16 +1165,12 @@ case OP_Move: { assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); - sqlite3VdbeMemRelease(pOut); - zMalloc = pOut->zMalloc; - memcpy(pOut, pIn1, sizeof(Mem)); + sqlite3VdbeMemMove(pOut, pIn1); #ifdef SQLITE_DEBUG if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrom<&aMem[p1+pOp->p3] ){ pOut->pScopyFrom += p1 - pOp->p2; } #endif - pIn1->flags = MEM_Undefined; - pIn1->zMalloc = zMalloc; REGISTER_TRACE(p2++, pOut); pIn1++; pOut++; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 8e4405437b..138e61d2c2 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -174,7 +174,9 @@ struct Mem { int n; /* Number of characters in string value, excluding '\0' */ char *z; /* String or BLOB value */ /* ShallowCopy only needs to copy the information above */ - char *zMalloc; /* Dynamic buffer allocated by sqlite3_malloc() */ + char *zMalloc; /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */ + int szMalloc; /* Size of the zMalloc allocation */ + int iPadding1; /* Padding for 8-byte alignment */ sqlite3 *db; /* The associated database connection */ void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */ #ifdef SQLITE_DEBUG diff --git a/src/vdbeapi.c b/src/vdbeapi.c index aad64aa64b..7d3ae9cb07 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -809,6 +809,8 @@ static const Mem *columnNullValue(void){ /* .n = */ 0, /* .z = */ 0, /* .zMalloc = */ 0, + /* .szMalloc = */ 0, + /* .iPadding1 = */ 0, /* .db = */ 0, /* .xDel = */ 0, #ifdef SQLITE_DEBUG diff --git a/src/vdbeaux.c b/src/vdbeaux.c index ad57f4ccd3..f990c40d41 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -698,7 +698,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ sqlite3ValueFree((sqlite3_value*)p4); }else{ Mem *p = (Mem*)p4; - sqlite3DbFree(db, p->zMalloc); + if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); sqlite3DbFree(db, p); } break; @@ -1231,7 +1231,7 @@ static void releaseMemArray(Mem *p, int N){ u8 malloc_failed = db->mallocFailed; if( db->pnBytesFreed ){ for(pEnd=&p[N]; pzMalloc); + if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); } return; } @@ -1257,9 +1257,9 @@ static void releaseMemArray(Mem *p, int N){ testcase( p->flags & MEM_RowSet ); if( p->flags&(MEM_Agg|MEM_Dyn|MEM_Frame|MEM_RowSet) ){ sqlite3VdbeMemRelease(p); - }else if( p->zMalloc ){ + }else if( p->szMalloc ){ sqlite3DbFree(db, p->zMalloc); - p->zMalloc = 0; + p->szMalloc = 0; } p->flags = MEM_Undefined; @@ -3167,7 +3167,7 @@ void sqlite3VdbeRecordUnpack( pMem->enc = pKeyInfo->enc; pMem->db = pKeyInfo->db; /* pMem->flags = 0; // sqlite3VdbeSerialGet() will set this for us */ - pMem->zMalloc = 0; + pMem->szMalloc = 0; d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); pMem++; if( (++u)>=p->nField ) break; @@ -3207,7 +3207,7 @@ static int vdbeRecordCompareDebug( mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; /* mem1.flags = 0; // Will be initialized by sqlite3VdbeSerialGet() */ - VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ + VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ /* Compilers may complain that mem1.u.i is potentially uninitialized. ** We could initialize it, as shown here, to silence those complaints. @@ -3250,7 +3250,7 @@ static int vdbeRecordCompareDebug( */ rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], pKeyInfo->aColl[i]); if( rc!=0 ){ - assert( mem1.zMalloc==0 ); /* See comment below */ + assert( mem1.szMalloc==0 ); /* See comment below */ if( pKeyInfo->aSortOrder[i] ){ rc = -rc; /* Invert the result for DESC sort order. */ } @@ -3263,7 +3263,7 @@ static int vdbeRecordCompareDebug( ** the following assert(). If the assert() fails, it indicates a ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). */ - assert( mem1.zMalloc==0 ); + assert( mem1.szMalloc==0 ); /* rc==0 here means that one of the keys ran out of fields and ** all the fields up to that point were equal. Return the default_rc @@ -3302,9 +3302,8 @@ static int vdbeCompareMemString( int n1, n2; Mem c1; Mem c2; - c1.db = c2.db = pMem1->db; - c1.flags = c2.flags = 0; - c1.zMalloc = c2.zMalloc = 0; + sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); + sqlite3VdbeMemInit(&c2, pMem1->db, MEM_Null); sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); @@ -3516,7 +3515,7 @@ static int vdbeRecordCompareWithSkip( i = 0; } - VVA_ONLY( mem1.zMalloc = 0; ) /* Only needed by assert() statements */ + VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ assert( pPKey2->pKeyInfo->nField+pPKey2->pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); assert( pPKey2->pKeyInfo->aSortOrder!=0 ); @@ -3639,7 +3638,7 @@ static int vdbeRecordCompareWithSkip( rc = -rc; } assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, rc) ); - assert( mem1.zMalloc==0 ); /* See comment below */ + assert( mem1.szMalloc==0 ); /* See comment below */ return rc; } @@ -3652,7 +3651,7 @@ static int vdbeRecordCompareWithSkip( /* No memory allocation is ever used on mem1. Prove this using ** the following assert(). If the assert() fails, it indicates a ** memory leak and a need to call sqlite3VdbeMemRelease(&mem1). */ - assert( mem1.zMalloc==0 ); + assert( mem1.szMalloc==0 ); /* rc==0 here means that one or both of the keys ran out of fields and ** all the fields up to that point were equal. Return the default_rc @@ -3937,7 +3936,7 @@ int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ /* Jump here if database corruption is detected after m has been ** allocated. Free the m object and return SQLITE_CORRUPT. */ idx_rowid_corruption: - testcase( m.zMalloc!=0 ); + testcase( m.szMalloc!=0 ); sqlite3VdbeMemRelease(&m); return SQLITE_CORRUPT_BKPT; } diff --git a/src/vdbemem.c b/src/vdbemem.c index 780bc5286b..4ea28f841f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -34,6 +34,10 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){ /* Cannot be both MEM_Int and MEM_Real at the same time */ assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) ); + /* The szMalloc field holds the correct memory allocation size */ + assert( p->szMalloc==0 + || p->szMalloc==sqlite3DbMallocSize(p->db,p->zMalloc) ); + /* If p holds a string or blob, the Mem.z must point to exactly ** one of the following: ** @@ -42,9 +46,9 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){ ** (3) An ephemeral string or blob ** (4) A static string or blob */ - if( (p->flags & (MEM_Str|MEM_Blob)) && p->z!=0 ){ + if( (p->flags & (MEM_Str|MEM_Blob)) && p->n>0 ){ assert( - ((p->z==p->zMalloc)? 1 : 0) + + ((p->szMalloc>0 && p->z==p->zMalloc)? 1 : 0) + ((p->flags&MEM_Dyn)!=0 ? 1 : 0) + ((p->flags&MEM_Ephem)!=0 ? 1 : 0) + ((p->flags&MEM_Static)!=0 ? 1 : 0) == 1 @@ -112,19 +116,24 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ assert( bPreserve==0 || pMem->flags&(MEM_Blob|MEM_Str) ); testcase( bPreserve && pMem->z==0 ); - if( pMem->zMalloc==0 || sqlite3DbMallocSize(pMem->db, pMem->zMalloc)szMalloc==0 + || pMem->szMalloc==sqlite3DbMallocSize(pMem->db, pMem->zMalloc) ); + if( pMem->szMallocz==pMem->zMalloc ){ + if( bPreserve && pMem->szMalloc>0 && pMem->z==pMem->zMalloc ){ pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); bPreserve = 0; }else{ - sqlite3DbFree(pMem->db, pMem->zMalloc); + if( pMem->szMalloc>0 ) sqlite3DbFree(pMem->db, pMem->zMalloc); pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); } if( pMem->zMalloc==0 ){ sqlite3VdbeMemSetNull(pMem); pMem->z = 0; + pMem->szMalloc = 0; return SQLITE_NOMEM; + }else{ + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); } } @@ -155,7 +164,7 @@ int sqlite3VdbeMemMakeWriteable(Mem *pMem){ assert( (pMem->flags&MEM_RowSet)==0 ); ExpandBlob(pMem); f = pMem->flags; - if( (f&(MEM_Str|MEM_Blob)) && pMem->z!=pMem->zMalloc ){ + if( (f&(MEM_Str|MEM_Blob)) && (pMem->szMalloc==0 || pMem->z!=pMem->zMalloc) ){ if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){ return SQLITE_NOMEM; } @@ -301,7 +310,7 @@ int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ ctx.pFunc = pFunc; pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ assert( (pMem->flags & MEM_Dyn)==0 ); - sqlite3DbFree(pMem->db, pMem->zMalloc); + if( pMem->szMalloc>0 ) sqlite3DbFree(pMem->db, pMem->zMalloc); memcpy(pMem, &t, sizeof(t)); rc = ctx.isError; } @@ -351,9 +360,9 @@ static SQLITE_NOINLINE void vdbeMemClear(Mem *p){ if( VdbeMemDynamic(p) ){ vdbeMemClearExternAndSetNull(p); } - if( p->zMalloc ){ + if( p->szMalloc ){ sqlite3DbFree(p->db, p->zMalloc); - p->zMalloc = 0; + p->szMalloc = 0; } p->z = 0; } @@ -370,7 +379,7 @@ static SQLITE_NOINLINE void vdbeMemClear(Mem *p){ */ void sqlite3VdbeMemRelease(Mem *p){ assert( sqlite3VdbeCheckMemInvariants(p) ); - if( VdbeMemDynamic(p) || p->zMalloc ){ + if( VdbeMemDynamic(p) || p->szMalloc ){ vdbeMemClear(p); } } @@ -592,7 +601,7 @@ void sqlite3VdbeMemInit(Mem *pMem, sqlite3 *db, u16 flags){ assert( (flags & ~MEM_TypeMask)==0 ); pMem->flags = flags; pMem->db = db; - pMem->zMalloc = 0; + pMem->szMalloc = 0; } @@ -683,10 +692,11 @@ void sqlite3VdbeMemSetRowSet(Mem *pMem){ pMem->zMalloc = sqlite3DbMallocRaw(db, 64); if( db->mallocFailed ){ pMem->flags = MEM_Null; + pMem->szMalloc = 0; }else{ assert( pMem->zMalloc ); - pMem->u.pRowSet = sqlite3RowSetInit(db, pMem->zMalloc, - sqlite3DbMallocSize(db, pMem->zMalloc)); + pMem->szMalloc = sqlite3DbMallocSize(db, pMem->zMalloc); + pMem->u.pRowSet = sqlite3RowSetInit(db, pMem->zMalloc, pMem->szMalloc); assert( pMem->u.pRowSet!=0 ); pMem->flags = MEM_RowSet; } @@ -789,7 +799,7 @@ void sqlite3VdbeMemMove(Mem *pTo, Mem *pFrom){ sqlite3VdbeMemRelease(pTo); memcpy(pTo, pFrom, sizeof(Mem)); pFrom->flags = MEM_Null; - pFrom->zMalloc = 0; + pFrom->szMalloc = 0; } /* @@ -863,6 +873,7 @@ int sqlite3VdbeMemSetStr( }else if( xDel==SQLITE_DYNAMIC ){ sqlite3VdbeMemRelease(pMem); pMem->zMalloc = pMem->z = (char *)z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); }else{ sqlite3VdbeMemRelease(pMem); pMem->z = (char *)z; @@ -1485,7 +1496,7 @@ void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ Mem *aMem = pRec->aMem; sqlite3 *db = aMem[0].db; for(i=0; ipKeyInfo); sqlite3DbFree(db, pRec); From 322f2852f29a2a315252a7f56d50a4723d9e7b46 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Sep 2014 00:43:39 +0000 Subject: [PATCH 352/710] Add the sqlite3VdbeMemClearAndResize() interface to be used in place of sqlite3VdbeMemGrow(). FossilOrigin-Name: 5b9b8987797abf7c68d2c3154f6657be9b8b1c8f --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/vdbe.c | 8 ++++---- src/vdbeInt.h | 1 + src/vdbeapi.c | 2 +- src/vdbeaux.c | 6 +++--- src/vdbemem.c | 35 +++++++++++++++++++++++++++++++---- src/vdbesort.c | 2 +- 8 files changed, 53 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 1ced1cbf05..28c4382ce9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sMem.szMalloc\selement\sto\sthe\sMem\sobject\sand\suse\sit\sto\skeep\strack\sof\nthe\ssize\sof\sthe\sMem.zMalloc\sallocation. -D 2014-09-18T21:25:33.845 +C Add\sthe\ssqlite3VdbeMemClearAndResize()\sinterface\sto\sbe\sused\sin\splace\sof\nsqlite3VdbeMemGrow(). +D 2014-09-19T00:43:39.899 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,14 +288,14 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c fb490f5b1b2ee2f33f60c7dc678c0d0b70f2e0cb +F src/vdbe.c 7bc115e540282b1bfff9b43205cbf6528e6398a6 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h 1ac536d1fa1260d72f81003ff5b283e8f3c40442 -F src/vdbeapi.c e088ed70b6cc42ed68985ab064397ebd452286d6 -F src/vdbeaux.c b3230032238df611aefee5907ea792786362a55d +F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 +F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d +F src/vdbeaux.c ecf0bb835f7c809ba8e22ba7d3a587b7a087ac52 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 3aea3831a981378368ca058cd8fc700b1982772d -F src/vdbesort.c 09efa5e5098d1a159cd21f588eb118e4fe87cfde +F src/vdbemem.c 0678ba6214a810539e51c6d8f263634d405b990f +F src/vdbesort.c 75c66c2fc02d450b67b4816873fba8088feaf12c F src/vdbetrace.c 4f29b04edb0cec3d5fcd9b566d9f0e75c8984362 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 55879932116d373c95a5f32ec44b53a9c3f4db24 -R 508057a57cef866ce62608d1eb2d9e8e +P 9c09ac353df6041808cace41880f4729ee73f5e1 +R eeadad19cadf3315e345f0babe4222b0 U drh -Z 7d8fd1db974daf115bf085d0714b6de9 +Z fd83cfbfc71d89fd149fcf4e06924224 diff --git a/manifest.uuid b/manifest.uuid index 77cd05df09..a4139a3651 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9c09ac353df6041808cace41880f4729ee73f5e1 \ No newline at end of file +5b9b8987797abf7c68d2c3154f6657be9b8b1c8f \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 4a63f30af8..25a33a63f7 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -209,7 +209,7 @@ static VdbeCursor *allocateCursor( sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; } - if( SQLITE_OK==sqlite3VdbeMemGrow(pMem, nByte, 0) ){ + if( SQLITE_OK==sqlite3VdbeMemClearAndResize(pMem, nByte) ){ p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z; memset(pCx, 0, sizeof(VdbeCursor)); pCx->iDb = iDb; @@ -2633,9 +2633,9 @@ case OP_MakeRecord: { /* Make sure the output register has a buffer large enough to store ** the new record. The output register (pOp->p3) is not allowed to ** be one of the input registers (because the following call to - ** sqlite3VdbeMemGrow() could clobber the value before it is used). + ** sqlite3VdbeMemClearAndResize() could clobber the value before it is used). */ - if( sqlite3VdbeMemGrow(pOut, (int)nByte, 0) ){ + if( sqlite3VdbeMemClearAndResize(pOut, (int)nByte) ){ goto no_mem; } zNewRecord = (u8 *)pOut->z; @@ -4339,7 +4339,7 @@ case OP_RowData: { goto too_big; } } - if( sqlite3VdbeMemGrow(pOut, n, 0) ){ + if( sqlite3VdbeMemClearAndResize(pOut, n) ){ goto no_mem; } pOut->n = n; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 138e61d2c2..56b5db7d1a 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -437,6 +437,7 @@ void sqlite3VdbeMemRelease(Mem *p); int sqlite3VdbeMemFinalize(Mem*, FuncDef*); const char *sqlite3OpcodeName(int); int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); +int sqlite3VdbeMemClearAndResize(Mem *pMem, int n); int sqlite3VdbeCloseStatement(Vdbe *, int); void sqlite3VdbeFrameDelete(VdbeFrame*); int sqlite3VdbeFrameRestore(VdbeFrame *); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 7d3ae9cb07..dc38132382 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -665,7 +665,7 @@ static SQLITE_NOINLINE void *createAggContext(sqlite3_context *p, int nByte){ sqlite3VdbeMemSetNull(pMem); pMem->z = 0; }else{ - sqlite3VdbeMemGrow(pMem, nByte, 0); + sqlite3VdbeMemClearAndResize(pMem, nByte); pMem->flags = MEM_Agg; pMem->u.pDef = p->pFunc; if( pMem->z ){ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index f990c40d41..3ce373dffe 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1426,7 +1426,7 @@ int sqlite3VdbeList( pMem->u.i = pOp->p3; /* P3 */ pMem++; - if( sqlite3VdbeMemGrow(pMem, 32, 0) ){ /* P4 */ + if( sqlite3VdbeMemClearAndResize(pMem, 32) ){ /* P4 */ assert( p->db->mallocFailed ); return SQLITE_ERROR; } @@ -1442,7 +1442,7 @@ int sqlite3VdbeList( pMem++; if( p->explain==1 ){ - if( sqlite3VdbeMemGrow(pMem, 4, 0) ){ + if( sqlite3VdbeMemClearAndResize(pMem, 4) ){ assert( p->db->mallocFailed ); return SQLITE_ERROR; } @@ -1453,7 +1453,7 @@ int sqlite3VdbeList( pMem++; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - if( sqlite3VdbeMemGrow(pMem, 500, 0) ){ + if( sqlite3VdbeMemClearAndResize(pMem, 500) ){ assert( p->db->mallocFailed ); return SQLITE_ERROR; } diff --git a/src/vdbemem.c b/src/vdbemem.c index 4ea28f841f..ff9f5b6df5 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -107,7 +107,7 @@ int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ ** blob if bPreserve is true. If bPreserve is false, any prior content ** in pMem->z is discarded. */ -int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ +SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ assert( sqlite3VdbeCheckMemInvariants(pMem) ); assert( (pMem->flags&MEM_RowSet)==0 ); @@ -150,6 +150,33 @@ int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ return SQLITE_OK; } +/* +** Change the pMem->zMalloc allocation to be at least szNew bytes. +** If pMem->zMalloc already meets or exceeds the requested size, this +** routine is a no-op. +** +** Any prior string or blob content in the pMem object may be discarded. +** The pMem->xDel destructor is called, if it exists. +** +** The value of the pMem after this routine returns might be NULL or +** it might retain its prior value, depending on circumstances. The +** caller should make no assumptions about the state of pMem after this +** routine returns, except that pMem->zMalloc is at least the requested +** size and pMem->z==pMem->zMalloc. +** +** Return SQLITE_OK on success or an error code (probably SQLITE_NOMEM) +** if unable to complete the resizing. +*/ +int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ + if( pMem->szMallocflags & MEM_Dyn)!=0 ){ + return sqlite3VdbeMemGrow(pMem, szNew, 0); + } + pMem->z = pMem->zMalloc; + pMem->flags &= ~(MEM_Dyn|MEM_Ephem|MEM_Static); +// pMem->flags = MEM_Null; + return SQLITE_OK; +} + /* ** Make the given Mem object MEM_Dyn. In other words, make it so ** that any TEXT or BLOB content is stored in memory obtained from @@ -262,7 +289,7 @@ int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - if( sqlite3VdbeMemGrow(pMem, nByte, 0) ){ + if( sqlite3VdbeMemClearAndResize(pMem, nByte) ){ return SQLITE_NOMEM; } @@ -866,7 +893,7 @@ int sqlite3VdbeMemSetStr( if( nByte>iLimit ){ return SQLITE_TOOBIG; } - if( sqlite3VdbeMemGrow(pMem, nAlloc, 0) ){ + if( sqlite3VdbeMemClearAndResize(pMem, nAlloc) ){ return SQLITE_NOMEM; } memcpy(pMem->z, z, nAlloc); @@ -944,7 +971,7 @@ int sqlite3VdbeMemFromBtree( pMem->n = (int)amt; }else{ pMem->flags = MEM_Null; - if( SQLITE_OK==(rc = sqlite3VdbeMemGrow(pMem, amt+2, 0)) ){ + if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+2)) ){ if( key ){ rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); }else{ diff --git a/src/vdbesort.c b/src/vdbesort.c index 2d7bc8a7a4..3e82f6e822 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -2461,7 +2461,7 @@ int sqlite3VdbeSorterRowkey(const VdbeCursor *pCsr, Mem *pOut){ void *pKey; int nKey; /* Sorter key to copy into pOut */ pKey = vdbeSorterRowkey(pSorter, &nKey); - if( sqlite3VdbeMemGrow(pOut, nKey, 0) ){ + if( sqlite3VdbeMemClearAndResize(pOut, nKey) ){ return SQLITE_NOMEM; } pOut->n = nKey; From 137fd4fda27c3c0b64f80d8523996559dd20aebe Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Sep 2014 02:01:37 +0000 Subject: [PATCH 353/710] Make sure that the sorting-index pre-filter recognizes that a rowid reference might be sortable. This fixes a performance regression. FossilOrigin-Name: 72727b68cd07969165f1f0943cc7e1a265436653 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 1 + test/orderby1.test | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 28c4382ce9..786c774f55 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3VdbeMemClearAndResize()\sinterface\sto\sbe\sused\sin\splace\sof\nsqlite3VdbeMemGrow(). -D 2014-09-19T00:43:39.899 +C Make\ssure\sthat\sthe\ssorting-index\spre-filter\srecognizes\sthat\sa\srowid\sreference\nmight\sbe\ssortable.\s\sThis\sfixes\sa\sperformance\sregression. +D 2014-09-19T02:01:37.043 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -301,7 +301,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c dc276288039fb45ce23c80e4535980f5a152d8ec +F src/where.c 0888567c0e01a41b6001647e333f8ccfd3ae7d36 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -739,7 +739,7 @@ F test/notnull.test f8fcf58669ddba79274daa2770d61dfad8274f62 F test/null.test a8b09b8ed87852742343b33441a9240022108993 F test/numcast.test 5d126f7f581432e86a90d1e35cac625164aec4a1 F test/openv2.test 0d3040974bf402e19b7df4b783e447289d7ab394 -F test/orderby1.test 12426f99518cde45f34215ca6a0ebc0e9bc5c77a +F test/orderby1.test eb246e377612b21a418fbea57047ba8ea88aaa6b F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9c09ac353df6041808cace41880f4729ee73f5e1 -R eeadad19cadf3315e345f0babe4222b0 +P 5b9b8987797abf7c68d2c3154f6657be9b8b1c8f +R 15204d62056b16fb4766540f7ebb9528 U drh -Z fd83cfbfc71d89fd149fcf4e06924224 +Z d3f0dcf862181fb956350fa4a77ff638 diff --git a/manifest.uuid b/manifest.uuid index a4139a3651..06e5229d32 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b9b8987797abf7c68d2c3154f6657be9b8b1c8f \ No newline at end of file +72727b68cd07969165f1f0943cc7e1a265436653 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c7cb7bb67c..318065d486 100644 --- a/src/where.c +++ b/src/where.c @@ -4560,6 +4560,7 @@ static int indexMightHelpWithOrderBy( Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); if( pExpr->op!=TK_COLUMN ) return 0; if( pExpr->iTable==iCursor ){ + if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; } diff --git a/test/orderby1.test b/test/orderby1.test index e06c9f19a0..6674e32220 100644 --- a/test/orderby1.test +++ b/test/orderby1.test @@ -481,5 +481,19 @@ do_execsql_test 6.0 { FROM abc; } {hardware hardware hardware} +# Here is a test for a query-planner problem reported on the SQLite +# mailing list on 2014-09-18 by "Merike". Beginning with version 3.8.0, +# a separate sort was being used rather than using the single-column +# index. This was due to an oversight in the indexMightHelpWithOrderby() +# routine in where.c. +# +do_execsql_test 7.0 { + CREATE TABLE t7(a,b); + CREATE INDEX t7a ON t7(a); + CREATE INDEX t7ab ON t7(a,b); + EXPLAIN QUERY PLAN + SELECT * FROM t7 WHERE a=?1 ORDER BY rowid; +} {~/ORDER BY/} + finish_test From 7b5ebcaf26d6640f5b4997732c8c5161feda4c9e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Sep 2014 15:28:33 +0000 Subject: [PATCH 354/710] The OP_Column opcode runs faster and is smaller by manually in-lining the code that persists string values in the output register. FossilOrigin-Name: 36b613ccf0ddb764af90841994da91b7fcaa8f00 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/vdbe.c | 16 +++++++++++++++- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 013d54f55b..611616d0c2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\ssqlite3VdbeMemClearAndResize()\sfunction.\s\sFix\sa\ssorting-index\nprefilter\sproblem. -D 2014-09-19T04:42:38.074 +C The\sOP_Column\sopcode\sruns\sfaster\sand\sis\ssmaller\sby\smanually\sin-lining\sthe\ncode\sthat\spersists\sstring\svalues\sin\sthe\soutput\sregister. +D 2014-09-19T15:28:33.728 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,7 +288,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 7bc115e540282b1bfff9b43205cbf6528e6398a6 +F src/vdbe.c 9f2a0a2dfa06e99feabd754e9be3a436ac3cf97d F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -1198,8 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9c09ac353df6041808cace41880f4729ee73f5e1 72727b68cd07969165f1f0943cc7e1a265436653 -R a5960c77be61d5e92e67de5776336365 -T +closed 72727b68cd07969165f1f0943cc7e1a265436653 +P 987a7a211913b3949da20e43423af376f72a28ba +R 03eb020eed7bf3a8527e5a4e8f2d486b U drh -Z 88c7b85ca090541e4ecff878bb673522 +Z 0d995c472ae8295ead3b2ebd0d73fc53 diff --git a/manifest.uuid b/manifest.uuid index 6dc6c4d043..f8720261f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -987a7a211913b3949da20e43423af376f72a28ba \ No newline at end of file +36b613ccf0ddb764af90841994da91b7fcaa8f00 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 25a33a63f7..0baacef4c0 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2482,7 +2482,21 @@ case OP_Column: { pDest->enc = encoding; op_column_out: - Deephemeralize(pDest); + /* If the column value is an ephemeral string, go ahead and persist + ** that string in case the cursor moves before the column value is + ** used. The following code does the equivalent of Deephemeralize() + ** but does it faster. */ + if( (pDest->flags & MEM_Ephem)!=0 && pDest->z ){ + u16 f = pDest->flags & (MEM_Str|MEM_Blob); + assert( f!=0 ); + zData = (const u8*)pDest->z; + len = pDest->n; + if( sqlite3VdbeMemClearAndResize(pDest, len+2) ) goto no_mem; + memcpy(pDest->z, zData, len); + pDest->z[len] = 0; + pDest->z[len+1] = 0; + pDest->flags = f|MEM_Term; + } op_column_error: UPDATE_MAX_BLOBSIZE(pDest); REGISTER_TRACE(pOp->p3, pDest); From 7abda856074ec8fecc28ca7a5fb2fc630b403e23 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Sep 2014 16:02:06 +0000 Subject: [PATCH 355/710] Updates to comments. No code changes. FossilOrigin-Name: 9b42c3da6b2593a10b8fe4b2fcc3d650132625c1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 12 +++--------- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 611616d0c2..c87b875a6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sOP_Column\sopcode\sruns\sfaster\sand\sis\ssmaller\sby\smanually\sin-lining\sthe\ncode\sthat\spersists\sstring\svalues\sin\sthe\soutput\sregister. -D 2014-09-19T15:28:33.728 +C Updates\sto\scomments.\s\sNo\scode\schanges. +D 2014-09-19T16:02:06.389 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vdbe.c 9f2a0a2dfa06e99feabd754e9be3a436ac3cf97d F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d -F src/vdbeaux.c ecf0bb835f7c809ba8e22ba7d3a587b7a087ac52 +F src/vdbeaux.c 498b42510679767ea6c893c8c51a8fd935d7ab2d F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 5cd963730414a1a6ba53b8b340eba3f46ec2cb1d F src/vdbesort.c 75c66c2fc02d450b67b4816873fba8088feaf12c @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 987a7a211913b3949da20e43423af376f72a28ba -R 03eb020eed7bf3a8527e5a4e8f2d486b +P 36b613ccf0ddb764af90841994da91b7fcaa8f00 +R 080b311a7df7cfe0ac5886dd9047451e U drh -Z 0d995c472ae8295ead3b2ebd0d73fc53 +Z bac819561e5ec88b9538e526b2bc8493 diff --git a/manifest.uuid b/manifest.uuid index f8720261f1..66235033a3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -36b613ccf0ddb764af90841994da91b7fcaa8f00 \ No newline at end of file +9b42c3da6b2593a10b8fe4b2fcc3d650132625c1 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 3ce373dffe..f4d50406b9 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -10,9 +10,7 @@ ** ************************************************************************* ** This file contains code used for creating, destroying, and populating -** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) Prior -** to version 2.8.7, all this code was combined into the vdbe.c source file. -** But that file was getting too big so this subroutines were split out. +** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) */ #include "sqliteInt.h" #include "vdbeInt.h" @@ -1606,7 +1604,7 @@ void sqlite3VdbeRewind(Vdbe *p){ /* ** Prepare a virtual machine for execution for the first time after ** creating the virtual machine. This involves things such -** as allocating stack space and initializing the program counter. +** as allocating registers and initializing the program counter. ** After the VDBE has be prepped, it can be executed by one or more ** calls to sqlite3VdbeExec(). ** @@ -1818,11 +1816,7 @@ static void closeAllCursors(Vdbe *p){ } /* -** Clean up the VM after execution. -** -** This routine will automatically close any cursors, lists, and/or -** sorters that were left open. It also deletes the values of -** variables in the aVar[] array. +** Clean up the VM after a single run. */ static void Cleanup(Vdbe *p){ sqlite3 *db = p->db; From 069c23c947cd5b5e85b6d9bf10d6291547fbda4f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Sep 2014 16:13:12 +0000 Subject: [PATCH 356/710] Small size reduction and performance increase for releaseMemArray(). FossilOrigin-Name: 24cd32d681df58f687b2afbe4b13d579e3efdd4b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index c87b875a6b..ddd7321ef0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\scomments.\s\sNo\scode\schanges. -D 2014-09-19T16:02:06.389 +C Small\ssize\sreduction\sand\sperformance\sincrease\sfor\sreleaseMemArray(). +D 2014-09-19T16:13:12.685 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -292,7 +292,7 @@ F src/vdbe.c 9f2a0a2dfa06e99feabd754e9be3a436ac3cf97d F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d -F src/vdbeaux.c 498b42510679767ea6c893c8c51a8fd935d7ab2d +F src/vdbeaux.c a05adc3c96abdaf3db14768ddd63132fc9678060 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 5cd963730414a1a6ba53b8b340eba3f46ec2cb1d F src/vdbesort.c 75c66c2fc02d450b67b4816873fba8088feaf12c @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 36b613ccf0ddb764af90841994da91b7fcaa8f00 -R 080b311a7df7cfe0ac5886dd9047451e +P 9b42c3da6b2593a10b8fe4b2fcc3d650132625c1 +R 351b243201f0eb22d3adcd68e13fd4a5 U drh -Z bac819561e5ec88b9538e526b2bc8493 +Z 9538899cfe34e917b2f35980fa39505f diff --git a/manifest.uuid b/manifest.uuid index 66235033a3..5c4c41d72c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b42c3da6b2593a10b8fe4b2fcc3d650132625c1 \ No newline at end of file +24cd32d681df58f687b2afbe4b13d579e3efdd4b \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index f4d50406b9..87b14a6d87 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1224,16 +1224,16 @@ void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){ */ static void releaseMemArray(Mem *p, int N){ if( p && N ){ - Mem *pEnd; + Mem *pEnd = &p[N]; sqlite3 *db = p->db; u8 malloc_failed = db->mallocFailed; if( db->pnBytesFreed ){ - for(pEnd=&p[N]; pszMalloc ) sqlite3DbFree(db, p->zMalloc); - } + }while( (++p)flags = MEM_Undefined; - } + }while( (++p)mallocFailed = malloc_failed; } } From 0c8f760ab35068d339edc4a4b9aee070c0198b47 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Sep 2014 16:56:45 +0000 Subject: [PATCH 357/710] Remove a local variable from the OP_Column implementation, resulting in a modest size reduction and a performance increase. FossilOrigin-Name: 6199760d1340858d97c845177986b783da915d9e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 23 +++++++++++------------ 3 files changed, 18 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index ddd7321ef0..bb272d2568 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\ssize\sreduction\sand\sperformance\sincrease\sfor\sreleaseMemArray(). -D 2014-09-19T16:13:12.685 +C Remove\sa\slocal\svariable\sfrom\sthe\sOP_Column\simplementation,\sresulting\sin\sa\nmodest\ssize\sreduction\sand\sa\sperformance\sincrease. +D 2014-09-19T16:56:45.748 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,7 +288,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 9f2a0a2dfa06e99feabd754e9be3a436ac3cf97d +F src/vdbe.c c90b7ebe856beb75077cf4486efc1863de219f34 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9b42c3da6b2593a10b8fe4b2fcc3d650132625c1 -R 351b243201f0eb22d3adcd68e13fd4a5 +P 24cd32d681df58f687b2afbe4b13d579e3efdd4b +R dfacf0845df6af56f07b0acf2e71dc1f U drh -Z 9538899cfe34e917b2f35980fa39505f +Z 845eafc57b4e53556b2c7ee6a79ebdad diff --git a/manifest.uuid b/manifest.uuid index 5c4c41d72c..67c66e8f0f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24cd32d681df58f687b2afbe4b13d579e3efdd4b \ No newline at end of file +6199760d1340858d97c845177986b783da915d9e \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 0baacef4c0..9b679c534b 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2257,7 +2257,6 @@ case OP_Column: { int p2; /* column number to retrieve */ VdbeCursor *pC; /* The VDBE cursor */ BtCursor *pCrsr; /* The BTree cursor */ - u32 *aType; /* aType[i] holds the numeric type of the i-th column */ u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ int len; /* The length of the serialized data for the column */ int i; /* Loop counter */ @@ -2270,6 +2269,7 @@ case OP_Column: { u32 szField; /* Number of bytes in the content of a field */ u32 avail; /* Number of bytes of available data */ u32 t; /* A type code from the record header */ + u16 fx; /* pDest->flags value */ Mem *pReg; /* PseudoTable input register */ p2 = pOp->p2; @@ -2280,8 +2280,7 @@ case OP_Column: { pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( p2nField ); - aType = pC->aType; - aOffset = aType + pC->nField; + aOffset = pC->aType + pC->nField; #ifndef SQLITE_OMIT_VIRTUALTABLE assert( pC->pVtabCursor==0 ); /* OP_Column never called on virtual table */ #endif @@ -2362,7 +2361,7 @@ case OP_Column: { } /* Make sure at least the first p2+1 entries of the header have been - ** parsed and valid information is in aOffset[] and aType[]. + ** parsed and valid information is in aOffset[] and pC->aType[]. */ if( pC->nHdrParsed<=p2 ){ /* If there is more header available for parsing in the record, try @@ -2382,7 +2381,7 @@ case OP_Column: { zData = pC->aRow; } - /* Fill in aType[i] and aOffset[i] values through the p2-th field. */ + /* Fill in pC->aType[i] and aOffset[i] values through the p2-th field. */ i = pC->nHdrParsed; offset = aOffset[i]; zHdr = zData + pC->iHdrOffset; @@ -2395,7 +2394,7 @@ case OP_Column: { }else{ zHdr += sqlite3GetVarint32(zHdr, &t); } - aType[i] = t; + pC->aType[i] = t; szField = sqlite3VdbeSerialTypeLen(t); offset += szField; if( offsetaType[p2] are ** all valid. */ assert( p2nHdrParsed ); assert( rc==SQLITE_OK ); assert( sqlite3VdbeCheckMemInvariants(pDest) ); if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest); + t = pC->aType[p2]; if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ - sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], aType[p2], pDest); + sqlite3VdbeSerialGet(pC->aRow+aOffset[p2], t, pDest); }else{ /* This branch happens only when content is on overflow pages */ - t = aType[p2]; if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)) || (len = sqlite3VdbeSerialTypeLen(t))==0 @@ -2487,15 +2486,15 @@ op_column_out: ** used. The following code does the equivalent of Deephemeralize() ** but does it faster. */ if( (pDest->flags & MEM_Ephem)!=0 && pDest->z ){ - u16 f = pDest->flags & (MEM_Str|MEM_Blob); - assert( f!=0 ); + fx = pDest->flags & (MEM_Str|MEM_Blob); + assert( fx!=0 ); zData = (const u8*)pDest->z; len = pDest->n; if( sqlite3VdbeMemClearAndResize(pDest, len+2) ) goto no_mem; memcpy(pDest->z, zData, len); pDest->z[len] = 0; pDest->z[len+1] = 0; - pDest->flags = f|MEM_Term; + pDest->flags = fx|MEM_Term; } op_column_error: UPDATE_MAX_BLOBSIZE(pDest); From 142341cd2344c8c97e34d20db51c6e266c652e36 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Sep 2014 19:00:48 +0000 Subject: [PATCH 358/710] Make the "nolock" VFS on unix a version-3 VFS so that the sorter can use memory-mapped I/O. FossilOrigin-Name: 3db78d6100a1ecf58c18eec3abefa7d1250c649c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index bb272d2568..6127e2a58d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\slocal\svariable\sfrom\sthe\sOP_Column\simplementation,\sresulting\sin\sa\nmodest\ssize\sreduction\sand\sa\sperformance\sincrease. -D 2014-09-19T16:56:45.748 +C Make\sthe\s"nolock"\sVFS\son\sunix\sa\sversion-3\sVFS\sso\sthat\sthe\ssorter\scan\nuse\smemory-mapped\sI/O. +D 2014-09-19T19:00:48.687 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -211,7 +211,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c addd023b26c623fec4dedc110fc4370a65b4768c +F src/os_unix.c 9096a1b1449182e67e759f59994eee04113bc587 F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c caab007743821d96752597c9cfd7351654697b06 @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 24cd32d681df58f687b2afbe4b13d579e3efdd4b -R dfacf0845df6af56f07b0acf2e71dc1f +P 6199760d1340858d97c845177986b783da915d9e +R 747bc9b4978b6e3fd1fc3d52243dfaf2 U drh -Z 845eafc57b4e53556b2c7ee6a79ebdad +Z 26fea32b6bd594b310d5a7de197ed86f diff --git a/manifest.uuid b/manifest.uuid index 67c66e8f0f..146b5e3eb7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6199760d1340858d97c845177986b783da915d9e \ No newline at end of file +3db78d6100a1ecf58c18eec3abefa7d1250c649c \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 5e820260a4..fcd9e72d06 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4997,7 +4997,7 @@ IOMETHODS( IOMETHODS( nolockIoFinder, /* Finder function name */ nolockIoMethods, /* sqlite3_io_methods object name */ - 1, /* shared memory is disabled */ + 3, /* shared memory is disabled */ nolockClose, /* xClose method */ nolockLock, /* xLock method */ nolockUnlock, /* xUnlock method */ From d74a90eab8903192e29fc56851eeee0f0410904f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Sep 2014 19:43:20 +0000 Subject: [PATCH 359/710] Do not attempt to extend the temp file if VFS version 3 is not supported and hence memory mapped I/O is unavailable. FossilOrigin-Name: 3ab20ba14f0204efeec62c7dbb87cb3f60e2497f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6127e2a58d..4f686333df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\s"nolock"\sVFS\son\sunix\sa\sversion-3\sVFS\sso\sthat\sthe\ssorter\scan\nuse\smemory-mapped\sI/O. -D 2014-09-19T19:00:48.687 +C Do\snot\sattempt\sto\sextend\sthe\stemp\sfile\sif\sVFS\sversion\s3\sis\snot\ssupported\sand\nhence\smemory\smapped\sI/O\sis\sunavailable. +D 2014-09-19T19:43:20.458 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -295,7 +295,7 @@ F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d F src/vdbeaux.c a05adc3c96abdaf3db14768ddd63132fc9678060 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 5cd963730414a1a6ba53b8b340eba3f46ec2cb1d -F src/vdbesort.c 75c66c2fc02d450b67b4816873fba8088feaf12c +F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef F src/vdbetrace.c 4f29b04edb0cec3d5fcd9b566d9f0e75c8984362 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6199760d1340858d97c845177986b783da915d9e -R 747bc9b4978b6e3fd1fc3d52243dfaf2 +P 3db78d6100a1ecf58c18eec3abefa7d1250c649c +R 818146e215cbcd83bc222da6a20cc822 U drh -Z 26fea32b6bd594b310d5a7de197ed86f +Z 7ba13d68a7d8baeb2d8a716a6d033daa diff --git a/manifest.uuid b/manifest.uuid index 146b5e3eb7..ab36f8e2a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3db78d6100a1ecf58c18eec3abefa7d1250c649c \ No newline at end of file +3ab20ba14f0204efeec62c7dbb87cb3f60e2497f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 3e82f6e822..d9679caa06 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1124,9 +1124,9 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ ** the specific VFS implementation. */ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ - if( nByte<=(i64)(db->nMaxSorterMmap) ){ + if( nByte<=(i64)(db->nMaxSorterMmap) && pFd->pMethods->iVersion>=3 ){ int rc = sqlite3OsTruncate(pFd, nByte); - if( rc==SQLITE_OK && pFd->pMethods->iVersion>=3 ){ + if( rc==SQLITE_OK ){ void *p = 0; sqlite3OsFetch(pFd, 0, (int)nByte, &p); sqlite3OsUnfetch(pFd, 0, p); From 4583c37ca0361ce9bdc19dd0bae8998f5b576466 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Sep 2014 20:13:25 +0000 Subject: [PATCH 360/710] Fix the affinity on inserts into the ANALYZE tables. Change the affinity characters to be upper case, to make the P5 parameter of comparison operators easier to read. FossilOrigin-Name: 3f3ca76aea38d566a574f4403b375bdac32854ed --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 6 ++++-- src/insert.c | 22 +++++++++++----------- src/sqliteInt.h | 20 ++++++++++---------- 5 files changed, 34 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index 4f686333df..3bf4c484b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sattempt\sto\sextend\sthe\stemp\sfile\sif\sVFS\sversion\s3\sis\snot\ssupported\sand\nhence\smemory\smapped\sI/O\sis\sunavailable. -D 2014-09-19T19:43:20.458 +C Fix\sthe\saffinity\son\sinserts\sinto\sthe\sANALYZE\stables.\s\sChange\sthe\saffinity\ncharacters\sto\sbe\supper\scase,\sto\smake\sthe\sP5\sparameter\sof\scomparison\soperators\neasier\sto\sread. +D 2014-09-19T20:13:25.422 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -165,7 +165,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 79383a54fee3b7f1fb03dd4c8c8115583f506de5 +F src/analyze.c 6290a109be876daaa242cd7216f97240f5401776 F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -188,7 +188,7 @@ F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 0b073fade178d9dbd990bbb32b4438e50b884a06 +F src/insert.c 5b9243a33726008cc4132897d2be371db12a13be F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 @@ -231,7 +231,7 @@ F src/shell.c c00220cdd7f2027780bc25b78376c16dc24e4b7d F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 9bb8f655b076e1b9ed7cfe0b8c181e758d937369 +F src/sqliteInt.h 686e6a49ebcea813eafa24752d19751ffa6a1b93 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3db78d6100a1ecf58c18eec3abefa7d1250c649c -R 818146e215cbcd83bc222da6a20cc822 +P 3ab20ba14f0204efeec62c7dbb87cb3f60e2497f +R 6403cad332b96b991c6e79d4338c8e0a U drh -Z 7ba13d68a7d8baeb2d8a716a6d033daa +Z a6ee4447c2926dae75a9d9931c4a5376 diff --git a/manifest.uuid b/manifest.uuid index ab36f8e2a0..5b0f66cb3c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3ab20ba14f0204efeec62c7dbb87cb3f60e2497f \ No newline at end of file +3f3ca76aea38d566a574f4403b375bdac32854ed \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index d5a11a3ed6..aec1f021ea 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1201,7 +1201,8 @@ static void analyzeOneTable( /* Add the entry to the stat1 table. */ callStatGet(v, regStat4, STAT_GET_STAT1, regStat1); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "aaa", 0); + assert( "BBB"[0]==SQLITE_AFF_TEXT ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -1264,7 +1265,8 @@ static void analyzeOneTable( sqlite3VdbeAddOp2(v, OP_Count, iTabCur, regStat1); jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "aaa", 0); + assert( "BBB"[0]==SQLITE_AFF_TEXT ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); diff --git a/src/insert.c b/src/insert.c index 6a3ab8edae..a5c3f3e92d 100644 --- a/src/insert.c +++ b/src/insert.c @@ -56,13 +56,13 @@ void sqlite3OpenTable( ** ** Character Column affinity ** ------------------------------ -** 'a' TEXT -** 'b' NONE -** 'c' NUMERIC -** 'd' INTEGER -** 'e' REAL +** 'A' NONE +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'F' REAL ** -** An extra 'd' is appended to the end of the string to cover the +** An extra 'D' is appended to the end of the string to cover the ** rowid that appears as the last column in every index. ** ** Memory for the buffer containing the column index affinity string @@ -111,11 +111,11 @@ const char *sqlite3IndexAffinityStr(Vdbe *v, Index *pIdx){ ** ** Character Column affinity ** ------------------------------ -** 'a' TEXT -** 'b' NONE -** 'c' NUMERIC -** 'd' INTEGER -** 'e' REAL +** 'A' NONE +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'E' REAL */ void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ int i; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ea8ad4573f..92efd9c087 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1422,18 +1422,18 @@ struct CollSeq { ** 't' for SQLITE_AFF_TEXT. But we can save a little space and improve ** the speed a little by numbering the values consecutively. ** -** But rather than start with 0 or 1, we begin with 'a'. That way, +** But rather than start with 0 or 1, we begin with 'A'. That way, ** when multiple affinity types are concatenated into a string and ** used as the P4 operand, they will be more readable. ** ** Note also that the numeric types are grouped together so that testing ** for a numeric type is a single comparison. And the NONE type is first. */ -#define SQLITE_AFF_NONE 'a' -#define SQLITE_AFF_TEXT 'b' -#define SQLITE_AFF_NUMERIC 'c' -#define SQLITE_AFF_INTEGER 'd' -#define SQLITE_AFF_REAL 'e' +#define SQLITE_AFF_NONE 'A' +#define SQLITE_AFF_TEXT 'B' +#define SQLITE_AFF_NUMERIC 'C' +#define SQLITE_AFF_INTEGER 'D' +#define SQLITE_AFF_REAL 'E' #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) @@ -1441,7 +1441,7 @@ struct CollSeq { ** The SQLITE_AFF_MASK values masks off the significant bits of an ** affinity value. */ -#define SQLITE_AFF_MASK 0x67 +#define SQLITE_AFF_MASK 0x47 /* ** Additional bit values that can be ORed with an affinity without @@ -1452,10 +1452,10 @@ struct CollSeq { ** operator is NULL. It is added to certain comparison operators to ** prove that the operands are always NOT NULL. */ -#define SQLITE_JUMPIFNULL 0x08 /* jumps if either operand is NULL */ -#define SQLITE_STOREP2 0x10 /* Store result in reg[P2] rather than jump */ +#define SQLITE_JUMPIFNULL 0x10 /* jumps if either operand is NULL */ +#define SQLITE_STOREP2 0x20 /* Store result in reg[P2] rather than jump */ #define SQLITE_NULLEQ 0x80 /* NULL=NULL */ -#define SQLITE_NOTNULL 0x88 /* Assert that operands are never NULL */ +#define SQLITE_NOTNULL 0x90 /* Assert that operands are never NULL */ /* ** An object of this type is created for each virtual table present in From 11a6eee8e1a3f74fb319ae69620d3c53658268b3 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Sep 2014 22:01:54 +0000 Subject: [PATCH 361/710] Tighten the conditions under which applyNumericAffinity() be called and add assert() statements to prove that it is never called otherwise. FossilOrigin-Name: e996ca32cb643c558b616c0dd872f3351b6aa3ef --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 7 +++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 3bf4c484b9..e2e61762cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\saffinity\son\sinserts\sinto\sthe\sANALYZE\stables.\s\sChange\sthe\saffinity\ncharacters\sto\sbe\supper\scase,\sto\smake\sthe\sP5\sparameter\sof\scomparison\soperators\neasier\sto\sread. -D 2014-09-19T20:13:25.422 +C Tighten\sthe\sconditions\sunder\swhich\sapplyNumericAffinity()\sbe\scalled\sand\sadd\nassert()\sstatements\sto\sprove\sthat\sit\sis\snever\scalled\sotherwise. +D 2014-09-19T22:01:54.366 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,7 +288,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c c90b7ebe856beb75077cf4486efc1863de219f34 +F src/vdbe.c 16efd1ae26d877827cd6669f5f19afd8d8903d08 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3ab20ba14f0204efeec62c7dbb87cb3f60e2497f -R 6403cad332b96b991c6e79d4338c8e0a +P 3f3ca76aea38d566a574f4403b375bdac32854ed +R fc5d306f71c9e96ead7521f7177e02ff U drh -Z a6ee4447c2926dae75a9d9931c4a5376 +Z cb569a0dfcbf7ed8e3897eb9a80bb15e diff --git a/manifest.uuid b/manifest.uuid index 5b0f66cb3c..3fc1bf7213 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3f3ca76aea38d566a574f4403b375bdac32854ed \ No newline at end of file +e996ca32cb643c558b616c0dd872f3351b6aa3ef \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 9b679c534b..136b7abae1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -242,8 +242,7 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ double rValue; i64 iValue; u8 enc = pRec->enc; - if( (pRec->flags&MEM_Str)==0 ) return; - if( (pRec->flags&(MEM_Int|MEM_Real))!=0 ) return; + assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real))==MEM_Str ); if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ pRec->u.i = iValue; @@ -283,7 +282,7 @@ static void applyAffinity( || affinity==SQLITE_AFF_NUMERIC ); if( (pRec->flags & MEM_Int)==0 ){ if( (pRec->flags & MEM_Real)==0 ){ - applyNumericAffinity(pRec,1); + if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1); }else{ sqlite3VdbeIntegerAffinity(pRec); } @@ -3558,7 +3557,7 @@ case OP_SeekGT: { /* jump, in3 */ ** blob, or NULL. But it needs to be an integer before we can do ** the seek, so convert it. */ pIn3 = &aMem[pOp->p3]; - if( (pIn3->flags & (MEM_Int|MEM_Real))==0 ){ + if( (pIn3->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3, 0); } iKey = sqlite3VdbeIntValue(pIn3); From 1eda9f7d8702b8a81023e43ce153938f69dd7f62 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Sep 2014 22:30:49 +0000 Subject: [PATCH 362/710] Recognize the invariant that a Mem object cannot be MEM_Dyn and have a non-zero szMalloc at the same time. Enforce this with assert()s and exploit it in the sqlite3VdbeMemClearAndResize() routine for a performance increase. FossilOrigin-Name: 3b21cf2b284048da4b728a5d6ec89e5c330144d4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 13 ++++++++----- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index e2e61762cf..6625f6c19a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tighten\sthe\sconditions\sunder\swhich\sapplyNumericAffinity()\sbe\scalled\sand\sadd\nassert()\sstatements\sto\sprove\sthat\sit\sis\snever\scalled\sotherwise. -D 2014-09-19T22:01:54.366 +C Recognize\sthe\sinvariant\sthat\sa\sMem\sobject\scannot\sbe\sMEM_Dyn\sand\shave\s\na\snon-zero\sszMalloc\sat\sthe\ssame\stime.\s\sEnforce\sthis\swith\sassert()s\sand\nexploit\sit\sin\sthe\ssqlite3VdbeMemClearAndResize()\sroutine\sfor\sa\sperformance\nincrease. +D 2014-09-19T22:30:49.809 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -294,7 +294,7 @@ F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d F src/vdbeaux.c a05adc3c96abdaf3db14768ddd63132fc9678060 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 5cd963730414a1a6ba53b8b340eba3f46ec2cb1d +F src/vdbemem.c 1e105dacf5190fc85a8ec2107c0dcc1884e75099 F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef F src/vdbetrace.c 4f29b04edb0cec3d5fcd9b566d9f0e75c8984362 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3f3ca76aea38d566a574f4403b375bdac32854ed -R fc5d306f71c9e96ead7521f7177e02ff +P e996ca32cb643c558b616c0dd872f3351b6aa3ef +R 5148a059197bc43e5fc9e398f77834ac U drh -Z cb569a0dfcbf7ed8e3897eb9a80bb15e +Z 24b5c2935cab55558784e01172e53731 diff --git a/manifest.uuid b/manifest.uuid index 3fc1bf7213..d94cacb1fd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e996ca32cb643c558b616c0dd872f3351b6aa3ef \ No newline at end of file +3b21cf2b284048da4b728a5d6ec89e5c330144d4 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 10c91dfd75..36db80fa18 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -31,6 +31,9 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){ */ assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 ); + /* MEM_Dyn may only be set if Mem.szMalloc==0 */ + assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 ); + /* Cannot be both MEM_Int and MEM_Real at the same time */ assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) ); @@ -164,19 +167,19 @@ SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ ** if unable to complete the resizing. */ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ - if( pMem->szMallocflags & MEM_Dyn)!=0 ){ + assert( szNew>=0 ); + if( pMem->szMallocflags & MEM_Dyn)==0 ); pMem->z = pMem->zMalloc; pMem->flags &= (MEM_Null|MEM_Int|MEM_Real); return SQLITE_OK; } /* -** Make the given Mem object MEM_Dyn. In other words, make it so -** that any TEXT or BLOB content is stored in memory obtained from -** malloc(). In this way, we know that the memory is safe to be -** overwritten or altered. +** Change pMem so that its MEM_Str or MEM_Blob value is stored in +** MEM.zMalloc, where it can be safely written. ** ** Return SQLITE_OK on success or SQLITE_NOMEM if malloc fails. */ From e7a3466458a175a88fa82c64c7a432825622c1d8 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Sep 2014 22:44:20 +0000 Subject: [PATCH 363/710] Simplify two conditionals and add testcase() macros to the affinity transform logic in the comparison operators. FossilOrigin-Name: 544664cadfb4e504bc0b321c865d1ecb8a831e20 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 8 ++++++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 6625f6c19a..fd932fa1b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Recognize\sthe\sinvariant\sthat\sa\sMem\sobject\scannot\sbe\sMEM_Dyn\sand\shave\s\na\snon-zero\sszMalloc\sat\sthe\ssame\stime.\s\sEnforce\sthis\swith\sassert()s\sand\nexploit\sit\sin\sthe\ssqlite3VdbeMemClearAndResize()\sroutine\sfor\sa\sperformance\nincrease. -D 2014-09-19T22:30:49.809 +C Simplify\stwo\sconditionals\sand\sadd\stestcase()\smacros\sto\sthe\saffinity\stransform\nlogic\sin\sthe\scomparison\soperators. +D 2014-09-19T22:44:20.033 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -288,7 +288,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 16efd1ae26d877827cd6669f5f19afd8d8903d08 +F src/vdbe.c de1af1795bebdad20c23e82bafa2f531e9893198 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e996ca32cb643c558b616c0dd872f3351b6aa3ef -R 5148a059197bc43e5fc9e398f77834ac +P 3b21cf2b284048da4b728a5d6ec89e5c330144d4 +R c321e5b6443015b26d22f3eade4b771c U drh -Z 24b5c2935cab55558784e01172e53731 +Z b0d83e95571a3002a17dc014110d23f5 diff --git a/manifest.uuid b/manifest.uuid index d94cacb1fd..aee58dcbcb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3b21cf2b284048da4b728a5d6ec89e5c330144d4 \ No newline at end of file +544664cadfb4e504bc0b321c865d1ecb8a831e20 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 136b7abae1..26ca72b9f4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1900,17 +1900,21 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ /* Neither operand is NULL. Do a comparison. */ affinity = pOp->p5 & SQLITE_AFF_MASK; if( affinity>=SQLITE_AFF_NUMERIC ){ - if( (pIn1->flags & (MEM_Int|MEM_Real))==0 && (pIn1->flags&MEM_Str)!=0 ){ + if( (pIn1->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn1,0); } - if( (pIn3->flags & (MEM_Int|MEM_Real))==0 && (pIn3->flags&MEM_Str)!=0 ){ + if( (pIn3->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3,0); } }else if( affinity==SQLITE_AFF_TEXT ){ if( (pIn1->flags & MEM_Str)==0 && (pIn1->flags & (MEM_Int|MEM_Real))!=0 ){ + testcase( pIn1->flags & MEM_Int ); + testcase( pIn1->flags & MEM_Real ); sqlite3VdbeMemStringify(pIn1, encoding, 1); } if( (pIn3->flags & MEM_Str)==0 && (pIn3->flags & (MEM_Int|MEM_Real))!=0 ){ + testcase( pIn3->flags & MEM_Int ); + testcase( pIn3->flags & MEM_Real ); sqlite3VdbeMemStringify(pIn3, encoding, 1); } } From 33ac4c8bf5de82ab0e496b1da86fc1f1d1ba8f3f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 20 Sep 2014 00:02:23 +0000 Subject: [PATCH 364/710] Revise macro usage in 'sqliteInt.h'. FossilOrigin-Name: 35db3e2f350ca2bc6bb9e1a647aec7f93bfb7065 --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/printf.c | 2 +- src/sqliteInt.h | 18 +++++++++--------- 4 files changed, 22 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index fd932fa1b7..9125f11713 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\stwo\sconditionals\sand\sadd\stestcase()\smacros\sto\sthe\saffinity\stransform\nlogic\sin\sthe\scomparison\soperators. -D 2014-09-19T22:44:20.033 +C Revise\smacro\susage\sin\s'sqliteInt.h'. +D 2014-09-20T00:02:23.616 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 19e3e81addf593195369ec8d487ed063ad3170bb +F src/printf.c 901a2b924f10db42b7c32936eda80feb3a769aca F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -231,7 +231,7 @@ F src/shell.c c00220cdd7f2027780bc25b78376c16dc24e4b7d F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 686e6a49ebcea813eafa24752d19751ffa6a1b93 +F src/sqliteInt.h 6fd801cac974d310949ddefd6a6c6982c9778f3f F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -1198,7 +1198,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3b21cf2b284048da4b728a5d6ec89e5c330144d4 -R c321e5b6443015b26d22f3eade4b771c -U drh -Z b0d83e95571a3002a17dc014110d23f5 +P 544664cadfb4e504bc0b321c865d1ecb8a831e20 +R 5186d6949cf27e738e34796d88aa31e5 +T *branch * sqliteIntMacros +T *sym-sqliteIntMacros * +T -sym-trunk * +U mistachkin +Z 51ca01ef3bf4ffdaf2fde2b29bc4c756 diff --git a/manifest.uuid b/manifest.uuid index aee58dcbcb..4d3ec94e76 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -544664cadfb4e504bc0b321c865d1ecb8a831e20 \ No newline at end of file +35db3e2f350ca2bc6bb9e1a647aec7f93bfb7065 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 03e39085b9..9173cf7332 100644 --- a/src/printf.c +++ b/src/printf.c @@ -21,7 +21,7 @@ ** the glibc version so the glibc version is definitely preferred. */ #if !defined(HAVE_STRCHRNUL) -# if defined(__linux__) && defined(_GNU_SOURCE) +# if defined(_GNU_SOURCE) # define HAVE_STRCHRNUL 1 # else # define HAVE_STRCHRNUL 0 diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 92efd9c087..75cfeefa0d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -47,6 +47,15 @@ # define _LARGEFILE_SOURCE 1 #endif +/* Needed for various definitions... */ +#if defined(__GNUC__) && !defined(_GNU_SOURCE) +# define _GNU_SOURCE +#endif + +#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) +# define _BSD_SOURCE +#endif + /* ** For MinGW, check to see if we can include the header file containing its ** version information, among other things. Normally, this internal MinGW @@ -104,15 +113,6 @@ #pragma warn -spa /* Suspicious pointer arithmetic */ #endif -/* Needed for various definitions... */ -#ifndef _GNU_SOURCE -# define _GNU_SOURCE -#endif - -#if defined(__OpenBSD__) && !defined(_BSD_SOURCE) -# define _BSD_SOURCE -#endif - /* ** Include standard header files as necessary */ From 4e7a479597ea919eca9fa82e304bd424dfd9eccf Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 20 Sep 2014 00:29:20 +0000 Subject: [PATCH 365/710] Only enable HAVE_STRCHRNUL by default on linux, as that is the only place it appears to work by default. FossilOrigin-Name: 0fac2c045f47c7735af4eb68ced81d8b43622a1f --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/printf.c | 2 +- 3 files changed, 9 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 9125f11713..aa9b7846e6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Revise\smacro\susage\sin\s'sqliteInt.h'. -D 2014-09-20T00:02:23.616 +C Only\senable\sHAVE_STRCHRNUL\sby\sdefault\son\slinux,\sas\sthat\sis\sthe\sonly\splace\nit\sappears\sto\swork\sby\sdefault. +D 2014-09-20T00:29:20.392 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 901a2b924f10db42b7c32936eda80feb3a769aca +F src/printf.c 3a47f526b173813d9a7f4e7044007771ba68cde1 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -1198,10 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 544664cadfb4e504bc0b321c865d1ecb8a831e20 -R 5186d6949cf27e738e34796d88aa31e5 -T *branch * sqliteIntMacros -T *sym-sqliteIntMacros * -T -sym-trunk * -U mistachkin -Z 51ca01ef3bf4ffdaf2fde2b29bc4c756 +P 35db3e2f350ca2bc6bb9e1a647aec7f93bfb7065 +R 8acbfeff4e0806bab5948d89d293bcf4 +U drh +Z 6fc488f20a05aa47a5cf4ff95476cc98 diff --git a/manifest.uuid b/manifest.uuid index 4d3ec94e76..da30674aba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35db3e2f350ca2bc6bb9e1a647aec7f93bfb7065 \ No newline at end of file +0fac2c045f47c7735af4eb68ced81d8b43622a1f \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 9173cf7332..6d4b1b4ac7 100644 --- a/src/printf.c +++ b/src/printf.c @@ -21,7 +21,7 @@ ** the glibc version so the glibc version is definitely preferred. */ #if !defined(HAVE_STRCHRNUL) -# if defined(_GNU_SOURCE) +# if defined(linux) # define HAVE_STRCHRNUL 1 # else # define HAVE_STRCHRNUL 0 From abd4c72357435db2f4a46d0acf8455ba2ed98fcc Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 20 Sep 2014 18:18:33 +0000 Subject: [PATCH 366/710] Enable SELECT query planning tracing when compiled with SQLITE_ENABLE_SELECTTRACE and either SQLITE_DEBUG or SQLITE_TEST. FossilOrigin-Name: cbe0cf9ddf46f0a678c85d49bfa74e3b7712e1a1 --- manifest | 19 +++++++++---------- manifest.uuid | 2 +- src/parse.y | 26 +++++++++++++++++++++++++- src/select.c | 13 +++++++++++++ src/shell.c | 9 +++++++++ src/sqliteInt.h | 17 +++++++++++++++++ 6 files changed, 74 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 34532191e9..fd7616c7ee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\susage\sof\sthe\s_GNU_SOURCE\sand\s_BSD_SOURCE\smacros\sin\sthe\smain\ninternal\sheader\sfile,\ssqliteInt.h.\s\sSet\sHAVE_STRCHRNUL\sto\s1\sby\sdefault\son\nLinux\sonly. -D 2014-09-20T00:35:05.099 +C Enable\sSELECT\squery\splanning\stracing\swhen\scompiled\swith\s\nSQLITE_ENABLE_SELECTTRACE\sand\seither\sSQLITE_DEBUG\sor\sSQLITE_TEST. +D 2014-09-20T18:18:33.584 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -216,7 +216,7 @@ F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c caab007743821d96752597c9cfd7351654697b06 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 +F src/parse.y 1976d28f168c63c6c14008e9a896620e0c76c25e F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa @@ -226,12 +226,12 @@ F src/printf.c 3a47f526b173813d9a7f4e7044007771ba68cde1 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 0cd6706fd52ae5db229e9041094db6ec27195335 -F src/shell.c c00220cdd7f2027780bc25b78376c16dc24e4b7d +F src/select.c 4e00e042994ae38e60576921f78e45311eead49e +F src/shell.c dad23987c34faddb061a339da3e92e05ccc6935e F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 6fd801cac974d310949ddefd6a6c6982c9778f3f +F src/sqliteInt.h 59b0796cd2fa201510ae9850b3b407fa9f997512 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -1198,8 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 544664cadfb4e504bc0b321c865d1ecb8a831e20 0fac2c045f47c7735af4eb68ced81d8b43622a1f -R 8acbfeff4e0806bab5948d89d293bcf4 -T +closed 0fac2c045f47c7735af4eb68ced81d8b43622a1f +P 59e2c9df02d7e988c5ad44c560ead1e5288b12e7 +R e5319fc7a5c9cf5efe2dc73e6ab8c7e0 U drh -Z 03e6ed8cf11426837aa81912fc8917d5 +Z 7cccf9f838c01602cd52161c3ac81285 diff --git a/manifest.uuid b/manifest.uuid index 299d8934f2..d3fdcfe339 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59e2c9df02d7e988c5ad44c560ead1e5288b12e7 \ No newline at end of file +cbe0cf9ddf46f0a678c85d49bfa74e3b7712e1a1 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index dbc129ce63..45e6fae6ad 100644 --- a/src/parse.y +++ b/src/parse.y @@ -459,9 +459,33 @@ multiselect_op(A) ::= UNION(OP). {A = @OP;} multiselect_op(A) ::= UNION ALL. {A = TK_ALL;} multiselect_op(A) ::= EXCEPT|INTERSECT(OP). {A = @OP;} %endif SQLITE_OMIT_COMPOUND_SELECT -oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y) +oneselect(A) ::= SELECT(S) distinct(D) selcollist(W) from(X) where_opt(Y) groupby_opt(P) having_opt(Q) orderby_opt(Z) limit_opt(L). { A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L.pLimit,L.pOffset); +#if SELECTTRACE_ENABLED + /* Populate the Select.zSelLabel[] string that is used to help with + ** query planner debugging, to differentiate between multiple Select + ** objects in a complex query. + ** + ** If the SELECT keyword is immediately followed by a C-style comment + ** then extract the first few alphanumeric characters from within that + ** comment to be the zSelLabel value. Otherwise, the label is #N where + ** is an integer that is incremented with each SELECT statement seen. + */ + if( A!=0 ){ + const char *z = S.z+6; + int i; + sqlite3_snprintf(sizeof(A->zSelLabel), A->zSelLabel, "#%d", + ++pParse->nSelect); + while( z[0]==' ' ) z++; + if( z[0]=='/' && z[1]=='*' ){ + z += 2; + while( z[0]==' ' ) z++; + for(i=0; sqlite3Isalnum(z[i]); i++){} + sqlite3_snprintf(sizeof(A->zSelLabel), A->zSelLabel, "%.*s", i, z); + } + } +#endif /* SELECTRACE_ENABLED */ } oneselect(A) ::= values(X). {A = X;} diff --git a/src/select.c b/src/select.c index d3ffaf451a..a0f6f35a7f 100644 --- a/src/select.c +++ b/src/select.c @@ -14,6 +14,17 @@ */ #include "sqliteInt.h" +/* +** Trace output macros +*/ +#if SELECTTRACE_ENABLED +/***/ int sqlite3SelectTrace = 0; +# define SELECTTRACE(K,X) if(sqlite3SelectTrace&(K)) sqlite3DebugPrintf X +#else +# define SELECTTRACE(K,X) +#endif + + /* ** An instance of the following object is used to record information about ** how to process the DISTINCT keyword, to simplify passing that information @@ -3355,6 +3366,8 @@ static int flattenSubquery( } /***** If we reach this point, flattening is permitted. *****/ + SELECTTRACE(1, ("flatten %s (term %d) into %s\n", + pSub->zSelLabel, iFrom, p->zSelLabel)); /* Authorize the subquery */ pParse->zAuthContext = pSubitem->zName; diff --git a/src/shell.c b/src/shell.c index ec83b13910..9745e0ff25 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3101,6 +3101,15 @@ static int do_meta_command(char *zLine, ShellState *p){ } }else + +#if defined(SQLITE_DEBUG) && defined(SQLITE_ENABLE_SELECTTRACE) + if( c=='s' && n==11 && strncmp(azArg[0], "selecttrace", n)==0 ){ + extern int sqlite3SelectTrace; + sqlite3SelectTrace = nArg>=2 ? booleanValue(azArg[1]) : 0xff; + }else +#endif + + #ifdef SQLITE_DEBUG /* Undocumented commands for internal testing. Subject to change ** without notice. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 75cfeefa0d..2d63228edd 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -706,6 +706,17 @@ extern const int sqlite3one; # undef SQLITE_ENABLE_STAT3_OR_STAT4 #endif +/* +** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not +** the Select query generator tracing logic is turned on. +*/ +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_SELECTTRACE)) +# define SELECTTRACE_ENABLED 1 +#else +# define SELECTTRACE_ENABLED 0 +#endif + /* ** An instance of the following structure is used to store the busy-handler ** callback for a given sqlite handle. @@ -2300,6 +2311,9 @@ struct Select { u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ u16 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ +#if SELECTTRACE_ENABLED + char zSelLabel[12]; /* Text in comment following SELECT keyword */ +#endif int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ u64 nSelectRow; /* Estimated number of result rows */ SrcList *pSrc; /* The FROM clause */ @@ -2558,6 +2572,9 @@ struct Parse { int regRowid; /* Register holding rowid of CREATE TABLE entry */ int regRoot; /* Register holding root page number for new objects */ int nMaxArg; /* Max args passed to user function by sub-program */ +#if SELECTTRACE_ENABLED + int nSelect; /* Number of SELECT statements seen */ +#endif #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ TableLock *aTableLock; /* Required table locks for shared-cache mode */ From 9300adbc79fc7951cd58b9cda9246453381d4344 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 20 Sep 2014 20:24:49 +0000 Subject: [PATCH 367/710] Fix the SELECTTRACE_ENABLE macro so that it doesn't cause problems for testfixture. Add new SELECTTRACE() calls. FossilOrigin-Name: f1ba68f131d2f03e4a7bc50cde23a7609d384279 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 3 +++ src/select.c | 2 ++ src/sqliteInt.h | 3 +-- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index fd7616c7ee..70f1c0adaf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sSELECT\squery\splanning\stracing\swhen\scompiled\swith\s\nSQLITE_ENABLE_SELECTTRACE\sand\seither\sSQLITE_DEBUG\sor\sSQLITE_TEST. -D 2014-09-20T18:18:33.584 +C Fix\sthe\sSELECTTRACE_ENABLE\smacro\sso\sthat\sit\sdoesn't\scause\sproblems\sfor\ntestfixture.\s\sAdd\snew\sSELECTTRACE()\scalls. +D 2014-09-20T20:24:49.725 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f -F src/expr.c 19392d98e089640c3336e65b4254cc337efef7d1 +F src/expr.c 51dfaa60c0ec9db231535c98ae9ad5ab1409fd88 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c 1629ccdd8ef3f19d7accc9d9287190489469ff81 @@ -226,12 +226,12 @@ F src/printf.c 3a47f526b173813d9a7f4e7044007771ba68cde1 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 4e00e042994ae38e60576921f78e45311eead49e +F src/select.c a2aac0a26b122b057f02464d6bfeeae3063583e7 F src/shell.c dad23987c34faddb061a339da3e92e05ccc6935e F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 59b0796cd2fa201510ae9850b3b407fa9f997512 +F src/sqliteInt.h 1020906859d2c369d214fd43f52c94385bbd38cc F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 59e2c9df02d7e988c5ad44c560ead1e5288b12e7 -R e5319fc7a5c9cf5efe2dc73e6ab8c7e0 +P cbe0cf9ddf46f0a678c85d49bfa74e3b7712e1a1 +R 0e38789ee4b65d4c643d61f203d35efb U drh -Z 7cccf9f838c01602cd52161c3ac81285 +Z 34fd21290591a3034ab211fbb42794da diff --git a/manifest.uuid b/manifest.uuid index d3fdcfe339..5a4cc52a81 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cbe0cf9ddf46f0a678c85d49bfa74e3b7712e1a1 \ No newline at end of file +f1ba68f131d2f03e4a7bc50cde23a7609d384279 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index c6d8b9e5f3..a1759374c9 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1069,6 +1069,9 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = p->nSelectRow; pNew->pWith = withDup(db, p->pWith); +#if SELECTTRACE_ENABLED + memcpy(pNew->zSelLabel, p->zSelLabel, sizeof(p->zSelLabel)); +#endif return pNew; } #else diff --git a/src/select.c b/src/select.c index a0f6f35a7f..7b2dc4ab9a 100644 --- a/src/select.c +++ b/src/select.c @@ -4612,6 +4612,7 @@ int sqlite3Select( } if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; memset(&sAggInfo, 0, sizeof(sAggInfo)); + SELECTTRACE(1, ("begin processing %s\n", p->zSelLabel)); assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); @@ -5367,6 +5368,7 @@ select_end: sqlite3DbFree(db, sAggInfo.aCol); sqlite3DbFree(db, sAggInfo.aFunc); + SELECTTRACE(1, ("end processing %s\n", p->zSelLabel)); return rc; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 2d63228edd..ac7f541246 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -710,8 +710,7 @@ extern const int sqlite3one; ** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not ** the Select query generator tracing logic is turned on. */ -#if defined(SQLITE_DEBUG) \ - && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_SELECTTRACE)) +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_SELECTTRACE) # define SELECTTRACE_ENABLED 1 #else # define SELECTTRACE_ENABLED 0 From 7c0a4720ca88e6f9b9c1a423bd74851ab0fbf8fe Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 20 Sep 2014 20:38:48 +0000 Subject: [PATCH 368/710] Candidate fix for [d11a6e908f]. FossilOrigin-Name: 89398880bcfff96e91d2a9c45774f5fb3209ffc1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 17 ++++++++++++++++- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 70f1c0adaf..bf6dc5e87d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sSELECTTRACE_ENABLE\smacro\sso\sthat\sit\sdoesn't\scause\sproblems\sfor\ntestfixture.\s\sAdd\snew\sSELECTTRACE()\scalls. -D 2014-09-20T20:24:49.725 +C Candidate\sfix\sfor\s[d11a6e908f]. +D 2014-09-20T20:38:48.213 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -226,7 +226,7 @@ F src/printf.c 3a47f526b173813d9a7f4e7044007771ba68cde1 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c a2aac0a26b122b057f02464d6bfeeae3063583e7 +F src/select.c 3108c73dff614eb8771cd251bed41e3fa5dfe33f F src/shell.c dad23987c34faddb061a339da3e92e05ccc6935e F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cbe0cf9ddf46f0a678c85d49bfa74e3b7712e1a1 -R 0e38789ee4b65d4c643d61f203d35efb -U drh -Z 34fd21290591a3034ab211fbb42794da +P f1ba68f131d2f03e4a7bc50cde23a7609d384279 +R 9363b1bc0cd8b39c47583b734f41ef37 +U dan +Z 365a1de296dc4c896c5c2cb92f759a75 diff --git a/manifest.uuid b/manifest.uuid index 5a4cc52a81..3be54e6571 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f1ba68f131d2f03e4a7bc50cde23a7609d384279 \ No newline at end of file +89398880bcfff96e91d2a9c45774f5fb3209ffc1 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 7b2dc4ab9a..92d4c01b75 100644 --- a/src/select.c +++ b/src/select.c @@ -3561,8 +3561,23 @@ static int flattenSubquery( pParent->pHaving = substExpr(db, pParent->pHaving, iParent, pSub->pEList); } if( pSub->pOrderBy ){ + /* At this point, any non-zero iOrderByCol values indicate that the + ** ORDER BY column expression is identical to the iOrderByCol'th + ** expression returned by SELECT statement pSub. Since these values + ** do not necessarily correspond to columns in SELECT statement pParent, + ** zero them before transfering the ORDER BY clause. + ** + ** Not doing this may cause an error if a subsequent call to this + ** function attempts to flatten a compound sub-query into pParent + ** (the only way this can happen is if the compound sub-query is + ** currently part of pSub->pSrc). See ticket [d11a6e908f]. */ + ExprList *pOrderBy = pSub->pOrderBy; + for(i=0; inExpr; i++){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } assert( pParent->pOrderBy==0 ); - pParent->pOrderBy = pSub->pOrderBy; + assert( pSub->pPrior==0 ); + pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; }else if( pParent->pOrderBy ){ substExprList(db, pParent->pOrderBy, iParent, pSub->pEList); From eb9b884c2db6b81a22b03613d65db7ce2434bda2 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 21 Sep 2014 00:27:26 +0000 Subject: [PATCH 369/710] Improved ".selecttrace" output. FossilOrigin-Name: c0b61f7092a7fd2c5f51db26ce7a7a5c75c227fe --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/expr.c | 4 +--- src/parse.y | 8 ++++---- src/select.c | 42 ++++++++++++++++++++++++++++++++++++------ src/sqliteInt.h | 8 +++++++- 6 files changed, 59 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index bf6dc5e87d..52365602a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Candidate\sfix\sfor\s[d11a6e908f]. -D 2014-09-20T20:38:48.213 +C Improved\s".selecttrace"\soutput. +D 2014-09-21T00:27:26.734 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f -F src/expr.c 51dfaa60c0ec9db231535c98ae9ad5ab1409fd88 +F src/expr.c 4f101c8ddc6d5a22303c88278069f5261562a9a8 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c 1629ccdd8ef3f19d7accc9d9287190489469ff81 @@ -216,7 +216,7 @@ F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c caab007743821d96752597c9cfd7351654697b06 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 1976d28f168c63c6c14008e9a896620e0c76c25e +F src/parse.y b98772da2bb5415970085b707203f92569400aa8 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa @@ -226,12 +226,12 @@ F src/printf.c 3a47f526b173813d9a7f4e7044007771ba68cde1 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 3108c73dff614eb8771cd251bed41e3fa5dfe33f +F src/select.c a83ed8bc2a31c131e3addb6f0488b68334085e7b F src/shell.c dad23987c34faddb061a339da3e92e05ccc6935e F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 1020906859d2c369d214fd43f52c94385bbd38cc +F src/sqliteInt.h 5ecde2191721a94cdce0d5248e26a373e0b17a70 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -1198,7 +1198,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f1ba68f131d2f03e4a7bc50cde23a7609d384279 -R 9363b1bc0cd8b39c47583b734f41ef37 -U dan -Z 365a1de296dc4c896c5c2cb92f759a75 +P 89398880bcfff96e91d2a9c45774f5fb3209ffc1 +R 279ae11a33db61e53f043e359f81738d +U drh +Z f68417c1797c667ad48e7e326a25e63f diff --git a/manifest.uuid b/manifest.uuid index 3be54e6571..e355bae159 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89398880bcfff96e91d2a9c45774f5fb3209ffc1 \ No newline at end of file +c0b61f7092a7fd2c5f51db26ce7a7a5c75c227fe \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index a1759374c9..c8e8e78268 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1069,9 +1069,7 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = p->nSelectRow; pNew->pWith = withDup(db, p->pWith); -#if SELECTTRACE_ENABLED - memcpy(pNew->zSelLabel, p->zSelLabel, sizeof(p->zSelLabel)); -#endif + sqlite3SelectSetName(pNew, p->zSelName); return pNew; } #else diff --git a/src/parse.y b/src/parse.y index 45e6fae6ad..30a6dc5ff4 100644 --- a/src/parse.y +++ b/src/parse.y @@ -463,26 +463,26 @@ oneselect(A) ::= SELECT(S) distinct(D) selcollist(W) from(X) where_opt(Y) groupby_opt(P) having_opt(Q) orderby_opt(Z) limit_opt(L). { A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L.pLimit,L.pOffset); #if SELECTTRACE_ENABLED - /* Populate the Select.zSelLabel[] string that is used to help with + /* Populate the Select.zSelName[] string that is used to help with ** query planner debugging, to differentiate between multiple Select ** objects in a complex query. ** ** If the SELECT keyword is immediately followed by a C-style comment ** then extract the first few alphanumeric characters from within that - ** comment to be the zSelLabel value. Otherwise, the label is #N where + ** comment to be the zSelName value. Otherwise, the label is #N where ** is an integer that is incremented with each SELECT statement seen. */ if( A!=0 ){ const char *z = S.z+6; int i; - sqlite3_snprintf(sizeof(A->zSelLabel), A->zSelLabel, "#%d", + sqlite3_snprintf(sizeof(A->zSelName), A->zSelName, "#%d", ++pParse->nSelect); while( z[0]==' ' ) z++; if( z[0]=='/' && z[1]=='*' ){ z += 2; while( z[0]==' ' ) z++; for(i=0; sqlite3Isalnum(z[i]); i++){} - sqlite3_snprintf(sizeof(A->zSelLabel), A->zSelLabel, "%.*s", i, z); + sqlite3_snprintf(sizeof(A->zSelName), A->zSelName, "%.*s", i, z); } } #endif /* SELECTRACE_ENABLED */ diff --git a/src/select.c b/src/select.c index 92d4c01b75..7820833643 100644 --- a/src/select.c +++ b/src/select.c @@ -19,9 +19,12 @@ */ #if SELECTTRACE_ENABLED /***/ int sqlite3SelectTrace = 0; -# define SELECTTRACE(K,X) if(sqlite3SelectTrace&(K)) sqlite3DebugPrintf X +# define SELECTTRACE(K,P,S,X) \ + if(sqlite3SelectTrace&(K)) \ + sqlite3DebugPrintf("%*s%s.%p: ",(P)->nSelectIndent*2-2,"",(S)->zSelName,(S)),\ + sqlite3DebugPrintf X #else -# define SELECTTRACE(K,X) +# define SELECTTRACE(K,P,S,X) #endif @@ -137,6 +140,18 @@ Select *sqlite3SelectNew( return pNew; } +#if SELECTTRACE_ENABLED +/* +** Set the name of a Select object +*/ +void sqlite3SelectSetName(Select *p, const char *zName){ + if( p && zName ){ + sqlite3_snprintf(sizeof(p->zSelName), p->zSelName, "%s", zName); + } +} +#endif + + /* ** Delete the given Select structure and all of its substructures. */ @@ -3366,8 +3381,8 @@ static int flattenSubquery( } /***** If we reach this point, flattening is permitted. *****/ - SELECTTRACE(1, ("flatten %s (term %d) into %s\n", - pSub->zSelLabel, iFrom, p->zSelLabel)); + SELECTTRACE(1,pParse,p,("flatten %s.%p from term %d\n", + pSub->zSelName, pSub, iFrom)); /* Authorize the subquery */ pParse->zAuthContext = pSubitem->zName; @@ -3420,6 +3435,7 @@ static int flattenSubquery( p->pLimit = 0; p->pOffset = 0; pNew = sqlite3SelectDup(db, p, 0); + sqlite3SelectSetName(pNew, pSub->zSelName); p->pOffset = pOffset; p->pLimit = pLimit; p->pOrderBy = pOrderBy; @@ -3432,6 +3448,9 @@ static int flattenSubquery( if( pPrior ) pPrior->pNext = pNew; pNew->pNext = p; p->pPrior = pNew; + SELECTTRACE(2,pParse,p, + ("compound-subquery flattener creates %s.%p as peer\n", + pNew->zSelName, pNew)); } if( db->mallocFailed ) return 1; } @@ -4093,6 +4112,7 @@ static int selectExpander(Walker *pWalker, Select *p){ if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; assert( pFrom->pSelect==0 ); pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); + sqlite3SelectSetName(pFrom->pSelect, pTab->zName); sqlite3WalkSelect(pWalker, pFrom->pSelect); } #endif @@ -4627,7 +4647,10 @@ int sqlite3Select( } if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; memset(&sAggInfo, 0, sizeof(sAggInfo)); - SELECTTRACE(1, ("begin processing %s\n", p->zSelLabel)); +#if SELECTTRACE_ENABLED + pParse->nSelectIndent++; + SELECTTRACE(1,pParse,p, ("begin processing\n")); +#endif assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); @@ -4784,6 +4807,10 @@ int sqlite3Select( if( p->pPrior ){ rc = multiSelect(pParse, p, pDest); explainSetInteger(pParse->iSelectId, iRestoreSelectId); +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p,("end compound-select processing\n")); + pParse->nSelectIndent--; +#endif return rc; } #endif @@ -5383,7 +5410,10 @@ select_end: sqlite3DbFree(db, sAggInfo.aCol); sqlite3DbFree(db, sAggInfo.aFunc); - SELECTTRACE(1, ("end processing %s\n", p->zSelLabel)); +#if SELECTTRACE_ENABLED + SELECTTRACE(1,pParse,p,("end processing\n")); + pParse->nSelectIndent--; +#endif return rc; } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ac7f541246..9a9675b0ab 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2311,7 +2311,7 @@ struct Select { u16 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ #if SELECTTRACE_ENABLED - char zSelLabel[12]; /* Text in comment following SELECT keyword */ + char zSelName[12]; /* Symbolic name of this SELECT use for debugging */ #endif int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ u64 nSelectRow; /* Estimated number of result rows */ @@ -2573,6 +2573,7 @@ struct Parse { int nMaxArg; /* Max args passed to user function by sub-program */ #if SELECTTRACE_ENABLED int nSelect; /* Number of SELECT statements seen */ + int nSelectIndent; /* How far to indent SELECTTRACE() output */ #endif #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ @@ -3308,6 +3309,11 @@ ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int); SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); IdList *sqlite3IdListDup(sqlite3*,IdList*); Select *sqlite3SelectDup(sqlite3*,Select*,int); +#if SELECTTRACE_ENABLED +void sqlite3SelectSetName(Select*,const char*); +#else +# define sqlite3SelectSetName(A,B) +#endif void sqlite3FuncDefInsert(FuncDefHash*, FuncDef*); FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,int,u8,u8); void sqlite3RegisterBuiltinFunctions(sqlite3*); From 249489331cddbdd50e3b6c8dd6ae396b14fe94a0 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 21 Sep 2014 17:51:37 +0000 Subject: [PATCH 370/710] Add the "showauth" extension in ext/misc. FossilOrigin-Name: 28d52c1c38d849f099bc777f5987d1ef89680c2a --- ext/misc/showauth.c | 103 ++++++++++++++++++++++++++++++++++++++++++++ manifest | 11 ++--- manifest.uuid | 2 +- 3 files changed, 110 insertions(+), 6 deletions(-) create mode 100644 ext/misc/showauth.c diff --git a/ext/misc/showauth.c b/ext/misc/showauth.c new file mode 100644 index 0000000000..87a9a6843c --- /dev/null +++ b/ext/misc/showauth.c @@ -0,0 +1,103 @@ +/* +** 2014-09-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension adds a debug "authorizer" callback to the database +** connection. The callback merely writes the authorization request to +** standard output and returns SQLITE_OK. +** +** This extension can be used (for example) in the command-line shell to +** trace the operation of the authorizer. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** Display the authorization request +*/ +static int authCallback( + void *pClientData, + int op, + const char *z1, + const char *z2, + const char *z3, + const char *z4 +){ + const char *zOp; + char zOpSpace[50]; + switch( op ){ + case SQLITE_CREATE_INDEX: zOp = "CREATE_INDEX"; break; + case SQLITE_CREATE_TABLE: zOp = "CREATE_TABLE"; break; + case SQLITE_CREATE_TEMP_INDEX: zOp = "CREATE_TEMP_INDEX"; break; + case SQLITE_CREATE_TEMP_TABLE: zOp = "CREATE_TEMP_TABLE"; break; + case SQLITE_CREATE_TEMP_TRIGGER: zOp = "CREATE_TEMP_TRIGGER"; break; + case SQLITE_CREATE_TEMP_VIEW: zOp = "CREATE_TEMP_VIEW"; break; + case SQLITE_CREATE_TRIGGER: zOp = "CREATE_TRIGGER"; break; + case SQLITE_CREATE_VIEW: zOp = "CREATE_VIEW"; break; + case SQLITE_DELETE: zOp = "DELETE"; break; + case SQLITE_DROP_INDEX: zOp = "DROP_INDEX"; break; + case SQLITE_DROP_TABLE: zOp = "DROP_TABLE"; break; + case SQLITE_DROP_TEMP_INDEX: zOp = "DROP_TEMP_INDEX"; break; + case SQLITE_DROP_TEMP_TABLE: zOp = "DROP_TEMP_TABLE"; break; + case SQLITE_DROP_TEMP_TRIGGER: zOp = "DROP_TEMP_TRIGGER"; break; + case SQLITE_DROP_TEMP_VIEW: zOp = "DROP_TEMP_VIEW"; break; + case SQLITE_DROP_TRIGGER: zOp = "DROP_TRIGGER"; break; + case SQLITE_DROP_VIEW: zOp = "DROP_VIEW"; break; + case SQLITE_INSERT: zOp = "INSERT"; break; + case SQLITE_PRAGMA: zOp = "PRAGMA"; break; + case SQLITE_READ: zOp = "READ"; break; + case SQLITE_SELECT: zOp = "SELECT"; break; + case SQLITE_TRANSACTION: zOp = "TRANSACTION"; break; + case SQLITE_UPDATE: zOp = "UPDATE"; break; + case SQLITE_ATTACH: zOp = "ATTACH"; break; + case SQLITE_DETACH: zOp = "DETACH"; break; + case SQLITE_ALTER_TABLE: zOp = "ALTER_TABLE"; break; + case SQLITE_REINDEX: zOp = "REINDEX"; break; + case SQLITE_ANALYZE: zOp = "ANALYZE"; break; + case SQLITE_CREATE_VTABLE: zOp = "CREATE_VTABLE"; break; + case SQLITE_DROP_VTABLE: zOp = "DROP_VTABLE"; break; + case SQLITE_FUNCTION: zOp = "FUNCTION"; break; + case SQLITE_SAVEPOINT: zOp = "SAVEPOINT"; break; + case SQLITE_COPY: zOp = "COPY"; break; + case SQLITE_RECURSIVE: zOp = "RECURSIVE"; break; + + + default: { + sqlite3_snprintf(sizeof(zOpSpace), zOpSpace, "%d", op); + zOp = zOpSpace; + break; + } + } + if( z1==0 ) z1 = "NULL"; + if( z2==0 ) z2 = "NULL"; + if( z3==0 ) z3 = "NULL"; + if( z4==0 ) z4 = "NULL"; + printf("AUTH: %s,%s,%s,%s,%s\n", zOp, z1, z2, z3, z4); + return SQLITE_OK; +} + + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_showauth_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_set_authorizer(db, authCallback, 0); + return rc; +} diff --git a/manifest b/manifest index 52365602a0..87ce18bc33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\s".selecttrace"\soutput. -D 2014-09-21T00:27:26.734 +C Add\sthe\s"showauth"\sextension\sin\sext/misc. +D 2014-09-21T17:51:37.899 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -116,6 +116,7 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a +F ext/misc/showauth.c 732578f0fe4ce42d577e1c86dc89dd14a006ab52 F ext/misc/spellfix.c 56739fab8c2ed6a9e2dac5592a88d281a999c43b F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 @@ -1198,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 89398880bcfff96e91d2a9c45774f5fb3209ffc1 -R 279ae11a33db61e53f043e359f81738d +P c0b61f7092a7fd2c5f51db26ce7a7a5c75c227fe +R 90118272bfe5d13644b89a2578b8154b U drh -Z f68417c1797c667ad48e7e326a25e63f +Z 3c4b3bea487974dee315367e89ebebc8 diff --git a/manifest.uuid b/manifest.uuid index e355bae159..4be4543ac6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c0b61f7092a7fd2c5f51db26ce7a7a5c75c227fe \ No newline at end of file +28d52c1c38d849f099bc777f5987d1ef89680c2a \ No newline at end of file From d7643037e6336191c5fe5bb009e1ab2b88677701 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 21 Sep 2014 20:31:26 +0000 Subject: [PATCH 371/710] Add test cases for ticket [89398880bcfff9]. FossilOrigin-Name: 9683e001ed38b41979220eef0bdfcb54df5f3191 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/subquery2.test | 45 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 87ce18bc33..b1705d3f7d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"showauth"\sextension\sin\sext/misc. -D 2014-09-21T17:51:37.899 +C Add\stest\scases\sfor\sticket\s[89398880bcfff9]. +D 2014-09-21T20:31:26.194 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -862,7 +862,7 @@ F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 F test/subquery.test 666fdecceac258f5fd84bed09a64e49d9f37edd9 -F test/subquery2.test 91e1e364072aeff431d1f9689b15147e421d88c7 +F test/subquery2.test 438f8a7da1457277b22e4176510f7659b286995f F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 @@ -1199,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c0b61f7092a7fd2c5f51db26ce7a7a5c75c227fe -R 90118272bfe5d13644b89a2578b8154b +P 28d52c1c38d849f099bc777f5987d1ef89680c2a +R df45958fe8f93552efb788581c98f0f7 U drh -Z 3c4b3bea487974dee315367e89ebebc8 +Z e77b30925e1036bea887cc5097672af2 diff --git a/manifest.uuid b/manifest.uuid index 4be4543ac6..f4ca445f31 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28d52c1c38d849f099bc777f5987d1ef89680c2a \ No newline at end of file +9683e001ed38b41979220eef0bdfcb54df5f3191 \ No newline at end of file diff --git a/test/subquery2.test b/test/subquery2.test index 4406efccf2..de637d5d25 100644 --- a/test/subquery2.test +++ b/test/subquery2.test @@ -103,5 +103,50 @@ do_execsql_test 2.2 { LIMIT (SELECT a FROM t5) } {2 3 3 6 4 10} +############################################################################ +# Ticket http://www.sqlite.org/src/info/d11a6e908f (2014-09-20) +# Query planner fault on three-way nested join with compound inner SELECT +# +do_execsql_test 3.0 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1 (id INTEGER PRIMARY KEY, data TEXT); + INSERT INTO t1(id,data) VALUES(9,'nine-a'); + INSERT INTO t1(id,data) VALUES(10,'ten-a'); + INSERT INTO t1(id,data) VALUES(11,'eleven-a'); + CREATE TABLE t2 (id INTEGER PRIMARY KEY, data TEXT); + INSERT INTO t2(id,data) VALUES(9,'nine-b'); + INSERT INTO t2(id,data) VALUES(10,'ten-b'); + INSERT INTO t2(id,data) VALUES(11,'eleven-b'); + + SELECT id FROM ( + SELECT id,data FROM ( + SELECT * FROM t1 UNION ALL SELECT * FROM t2 + ) + WHERE id=10 ORDER BY data + ); +} {10 10} +do_execsql_test 3.1 { + SELECT data FROM ( + SELECT 'dummy', data FROM ( + SELECT data FROM t1 UNION ALL SELECT data FROM t1 + ) ORDER BY data + ); +} {eleven-a eleven-a nine-a nine-a ten-a ten-a} +do_execsql_test 3.2 { + DROP TABLE IF EXISTS t3; + DROP TABLE IF EXISTS t4; + CREATE TABLE t3(id INTEGER, data TEXT); + CREATE TABLE t4(id INTEGER, data TEXT); + INSERT INTO t3 VALUES(4, 'a'),(2,'c'); + INSERT INTO t4 VALUES(3, 'b'),(1,'d'); + + SELECT data, id FROM ( + SELECT id, data FROM ( + SELECT * FROM t3 UNION ALL SELECT * FROM t4 + ) ORDER BY data + ); +} {a 4 b 3 c 2 d 1} + finish_test From d9f9441d7d4435bcd714b2cbda597c6da45c535a Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Sep 2014 03:22:27 +0000 Subject: [PATCH 372/710] Disable shared memory operations using the unix-nolock VFS. FossilOrigin-Name: 10a6e510497b471d67ac3dfb19ff256a7d18adf4 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/os_unix.c | 28 ++++++++++++++++++---------- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index ddf26d66a1..b51e6af8f7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Correctly\shandle\san\sORDER\sBY\sclause\son\san\souter\squery\swhen\sapplying\nthe\scompound-subquery\sflattening\soptimization.\s\sTicket\s[89398880bcfff].\nAlso\sadd\sthe\sSQLITE_ENABLE_SELECTTRACE\soption\sfor\sadditional\sdebugging\nand\sanalysis\sinformation\sabout\sselect\sstatement\sprocessing. -D 2014-09-21T22:31:52.593 +C Disable\sshared\smemory\soperations\susing\sthe\sunix-nolock\sVFS. +D 2014-09-22T03:22:27.824 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -212,7 +212,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 9096a1b1449182e67e759f59994eee04113bc587 +F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c caab007743821d96752597c9cfd7351654697b06 @@ -1199,8 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 59e2c9df02d7e988c5ad44c560ead1e5288b12e7 9683e001ed38b41979220eef0bdfcb54df5f3191 -R df45958fe8f93552efb788581c98f0f7 -T +closed 9683e001ed38b41979220eef0bdfcb54df5f3191 +P d5880abd63c83c88e135257373afa0a3fd88297e +R 78a2249f9b04bb5d20ea2c24b3aed300 U drh -Z d861c46be738b6f104b52fa4441ce204 +Z 8d87b3613ef02056468b7e3fc8779d9f diff --git a/manifest.uuid b/manifest.uuid index 7cf6d685c3..87021972ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5880abd63c83c88e135257373afa0a3fd88297e \ No newline at end of file +10a6e510497b471d67ac3dfb19ff256a7d18adf4 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index fcd9e72d06..a9344ee830 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4951,7 +4951,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ ** * An I/O method finder function called FINDER that returns a pointer ** to the METHOD object in the previous bullet. */ -#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK) \ +#define IOMETHODS(FINDER, METHOD, VERSION, CLOSE, LOCK, UNLOCK, CKLOCK, SHMMAP) \ static const sqlite3_io_methods METHOD = { \ VERSION, /* iVersion */ \ CLOSE, /* xClose */ \ @@ -4966,7 +4966,7 @@ static const sqlite3_io_methods METHOD = { \ unixFileControl, /* xFileControl */ \ unixSectorSize, /* xSectorSize */ \ unixDeviceCharacteristics, /* xDeviceCapabilities */ \ - unixShmMap, /* xShmMap */ \ + SHMMAP, /* xShmMap */ \ unixShmLock, /* xShmLock */ \ unixShmBarrier, /* xShmBarrier */ \ unixShmUnmap, /* xShmUnmap */ \ @@ -4992,7 +4992,8 @@ IOMETHODS( unixClose, /* xClose method */ unixLock, /* xLock method */ unixUnlock, /* xUnlock method */ - unixCheckReservedLock /* xCheckReservedLock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + unixShmMap /* xShmMap method */ ) IOMETHODS( nolockIoFinder, /* Finder function name */ @@ -5001,7 +5002,8 @@ IOMETHODS( nolockClose, /* xClose method */ nolockLock, /* xLock method */ nolockUnlock, /* xUnlock method */ - nolockCheckReservedLock /* xCheckReservedLock method */ + nolockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) IOMETHODS( dotlockIoFinder, /* Finder function name */ @@ -5010,7 +5012,8 @@ IOMETHODS( dotlockClose, /* xClose method */ dotlockLock, /* xLock method */ dotlockUnlock, /* xUnlock method */ - dotlockCheckReservedLock /* xCheckReservedLock method */ + dotlockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #if SQLITE_ENABLE_LOCKING_STYLE && !OS_VXWORKS @@ -5021,7 +5024,8 @@ IOMETHODS( flockClose, /* xClose method */ flockLock, /* xLock method */ flockUnlock, /* xUnlock method */ - flockCheckReservedLock /* xCheckReservedLock method */ + flockCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5033,7 +5037,8 @@ IOMETHODS( semClose, /* xClose method */ semLock, /* xLock method */ semUnlock, /* xUnlock method */ - semCheckReservedLock /* xCheckReservedLock method */ + semCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5045,7 +5050,8 @@ IOMETHODS( afpClose, /* xClose method */ afpLock, /* xLock method */ afpUnlock, /* xUnlock method */ - afpCheckReservedLock /* xCheckReservedLock method */ + afpCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5070,7 +5076,8 @@ IOMETHODS( proxyClose, /* xClose method */ proxyLock, /* xLock method */ proxyUnlock, /* xUnlock method */ - proxyCheckReservedLock /* xCheckReservedLock method */ + proxyCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif @@ -5083,7 +5090,8 @@ IOMETHODS( unixClose, /* xClose method */ unixLock, /* xLock method */ nfsUnlock, /* xUnlock method */ - unixCheckReservedLock /* xCheckReservedLock method */ + unixCheckReservedLock, /* xCheckReservedLock method */ + 0 /* xShmMap method */ ) #endif From 7e07433fe72598b932a2115d2774a98e12f50789 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Sep 2014 14:30:51 +0000 Subject: [PATCH 373/710] Tune the query planner to be more aggressive about using automatic indexes on views and subqueries for which there is not opportunity to declare a persistent schema index. FossilOrigin-Name: 41de1643bfc9ae25e20790d707e2789b665baa2b --- manifest | 14 +++---- manifest.uuid | 2 +- src/where.c | 14 +++++-- test/autoindex1.test | 97 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 116 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b51e6af8f7..926eedc817 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sshared\smemory\soperations\susing\sthe\sunix-nolock\sVFS. -D 2014-09-22T03:22:27.824 +C Tune\sthe\squery\splanner\sto\sbe\smore\saggressive\sabout\susing\sautomatic\sindexes\non\sviews\sand\ssubqueries\sfor\swhich\sthere\sis\snot\sopportunity\sto\sdeclare\sa\npersistent\sschema\sindex. +D 2014-09-22T14:30:51.911 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 0888567c0e01a41b6001647e333f8ccfd3ae7d36 +F src/where.c 3f859ecfada8643ce8255f20d481b5a3e0921662 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -341,7 +341,7 @@ F test/auth.test 855233ef26eb3601b6886567ea4e326c72959360 F test/auth2.test 264c6af53cad9aba5218c68bbe18036e39007bfa F test/auth3.test 5cfa94ed90c6617c42b7ba4b133fd79678b251c7 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 -F test/autoindex1.test 762ff3f8e25d852aae55c6462ca166a80c0cde61 +F test/autoindex1.test 6ff78b94f43a59616c06c11c55b12935173506d7 F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de F test/autoindex3.test 8254f689c3241081fad52b7bea18ba53e07e14a2 F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 @@ -1199,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d5880abd63c83c88e135257373afa0a3fd88297e -R 78a2249f9b04bb5d20ea2c24b3aed300 +P 10a6e510497b471d67ac3dfb19ff256a7d18adf4 +R f6ce79b0bd0352edf4a2c65db2a400a7 U drh -Z 8d87b3613ef02056468b7e3fc8779d9f +Z b1246ec5b6570c51d45a58350d46840c diff --git a/manifest.uuid b/manifest.uuid index 87021972ad..a03afa027d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -10a6e510497b471d67ac3dfb19ff256a7d18adf4 \ No newline at end of file +41de1643bfc9ae25e20790d707e2789b665baa2b \ No newline at end of file diff --git a/src/where.c b/src/where.c index 318065d486..d1862b499a 100644 --- a/src/where.c +++ b/src/where.c @@ -4718,9 +4718,17 @@ static int whereLoopAddBtree( pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; /* TUNING: One-time cost for computing the automatic index is - ** approximately 7*N*log2(N) where N is the number of rows in - ** the table being indexed. */ - pNew->rSetup = rLogSize + rSize + 28; assert( 28==sqlite3LogEst(7) ); + ** estimated to be X*N*log2(N) where N is the number of rows in + ** the table being indexed and where X is 7 (LogEst=28) for normal + ** tables or 1.375 (LogEst=4) for views and subqueries. The value + ** of X is smaller for views and subqueries so that the query planner + ** will be more aggressive about generating automatic indexes for + ** those objects, since there is no opportunity to add schema + ** indexes on subqueries and views. */ + pNew->rSetup = rLogSize + rSize + 4; + if( pTab->pSelect==0 && (pTab->tabFlags & TF_Ephemeral)==0 ){ + pNew->rSetup += 24; + } ApplyCostMultiplier(pNew->rSetup, pTab->costMult); /* TUNING: Each index lookup yields 20 rows in the table. This ** is more than the usual guess of 10 rows, since we have no way diff --git a/test/autoindex1.test b/test/autoindex1.test index 6cb0ab146a..bcde5bc2e7 100644 --- a/test/autoindex1.test +++ b/test/autoindex1.test @@ -413,4 +413,101 @@ do_execsql_test autoindex1-801 { WHERE mimetypes._id=10 AND data14 IS NOT NULL; } {/SEARCH TABLE data .*SEARCH TABLE raw_contacts/} +# Another test case from an important user of SQLite. The key feature of +# this test is that the "aggindex" subquery should make use of an +# automatic index. If it does, the query is fast. If it does not, the +# query is deathly slow. It worked OK in 3.7.17 but started going slow +# with version 3.8.0. The problem was fixed for 3.8.7 by reducing the +# cost estimate for automatic indexes on views and subqueries. +# +db close +forcedelete test.db +sqlite3 db test.db +do_execsql_test autoindex1-900 { + CREATE TABLE messages (ROWID INTEGER PRIMARY KEY AUTOINCREMENT, message_id, document_id BLOB, in_reply_to, remote_id INTEGER, sender INTEGER, subject_prefix, subject INTEGER, date_sent INTEGER, date_received INTEGER, date_created INTEGER, date_last_viewed INTEGER, mailbox INTEGER, remote_mailbox INTEGER, original_mailbox INTEGER, flags INTEGER, read, flagged, size INTEGER, color, encoding, type INTEGER, pad, conversation_id INTEGER DEFAULT -1, snippet TEXT DEFAULT NULL, fuzzy_ancestor INTEGER DEFAULT NULL, automated_conversation INTEGER DEFAULT 0, root_status INTEGER DEFAULT -1, conversation_position INTEGER DEFAULT -1); + CREATE INDEX date_index ON messages(date_received); + CREATE INDEX date_last_viewed_index ON messages(date_last_viewed); + CREATE INDEX date_created_index ON messages(date_created); + CREATE INDEX message_message_id_mailbox_index ON messages(message_id, mailbox); + CREATE INDEX message_document_id_index ON messages(document_id); + CREATE INDEX message_read_index ON messages(read); + CREATE INDEX message_flagged_index ON messages(flagged); + CREATE INDEX message_mailbox_index ON messages(mailbox, date_received); + CREATE INDEX message_remote_mailbox_index ON messages(remote_mailbox, remote_id); + CREATE INDEX message_type_index ON messages(type); + CREATE INDEX message_conversation_id_conversation_position_index ON messages(conversation_id, conversation_position); + CREATE INDEX message_fuzzy_ancestor_index ON messages(fuzzy_ancestor); + CREATE INDEX message_subject_fuzzy_ancestor_index ON messages(subject, fuzzy_ancestor); + CREATE INDEX message_sender_subject_automated_conversation_index ON messages(sender, subject, automated_conversation); + CREATE INDEX message_sender_index ON messages(sender); + CREATE INDEX message_root_status ON messages(root_status); + CREATE TABLE subjects (ROWID INTEGER PRIMARY KEY, subject COLLATE RTRIM, normalized_subject COLLATE RTRIM); + CREATE INDEX subject_subject_index ON subjects(subject); + CREATE INDEX subject_normalized_subject_index ON subjects(normalized_subject); + CREATE TABLE addresses (ROWID INTEGER PRIMARY KEY, address COLLATE NOCASE, comment, UNIQUE(address, comment)); + CREATE INDEX addresses_address_index ON addresses(address); + CREATE TABLE mailboxes (ROWID INTEGER PRIMARY KEY, url UNIQUE, total_count INTEGER DEFAULT 0, unread_count INTEGER DEFAULT 0, unseen_count INTEGER DEFAULT 0, deleted_count INTEGER DEFAULT 0, unread_count_adjusted_for_duplicates INTEGER DEFAULT 0, change_identifier, source INTEGER, alleged_change_identifier); + CREATE INDEX mailboxes_source_index ON mailboxes(source); + CREATE TABLE labels (ROWID INTEGER PRIMARY KEY, message_id INTEGER NOT NULL, mailbox_id INTEGER NOT NULL, UNIQUE(message_id, mailbox_id)); + CREATE INDEX labels_message_id_mailbox_id_index ON labels(message_id, mailbox_id); + CREATE INDEX labels_mailbox_id_index ON labels(mailbox_id); + + explain query plan + SELECT messages.ROWID, + messages.message_id, + messages.remote_id, + messages.date_received, + messages.date_sent, + messages.flags, + messages.size, + messages.color, + messages.date_last_viewed, + messages.subject_prefix, + subjects.subject, + sender.comment, + sender.address, + NULL, + messages.mailbox, + messages.original_mailbox, + NULL, + NULL, + messages.type, + messages.document_id, + sender, + NULL, + messages.conversation_id, + messages.conversation_position, + agglabels.labels + FROM mailboxes AS mailbox + JOIN messages ON mailbox.ROWID = messages.mailbox + LEFT OUTER JOIN subjects ON messages.subject = subjects.ROWID + LEFT OUTER JOIN addresses AS sender ON messages.sender = sender.ROWID + LEFT OUTER JOIN ( + SELECT message_id, group_concat(mailbox_id) as labels + FROM labels GROUP BY message_id + ) AS agglabels ON messages.ROWID = agglabels.message_id + WHERE (mailbox.url = 'imap://email.app@imap.gmail.com/%5BGmail%5D/All%20Mail') + AND (messages.ROWID IN ( + SELECT labels.message_id + FROM labels JOIN mailboxes ON labels.mailbox_id = mailboxes.ROWID + WHERE mailboxes.url = 'imap://email.app@imap.gmail.com/INBOX')) + AND messages.mailbox in (6,12,18,24,30,36,42,1,7,13,19,25,31,37,43,2,8, + 14,20,26,32,38,3,9,15,21,27,33,39,4,10,16,22,28, + 34,40,5,11,17,23,35,41) + ORDER BY date_received DESC; +} {/agglabels USING AUTOMATIC COVERING INDEX/} + +# A test case for VIEWs +# +do_execsql_test autoindex1-901 { + CREATE TABLE t1(x INTEGER PRIMARY KEY, y, z); + CREATE TABLE t2(a, b); + CREATE VIEW agg2 AS SELECT a, sum(b) AS m FROM t2 GROUP BY a; + EXPLAIN QUERY PLAN + SELECT t1.z, agg2.m + FROM t1 JOIN agg2 ON t1.y=agg2.m + WHERE t1.x IN (1,2,3); +} {/USING AUTOMATIC COVERING INDEX/} + + finish_test From dc41d60aaa9add6f5909cbd3d91c842f365ec773 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Sep 2014 19:51:35 +0000 Subject: [PATCH 374/710] Size reduction and substantial performance increase for cellSizePtr(). FossilOrigin-Name: bc8bbf3207288d160287519c3b7123997996b440 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 25 +++++++++++++++---------- 3 files changed, 22 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 926eedc817..d9984e2d8c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tune\sthe\squery\splanner\sto\sbe\smore\saggressive\sabout\susing\sautomatic\sindexes\non\sviews\sand\ssubqueries\sfor\swhich\sthere\sis\snot\sopportunity\sto\sdeclare\sa\npersistent\sschema\sindex. -D 2014-09-22T14:30:51.911 +C Size\sreduction\sand\ssubstantial\sperformance\sincrease\sfor\scellSizePtr(). +D 2014-09-22T19:51:35.432 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 6aa61c0e3d20d1d1acc8fb33d8f0ebd675305d3c +F src/btree.c 86cc6efed093b80360489acac4d2daf064a1ad58 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 @@ -1199,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 10a6e510497b471d67ac3dfb19ff256a7d18adf4 -R f6ce79b0bd0352edf4a2c65db2a400a7 +P 41de1643bfc9ae25e20790d707e2789b665baa2b +R 20814d8e71474a3ca52334b7cc5e7228 U drh -Z b1246ec5b6570c51d45a58350d46840c +Z 93d80b8ba28615b25f3f6e1541988990 diff --git a/manifest.uuid b/manifest.uuid index a03afa027d..63137ebd34 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -41de1643bfc9ae25e20790d707e2789b665baa2b \ No newline at end of file +bc8bbf3207288d160287519c3b7123997996b440 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 522e945ac2..46eb5be303 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1057,6 +1057,7 @@ static void btreeParseCell( */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ u8 *pIter = &pCell[pPage->childPtrSize]; + u8 *pEnd; u32 nSize; #ifdef SQLITE_DEBUG @@ -1068,21 +1069,25 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ btreeParseCellPtr(pPage, pCell, &debuginfo); #endif - if( pPage->intKey ){ - u8 *pEnd; - if( pPage->hasData ){ - pIter += getVarint32(pIter, nSize); - }else{ - nSize = 0; + if( pPage->intKey==0 || pPage->hasData ){ + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[9]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIter<&pCell[6] ); } - + pIter++; + }else{ + nSize = 0; + } + if( pPage->intKey ){ /* pIter now points at the 64-bit integer key value, a variable length ** integer. The following block moves pIter to point at the first byte ** past the end of the key value. */ pEnd = &pIter[9]; while( (*pIter++)&0x80 && pItermaxLocal ); @@ -1104,7 +1109,7 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ nSize = 4; } - assert( nSize==debuginfo.nSize ); + assert( nSize==debuginfo.nSize || CORRUPT_DB ); return (u16)nSize; } From a4bb5b0726772476cdf36b4a0182e27a73fab557 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Sep 2014 20:38:10 +0000 Subject: [PATCH 375/710] Fix to payload size overflow detection in the cellSizePtr() change of the previous check-in. FossilOrigin-Name: 7609744014c6a84a8379794a0351a2e9626ec86b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d9984e2d8c..625787bf08 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Size\sreduction\sand\ssubstantial\sperformance\sincrease\sfor\scellSizePtr(). -D 2014-09-22T19:51:35.432 +C Fix\sto\spayload\ssize\soverflow\sdetection\sin\sthe\scellSizePtr()\schange\sof\nthe\sprevious\scheck-in. +D 2014-09-22T20:38:10.316 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 86cc6efed093b80360489acac4d2daf064a1ad58 +F src/btree.c c2645014c525c0b4a8327971c331f55b8747b443 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 @@ -1199,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 41de1643bfc9ae25e20790d707e2789b665baa2b -R 20814d8e71474a3ca52334b7cc5e7228 +P bc8bbf3207288d160287519c3b7123997996b440 +R 5a3408a8ae29b6451a1037ab47048e78 U drh -Z 93d80b8ba28615b25f3f6e1541988990 +Z 93e96edbb124ae02212644cc0f189540 diff --git a/manifest.uuid b/manifest.uuid index 63137ebd34..b44f538e96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bc8bbf3207288d160287519c3b7123997996b440 \ No newline at end of file +7609744014c6a84a8379794a0351a2e9626ec86b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 46eb5be303..1856ecb1a5 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1076,7 +1076,7 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ nSize &= 0x7f; do{ nSize = (nSize<<7) | (*++pIter & 0x7f); - }while( *(pIter)>=0x80 && pIter<&pCell[6] ); + }while( *(pIter)>=0x80 && pIter Date: Tue, 23 Sep 2014 01:40:59 +0000 Subject: [PATCH 376/710] Adjust skip-scan cost estimates slightly so that a full table scan is preferred over a skip-scan to a column with only two distinct values. FossilOrigin-Name: ae9a42b268ad3f7d21a5813bb931e795c6917014 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 5 ++++- test/skipscan1.test | 28 +++++++++++++++++++++++++++- test/skipscan5.test | 6 +----- 5 files changed, 41 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 625787bf08..aeea9bdb33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sto\spayload\ssize\soverflow\sdetection\sin\sthe\scellSizePtr()\schange\sof\nthe\sprevious\scheck-in. -D 2014-09-22T20:38:10.316 +C Adjust\sskip-scan\scost\sestimates\sslightly\sso\sthat\sa\sfull\stable\sscan\sis\npreferred\sover\sa\sskip-scan\sto\sa\scolumn\swith\sonly\stwo\sdistinct\svalues. +D 2014-09-23T01:40:59.122 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 3f859ecfada8643ce8255f20d481b5a3e0921662 +F src/where.c a14d3d8042adeb51f81731c1b47b3e481d1cc23a F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -836,10 +836,10 @@ F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test 28c7faa41a0d7265040ecb0a0abd90c0904270b2 +F test/skipscan1.test 7e15e1cc524524e7b2c4595ec85c75501d22f4ff F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 -F test/skipscan5.test d8b9692b702745a0e41c23f9da6beac81df01196 +F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test 15e1d3014abc3f6d4357ed81b93b82117aefd235 @@ -1199,7 +1199,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bc8bbf3207288d160287519c3b7123997996b440 -R 5a3408a8ae29b6451a1037ab47048e78 +P 7609744014c6a84a8379794a0351a2e9626ec86b +R cb94e4a1db7c56387373fb77d2698ee3 U drh -Z 93e96edbb124ae02212644cc0f189540 +Z a13a3c1c4006b80e078cd097de2cdb1a diff --git a/manifest.uuid b/manifest.uuid index b44f538e96..c811540af6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7609744014c6a84a8379794a0351a2e9626ec86b \ No newline at end of file +ae9a42b268ad3f7d21a5813bb931e795c6917014 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d1862b499a..40f95acc49 100644 --- a/src/where.c +++ b/src/where.c @@ -4359,11 +4359,14 @@ static int whereLoopAddBtreeIndex( nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; if( pTerm ){ /* TUNING: When estimating skip-scan for a term that is also indexable, - ** increase the cost of the skip-scan by 2x, to make it a little less + ** multiply the cost of the skip-scan by 2.0, to make it a little less ** desirable than the regular index lookup. */ nIter += 10; assert( 10==sqlite3LogEst(2) ); } pNew->nOut -= nIter; + /* TUNING: Because uncertainties in the estimates for skip-scan queries, + ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ + nIter += 5; whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); pNew->nOut = saved_nOut; pNew->u.btree.nEq = saved_nEq; diff --git a/test/skipscan1.test b/test/skipscan1.test index 8150b012f6..6b9f1209a5 100644 --- a/test/skipscan1.test +++ b/test/skipscan1.test @@ -245,6 +245,32 @@ do_execsql_test skipscan1-5.3 { SELECT xh, loc FROM t5 WHERE loc >= 'M' AND loc < 'N'; } {/.*COVERING INDEX t5i1 .*/} - +# The column used by the skip-scan needs to be sufficiently selective. +# See the private email from Adi Zaimi to drh@sqlite.org on 2014-09-22. +# +db close +forcedelete test.db +sqlite3 db test.db +do_execsql_test skipscan1-6.1 { + CREATE TABLE t1(a,b,c,d,e,f,g,h varchar(300)); + CREATE INDEX t1ab ON t1(a,b); + ANALYZE sqlite_master; + -- Only two distinct values for the skip-scan column. Skip-scan is not used. + INSERT INTO sqlite_stat1 VALUES('t1','t1ab','500000 250000 125000'); + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {~/ANY/} +do_execsql_test skipscan1-6.2 { + -- Four distinct values for the skip-scan column. Skip-scan is used. + UPDATE sqlite_stat1 SET stat='500000 250000 62500'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {/ANY.a. AND b=/} +do_execsql_test skipscan1-6.3 { + -- Two distinct values for the skip-scan column again. Skip-scan is not used. + UPDATE sqlite_stat1 SET stat='500000 125000 62500'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {~/ANY/} finish_test diff --git a/test/skipscan5.test b/test/skipscan5.test index 5d6d392998..7c3b166a8c 100644 --- a/test/skipscan5.test +++ b/test/skipscan5.test @@ -108,7 +108,7 @@ foreach {tn dbenc coll} { 3 { c > 'q' } {/*ANY(a) AND ANY(b) AND c>?*/} 4 { c > 'e' } {/*SCAN TABLE t2*/} 5 { c < 'q' } {/*SCAN TABLE t2*/} - 4 { c < 'e' } {/*ANY(a) AND ANY(b) AND c Date: Tue, 23 Sep 2014 18:30:00 +0000 Subject: [PATCH 377/710] Add the "multiplex_truncate" PRAGMA to the multiplexor extension, for querying and setting the truncate flag on a database connection. FossilOrigin-Name: d2962a5f388f30a02429e0c8b87399f482b5604c --- manifest | 13 ++--- manifest.uuid | 2 +- src/test_multiplex.c | 20 ++++++++ test/multiplex4.test | 114 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 142 insertions(+), 7 deletions(-) create mode 100644 test/multiplex4.test diff --git a/manifest b/manifest index aeea9bdb33..7873e80eab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sskip-scan\scost\sestimates\sslightly\sso\sthat\sa\sfull\stable\sscan\sis\npreferred\sover\sa\sskip-scan\sto\sa\scolumn\swith\sonly\stwo\sdistinct\svalues. -D 2014-09-23T01:40:59.122 +C Add\sthe\s"multiplex_truncate"\sPRAGMA\sto\sthe\smultiplexor\sextension,\sfor\nquerying\sand\ssetting\sthe\struncate\sflag\son\sa\sdatabase\sconnection. +D 2014-09-23T18:30:00.961 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -262,7 +262,7 @@ F src/test_intarray.h 9dc57417fb65bc7835cc18548852cc08cc062202 F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c a5251f956ab6af21e138dc1f9c0399394a510cb4 F src/test_malloc.c ba34143f941a9d74b30bbffc8818389bb73a1ca2 -F src/test_multiplex.c ca90057438b63bf0840ebb84d0ef050624519a76 +F src/test_multiplex.c caadb62cc777268b4f8fb94d5b27b80156c8f7c0 F src/test_multiplex.h c08e4e8f8651f0c5e0509b138ff4d5b43ed1f5d3 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f F src/test_onefile.c 0396f220561f3b4eedc450cef26d40c593c69a25 @@ -729,6 +729,7 @@ F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 F test/multiplex.test efd015ca0b5b4a57dc9535b8feb1273eebeadb60 F test/multiplex2.test 580ca5817c7edbe4cc68fa150609c9473393003a F test/multiplex3.test d228f59eac91839a977eac19f21d053f03e4d101 +F test/multiplex4.test d3e8a5a522c51cbf3ed1c5b0bd496be02c29d7b1 F test/mutex1.test 78b2b9bb320e51d156c4efdb71b99b051e7a4b41 F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a @@ -1199,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7609744014c6a84a8379794a0351a2e9626ec86b -R cb94e4a1db7c56387373fb77d2698ee3 +P ae9a42b268ad3f7d21a5813bb931e795c6917014 +R 38a39c43ba2ffbcd445b1883953d7c57 U drh -Z a13a3c1c4006b80e078cd097de2cdb1a +Z 9797dd2e522d2cafa5364dec0ef721e8 diff --git a/manifest.uuid b/manifest.uuid index c811540af6..a1ae8eed66 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae9a42b268ad3f7d21a5813bb931e795c6917014 \ No newline at end of file +d2962a5f388f30a02429e0c8b87399f482b5604c \ No newline at end of file diff --git a/src/test_multiplex.c b/src/test_multiplex.c index 427cc65ad7..99819371ce 100644 --- a/src/test_multiplex.c +++ b/src/test_multiplex.c @@ -1002,6 +1002,26 @@ static int multiplexFileControl(sqlite3_file *pConn, int op, void *pArg){ /* no-op these */ rc = SQLITE_OK; break; + case SQLITE_FCNTL_PRAGMA: { + char **aFcntl = (char**)pArg; + if( aFcntl[1] && sqlite3_stricmp(aFcntl[1],"multiplex_truncate")==0 ){ + if( aFcntl[2] && aFcntl[2][0] ){ + if( sqlite3_stricmp(aFcntl[2], "on")==0 + || sqlite3_stricmp(aFcntl[2], "1")==0 ){ + pGroup->bTruncate = 1; + }else + if( sqlite3_stricmp(aFcntl[2], "off")==0 + || sqlite3_stricmp(aFcntl[2], "0")==0 ){ + pGroup->bTruncate = 0; + } + } + aFcntl[0] = sqlite3_mprintf(pGroup->bTruncate ? "on" : "off"); + rc = SQLITE_OK; + break; + } + /* If the multiplexor does not handle the pragma, pass it through + ** into the default case. */ + } default: pSubOpen = multiplexSubOpen(pGroup, 0, &rc, NULL, 0); if( pSubOpen ){ diff --git a/test/multiplex4.test b/test/multiplex4.test new file mode 100644 index 0000000000..9c304c314d --- /dev/null +++ b/test/multiplex4.test @@ -0,0 +1,114 @@ +# 2014-09-25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the "truncate" option in the multiplexor. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix multiplex4 + +db close +sqlite3_shutdown +sqlite3_multiplex_initialize {} 0 + +# delete all filesl with the base name of $basename +# +proc multiplex_delete_db {basename} { + foreach file [glob -nocomplain $basename.*] { + forcedelete $file + } +} + +# Return a sorted list of all files with the base name of $basename. +# Except, delete all text from the end of $basename through the NNN +# suffix on the end of the filename. +# +proc multiplex_file_list {basename} { + set x {} + foreach file [glob -nocomplain $basename.*] { + regsub "^$basename\\..*(\\d\\d\\d)\$" $file $basename.\\1 file + lappend x $file + } + return [lsort $x] +} + +do_test multiplex4-1.0 { + multiplex_delete_db mx4test + sqlite3 db {file:mx4test.db?chunksize=10&truncate=1} -uri 1 -vfs multiplex + db eval { + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES(randomblob(250000)); + } + multiplex_file_list mx4test +} {mx4test.001 mx4test.db} + +do_test multiplex4-1.1 { + db eval { + DELETE FROM t1; + VACUUM; + } + multiplex_file_list mx4test +} {mx4test.db} + +do_test multiplex4-1.2 { + db eval {PRAGMA multiplex_truncate} +} {on} +do_test multiplex4-1.3 { + db eval {PRAGMA multiplex_truncate=off} +} {off} +do_test multiplex4-1.4 { + db eval {PRAGMA multiplex_truncate} +} {off} +do_test multiplex4-1.5 { + db eval {PRAGMA multiplex_truncate=on} +} {on} +do_test multiplex4-1.6 { + db eval {PRAGMA multiplex_truncate} +} {on} +do_test multiplex4-1.7 { + db eval {PRAGMA multiplex_truncate=0} +} {off} +do_test multiplex4-1.8 { + db eval {PRAGMA multiplex_truncate=1} +} {on} +do_test multiplex4-1.9 { + db eval {PRAGMA multiplex_truncate=0} +} {off} + +do_test multiplex4-1.10 { + db eval { + INSERT INTO t1(x) VALUES(randomblob(250000)); + } + multiplex_file_list mx4test +} {mx4test.001 mx4test.db} + +do_test multiplex4-1.11 { + db eval { + DELETE FROM t1; + VACUUM; + } + multiplex_file_list mx4test +} {mx4test.001 mx4test.db} + +do_test multiplex4-1.12 { + db eval { + PRAGMA multiplex_truncate=ON; + DROP TABLE t1; + VACUUM; + } + multiplex_file_list mx4test +} {mx4test.db} + +catch { db close } +forcedelete mx4test.db +sqlite3_multiplex_shutdown +finish_test From ab1cc58b7018a2bcfd9a3b2107bd2bbb6cbbf0f8 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Sep 2014 21:25:19 +0000 Subject: [PATCH 378/710] Simplify the CellInfo structure for a size reduction and performance improvement. FossilOrigin-Name: bf59df66b3613c38cfb13a68091b8328ebb22c78 --- manifest | 14 ++++++------ manifest.uuid | 2 +- src/btree.c | 60 ++++++++++++++++++++++---------------------------- src/btreeInt.h | 10 ++++----- 4 files changed, 38 insertions(+), 48 deletions(-) diff --git a/manifest b/manifest index 7873e80eab..70db7cda92 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"multiplex_truncate"\sPRAGMA\sto\sthe\smultiplexor\sextension,\sfor\nquerying\sand\ssetting\sthe\struncate\sflag\son\sa\sdatabase\sconnection. -D 2014-09-23T18:30:00.961 +C Simplify\sthe\sCellInfo\sstructure\sfor\sa\ssize\sreduction\sand\sperformance\nimprovement. +D 2014-09-23T21:25:19.710 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,9 +172,9 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c c2645014c525c0b4a8327971c331f55b8747b443 +F src/btree.c 2a7c67d474624732612f97a89e34cf85f8cd4905 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 -F src/btreeInt.h e0ecb5dba292722039a7540beb3fc448103273cc +F src/btreeInt.h a5a869ec2c3e56ee9e214ee748d7942716be0340 F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ae9a42b268ad3f7d21a5813bb931e795c6917014 -R 38a39c43ba2ffbcd445b1883953d7c57 +P d2962a5f388f30a02429e0c8b87399f482b5604c +R 3633e95932853227476c5d6e8f02e193 U drh -Z 9797dd2e522d2cafa5364dec0ef721e8 +Z 6131f39965fefa27be7e69636d8311d7 diff --git a/manifest.uuid b/manifest.uuid index a1ae8eed66..24ab1f920d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2962a5f388f30a02429e0c8b87399f482b5604c \ No newline at end of file +bf59df66b3613c38cfb13a68091b8328ebb22c78 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 1856ecb1a5..233f674f0f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -977,38 +977,33 @@ static void btreeParseCellPtr( u8 *pCell, /* Pointer to the cell text. */ CellInfo *pInfo /* Fill in this structure */ ){ - u16 n; /* Number bytes in cell content header */ + u8 *pIter = &pCell[pPage->childPtrSize]; u32 nPayload; /* Number of bytes of cell payload */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - - pInfo->pCell = pCell; assert( pPage->leaf==0 || pPage->leaf==1 ); - n = pPage->childPtrSize; - assert( n==4-4*pPage->leaf ); if( pPage->intKey ){ if( pPage->hasData ){ - assert( n==0 ); - n = getVarint32(pCell, nPayload); + assert( pIter==pCell ); + pIter += getVarint32(pIter, nPayload); }else{ nPayload = 0; } - n += getVarint(&pCell[n], (u64*)&pInfo->nKey); - pInfo->nData = nPayload; + pIter += getVarint(pIter, (u64*)&pInfo->nKey); }else{ - pInfo->nData = 0; - n += getVarint32(&pCell[n], nPayload); + pIter += getVarint32(pIter, nPayload); pInfo->nKey = nPayload; } pInfo->nPayload = nPayload; - pInfo->nHeader = n; + pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); testcase( nPayload==pPage->maxLocal+1 ); - if( likely(nPayload<=pPage->maxLocal) ){ + if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. */ - if( (pInfo->nSize = (u16)(n+nPayload))<4 ) pInfo->nSize = 4; + pInfo->nSize = nPayload + (u16)(pIter - pCell); + if( pInfo->nSize<4 ) pInfo->nSize = 4; pInfo->nLocal = (u16)nPayload; pInfo->iOverflow = 0; }else{ @@ -1035,7 +1030,7 @@ static void btreeParseCellPtr( }else{ pInfo->nLocal = (u16)minLocal; } - pInfo->iOverflow = (u16)(pInfo->nLocal + n); + pInfo->iOverflow = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell); pInfo->nSize = pInfo->iOverflow + 4; } } @@ -1132,7 +1127,6 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ if( *pRC ) return; assert( pCell!=0 ); btreeParseCellPtr(pPage, pCell, &info); - assert( (info.nData+(pPage->intKey?0:info.nKey))==info.nPayload ); if( info.iOverflow ){ Pgno ovfl = get4byte(&pCell[info.iOverflow]); ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); @@ -3866,8 +3860,9 @@ int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); + assert( pCur->apPage[pCur->iPage]->intKey==1 ); getCellInfo(pCur); - *pSize = pCur->info.nData; + *pSize = pCur->info.nPayload; return SQLITE_OK; } @@ -4018,7 +4013,6 @@ static int accessPayload( ){ unsigned char *aPayload; int rc = SQLITE_OK; - u32 nKey; int iIdx = 0; MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ @@ -4033,15 +4027,13 @@ static int accessPayload( assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ getCellInfo(pCur); - aPayload = pCur->info.pCell + pCur->info.nHeader; - nKey = (pPage->intKey ? 0 : (int)pCur->info.nKey); + aPayload = pCur->info.pPayload; #ifdef SQLITE_DIRECT_OVERFLOW_READ - bEnd = (offset+amt==nKey+pCur->info.nData); + bEnd = offset+amt==pCur->info.nPayload; #endif + assert( offset+amt <= pCur->info.nPayload ); - if( NEVER(offset+amt > nKey+pCur->info.nData) - || &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] - ){ + if( &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] ){ /* Trying to read or write past the end of the data is an error */ return SQLITE_CORRUPT_BKPT; } @@ -4275,7 +4267,7 @@ static const void *fetchPayload( assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); assert( pCur->info.nSize>0 ); *pAmt = pCur->info.nLocal; - return (void*)(pCur->info.pCell + pCur->info.nHeader); + return (void*)pCur->info.pPayload; } @@ -5640,9 +5632,10 @@ static int fillInCell( } nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); btreeParseCellPtr(pPage, pCell, &info); - assert( info.nHeader==nHeader ); + assert( nHeader=(int)(info.pPayload - pCell) ); assert( info.nKey==nKey ); - assert( info.nData==(u32)(nData+nZero) ); + assert( pPage->intKey==0 || info.nPayload==(u32)(nData+nZero) ); + assert( pPage->intKey==1 || info.nPayload==nKey ); /* Fill in the payload */ nPayload = nData + nZero; @@ -8086,19 +8079,18 @@ static int checkTreePage( "On tree page %d cell %d: ", iPage, i); pCell = findCell(pPage,i); btreeParseCellPtr(pPage, pCell, &info); - sz = info.nData; - if( !pPage->intKey ) sz += (int)info.nKey; + sz = info.nPayload; /* For intKey pages, check that the keys are in order. */ - else if( i==0 ) nMinKey = nMaxKey = info.nKey; - else{ - if( info.nKey <= nMaxKey ){ + if( pPage->intKey ){ + if( i==0 ){ + nMinKey = nMaxKey = info.nKey; + }else if( info.nKey <= nMaxKey ){ checkAppendMsg(pCheck, zContext, - "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); + "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); } nMaxKey = info.nKey; } - assert( sz==info.nPayload ); if( (sz>info.nLocal) && (&pCell[info.iOverflow]<=&pPage->aData[pBt->usableSize]) ){ diff --git a/src/btreeInt.h b/src/btreeInt.h index fbfe47f6bc..c9ff79c90f 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -456,12 +456,10 @@ struct BtShared { */ typedef struct CellInfo CellInfo; struct CellInfo { - i64 nKey; /* The key for INTKEY tables, or number of bytes in key */ - u8 *pCell; /* Pointer to the start of cell content */ - u32 nData; /* Number of bytes of data */ - u32 nPayload; /* Total amount of payload */ - u16 nHeader; /* Size of the cell content header in bytes */ - u16 nLocal; /* Amount of payload held locally */ + i64 nKey; /* The key for INTKEY tables, or nPayload otherwise */ + u8 *pPayload; /* Pointer to the start of payload */ + u32 nPayload; /* Bytes of payload */ + u16 nLocal; /* Amount of payload held locally, not on overflow */ u16 iOverflow; /* Offset to overflow page number. Zero if no overflow */ u16 nSize; /* Size of the cell content on the main b-tree page */ }; From 6200c88123e5933dc81ec132d0c03f8bff86a07b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Sep 2014 22:36:25 +0000 Subject: [PATCH 379/710] Avoid calling btreeParseCellPtr() from within fillInCell() since most of what btreeParseCellPtr() computes is ignored by fillInCell(). Instead, have fillInCell() compute the values it needs inline. Performance improvement. FossilOrigin-Name: 4147f6671e3faa8ddffab8387a6c7d9b5b962fc8 --- manifest | 12 ++++----- manifest.uuid | 2 +- src/btree.c | 69 +++++++++++++++++++++++++++++++++++++-------------- 3 files changed, 58 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 70db7cda92..2923bfbe6d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\sCellInfo\sstructure\sfor\sa\ssize\sreduction\sand\sperformance\nimprovement. -D 2014-09-23T21:25:19.710 +C Avoid\scalling\sbtreeParseCellPtr()\sfrom\swithin\sfillInCell()\ssince\smost\sof\nwhat\sbtreeParseCellPtr()\scomputes\sis\signored\sby\sfillInCell().\s\sInstead,\shave\nfillInCell()\scompute\sthe\svalues\sit\sneeds\sinline.\s\sPerformance\simprovement. +D 2014-09-23T22:36:25.858 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 2a7c67d474624732612f97a89e34cf85f8cd4905 +F src/btree.c 11cf36074a9829bad4506e8486bfd2431168bb54 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h a5a869ec2c3e56ee9e214ee748d7942716be0340 F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d2962a5f388f30a02429e0c8b87399f482b5604c -R 3633e95932853227476c5d6e8f02e193 +P bf59df66b3613c38cfb13a68091b8328ebb22c78 +R 7c25e7d21782a499afecd58eb6881944 U drh -Z 6131f39965fefa27be7e69636d8311d7 +Z 9128d4e264c0fc76a3ba82d24c937cbe diff --git a/manifest.uuid b/manifest.uuid index 24ab1f920d..9320755605 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bf59df66b3613c38cfb13a68091b8328ebb22c78 \ No newline at end of file +4147f6671e3faa8ddffab8387a6c7d9b5b962fc8 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 233f674f0f..5d05e98d85 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5611,7 +5611,6 @@ static int fillInCell( BtShared *pBt = pPage->pBt; Pgno pgnoOvfl = 0; int nHeader; - CellInfo info; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); @@ -5621,24 +5620,18 @@ static int fillInCell( || sqlite3PagerIswriteable(pPage->pDbPage) ); /* Fill in the header. */ - nHeader = 0; - if( !pPage->leaf ){ - nHeader += 4; - } + nHeader = pPage->childPtrSize; + nPayload = nData + nZero; if( pPage->hasData ){ - nHeader += putVarint32(&pCell[nHeader], nData+nZero); + assert( pPage->intKey ); + nHeader += putVarint32(&pCell[nHeader], nPayload); }else{ - nData = nZero = 0; + assert( nData==0 ); + assert( nZero==0 ); } nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); - btreeParseCellPtr(pPage, pCell, &info); - assert( nHeader=(int)(info.pPayload - pCell) ); - assert( info.nKey==nKey ); - assert( pPage->intKey==0 || info.nPayload==(u32)(nData+nZero) ); - assert( pPage->intKey==1 || info.nPayload==nKey ); - /* Fill in the payload */ - nPayload = nData + nZero; + /* Fill in the payload size */ if( pPage->intKey ){ pSrc = pData; nSrc = nData; @@ -5647,15 +5640,55 @@ static int fillInCell( if( NEVER(nKey>0x7fffffff || pKey==0) ){ return SQLITE_CORRUPT_BKPT; } - nPayload += (int)nKey; + nPayload = (int)nKey; pSrc = pKey; nSrc = (int)nKey; } - *pnSize = info.nSize; - spaceLeft = info.nLocal; + if( nPayload<=pPage->maxLocal ){ + n = nHeader + nPayload; + testcase( n==3 ); + testcase( n==4 ); + if( n<4 ) n = 4; + *pnSize = n; + spaceLeft = nPayload; + pPrior = pCell; + }else{ + int mn = pPage->minLocal; + n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4); + testcase( n==pPage->maxLocal ); + testcase( n==pPage->maxLocal+1 ); + if( n > pPage->maxLocal ) n = mn; + spaceLeft = n; + *pnSize = n + nHeader + 4; + pPrior = &pCell[nHeader+n]; + } pPayload = &pCell[nHeader]; - pPrior = &pCell[info.iOverflow]; + /* At this point variables should be set as follows: + ** + ** nPayload Total payload size in bytes + ** pPayload Begin writing payload here + ** spaceLeft Space available at pPayload. If nPayload>spaceLeft, + ** that means content must spill into overflow pages. + ** *pnSize Size of the local cell (not counting overflow pages) + ** pPrior Where to write the pgno of the first overflow page + ** + ** Use a call to btreeParseCellPtr() to verify that the values above + ** were computed correctly. + */ +#if SQLITE_DEBUG + { + CellInfo info; + btreeParseCellPtr(pPage, pCell, &info); + assert( nHeader=(int)(info.pPayload - pCell) ); + assert( info.nKey==nKey ); + assert( *pnSize == info.nSize ); + assert( spaceLeft == info.nLocal ); + assert( pPrior == &pCell[info.iOverflow] ); + } +#endif + + /* Write the payload into the local Cell and any extra into overflow pages */ while( nPayload>0 ){ if( spaceLeft==0 ){ #ifndef SQLITE_OMIT_AUTOVACUUM From c46838309c525b1b441f12f42b1cb493cdca89bc Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Sep 2014 23:12:53 +0000 Subject: [PATCH 380/710] Remove an unused C-preprocessor macro. No functional changes to the code. FossilOrigin-Name: f480582ccae0e9a917d4523191025bd16016ba64 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 7 +------ 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 2923bfbe6d..b80dfe9ce8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scalling\sbtreeParseCellPtr()\sfrom\swithin\sfillInCell()\ssince\smost\sof\nwhat\sbtreeParseCellPtr()\scomputes\sis\signored\sby\sfillInCell().\s\sInstead,\shave\nfillInCell()\scompute\sthe\svalues\sit\sneeds\sinline.\s\sPerformance\simprovement. -D 2014-09-23T22:36:25.858 +C Remove\san\sunused\sC-preprocessor\smacro.\s\sNo\sfunctional\schanges\sto\sthe\scode. +D 2014-09-23T23:12:53.118 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 11cf36074a9829bad4506e8486bfd2431168bb54 +F src/btree.c b10d4c349e62faca4ff86f168a02ca9988d1fd6f F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h a5a869ec2c3e56ee9e214ee748d7942716be0340 F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bf59df66b3613c38cfb13a68091b8328ebb22c78 -R 7c25e7d21782a499afecd58eb6881944 +P 4147f6671e3faa8ddffab8387a6c7d9b5b962fc8 +R ff3738f15db60d92a05b3981ec1f9ca5 U drh -Z 9128d4e264c0fc76a3ba82d24c937cbe +Z 53dcb91c2d5e4a116ccba455c707085a diff --git a/manifest.uuid b/manifest.uuid index 9320755605..71bf446c8d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4147f6671e3faa8ddffab8387a6c7d9b5b962fc8 \ No newline at end of file +f480582ccae0e9a917d4523191025bd16016ba64 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 5d05e98d85..5a0fd4e184 100644 --- a/src/btree.c +++ b/src/btree.c @@ -968,9 +968,6 @@ static u8 *findOverflowCell(MemPage *pPage, int iCell){ ** are two versions of this function. btreeParseCell() takes a ** cell index as the second argument and btreeParseCellPtr() ** takes a pointer to the body of the cell as its second argument. -** -** Within this file, the parseCell() macro can be called instead of -** btreeParseCellPtr(). Using some compilers, this will be faster. */ static void btreeParseCellPtr( MemPage *pPage, /* Page containing the cell */ @@ -1034,14 +1031,12 @@ static void btreeParseCellPtr( pInfo->nSize = pInfo->iOverflow + 4; } } -#define parseCell(pPage, iCell, pInfo) \ - btreeParseCellPtr((pPage), findCell((pPage), (iCell)), (pInfo)) static void btreeParseCell( MemPage *pPage, /* Page containing the cell */ int iCell, /* The cell index. First cell is 0 */ CellInfo *pInfo /* Fill in this structure */ ){ - parseCell(pPage, iCell, pInfo); + btreeParseCellPtr(pPage, findCell(pPage, iCell), pInfo); } /* From 3e28ff5cb53f51f2903d5ec056a71040bdddf5de Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Sep 2014 00:59:08 +0000 Subject: [PATCH 381/710] Add the MemPage.noPayload boolean and use it to help cellSizePtr() and btreeParseCellPtr() run faster. FossilOrigin-Name: 8e3375313ebbf26b68561f3ed31d2a488222e5d0 --- manifest | 14 ++++----- manifest.uuid | 2 +- src/btree.c | 85 ++++++++++++++++++++++++++------------------------ src/btreeInt.h | 7 +++-- 4 files changed, 56 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index b80dfe9ce8..41868133a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\sC-preprocessor\smacro.\s\sNo\sfunctional\schanges\sto\sthe\scode. -D 2014-09-23T23:12:53.118 +C Add\sthe\sMemPage.noPayload\sboolean\sand\suse\sit\sto\shelp\ncellSizePtr()\sand\sbtreeParseCellPtr()\srun\sfaster. +D 2014-09-24T00:59:08.082 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,9 +172,9 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c b10d4c349e62faca4ff86f168a02ca9988d1fd6f +F src/btree.c d64b3c5569bda0e7fbe9bc9388e17eaf70d63dec F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 -F src/btreeInt.h a5a869ec2c3e56ee9e214ee748d7942716be0340 +F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4147f6671e3faa8ddffab8387a6c7d9b5b962fc8 -R ff3738f15db60d92a05b3981ec1f9ca5 +P f480582ccae0e9a917d4523191025bd16016ba64 +R a8f50a441085b36b058ddd2aec76446f U drh -Z 53dcb91c2d5e4a116ccba455c707085a +Z b0d06d7ed20970cccde4605a2a98dc92 diff --git a/manifest.uuid b/manifest.uuid index 71bf446c8d..dae8a6ddc0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f480582ccae0e9a917d4523191025bd16016ba64 \ No newline at end of file +8e3375313ebbf26b68561f3ed31d2a488222e5d0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 5a0fd4e184..8644538abb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -974,20 +974,25 @@ static void btreeParseCellPtr( u8 *pCell, /* Pointer to the cell text. */ CellInfo *pInfo /* Fill in this structure */ ){ - u8 *pIter = &pCell[pPage->childPtrSize]; + u8 *pIter; /* For scanning through pCell */ u32 nPayload; /* Number of bytes of cell payload */ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( pPage->leaf==0 || pPage->leaf==1 ); - if( pPage->intKey ){ - if( pPage->hasData ){ - assert( pIter==pCell ); - pIter += getVarint32(pIter, nPayload); - }else{ - nPayload = 0; - } + if( pPage->intKeyLeaf ){ + assert( pPage->childPtrSize==0 ); + pIter = pCell + getVarint32(pCell, nPayload); pIter += getVarint(pIter, (u64*)&pInfo->nKey); + }else if( pPage->noPayload ){ + assert( pPage->childPtrSize==4 ); + pInfo->nSize = 4 + getVarint(&pCell[4], (u64*)&pInfo->nKey); + pInfo->nPayload = 0; + pInfo->nLocal = 0; + pInfo->iOverflow = 0; + pInfo->pPayload = 0; + return; }else{ + pIter = pCell + pPage->childPtrSize; pIter += getVarint32(pIter, nPayload); pInfo->nKey = nPayload; } @@ -1046,9 +1051,9 @@ static void btreeParseCell( ** the space used by the cell pointer. */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ - u8 *pIter = &pCell[pPage->childPtrSize]; - u8 *pEnd; - u32 nSize; + u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ #ifdef SQLITE_DEBUG /* The value returned by this function should always be the same as @@ -1059,19 +1064,21 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ btreeParseCellPtr(pPage, pCell, &debuginfo); #endif - if( pPage->intKey==0 || pPage->hasData ){ - nSize = *pIter; - if( nSize>=0x80 ){ - pEnd = &pIter[9]; - nSize &= 0x7f; - do{ - nSize = (nSize<<7) | (*++pIter & 0x7f); - }while( *(pIter)>=0x80 && pIternoPayload ){ + pEnd = &pIter[9]; + while( (*pIter++)&0x80 && pIterchildPtrSize==4 ); + return (u16)(pIter - pCell); } + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[9]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pIterintKey ){ /* pIter now points at the 64-bit integer key value, a variable length ** integer. The following block moves pIter to point at the first byte @@ -1079,10 +1086,12 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ pEnd = &pIter[9]; while( (*pIter++)&0x80 && pItermaxLocal ); testcase( nSize==pPage->maxLocal+1 ); - if( nSize>pPage->maxLocal ){ + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + if( nSize<4 ) nSize = 4; + }else{ int minLocal = pPage->minLocal; nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); testcase( nSize==pPage->maxLocal ); @@ -1090,15 +1099,8 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ if( nSize>pPage->maxLocal ){ nSize = minLocal; } - nSize += 4; + nSize += 4 + (u16)(pIter - pCell); } - nSize += (u32)(pIter - pCell); - - /* The minimum size of any cell is 4 bytes. */ - if( nSize<4 ){ - nSize = 4; - } - assert( nSize==debuginfo.nSize || CORRUPT_DB ); return (u16)nSize; } @@ -1442,12 +1444,14 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pBt = pPage->pBt; if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ pPage->intKey = 1; - pPage->hasData = pPage->leaf; + pPage->intKeyLeaf = pPage->leaf; + pPage->noPayload = !pPage->leaf; pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; }else if( flagByte==PTF_ZERODATA ){ pPage->intKey = 0; - pPage->hasData = 0; + pPage->intKeyLeaf = 0; + pPage->noPayload = 0; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; }else{ @@ -3855,7 +3859,7 @@ int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - assert( pCur->apPage[pCur->iPage]->intKey==1 ); + assert( pCur->apPage[pCur->iPage]->intKeyLeaf==1 ); getCellInfo(pCur); *pSize = pCur->info.nPayload; return SQLITE_OK; @@ -4690,7 +4694,7 @@ int sqlite3BtreeMovetoUnpacked( for(;;){ i64 nCellKey; pCell = findCell(pPage, idx) + pPage->childPtrSize; - if( pPage->hasData ){ + if( pPage->intKeyLeaf ){ while( 0x80 <= *(pCell++) ){ if( pCell>=pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; } @@ -5617,8 +5621,7 @@ static int fillInCell( /* Fill in the header. */ nHeader = pPage->childPtrSize; nPayload = nData + nZero; - if( pPage->hasData ){ - assert( pPage->intKey ); + if( pPage->intKeyLeaf ){ nHeader += putVarint32(&pCell[nHeader], nPayload); }else{ assert( nData==0 ); @@ -6391,7 +6394,7 @@ static int balance_nonroot( ** leafData: 1 if pPage holds key+data and pParent holds only keys. */ leafCorrection = apOld[0]->leaf*4; - leafData = apOld[0]->hasData; + leafData = apOld[0]->intKeyLeaf; for(i=0; ipDbPage); if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_QUICKBALANCE - if( pPage->hasData + if( pPage->intKeyLeaf && pPage->nOverflow==1 && pPage->aiOvfl[0]==pPage->nCell && pParent->pgno!=1 diff --git a/src/btreeInt.h b/src/btreeInt.h index c9ff79c90f..df9684e8e9 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -273,9 +273,10 @@ typedef struct BtLock BtLock; struct MemPage { u8 isInit; /* True if previously initialized. MUST BE FIRST! */ u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ - u8 intKey; /* True if intkey flag is set */ - u8 leaf; /* True if leaf flag is set */ - u8 hasData; /* True if this page stores data */ + u8 intKey; /* True if table b-trees. False for index b-trees */ + u8 intKeyLeaf; /* True if the leaf of an intKey table */ + u8 noPayload; /* True if internal intKey page (thus w/o data) */ + u8 leaf; /* True if a leaf page */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ u8 max1bytePayload; /* min(maxLocal,127) */ From 3f3874030cd8cf62f026ba46fe9c5abbef0d8a44 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Sep 2014 01:23:00 +0000 Subject: [PATCH 382/710] Shorten all lines of source code in btree.c to at most 80 characters. No logical changes. FossilOrigin-Name: 5dd41cdbfebdd088ebd9a90a394ee296c207ad90 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 54 ++++++++++++++++++++++++++++----------------------- 3 files changed, 37 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 41868133a8..a69f68fbf0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sMemPage.noPayload\sboolean\sand\suse\sit\sto\shelp\ncellSizePtr()\sand\sbtreeParseCellPtr()\srun\sfaster. -D 2014-09-24T00:59:08.082 +C Shorten\sall\slines\sof\ssource\scode\sin\sbtree.c\sto\sat\smost\s80\scharacters.\nNo\slogical\schanges. +D 2014-09-24T01:23:00.817 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c d64b3c5569bda0e7fbe9bc9388e17eaf70d63dec +F src/btree.c 3732a278d80867b06f8ed3dfa95338d021f874b0 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f480582ccae0e9a917d4523191025bd16016ba64 -R a8f50a441085b36b058ddd2aec76446f +P 8e3375313ebbf26b68561f3ed31d2a488222e5d0 +R 7f460f85115b9f5ff284729f30ee857e U drh -Z b0d06d7ed20970cccde4605a2a98dc92 +Z 15056c33153d13d2de1255f45fb692fc diff --git a/manifest.uuid b/manifest.uuid index dae8a6ddc0..57ede99673 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e3375313ebbf26b68561f3ed31d2a488222e5d0 \ No newline at end of file +5dd41cdbfebdd088ebd9a90a394ee296c207ad90 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 8644538abb..b943751253 100644 --- a/src/btree.c +++ b/src/btree.c @@ -487,7 +487,9 @@ static void invalidateIncrblobCursors( BtShared *pBt = pBtree->pBt; assert( sqlite3BtreeHoldsMutex(pBtree) ); for(p=pBt->pCursor; p; p=p->pNext){ - if( (p->curFlags & BTCF_Incrblob)!=0 && (isClearTable || p->info.nKey==iRow) ){ + if( (p->curFlags & BTCF_Incrblob)!=0 + && (isClearTable || p->info.nKey==iRow) + ){ p->eState = CURSOR_INVALID; } } @@ -660,9 +662,9 @@ static int saveAllCursors(BtShared *pBt, Pgno iRoot, BtCursor *pExcept){ ** broken out from its caller to avoid unnecessary stack pointer movement. */ static int SQLITE_NOINLINE saveCursorsOnList( - BtCursor *p, /* The first cursor that needs saving */ - Pgno iRoot, /* Only save cursor with this iRoot. Save all if zero */ - BtCursor *pExcept /* Do not save this cursor */ + BtCursor *p, /* The first cursor that needs saving */ + Pgno iRoot, /* Only save cursor with this iRoot. Save all if zero */ + BtCursor *pExcept /* Do not save this cursor */ ){ do{ if( p!=pExcept && (0==iRoot || p->pgnoRoot==iRoot) ){ @@ -1051,9 +1053,9 @@ static void btreeParseCell( ** the space used by the cell pointer. */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ - u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ - u8 *pEnd; /* End mark for a varint */ - u32 nSize; /* Size value to return */ + u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ #ifdef SQLITE_DEBUG /* The value returned by this function should always be the same as @@ -1336,7 +1338,7 @@ defragment_page: ** routine and return SQLITE_CORRUPT if any problems are found. */ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ - u16 iPtr; /* Address of pointer to next freeblock */ + u16 iPtr; /* Address of ptr to next freeblock */ u16 iFreeBlk; /* Address of the next freeblock */ u8 hdr; /* Page header size. 0 or 100 */ u8 nFrag = 0; /* Reduction in fragmentation */ @@ -1388,9 +1390,9 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ iFreeBlk = get2byte(&data[iFreeBlk]); } - /* If iPtr is another freeblock (that is, if iPtr is not the freelist pointer - ** in the page header) then check to see if iStart should be coalesced - ** onto the end of iPtr. + /* If iPtr is another freeblock (that is, if iPtr is not the freelist + ** pointer in the page header) then check to see if iStart should be + ** coalesced onto the end of iPtr. */ if( iPtr>hdr+1 ){ int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); @@ -4016,14 +4018,14 @@ static int accessPayload( MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ #ifdef SQLITE_DIRECT_OVERFLOW_READ - int bEnd; /* True if reading to end of data */ + int bEnd; /* True if reading to end of data */ #endif assert( pPage ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->aiIdx[pCur->iPage]nCell ); assert( cursorHoldsMutex(pCur) ); - assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ + assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ getCellInfo(pCur); aPayload = pCur->info.pPayload; @@ -4088,7 +4090,9 @@ static int accessPayload( ** entry for the first required overflow page is valid, skip ** directly to it. */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 && pCur->aOverflow[offset/ovflSize] ){ + if( (pCur->curFlags & BTCF_ValidOvfl)!=0 + && pCur->aOverflow[offset/ovflSize] + ){ iIdx = (offset/ovflSize); nextPage = pCur->aOverflow[iIdx]; offset = (offset%ovflSize); @@ -4953,9 +4957,9 @@ int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** ** The main entry point is sqlite3BtreePrevious(). That routine is optimized ** for the common case of merely decrementing the cell counter BtCursor.aiIdx -** to the previous cell on the current page. The (slower) btreePrevious() helper -** routine is called when it is necessary to move to a different page or -** to restore the cursor. +** to the previous cell on the current page. The (slower) btreePrevious() +** helper routine is called when it is necessary to move to a different page +** or to restore the cursor. ** ** The calling function will set *pRes to 0 or 1. The initial *pRes value ** will be 1 if the cursor being stepped corresponds to an SQL index and @@ -5283,7 +5287,7 @@ static int allocateBtreePage( memcpy(&aData[8+closest*4], &aData[4+k*4], 4); } put4byte(&aData[4], k-1); - noContent = !btreeGetHasContent(pBt, *pPgno) ? PAGER_GET_NOCONTENT : 0; + noContent = !btreeGetHasContent(pBt, *pPgno)? PAGER_GET_NOCONTENT : 0; rc = btreeGetPage(pBt, *pPgno, ppPage, noContent); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite((*ppPage)->pDbPage); @@ -5316,7 +5320,7 @@ static int allocateBtreePage( ** here are confined to those pages that lie between the end of the ** database image and the end of the database file. */ - int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate)) ? PAGER_GET_NOCONTENT : 0; + int bNoContent = (0==IfNotOmitAV(pBt->bDoTruncate))? PAGER_GET_NOCONTENT:0; rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); if( rc ) return rc; @@ -7089,7 +7093,8 @@ int sqlite3BtreeInsert( } assert( cursorHoldsMutex(pCur) ); - assert( (pCur->curFlags & BTCF_WriteFlag)!=0 && pBt->inTransaction==TRANS_WRITE + assert( (pCur->curFlags & BTCF_WriteFlag)!=0 + && pBt->inTransaction==TRANS_WRITE && (pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); @@ -7122,7 +7127,8 @@ int sqlite3BtreeInsert( /* If the cursor is currently on the last row and we are appending a ** new row onto the end, set the "loc" to avoid an unnecessary btreeMoveto() ** call */ - if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 && pCur->info.nKey==nKey-1 ){ + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 + && pCur->info.nKey==nKey-1 ){ loc = -1; } } @@ -8144,7 +8150,7 @@ static int checkTreePage( checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); } #endif - d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0 ? NULL : &nMaxKey); + d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0?NULL:&nMaxKey); if( i>0 && d2!=depth ){ checkAppendMsg(pCheck, zContext, "Child page depth differs"); } @@ -8161,7 +8167,7 @@ static int checkTreePage( checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); } #endif - checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell ? NULL : &nMaxKey); + checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell?NULL:&nMaxKey); } /* For intKey leaf pages, check that the min/max keys are in order @@ -8554,7 +8560,7 @@ int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void *z){ ** required in case any of them are holding references to an xFetch ** version of the b-tree page modified by the accessPayload call below. ** - ** Note that pCsr must be open on a BTREE_INTKEY table and saveCursorPosition() + ** Note that pCsr must be open on a INTKEY table and saveCursorPosition() ** and hence saveAllCursors() cannot fail on a BTREE_INTKEY table, hence ** saveAllCursors can only return SQLITE_OK. */ From 9bfdc250622df58dd87e34566ccb23d5910b40a6 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Sep 2014 02:05:41 +0000 Subject: [PATCH 383/710] Have the clearCell() routine return the cell size to the caller, rather than have the caller make a separate call to cellSizePtr(). FossilOrigin-Name: f21d217583c205dc17f98bb4877fd4ed98cefcb1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 22 +++++++++++++++------- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index a69f68fbf0..209072a802 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Shorten\sall\slines\sof\ssource\scode\sin\sbtree.c\sto\sat\smost\s80\scharacters.\nNo\slogical\schanges. -D 2014-09-24T01:23:00.817 +C Have\sthe\sclearCell()\sroutine\sreturn\sthe\scell\ssize\sto\sthe\scaller,\srather\nthan\shave\sthe\scaller\smake\sa\sseparate\scall\sto\scellSizePtr(). +D 2014-09-24T02:05:41.968 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 3732a278d80867b06f8ed3dfa95338d021f874b0 +F src/btree.c 183d62b37358f95d2ffac796f8491591a3456362 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8e3375313ebbf26b68561f3ed31d2a488222e5d0 -R 7f460f85115b9f5ff284729f30ee857e +P 5dd41cdbfebdd088ebd9a90a394ee296c207ad90 +R c3c34442c09fda273806b002932cda25 U drh -Z 15056c33153d13d2de1255f45fb692fc +Z 4d0ec3d41d82b7fff44b33f0ea314f2b diff --git a/manifest.uuid b/manifest.uuid index 57ede99673..1ee8b65c87 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5dd41cdbfebdd088ebd9a90a394ee296c207ad90 \ No newline at end of file +f21d217583c205dc17f98bb4877fd4ed98cefcb1 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b943751253..9af99dd8ad 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5519,9 +5519,15 @@ static void freePage(MemPage *pPage, int *pRC){ } /* -** Free any overflow pages associated with the given Cell. +** Free any overflow pages associated with the given Cell. Write the +** local Cell size (the number of bytes on the original page, omitting +** overflow) into *pnSize. */ -static int clearCell(MemPage *pPage, unsigned char *pCell){ +static int clearCell( + MemPage *pPage, /* The page that contains the Cell */ + unsigned char *pCell, /* First byte of the Cell */ + u16 *pnSize /* Write the size of the Cell here */ +){ BtShared *pBt = pPage->pBt; CellInfo info; Pgno ovflPgno; @@ -5531,6 +5537,7 @@ static int clearCell(MemPage *pPage, unsigned char *pCell){ assert( sqlite3_mutex_held(pPage->pBt->mutex) ); btreeParseCellPtr(pPage, pCell, &info); + *pnSize = info.nSize; if( info.iOverflow==0 ){ return SQLITE_OK; /* No overflow pages. Return without doing anything */ } @@ -7166,8 +7173,7 @@ int sqlite3BtreeInsert( if( !pPage->leaf ){ memcpy(newCell, oldCell, 4); } - szOld = cellSizePtr(pPage, oldCell); - rc = clearCell(pPage, oldCell); + rc = clearCell(pPage, oldCell, &szOld); dropCell(pPage, idx, szOld, &rc); if( rc ) goto end_insert; }else if( loc<0 && pPage->nCell>0 ){ @@ -7229,6 +7235,7 @@ int sqlite3BtreeDelete(BtCursor *pCur){ unsigned char *pCell; /* Pointer to cell to delete */ int iCellIdx; /* Index of cell to delete */ int iCellDepth; /* Depth of node containing pCell */ + u16 szCell; /* Size of the cell being deleted */ assert( cursorHoldsMutex(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -7277,8 +7284,8 @@ int sqlite3BtreeDelete(BtCursor *pCur){ rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; - rc = clearCell(pPage, pCell); - dropCell(pPage, iCellIdx, cellSizePtr(pPage, pCell), &rc); + rc = clearCell(pPage, pCell, &szCell); + dropCell(pPage, iCellIdx, szCell, &rc); if( rc ) return rc; /* If the cell deleted was not located on a leaf page, then the cursor @@ -7510,6 +7517,7 @@ static int clearDatabasePage( unsigned char *pCell; int i; int hdr; + u16 szCell; assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ @@ -7525,7 +7533,7 @@ static int clearDatabasePage( rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); if( rc ) goto cleardatabasepage_out; } - rc = clearCell(pPage, pCell); + rc = clearCell(pPage, pCell, &szCell); if( rc ) goto cleardatabasepage_out; } if( !pPage->leaf ){ From feada2df39af475b18055c117d4f3c75212dbbe5 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Sep 2014 13:20:22 +0000 Subject: [PATCH 384/710] Do not allow parameters in a DEFAULT clause of a CREATE TABLE statement. Ticket [78c0c8c3c9f7c1]. FossilOrigin-Name: 1ad2bc1ed4c4ac81ac67a9660761f0eeb47c7fef --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/build.c | 2 +- src/expr.c | 44 +++++++++++++++++++++++++++++++++----------- src/sqliteInt.h | 2 +- test/default.test | 28 ++++++++++++++++++++++++++++ 6 files changed, 73 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 209072a802..01e1d67d9f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sthe\sclearCell()\sroutine\sreturn\sthe\scell\ssize\sto\sthe\scaller,\srather\nthan\shave\sthe\scaller\smake\sa\sseparate\scall\sto\scellSizePtr(). -D 2014-09-24T02:05:41.968 +C Do\snot\sallow\sparameters\sin\sa\sDEFAULT\sclause\sof\sa\sCREATE\sTABLE\sstatement.\nTicket\s[78c0c8c3c9f7c1]. +D 2014-09-24T13:20:22.559 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,13 +175,13 @@ F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c 183d62b37358f95d2ffac796f8491591a3456362 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef -F src/build.c 8dbca25988045fbf2a33c9631c42706fa6449e60 +F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f -F src/expr.c 4f101c8ddc6d5a22303c88278069f5261562a9a8 +F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c 1629ccdd8ef3f19d7accc9d9287190489469ff81 @@ -232,7 +232,7 @@ F src/shell.c dad23987c34faddb061a339da3e92e05ccc6935e F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 5ecde2191721a94cdce0d5248e26a373e0b17a70 +F src/sqliteInt.h ca777ba045b1849d36a44dee1c71c652ae915093 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -438,7 +438,7 @@ F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test 42973251b9429f2c41b77eb98a7b0b0ba2d3b2c0 F test/dbstatus.test 8de104bb5606f19537d23cd553b41349b5ab1204 F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 -F test/default.test 792c3c70836f1901e2a8cb34fa0880ed71e2c1a9 +F test/default.test 0cb49b1c315a0d81c81d775e407f66906a2a604d F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa F test/delete3.test 555e84a00a99230b7d049d477a324a631126a6ab @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5dd41cdbfebdd088ebd9a90a394ee296c207ad90 -R c3c34442c09fda273806b002932cda25 +P f21d217583c205dc17f98bb4877fd4ed98cefcb1 +R 0c35c4c692384a20d9dc945800977dcf U drh -Z 4d0ec3d41d82b7fff44b33f0ea314f2b +Z ea463bda0fe2fcb1086a67d4498e377c diff --git a/manifest.uuid b/manifest.uuid index 1ee8b65c87..61739fa20a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f21d217583c205dc17f98bb4877fd4ed98cefcb1 \ No newline at end of file +1ad2bc1ed4c4ac81ac67a9660761f0eeb47c7fef \ No newline at end of file diff --git a/src/build.c b/src/build.c index 0c02a56fe7..777831aab5 100644 --- a/src/build.c +++ b/src/build.c @@ -1236,7 +1236,7 @@ void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ p = pParse->pNewTable; if( p!=0 ){ pCol = &(p->aCol[p->nCol-1]); - if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr) ){ + if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr, db->init.busy) ){ sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", pCol->zName); }else{ diff --git a/src/expr.c b/src/expr.c index c8e8e78268..57e462ea6e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1212,32 +1212,40 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ /* ** These routines are Walker callbacks. Walker.u.pi is a pointer ** to an integer. These routines are checking an expression to see -** if it is a constant. Set *Walker.u.pi to 0 if the expression is +** if it is a constant. Set *Walker.u.i to 0 if the expression is ** not constant. ** ** These callback routines are used to implement the following: ** -** sqlite3ExprIsConstant() -** sqlite3ExprIsConstantNotJoin() -** sqlite3ExprIsConstantOrFunction() +** sqlite3ExprIsConstant() pWalker->u.i==1 +** sqlite3ExprIsConstantNotJoin() pWalker->u.i==2 +** sqlite3ExprIsConstantOrFunction() pWalker->u.i==3 or 4 ** +** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions +** in a CREATE TABLE statement. The Walker.u.i value is 4 when parsing +** an existing schema and 3 when processing a new statement. A bound +** parameter raises an error for new statements, but is silently converted +** to NULL for existing schemas. This allows sqlite_master tables that +** contain a bound parameter because they were generated by older versions +** of SQLite to be parsed by newer versions of SQLite without raising a +** malformed schema error. */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ - /* If pWalker->u.i is 3 then any term of the expression that comes from + /* If pWalker->u.i is 2 then any term of the expression that comes from ** the ON or USING clauses of a join disqualifies the expression ** from being considered constant. */ - if( pWalker->u.i==3 && ExprHasProperty(pExpr, EP_FromJoin) ){ + if( pWalker->u.i==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ pWalker->u.i = 0; return WRC_Abort; } switch( pExpr->op ){ /* Consider functions to be constant if all their arguments are constant - ** and either pWalker->u.i==2 or the function as the SQLITE_FUNC_CONST + ** and either pWalker->u.i==3 or 4 or the function as the SQLITE_FUNC_CONST ** flag. */ case TK_FUNCTION: - if( pWalker->u.i==2 || ExprHasProperty(pExpr,EP_Constant) ){ + if( pWalker->u.i>=3 || ExprHasProperty(pExpr,EP_Constant) ){ return WRC_Continue; } /* Fall through */ @@ -1251,6 +1259,19 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_AGG_COLUMN ); pWalker->u.i = 0; return WRC_Abort; + case TK_VARIABLE: + if( pWalker->u.i==4 ){ + /* Silently convert bound parameters that appear inside of CREATE + ** statements into a NULL when parsing the CREATE statement text out + ** of the sqlite_master table */ + pExpr->op = TK_NULL; + }else if( pWalker->u.i==3 ){ + /* A bound parameter in a CREATE statement that originates from + ** sqlite3_prepare() causes an error */ + pWalker->u.i = 0; + return WRC_Abort; + } + /* Fall through */ default: testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */ testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */ @@ -1291,7 +1312,7 @@ int sqlite3ExprIsConstant(Expr *p){ ** an ON or USING clause. */ int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 3); + return exprIsConst(p, 2); } /* @@ -1303,8 +1324,9 @@ int sqlite3ExprIsConstantNotJoin(Expr *p){ ** is considered a variable but a single-quoted string (ex: 'abc') is ** a constant. */ -int sqlite3ExprIsConstantOrFunction(Expr *p){ - return exprIsConst(p, 2); +int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ + assert( isInit==0 || isInit==1 ); + return exprIsConst(p, 3+isInit); } /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9a9675b0ab..414f30b66c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3285,7 +3285,7 @@ void sqlite3CloseSavepoints(sqlite3 *); void sqlite3LeaveMutexAndCloseZombie(sqlite3*); int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); -int sqlite3ExprIsConstantOrFunction(Expr*); +int sqlite3ExprIsConstantOrFunction(Expr*, u8); int sqlite3ExprIsInteger(Expr*, int*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); diff --git a/test/default.test b/test/default.test index d6b6f97d98..406eb53677 100644 --- a/test/default.test +++ b/test/default.test @@ -99,4 +99,32 @@ do_execsql_test default-3.3 { SELECT * FROM t300; } {2147483647 2147483648 9223372036854775807 -2147483647 -2147483648 -9223372036854775808 9.22337203685478e+18 9223372036854775807} +# Do now allow bound parameters in new DEFAULT values. +# Silently convert bound parameters to NULL in DEFAULT causes +# in the sqlite_master table, for backwards compatibility. +# +db close +forcedelete test.db +sqlite3 db test.db +do_execsql_test default-4.0 { + CREATE TABLE t1(a TEXT, b TEXT DEFAULT(99)); + PRAGMA writable_schema=ON; + UPDATE sqlite_master SET sql='CREATE TABLE t1(a TEXT, b TEXT DEFAULT(:xyz))'; +} {} +db close +sqlite3 db test.db +do_execsql_test default-4.1 { + INSERT INTO t1(a) VALUES('xyzzy'); + SELECT a, quote(b) FROM t1; +} {xyzzy NULL} +do_catchsql_test default-4.2 { + CREATE TABLE t2(a TEXT, b TEXT DEFAULT(:xyz)); +} {1 {default value of column [b] is not constant}} +do_catchsql_test default-4.3 { + CREATE TABLE t2(a TEXT, b TEXT DEFAULT(abs(:xyz))); +} {1 {default value of column [b] is not constant}} +do_catchsql_test default-4.4 { + CREATE TABLE t2(a TEXT, b TEXT DEFAULT(98+coalesce(5,:xyz))); +} {1 {default value of column [b] is not constant}} + finish_test From b2325b72df09777cf039ff5036d58047fd0bb2f7 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Sep 2014 18:31:07 +0000 Subject: [PATCH 385/710] Small performance and size optimization for btreeUnlockIfUnused(). FossilOrigin-Name: 13c746f85d254475b10c3dd58555acd3bbead0ce --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 01e1d67d9f..c64e305a7f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sallow\sparameters\sin\sa\sDEFAULT\sclause\sof\sa\sCREATE\sTABLE\sstatement.\nTicket\s[78c0c8c3c9f7c1]. -D 2014-09-24T13:20:22.559 +C Small\sperformance\sand\ssize\soptimization\sfor\sbtreeUnlockIfUnused(). +D 2014-09-24T18:31:07.339 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 183d62b37358f95d2ffac796f8491591a3456362 +F src/btree.c ee77b1d3a346dd0d581e3d729524243c68ba5b96 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f21d217583c205dc17f98bb4877fd4ed98cefcb1 -R 0c35c4c692384a20d9dc945800977dcf +P 1ad2bc1ed4c4ac81ac67a9660761f0eeb47c7fef +R d6b9d6f16b042c47207345c5727aac20 U drh -Z ea463bda0fe2fcb1086a67d4498e377c +Z 51a889840b720ee888395fd53da3c843 diff --git a/manifest.uuid b/manifest.uuid index 61739fa20a..838ae9d867 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1ad2bc1ed4c4ac81ac67a9660761f0eeb47c7fef \ No newline at end of file +13c746f85d254475b10c3dd58555acd3bbead0ce \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9af99dd8ad..d8bf076e85 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2627,11 +2627,11 @@ static void unlockBtreeIfUnused(BtShared *pBt){ assert( sqlite3_mutex_held(pBt->mutex) ); assert( countValidCursors(pBt,0)==0 || pBt->inTransaction>TRANS_NONE ); if( pBt->inTransaction==TRANS_NONE && pBt->pPage1!=0 ){ - assert( pBt->pPage1->aData ); + MemPage *pPage1 = pBt->pPage1; + assert( pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); - assert( pBt->pPage1->aData ); - releasePage(pBt->pPage1); pBt->pPage1 = 0; + releasePage(pPage1); } } From 3fbb022b98a05d985571a188003f3dd1fb2b94a5 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Sep 2014 19:47:27 +0000 Subject: [PATCH 386/710] Have each open database allocate its pTmpSpace when the first write cursor is opened, rather than on each insert or delete, for a small space savings and performance boost. FossilOrigin-Name: 99323552c001bc9173eb2a44542234c8ef7a9845 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 11 ++++++----- src/vdbe.c | 4 ---- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index c64e305a7f..9d16127977 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\sperformance\sand\ssize\soptimization\sfor\sbtreeUnlockIfUnused(). -D 2014-09-24T18:31:07.339 +C Have\seach\sopen\sdatabase\sallocate\sits\spTmpSpace\swhen\sthe\sfirst\swrite\scursor\nis\sopened,\srather\sthan\son\seach\sinsert\sor\sdelete,\sfor\sa\ssmall\sspace\ssavings\nand\sperformance\sboost. +D 2014-09-24T19:47:27.033 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c ee77b1d3a346dd0d581e3d729524243c68ba5b96 +F src/btree.c 4d5cdfeaea4a00f796c17af246dd5b48cd525d5e F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c de1af1795bebdad20c23e82bafa2f531e9893198 +F src/vdbe.c 23db9b11f0d0a022c42bf71c2b036d32c82a9abd F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1ad2bc1ed4c4ac81ac67a9660761f0eeb47c7fef -R d6b9d6f16b042c47207345c5727aac20 +P 13c746f85d254475b10c3dd58555acd3bbead0ce +R dd216235dc00cda1e50e4687ef88aee3 U drh -Z 51a889840b720ee888395fd53da3c843 +Z 5eb3b3c4f4a76619f8f4a6dc0df5e879 diff --git a/manifest.uuid b/manifest.uuid index 838ae9d867..c31ec3cb1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13c746f85d254475b10c3dd58555acd3bbead0ce \ No newline at end of file +99323552c001bc9173eb2a44542234c8ef7a9845 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index d8bf076e85..37aabbef77 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3672,6 +3672,10 @@ static int btreeCursor( if( NEVER(wrFlag && (pBt->btsFlags & BTS_READ_ONLY)!=0) ){ return SQLITE_READONLY; } + if( wrFlag ){ + allocateTempSpace(pBt); + if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM; + } if( iTable==1 && btreePagecount(pBt)==0 ){ assert( wrFlag==0 ); iTable = 0; @@ -7154,9 +7158,8 @@ int sqlite3BtreeInsert( pCur->pgnoRoot, nKey, nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); assert( pPage->isInit ); - allocateTempSpace(pBt); newCell = pBt->pTmpSpace; - if( newCell==0 ) return SQLITE_NOMEM; + assert( newCell!=0 ); rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); if( rc ) goto end_insert; assert( szNew==cellSizePtr(pPage, newCell) ); @@ -7302,10 +7305,8 @@ int sqlite3BtreeDelete(BtCursor *pCur){ pCell = findCell(pLeaf, pLeaf->nCell-1); nCell = cellSizePtr(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); - - allocateTempSpace(pBt); pTmp = pBt->pTmpSpace; - + assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); diff --git a/src/vdbe.c b/src/vdbe.c index 26ca72b9f4..4c2b75f641 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3287,10 +3287,6 @@ case OP_OpenWrite: { assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); sqlite3BtreeCursorHints(pCur->pCursor, (pOp->p5 & OPFLAG_BULKCSR)); - /* Since it performs no memory allocation or IO, the only value that - ** sqlite3BtreeCursor() may return is SQLITE_OK. */ - assert( rc==SQLITE_OK ); - /* Set the VdbeCursor.isTable variable. Previous versions of ** SQLite used to check if the root-page flags were sane at this point ** and report database corruption if they were not, but this check has From 328d913cbdb3aac68c244abbf433ef318f4593a4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 00:56:00 +0000 Subject: [PATCH 387/710] Size reduction and performance improvement in the LIKE and GLOB operators. FossilOrigin-Name: b2c89ef49cd19b8031a8149a2dc47cea07dd04e0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 11 +++++------ 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 9d16127977..6a57efd696 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\seach\sopen\sdatabase\sallocate\sits\spTmpSpace\swhen\sthe\sfirst\swrite\scursor\nis\sopened,\srather\sthan\son\seach\sinsert\sor\sdelete,\sfor\sa\ssmall\sspace\ssavings\nand\sperformance\sboost. -D 2014-09-24T19:47:27.033 +C Size\sreduction\sand\sperformance\simprovement\sin\sthe\sLIKE\sand\sGLOB\soperators. +D 2014-09-25T00:56:00.685 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -184,7 +184,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c 1629ccdd8ef3f19d7accc9d9287190489469ff81 +F src/func.c fd49097fdd74eecbc244e5e64fd288a303db20e9 F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 13c746f85d254475b10c3dd58555acd3bbead0ce -R dd216235dc00cda1e50e4687ef88aee3 +P 99323552c001bc9173eb2a44542234c8ef7a9845 +R 81650f65f515728b656a3f04c1aeaf34 U drh -Z 5eb3b3c4f4a76619f8f4a6dc0df5e879 +Z 2d4294ecf245e95fb6c7da3803927f1a diff --git a/manifest.uuid b/manifest.uuid index c31ec3cb1b..e8ae266ea4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -99323552c001bc9173eb2a44542234c8ef7a9845 \ No newline at end of file +b2c89ef49cd19b8031a8149a2dc47cea07dd04e0 \ No newline at end of file diff --git a/src/func.c b/src/func.c index e1961118fd..fc908ded36 100644 --- a/src/func.c +++ b/src/func.c @@ -622,10 +622,9 @@ static int patternCompare( u8 matchAll = pInfo->matchAll; u8 matchSet = pInfo->matchSet; u8 noCase = pInfo->noCase; - int prevEscape = 0; /* True if the previous character was 'escape' */ while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ - if( c==matchAll && !prevEscape ){ + if( c==matchAll ){ while( (c=sqlite3Utf8Read(&zPattern)) == matchAll || c == matchOne ){ if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ @@ -664,7 +663,7 @@ static int patternCompare( if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } return 0; - }else if( c==matchOne && !prevEscape ){ + }else if( c==matchOne ){ if( sqlite3Utf8Read(&zString)==0 ){ return 0; } @@ -700,10 +699,11 @@ static int patternCompare( if( c2==0 || (seen ^ invert)==0 ){ return 0; } - }else if( esc==c && !prevEscape ){ - prevEscape = 1; }else{ c2 = sqlite3Utf8Read(&zString); + if( c==esc ){ + c = sqlite3Utf8Read(&zPattern); + } if( noCase ){ GlobUpperToLower(c); GlobUpperToLower(c2); @@ -711,7 +711,6 @@ static int patternCompare( if( c!=c2 ){ return 0; } - prevEscape = 0; } } return *zString==0; From 97348b37c235f53df245e916a5d32652a837168e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 02:44:29 +0000 Subject: [PATCH 388/710] Change that might allow SQLite to build and work using the EBCDIC character set. FossilOrigin-Name: ef30e0352b3d4a29749cd0872c10e45a6649ec52 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/ctime.c | 2 +- src/sqliteInt.h | 2 ++ src/tokenize.c | 1 + 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 6a57efd696..de2a2bc27f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Size\sreduction\sand\sperformance\simprovement\sin\sthe\sLIKE\sand\sGLOB\soperators. -D 2014-09-25T00:56:00.685 +C Change\sthat\smight\sallow\sSQLite\sto\sbuild\sand\swork\susing\sthe\sEBCDIC\scharacter\nset. +D 2014-09-25T02:44:29.974 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -178,7 +178,7 @@ F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 -F src/ctime.c 16cd19215d9fd849ee2b7509b092f2e0bbd6a958 +F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 @@ -232,7 +232,7 @@ F src/shell.c dad23987c34faddb061a339da3e92e05ccc6935e F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h ca777ba045b1849d36a44dee1c71c652ae915093 +F src/sqliteInt.h 5e09fe04f999223680801ddf8fbae6b60751d613 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -283,7 +283,7 @@ F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 22dded4283dc4b25422f6444cdcb8d6b1ea0b5ff -F src/tokenize.c 3df63041994f55afeb168b463ec836e8f1c50e7c +F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 99323552c001bc9173eb2a44542234c8ef7a9845 -R 81650f65f515728b656a3f04c1aeaf34 +P b2c89ef49cd19b8031a8149a2dc47cea07dd04e0 +R 594413b3f59b58a61e72cebd73f880a5 U drh -Z 2d4294ecf245e95fb6c7da3803927f1a +Z 19c5c0df4d171b8ce37674c647553165 diff --git a/manifest.uuid b/manifest.uuid index e8ae266ea4..7b2fbde36d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b2c89ef49cd19b8031a8149a2dc47cea07dd04e0 \ No newline at end of file +ef30e0352b3d4a29749cd0872c10e45a6649ec52 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index 6f7ac8fcba..82a2f35204 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -395,7 +395,7 @@ int sqlite3_compileoption_used(const char *zOptName){ ** linear search is adequate. No need for a binary search. */ for(i=0; inChange */ +#define OPFLAG_EPHEM 0x01 /* OP_Column: Ephemeral output is ok */ #define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ @@ -2987,6 +2988,7 @@ int sqlite3CantopenError(int); # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) #endif +int sqlite3IsIdChar(u8); /* ** Internal function prototypes diff --git a/src/tokenize.c b/src/tokenize.c index 8a7894514c..5bb9155460 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -102,6 +102,7 @@ const char sqlite3IsEbcdicIdChar[] = { }; #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) #endif +int sqlite3IsIdChar(u8 c){ return IdChar(c); } /* From 88b3322f272341f4a8145583d6ba55b2d35da211 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 03:51:37 +0000 Subject: [PATCH 389/710] More performance optimization for the LIKE and GLOB operators. FossilOrigin-Name: 5ab1023d6cfe31fa8a194804b8216058977ac973 --- manifest | 12 ++--- manifest.uuid | 2 +- src/func.c | 146 +++++++++++++++++++++++++++----------------------- 3 files changed, 86 insertions(+), 74 deletions(-) diff --git a/manifest b/manifest index de2a2bc27f..37342c612c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthat\smight\sallow\sSQLite\sto\sbuild\sand\swork\susing\sthe\sEBCDIC\scharacter\nset. -D 2014-09-25T02:44:29.974 +C More\sperformance\soptimization\sfor\sthe\sLIKE\sand\sGLOB\soperators. +D 2014-09-25T03:51:37.139 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -184,7 +184,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c fd49097fdd74eecbc244e5e64fd288a303db20e9 +F src/func.c 610b18afde750686785cdad9196b9fb1b03dc9b3 F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b2c89ef49cd19b8031a8149a2dc47cea07dd04e0 -R 594413b3f59b58a61e72cebd73f880a5 +P ef30e0352b3d4a29749cd0872c10e45a6649ec52 +R 8a27395e5bbddced71b289eca8fe4771 U drh -Z 19c5c0df4d171b8ce37674c647553165 +Z c372796d929c11ef7d059e5ca2e0eb4a diff --git a/manifest.uuid b/manifest.uuid index 7b2fbde36d..035ef4eeb2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef30e0352b3d4a29749cd0872c10e45a6649ec52 \ No newline at end of file +5ab1023d6cfe31fa8a194804b8216058977ac973 \ No newline at end of file diff --git a/src/func.c b/src/func.c index fc908ded36..a2a5c1858d 100644 --- a/src/func.c +++ b/src/func.c @@ -567,10 +567,12 @@ struct compareInfo { ** whereas only characters less than 0x80 do in ASCII. */ #if defined(SQLITE_EBCDIC) -# define sqlite3Utf8Read(A) (*((*A)++)) -# define GlobUpperToLower(A) A = sqlite3UpperToLower[A] +# define sqlite3Utf8Read(A) (*((*A)++)) +# define GlobUpperToLower(A) A = sqlite3UpperToLower[A] +# define GlobUpperToLowerAscii(A) A = sqlite3UpperToLower[A] #else -# define GlobUpperToLower(A) if( !((A)&~0x7f) ){ A = sqlite3UpperToLower[A]; } +# define GlobUpperToLower(A) if( A<=0x7f ){ A = sqlite3UpperToLower[A]; } +# define GlobUpperToLowerAscii(A) A = sqlite3UpperToLower[A] #endif static const struct compareInfo globInfo = { '*', '?', '[', 0 }; @@ -618,10 +620,17 @@ static int patternCompare( u32 c, c2; int invert; int seen; - u8 matchOne = pInfo->matchOne; - u8 matchAll = pInfo->matchAll; - u8 matchSet = pInfo->matchSet; - u8 noCase = pInfo->noCase; + u32 matchOne = pInfo->matchOne; + u32 matchAll = pInfo->matchAll; + u32 matchOther; + u8 noCase = pInfo->noCase; + + /* The GLOB operator does not have an ESCAPE clause. And LIKE does not + ** have the matchSet operator. So we either have to look for one or + ** the other, never both. Hence the single variable matchOther is used + ** to store the one we have to look for. + */ + matchOther = esc ? esc : pInfo->matchSet; while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ if( c==matchAll ){ @@ -633,26 +642,26 @@ static int patternCompare( } if( c==0 ){ return 1; - }else if( c==esc ){ - c = sqlite3Utf8Read(&zPattern); - if( c==0 ){ - return 0; + }else if( c==matchOther ){ + if( esc ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return 0; + }else{ + assert( matchOther<0x80 ); /* '[' is a single-byte character */ + while( *zString + && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ + SQLITE_SKIP_UTF8(zString); + } + return *zString!=0; } - }else if( c==matchSet ){ - assert( esc==0 ); /* This is GLOB, not LIKE */ - assert( matchSet<0x80 ); /* '[' is a single-byte character */ - while( *zString && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ - SQLITE_SKIP_UTF8(zString); - } - return *zString!=0; } while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ - if( noCase ){ + if( noCase && c<0x80 ){ GlobUpperToLower(c2); - GlobUpperToLower(c); + GlobUpperToLowerAscii(c); while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(&zString); - GlobUpperToLower(c2); + do{ c2 = *(zString++); }while( c2>0x7f ); + GlobUpperToLowerAscii(c2); } }else{ while( c2 != 0 && c2 != c ){ @@ -663,55 +672,58 @@ static int patternCompare( if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } return 0; - }else if( c==matchOne ){ + } + if( c==matchOne ){ if( sqlite3Utf8Read(&zString)==0 ){ return 0; - } - }else if( c==matchSet ){ - u32 prior_c = 0; - assert( esc==0 ); /* This only occurs for GLOB, not LIKE */ - seen = 0; - invert = 0; - c = sqlite3Utf8Read(&zString); - if( c==0 ) return 0; - c2 = sqlite3Utf8Read(&zPattern); - if( c2=='^' ){ - invert = 1; - c2 = sqlite3Utf8Read(&zPattern); - } - if( c2==']' ){ - if( c==']' ) seen = 1; - c2 = sqlite3Utf8Read(&zPattern); - } - while( c2 && c2!=']' ){ - if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ - c2 = sqlite3Utf8Read(&zPattern); - if( c>=prior_c && c<=c2 ) seen = 1; - prior_c = 0; - }else{ - if( c==c2 ){ - seen = 1; - } - prior_c = c2; - } - c2 = sqlite3Utf8Read(&zPattern); - } - if( c2==0 || (seen ^ invert)==0 ){ - return 0; - } - }else{ - c2 = sqlite3Utf8Read(&zString); - if( c==esc ){ - c = sqlite3Utf8Read(&zPattern); - } - if( noCase ){ - GlobUpperToLower(c); - GlobUpperToLower(c2); - } - if( c!=c2 ){ - return 0; + }else{ + continue; } } + if( c==matchOther ){ + if( esc ){ + c = sqlite3Utf8Read(&zPattern); + if( c==0 ) return 0; + }else{ + u32 prior_c = 0; + seen = 0; + invert = 0; + c = sqlite3Utf8Read(&zString); + if( c==0 ) return 0; + c2 = sqlite3Utf8Read(&zPattern); + if( c2=='^' ){ + invert = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==']' ){ + if( c==']' ) seen = 1; + c2 = sqlite3Utf8Read(&zPattern); + } + while( c2 && c2!=']' ){ + if( c2=='-' && zPattern[0]!=']' && zPattern[0]!=0 && prior_c>0 ){ + c2 = sqlite3Utf8Read(&zPattern); + if( c>=prior_c && c<=c2 ) seen = 1; + prior_c = 0; + }else{ + if( c==c2 ){ + seen = 1; + } + prior_c = c2; + } + c2 = sqlite3Utf8Read(&zPattern); + } + if( c2==0 || (seen ^ invert)==0 ){ + return 0; + } + continue; + } + } + c2 = sqlite3Utf8Read(&zString); + if( c==c2 ) continue; + if( !noCase ) return 0; + GlobUpperToLower(c); + GlobUpperToLower(c2); + if( c!=c2 ) return 0; } return *zString==0; } From 9fdfdc893bb14459ed6cfc142ca7208ecdb3abc8 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 11:08:57 +0000 Subject: [PATCH 390/710] Still more performance enhancements to the LIKE and GLOB operators. FossilOrigin-Name: 6c8924cacc2b875270770fed2cc3b1884f57a655 --- manifest | 12 +++---- manifest.uuid | 2 +- src/func.c | 98 ++++++++++++++++++++++++++++++--------------------- 3 files changed, 65 insertions(+), 47 deletions(-) diff --git a/manifest b/manifest index 37342c612c..fb9c0ed010 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\sperformance\soptimization\sfor\sthe\sLIKE\sand\sGLOB\soperators. -D 2014-09-25T03:51:37.139 +C Still\smore\sperformance\senhancements\sto\sthe\sLIKE\sand\sGLOB\soperators. +D 2014-09-25T11:08:57.081 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -184,7 +184,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c 610b18afde750686785cdad9196b9fb1b03dc9b3 +F src/func.c 727a324e87a3392a47e44568b901d2fb96ba0ed4 F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef30e0352b3d4a29749cd0872c10e45a6649ec52 -R 8a27395e5bbddced71b289eca8fe4771 +P 5ab1023d6cfe31fa8a194804b8216058977ac973 +R 3c45ffccf6b8b9761c0cc8bd190b6e11 U drh -Z c372796d929c11ef7d059e5ca2e0eb4a +Z 88a762d5e9ae73a0ed674e2385a2b544 diff --git a/manifest.uuid b/manifest.uuid index 035ef4eeb2..9939e8087e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ab1023d6cfe31fa8a194804b8216058977ac973 \ No newline at end of file +6c8924cacc2b875270770fed2cc3b1884f57a655 \ No newline at end of file diff --git a/src/func.c b/src/func.c index a2a5c1858d..5b7056b401 100644 --- a/src/func.c +++ b/src/func.c @@ -585,7 +585,7 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; /* ** Compare two UTF-8 strings for equality where the first string can -** potentially be a "glob" expression. Return true (1) if they +** potentially be a "glob" or "like" expression. Return true (1) if they ** are the same and false (0) if they are different. ** ** Globbing rules: @@ -605,11 +605,18 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; ** "[a-z]" matches any single lower-case letter. To match a '-', make ** it the last character in the list. ** +** Like matching rules: +** +** '%' Matches any sequence of zero or more characters +** +*** '_' Matches any one character +** +** Ec Where E is the "esc" character and c is any other +** character, including '%', '_', and esc, match exactly c. +** +** The comments through this routine usually assume glob matching. +** ** This routine is usually quick, but can be N**2 in the worst case. -** -** Hints: to match '*' or '?', put them in "[]". Like this: -** -** abc[*]xyz Matches "abc*xyz" only */ static int patternCompare( const u8 *zPattern, /* The glob pattern */ @@ -617,13 +624,12 @@ static int patternCompare( const struct compareInfo *pInfo, /* Information about how to do the compare */ u32 esc /* The escape character */ ){ - u32 c, c2; - int invert; - int seen; - u32 matchOne = pInfo->matchOne; - u32 matchAll = pInfo->matchAll; - u32 matchOther; - u8 noCase = pInfo->noCase; + u32 c, c2; /* Next pattern and input string chars */ + u32 matchOne = pInfo->matchOne; /* "?" or "_" */ + u32 matchAll = pInfo->matchAll; /* "*" or "%" */ + u32 matchOther; /* "[" or the escape character */ + u8 noCase = pInfo->noCase; /* True if uppercase==lowercase */ + const u8 *zEscaped = 0; /* One past the last escaped input char */ /* The GLOB operator does not have an ESCAPE clause. And LIKE does not ** have the matchSet operator. So we either have to look for one or @@ -633,7 +639,10 @@ static int patternCompare( matchOther = esc ? esc : pInfo->matchSet; while( (c = sqlite3Utf8Read(&zPattern))!=0 ){ - if( c==matchAll ){ + if( c==matchAll ){ /* Match "*" */ + /* Skip over multiple "*" characters in the pattern. If there + ** are also "?" characters, skip those as well, but consume a + ** single character of the input string for each "?" skipped */ while( (c=sqlite3Utf8Read(&zPattern)) == matchAll || c == matchOne ){ if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ @@ -641,12 +650,14 @@ static int patternCompare( } } if( c==0 ){ - return 1; + return 1; /* "*" at the end of the pattern matches */ }else if( c==matchOther ){ if( esc ){ c = sqlite3Utf8Read(&zPattern); if( c==0 ) return 0; }else{ + /* "[...]" immediately follows the "*". We have to do a slow + ** recursive search in this case, but it is an unusual case. */ assert( matchOther<0x80 ); /* '[' is a single-byte character */ while( *zString && patternCompare(&zPattern[-1],zString,pInfo,esc)==0 ){ @@ -655,39 +666,45 @@ static int patternCompare( return *zString!=0; } } - while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ - if( noCase && c<0x80 ){ - GlobUpperToLower(c2); - GlobUpperToLowerAscii(c); - while( c2 != 0 && c2 != c ){ - do{ c2 = *(zString++); }while( c2>0x7f ); - GlobUpperToLowerAscii(c2); - } + + /* At this point variable c contains the first character of the + ** pattern string past the "*". Search in the input string for the + ** first matching character and recursively contine the match from + ** that point. + ** + ** For a case-insensitive search, set variable cx to be the same as + ** c but in the other case and search the input string for either + ** c or cx. + */ + if( c<=0x80 ){ + u32 cx; + if( noCase ){ + cx = sqlite3Toupper(c); + c = sqlite3Tolower(c); }else{ - while( c2 != 0 && c2 != c ){ - c2 = sqlite3Utf8Read(&zString); - } + cx = c; + } + while( (c2 = *(zString++))!=0 ){ + if( c2!=c && c2!=cx ) continue; + if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; + } + }else{ + while( (c2 = sqlite3Utf8Read(&zString))!=0 ){ + if( c2!=c ) continue; + if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } - if( c2==0 ) return 0; - if( patternCompare(zPattern,zString,pInfo,esc) ) return 1; } return 0; } - if( c==matchOne ){ - if( sqlite3Utf8Read(&zString)==0 ){ - return 0; - }else{ - continue; - } - } if( c==matchOther ){ if( esc ){ c = sqlite3Utf8Read(&zPattern); if( c==0 ) return 0; + zEscaped = zPattern; }else{ u32 prior_c = 0; - seen = 0; - invert = 0; + int seen = 0; + int invert = 0; c = sqlite3Utf8Read(&zString); if( c==0 ) return 0; c2 = sqlite3Utf8Read(&zPattern); @@ -720,10 +737,11 @@ static int patternCompare( } c2 = sqlite3Utf8Read(&zString); if( c==c2 ) continue; - if( !noCase ) return 0; - GlobUpperToLower(c); - GlobUpperToLower(c2); - if( c!=c2 ) return 0; + if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){ + continue; + } + if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue; + return 0; } return *zString==0; } From 2c4dc635a198787e14f2548ddd04d9fade2ab278 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 12:31:28 +0000 Subject: [PATCH 391/710] Simplification to the random rowid picking logic that begins running when the maximum possible rowid has already been used. FossilOrigin-Name: 1330c72e172324c68ab49e5bb2ceba985935ae01 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 22 ++++++---------------- test/rowid.test | 4 ++-- 4 files changed, 16 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index fb9c0ed010..658142eff5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Still\smore\sperformance\senhancements\sto\sthe\sLIKE\sand\sGLOB\soperators. -D 2014-09-25T11:08:57.081 +C Simplification\sto\sthe\srandom\srowid\spicking\slogic\sthat\sbegins\srunning\swhen\nthe\smaximum\spossible\srowid\shas\salready\sbeen\sused. +D 2014-09-25T12:31:28.476 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 23db9b11f0d0a022c42bf71c2b036d32c82a9abd +F src/vdbe.c 9fe630d05840aa151a5ba9039901478d9524120b F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -784,7 +784,7 @@ F test/releasetest.tcl a0df0dfc5e3ee83ade87b9cc96db41b52d590b9e F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 -F test/rowid.test b78b30afb9537a73788ca1233a23a32190a3bb1f +F test/rowid.test 9ffee168c4be901820bf5cf5fcbb2105117d0d45 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 F test/savepoint.test 51d3900dc071a7c2ad4248578a5925631b476313 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5ab1023d6cfe31fa8a194804b8216058977ac973 -R 3c45ffccf6b8b9761c0cc8bd190b6e11 +P 6c8924cacc2b875270770fed2cc3b1884f57a655 +R c9d002c28940b83ace850b74161b9f39 U drh -Z 88a762d5e9ae73a0ed674e2385a2b544 +Z a6a7b4b719a1f0df211ae0686c0976a9 diff --git a/manifest.uuid b/manifest.uuid index 9939e8087e..5e3b55899b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c8924cacc2b875270770fed2cc3b1884f57a655 \ No newline at end of file +1330c72e172324c68ab49e5bb2ceba985935ae01 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 4c2b75f641..27d7e74901 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4020,25 +4020,15 @@ case OP_NewRowid: { /* out2-prerelease */ ** it finds one that is not previously used. */ assert( pOp->p3==0 ); /* We cannot be in random rowid mode if this is ** an AUTOINCREMENT table. */ - /* on the first attempt, simply do one more than previous */ - v = lastRowid; - v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ - v++; /* ensure non-zero */ cnt = 0; - while( ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v, + do{ + sqlite3_randomness(sizeof(v), &v); + v &= (MAX_ROWID>>1); + v++; + }while( ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v, 0, &res))==SQLITE_OK) && (res==0) - && (++cnt<100)){ - /* collision - try another random rowid */ - sqlite3_randomness(sizeof(v), &v); - if( cnt<5 ){ - /* try "small" random rowids for the initial attempts */ - v &= 0xffffff; - }else{ - v &= (MAX_ROWID>>1); /* ensure doesn't go negative */ - } - v++; /* ensure non-zero */ - } + && (++cnt<100)); if( rc==SQLITE_OK && res==0 ){ rc = SQLITE_FULL; /* IMP: R-38219-53002 */ goto abort_due_to_error; diff --git a/test/rowid.test b/test/rowid.test index 6d068d79bb..d232139ea0 100644 --- a/test/rowid.test +++ b/test/rowid.test @@ -679,9 +679,9 @@ do_test rowid-12.2 { save_prng_state execsql { INSERT INTO t7 VALUES(NULL,'b'); - SELECT x, y FROM t7; + SELECT x, y FROM t7 ORDER BY x; } -} {1 b 9223372036854775807 a} +} {/\d+ b 9223372036854775807 a/} execsql {INSERT INTO t7 VALUES(2,'y');} for {set i 1} {$i<100} {incr i} { do_test rowid-12.3.$i { From a15cc47f81fc9fe637166e8ea475e8d81557e0c7 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 13:17:30 +0000 Subject: [PATCH 392/710] Simplifications to the SQL function and aggregate calling procedures. FossilOrigin-Name: 3467049a1705b49905ea88a5c6becb6fe318f2fa --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/func.c | 5 ++++- src/vdbe.c | 18 ++---------------- src/vdbeInt.h | 1 - 5 files changed, 15 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 658142eff5..0b87bd95d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplification\sto\sthe\srandom\srowid\spicking\slogic\sthat\sbegins\srunning\swhen\nthe\smaximum\spossible\srowid\shas\salready\sbeen\sused. -D 2014-09-25T12:31:28.476 +C Simplifications\sto\sthe\sSQL\sfunction\sand\saggregate\scalling\sprocedures. +D 2014-09-25T13:17:30.283 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -184,7 +184,7 @@ F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c 727a324e87a3392a47e44568b901d2fb96ba0ed4 +F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -289,9 +289,9 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 9fe630d05840aa151a5ba9039901478d9524120b +F src/vdbe.c 73eace757ead9fee63576e8c9f5337edb4c8c69d F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h f177bed1ec8d4eb5c7089f012aeb95f374745735 +F src/vdbeInt.h bb7f7ecfdead1a2ae0251b59f86f5724838d975c F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d F src/vdbeaux.c a05adc3c96abdaf3db14768ddd63132fc9678060 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6c8924cacc2b875270770fed2cc3b1884f57a655 -R c9d002c28940b83ace850b74161b9f39 +P 1330c72e172324c68ab49e5bb2ceba985935ae01 +R db6a54c99d97f1320af4a50fd756c282 U drh -Z a6a7b4b719a1f0df211ae0686c0976a9 +Z 1da03df9544819e54ad17b4099a36954 diff --git a/manifest.uuid b/manifest.uuid index 5e3b55899b..d95b1f3d63 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1330c72e172324c68ab49e5bb2ceba985935ae01 \ No newline at end of file +3467049a1705b49905ea88a5c6becb6fe318f2fa \ No newline at end of file diff --git a/src/func.c b/src/func.c index 5b7056b401..cf556e2439 100644 --- a/src/func.c +++ b/src/func.c @@ -22,7 +22,10 @@ ** Return the collating function associated with a function. */ static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ - return context->pColl; + VdbeOp *pOp = &context->pVdbe->aOp[context->iOp-1]; + assert( pOp->opcode==OP_CollSeq ); + assert( pOp->p4type==P4_COLLSEQ ); + return pOp->p4.pColl; } /* diff --git a/src/vdbe.c b/src/vdbe.c index 27d7e74901..8ae56416f2 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1557,17 +1557,8 @@ case OP_Function: { ctx.iOp = pc; ctx.pVdbe = p; MemSetTypeFlag(ctx.pOut, MEM_Null); - ctx.fErrorOrAux = 0; - if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - assert( pOp>aOp ); - assert( pOp[-1].p4type==P4_COLLSEQ ); - assert( pOp[-1].opcode==OP_CollSeq ); - ctx.pColl = pOp[-1].p4.pColl; - } - db->lastRowid = lastRowid; (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ - lastRowid = db->lastRowid; /* If the function returned an error, throw an exception */ if( ctx.fErrorOrAux ){ @@ -5624,14 +5615,9 @@ case OP_AggStep: { sqlite3VdbeMemInit(&t, db, MEM_Null); ctx.pOut = &t; ctx.isError = 0; - ctx.pColl = 0; + ctx.pVdbe = p; + ctx.iOp = pc; ctx.skipFlag = 0; - if( ctx.pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - assert( pOp>p->aOp ); - assert( pOp[-1].p4type==P4_COLLSEQ ); - assert( pOp[-1].opcode==OP_CollSeq ); - ctx.pColl = pOp[-1].p4.pColl; - } (ctx.pFunc->xStep)(&ctx, n, apVal); /* IMP: R-24505-23230 */ if( ctx.isError ){ sqlite3SetString(&p->zErrMsg, db, "%s", sqlite3_value_text(&t)); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 56b5db7d1a..f54c9c6d3b 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -272,7 +272,6 @@ struct sqlite3_context { Mem *pOut; /* The return value is stored here */ FuncDef *pFunc; /* Pointer to function information */ Mem *pMem; /* Memory cell used to store aggregate context */ - CollSeq *pColl; /* Collating sequence */ Vdbe *pVdbe; /* The VM that owns this context */ int iOp; /* Instruction number of OP_Function */ int isError; /* Error code returned by the function. */ From d86334619497f717dd81d70f04999a466f967646 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Sep 2014 17:42:41 +0000 Subject: [PATCH 393/710] Minor code reformatting and comment change, to improve clarity. No logic changes. FossilOrigin-Name: baeb72a356d73e6f624edacd2986ab766105e177 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 0b87bd95d0..513f9c3310 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplifications\sto\sthe\sSQL\sfunction\sand\saggregate\scalling\sprocedures. -D 2014-09-25T13:17:30.283 +C Minor\scode\sreformatting\sand\scomment\schange,\sto\simprove\sclarity.\nNo\slogic\schanges. +D 2014-09-25T17:42:41.230 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 73eace757ead9fee63576e8c9f5337edb4c8c69d +F src/vdbe.c e4df7191abdae132b89c57086d0dd1a0d6cb400b F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h bb7f7ecfdead1a2ae0251b59f86f5724838d975c F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1330c72e172324c68ab49e5bb2ceba985935ae01 -R db6a54c99d97f1320af4a50fd756c282 +P 3467049a1705b49905ea88a5c6becb6fe318f2fa +R ed3d6d35b7bb71031f12c63341a22b21 U drh -Z 1da03df9544819e54ad17b4099a36954 +Z c1b4b9aea98707eadcd6494d5019f9dc diff --git a/manifest.uuid b/manifest.uuid index d95b1f3d63..a290ea0464 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3467049a1705b49905ea88a5c6becb6fe318f2fa \ No newline at end of file +baeb72a356d73e6f624edacd2986ab766105e177 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 8ae56416f2..23d18b5031 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4014,8 +4014,7 @@ case OP_NewRowid: { /* out2-prerelease */ cnt = 0; do{ sqlite3_randomness(sizeof(v), &v); - v &= (MAX_ROWID>>1); - v++; + v &= (MAX_ROWID>>1); v++; /* Ensure that v is greater than zero */ }while( ((rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)v, 0, &res))==SQLITE_OK) && (res==0) From 3b130beb159f82cc1693fab0a1e924f5b7d3a586 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Sep 2014 01:10:02 +0000 Subject: [PATCH 394/710] If an SQL function makes a recursive call to do an INSERT into the same database, make sure that the last_insert_rowid() for that INSERT is recorded. FossilOrigin-Name: e93aecc090c2a1d3c231bb2bde044886eff0bdf7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 1 + test/rowid.test | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 513f9c3310..cb7c310836 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scode\sreformatting\sand\scomment\schange,\sto\simprove\sclarity.\nNo\slogic\schanges. -D 2014-09-25T17:42:41.230 +C If\san\sSQL\sfunction\smakes\sa\srecursive\scall\sto\sdo\san\sINSERT\sinto\sthe\ssame\ndatabase,\smake\ssure\sthat\sthe\slast_insert_rowid()\sfor\sthat\sINSERT\sis\srecorded. +D 2014-09-26T01:10:02.814 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c e4df7191abdae132b89c57086d0dd1a0d6cb400b +F src/vdbe.c 91b7e12bca7b6056574ce28935e3e3f4769ce3c4 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h bb7f7ecfdead1a2ae0251b59f86f5724838d975c F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -784,7 +784,7 @@ F test/releasetest.tcl a0df0dfc5e3ee83ade87b9cc96db41b52d590b9e F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 -F test/rowid.test 9ffee168c4be901820bf5cf5fcbb2105117d0d45 +F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 F test/savepoint.test 51d3900dc071a7c2ad4248578a5925631b476313 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3467049a1705b49905ea88a5c6becb6fe318f2fa -R ed3d6d35b7bb71031f12c63341a22b21 +P baeb72a356d73e6f624edacd2986ab766105e177 +R d5c71b678da0df654e28e5676923f229 U drh -Z c1b4b9aea98707eadcd6494d5019f9dc +Z 33ff9c5d364d8f96a21ad755540365a5 diff --git a/manifest.uuid b/manifest.uuid index a290ea0464..a544dd408b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -baeb72a356d73e6f624edacd2986ab766105e177 \ No newline at end of file +e93aecc090c2a1d3c231bb2bde044886eff0bdf7 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 23d18b5031..1a1e441cd4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1559,6 +1559,7 @@ case OP_Function: { MemSetTypeFlag(ctx.pOut, MEM_Null); ctx.fErrorOrAux = 0; (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ + lastRowid = db->lastRowid; /* Remember rowid changes made by xFunc */ /* If the function returned an error, throw an exception */ if( ctx.fErrorOrAux ){ diff --git a/test/rowid.test b/test/rowid.test index d232139ea0..b00b5287fd 100644 --- a/test/rowid.test +++ b/test/rowid.test @@ -701,5 +701,19 @@ do_test rowid-12.4 { } } {1 {database or disk is full}} +# INSERTs that happen inside of nested function calls are recorded +# by last_insert_rowid. +# +proc rowid_addrow_func {n} { + db eval {INSERT INTO t13(rowid,x) VALUES($n,$n*$n)} + return [db last_insert_rowid] +} +db function addrow rowid_addrow_func +do_execsql_test rowid-13.1 { + CREATE TABLE t13(x); + INSERT INTO t13(rowid,x) VALUES(1234,5); + SELECT rowid, x, addrow(rowid+1000), '|' FROM t13 LIMIT 3; + SELECT last_insert_rowid(); +} {1234 5 2234 | 2234 4990756 3234 | 3234 10458756 4234 | 4234} finish_test From 867db8315930541fa5f33da083066d56a6eb06b2 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 26 Sep 2014 02:41:05 +0000 Subject: [PATCH 395/710] Fix the "PRAGMA integrity_check" command so that it avoids formatting error message context messages until it actually needs to generate an error message. This avoids much formatting, and hence greatly improves the performance of "PRAGMA integrity_check" in the common case when there are no errors. It also makes the code a little smaller. FossilOrigin-Name: 83913515830aa850f9e38406f9422d7e88dcab66 --- manifest | 14 +++--- manifest.uuid | 2 +- src/btree.c | 129 ++++++++++++++++++++++++++++--------------------- src/btreeInt.h | 2 + 4 files changed, 84 insertions(+), 63 deletions(-) diff --git a/manifest b/manifest index cb7c310836..131d77b3a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\san\sSQL\sfunction\smakes\sa\srecursive\scall\sto\sdo\san\sINSERT\sinto\sthe\ssame\ndatabase,\smake\ssure\sthat\sthe\slast_insert_rowid()\sfor\sthat\sINSERT\sis\srecorded. -D 2014-09-26T01:10:02.814 +C Fix\sthe\s"PRAGMA\sintegrity_check"\scommand\sso\sthat\sit\savoids\sformatting\serror\nmessage\scontext\smessages\suntil\sit\sactually\sneeds\sto\sgenerate\san\serror\smessage.\nThis\savoids\smuch\sformatting,\sand\shence\sgreatly\simproves\sthe\sperformance\sof\n"PRAGMA\sintegrity_check"\sin\sthe\scommon\scase\swhen\sthere\sare\sno\serrors.\s\sIt\salso\nmakes\sthe\scode\sa\slittle\ssmaller. +D 2014-09-26T02:41:05.726 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,9 +172,9 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 4d5cdfeaea4a00f796c17af246dd5b48cd525d5e +F src/btree.c 59f03e421dad3cb6e27cc7d2393d3a7459be4b5e F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 -F src/btreeInt.h 9db0d303b203d18871dc9a1d78a3e1ae4d62c1ef +F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P baeb72a356d73e6f624edacd2986ab766105e177 -R d5c71b678da0df654e28e5676923f229 +P e93aecc090c2a1d3c231bb2bde044886eff0bdf7 +R 32be19747fff5e8f2465eed0f224b45d U drh -Z 33ff9c5d364d8f96a21ad755540365a5 +Z f56b9000203c19d0f3a8172e8374b279 diff --git a/manifest.uuid b/manifest.uuid index a544dd408b..b182c8c316 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e93aecc090c2a1d3c231bb2bde044886eff0bdf7 \ No newline at end of file +83913515830aa850f9e38406f9422d7e88dcab66 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 37aabbef77..135b40139f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7880,11 +7880,11 @@ Pager *sqlite3BtreePager(Btree *p){ */ static void checkAppendMsg( IntegrityCk *pCheck, - char *zMsg1, const char *zFormat, ... ){ va_list ap; + char zBuf[200]; if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; @@ -7892,8 +7892,9 @@ static void checkAppendMsg( if( pCheck->errMsg.nChar ){ sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); } - if( zMsg1 ){ - sqlite3StrAccumAppendAll(&pCheck->errMsg, zMsg1); + if( pCheck->zPfx ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, pCheck->zPfx, pCheck->v1, pCheck->v2); + sqlite3StrAccumAppendAll(&pCheck->errMsg, zBuf); } sqlite3VXPrintf(&pCheck->errMsg, 1, zFormat, ap); va_end(ap); @@ -7931,14 +7932,14 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ ** ** Also check that the page number is in bounds. */ -static int checkRef(IntegrityCk *pCheck, Pgno iPage, char *zContext){ +static int checkRef(IntegrityCk *pCheck, Pgno iPage){ if( iPage==0 ) return 1; if( iPage>pCheck->nPage ){ - checkAppendMsg(pCheck, zContext, "invalid page number %d", iPage); + checkAppendMsg(pCheck, "invalid page number %d", iPage); return 1; } if( getPageReferenced(pCheck, iPage) ){ - checkAppendMsg(pCheck, zContext, "2nd reference to page %d", iPage); + checkAppendMsg(pCheck, "2nd reference to page %d", iPage); return 1; } setPageReferenced(pCheck, iPage); @@ -7955,8 +7956,7 @@ static void checkPtrmap( IntegrityCk *pCheck, /* Integrity check context */ Pgno iChild, /* Child page number */ u8 eType, /* Expected pointer map type */ - Pgno iParent, /* Expected pointer map parent page number */ - char *zContext /* Context description (used for error msg) */ + Pgno iParent /* Expected pointer map parent page number */ ){ int rc; u8 ePtrmapType; @@ -7965,12 +7965,12 @@ static void checkPtrmap( rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->mallocFailed = 1; - checkAppendMsg(pCheck, zContext, "Failed to read ptrmap key=%d", iChild); + checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild); return; } if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", iChild, eType, iParent, ePtrmapType, iPtrmapParent); } @@ -7985,8 +7985,7 @@ static void checkList( IntegrityCk *pCheck, /* Integrity checking context */ int isFreeList, /* True for a freelist. False for overflow page list */ int iPage, /* Page number for first page in the list */ - int N, /* Expected number of pages in the list */ - char *zContext /* Context for error messages */ + int N /* Expected number of pages in the list */ ){ int i; int expected = N; @@ -7995,14 +7994,14 @@ static void checkList( DbPage *pOvflPage; unsigned char *pOvflData; if( iPage<1 ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "%d of %d pages missing from overflow list starting at %d", N+1, expected, iFirst); break; } - if( checkRef(pCheck, iPage, zContext) ) break; + if( checkRef(pCheck, iPage) ) break; if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage) ){ - checkAppendMsg(pCheck, zContext, "failed to get page %d", iPage); + checkAppendMsg(pCheck, "failed to get page %d", iPage); break; } pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); @@ -8010,11 +8009,11 @@ static void checkList( int n = get4byte(&pOvflData[4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){ - checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0, zContext); + checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0); } #endif if( n>(int)pCheck->pBt->usableSize/4-2 ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "freelist leaf count too big on page %d", iPage); N--; }else{ @@ -8022,10 +8021,10 @@ static void checkList( Pgno iFreePage = get4byte(&pOvflData[8+i*4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){ - checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0, zContext); + checkPtrmap(pCheck, iFreePage, PTRMAP_FREEPAGE, 0); } #endif - checkRef(pCheck, iFreePage, zContext); + checkRef(pCheck, iFreePage); } N -= n; } @@ -8038,7 +8037,7 @@ static void checkList( */ if( pCheck->pBt->autoVacuum && N>0 ){ i = get4byte(pOvflData); - checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage, zContext); + checkPtrmap(pCheck, i, PTRMAP_OVERFLOW2, iPage); } } #endif @@ -8070,7 +8069,6 @@ static void checkList( static int checkTreePage( IntegrityCk *pCheck, /* Context for the sanity check */ int iPage, /* Page number of the page to check */ - char *zParentContext, /* Parent context */ i64 *pnParentMinKey, i64 *pnParentMaxKey ){ @@ -8081,23 +8079,26 @@ static int checkTreePage( u8 *data; BtShared *pBt; int usableSize; - char zContext[100]; char *hit = 0; i64 nMinKey = 0; i64 nMaxKey = 0; - - sqlite3_snprintf(sizeof(zContext), zContext, "Page %d: ", iPage); + const char *saved_zPfx = pCheck->zPfx; + int saved_v1 = pCheck->v1; + int saved_v2 = pCheck->v2; /* Check that the page exists */ pBt = pCheck->pBt; usableSize = pBt->usableSize; if( iPage==0 ) return 0; - if( checkRef(pCheck, iPage, zParentContext) ) return 0; + if( checkRef(pCheck, iPage) ) return 0; + pCheck->zPfx = "Page %d: "; + pCheck->v1 = iPage; if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); - return 0; + depth = -1; + goto end_of_check; } /* Clear MemPage.isInit to make sure the corruption detection code in @@ -8105,10 +8106,11 @@ static int checkTreePage( pPage->isInit = 0; if( (rc = btreeInitPage(pPage))!=0 ){ assert( rc==SQLITE_CORRUPT ); /* The only possible error from InitPage */ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "btreeInitPage() returns error code %d", rc); releasePage(pPage); - return 0; + depth = -1; + goto end_of_check; } /* Check out all the cells. @@ -8121,8 +8123,9 @@ static int checkTreePage( /* Check payload overflow pages */ - sqlite3_snprintf(sizeof(zContext), zContext, - "On tree page %d cell %d: ", iPage, i); + pCheck->zPfx = "On tree page %d cell %d: "; + pCheck->v1 = iPage; + pCheck->v2 = i; pCell = findCell(pPage,i); btreeParseCellPtr(pPage, pCell, &info); sz = info.nPayload; @@ -8132,7 +8135,7 @@ static int checkTreePage( if( i==0 ){ nMinKey = nMaxKey = info.nKey; }else if( info.nKey <= nMaxKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (previous was %lld)", info.nKey, nMaxKey); } nMaxKey = info.nKey; @@ -8144,10 +8147,10 @@ static int checkTreePage( Pgno pgnoOvfl = get4byte(&pCell[info.iOverflow]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage, zContext); + checkPtrmap(pCheck, pgnoOvfl, PTRMAP_OVERFLOW1, iPage); } #endif - checkList(pCheck, 0, pgnoOvfl, nPage, zContext); + checkList(pCheck, 0, pgnoOvfl, nPage); } /* Check sanity of left child page. @@ -8156,12 +8159,12 @@ static int checkTreePage( pgno = get4byte(pCell); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif - d2 = checkTreePage(pCheck, pgno, zContext, &nMinKey, i==0?NULL:&nMaxKey); + d2 = checkTreePage(pCheck, pgno, &nMinKey, i==0?NULL:&nMaxKey); if( i>0 && d2!=depth ){ - checkAppendMsg(pCheck, zContext, "Child page depth differs"); + checkAppendMsg(pCheck, "Child page depth differs"); } depth = d2; } @@ -8169,37 +8172,39 @@ static int checkTreePage( if( !pPage->leaf ){ pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); - sqlite3_snprintf(sizeof(zContext), zContext, - "On page %d at right child: ", iPage); + pCheck->zPfx = "On page %d at right child: "; + pCheck->v1 = iPage; #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage, zContext); + checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif - checkTreePage(pCheck, pgno, zContext, NULL, !pPage->nCell?NULL:&nMaxKey); + checkTreePage(pCheck, pgno, NULL, !pPage->nCell?NULL:&nMaxKey); } /* For intKey leaf pages, check that the min/max keys are in order ** with any left/parent/right pages. */ + pCheck->zPfx = "Page %d: "; + pCheck->v1 = iPage; if( pPage->leaf && pPage->intKey ){ /* if we are a left child page */ if( pnParentMinKey ){ /* if we are the left most child page */ if( !pnParentMaxKey ){ if( nMaxKey > *pnParentMinKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (max larger than parent min of %lld)", nMaxKey, *pnParentMinKey); } }else{ if( nMinKey <= *pnParentMinKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (min less than parent min of %lld)", nMinKey, *pnParentMinKey); } if( nMaxKey > *pnParentMaxKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (max larger than parent max of %lld)", nMaxKey, *pnParentMaxKey); } @@ -8208,7 +8213,7 @@ static int checkTreePage( /* else if we're a right child page */ } else if( pnParentMaxKey ){ if( nMinKey <= *pnParentMaxKey ){ - checkAppendMsg(pCheck, zContext, + checkAppendMsg(pCheck, "Rowid %lld out of order (min less than parent max of %lld)", nMinKey, *pnParentMaxKey); } @@ -8220,6 +8225,7 @@ static int checkTreePage( data = pPage->aData; hdr = pPage->hdrOffset; hit = sqlite3PageMalloc( pBt->pageSize ); + pCheck->zPfx = 0; if( hit==0 ){ pCheck->mallocFailed = 1; }else{ @@ -8237,7 +8243,8 @@ static int checkTreePage( size = cellSizePtr(pPage, &data[pc]); } if( (int)(pc+size-1)>=usableSize ){ - checkAppendMsg(pCheck, 0, + pCheck->zPfx = 0; + checkAppendMsg(pCheck, "Corruption detected in cell %d on page %d",i,iPage); }else{ for(j=pc+size-1; j>=pc; j--) hit[j]++; @@ -8259,19 +8266,24 @@ static int checkTreePage( if( hit[i]==0 ){ cnt++; }else if( hit[i]>1 ){ - checkAppendMsg(pCheck, 0, + checkAppendMsg(pCheck, "Multiple uses for byte %d of page %d", i, iPage); break; } } if( cnt!=data[hdr+7] ){ - checkAppendMsg(pCheck, 0, + checkAppendMsg(pCheck, "Fragmentation of %d bytes reported as %d on page %d", cnt, data[hdr+7], iPage); } } sqlite3PageFree(hit); releasePage(pPage); + +end_of_check: + pCheck->zPfx = saved_zPfx; + pCheck->v1 = saved_v1; + pCheck->v2 = saved_v2; return depth+1; } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -8312,6 +8324,9 @@ char *sqlite3BtreeIntegrityCheck( sCheck.mxErr = mxErr; sCheck.nErr = 0; sCheck.mallocFailed = 0; + sCheck.zPfx = 0; + sCheck.v1 = 0; + sCheck.v2 = 0; *pnErr = 0; if( sCheck.nPage==0 ){ sqlite3BtreeLeave(p); @@ -8331,8 +8346,10 @@ char *sqlite3BtreeIntegrityCheck( /* Check the integrity of the freelist */ + sCheck.zPfx = "Main freelist: "; checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), - get4byte(&pBt->pPage1->aData[36]), "Main freelist: "); + get4byte(&pBt->pPage1->aData[36])); + sCheck.zPfx = 0; /* Check all the tables. */ @@ -8340,10 +8357,12 @@ char *sqlite3BtreeIntegrityCheck( if( aRoot[i]==0 ) continue; #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum && aRoot[i]>1 ){ - checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0, 0); + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); } #endif - checkTreePage(&sCheck, aRoot[i], "List of tree roots: ", NULL, NULL); + sCheck.zPfx = "List of tree roots: "; + checkTreePage(&sCheck, aRoot[i], NULL, NULL); + sCheck.zPfx = 0; } /* Make sure every page in the file is referenced @@ -8351,7 +8370,7 @@ char *sqlite3BtreeIntegrityCheck( for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM if( getPageReferenced(&sCheck, i)==0 ){ - checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %d is never used", i); } #else /* If the database supports auto-vacuum, make sure no tables contain @@ -8359,11 +8378,11 @@ char *sqlite3BtreeIntegrityCheck( */ if( getPageReferenced(&sCheck, i)==0 && (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, 0, "Page %d is never used", i); + checkAppendMsg(&sCheck, "Page %d is never used", i); } if( getPageReferenced(&sCheck, i)!=0 && (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, 0, "Pointer map page %d is referenced", i); + checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i); } #endif } @@ -8373,7 +8392,7 @@ char *sqlite3BtreeIntegrityCheck( ** of the integrity check. */ if( NEVER(nRef != sqlite3PagerRefcount(pBt->pPager)) ){ - checkAppendMsg(&sCheck, 0, + checkAppendMsg(&sCheck, "Outstanding page count goes from %d to %d during this analysis", nRef, sqlite3PagerRefcount(pBt->pPager) ); diff --git a/src/btreeInt.h b/src/btreeInt.h index df9684e8e9..9f648fceb0 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -657,6 +657,8 @@ struct IntegrityCk { int mxErr; /* Stop accumulating errors when this reaches zero */ int nErr; /* Number of messages written to zErrMsg so far */ int mallocFailed; /* A memory allocation error has occurred */ + const char *zPfx; /* Error message prefix */ + int v1, v2; /* Values for up to two %d fields in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ }; From 843e4cdea9418748de18aa76879d69de0e861027 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Fri, 26 Sep 2014 18:30:11 +0000 Subject: [PATCH 396/710] Add an assert() to verify the last-row-id for the database just prior to calling a SQL function. FossilOrigin-Name: d026f0c944ce812732d3595eaa3c5d432a86c7dd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 131d77b3a9..63e97cbf9f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s"PRAGMA\sintegrity_check"\scommand\sso\sthat\sit\savoids\sformatting\serror\nmessage\scontext\smessages\suntil\sit\sactually\sneeds\sto\sgenerate\san\serror\smessage.\nThis\savoids\smuch\sformatting,\sand\shence\sgreatly\simproves\sthe\sperformance\sof\n"PRAGMA\sintegrity_check"\sin\sthe\scommon\scase\swhen\sthere\sare\sno\serrors.\s\sIt\salso\nmakes\sthe\scode\sa\slittle\ssmaller. -D 2014-09-26T02:41:05.726 +C Add\san\sassert()\sto\sverify\sthe\slast-row-id\sfor\sthe\sdatabase\sjust\sprior\sto\scalling\sa\sSQL\sfunction. +D 2014-09-26T18:30:11.093 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 91b7e12bca7b6056574ce28935e3e3f4769ce3c4 +F src/vdbe.c 93eeb6f9c3a3084133225a196f220454d71cca10 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h bb7f7ecfdead1a2ae0251b59f86f5724838d975c F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e93aecc090c2a1d3c231bb2bde044886eff0bdf7 -R 32be19747fff5e8f2465eed0f224b45d -U drh -Z f56b9000203c19d0f3a8172e8374b279 +P 83913515830aa850f9e38406f9422d7e88dcab66 +R dc4da31df9102dea4f18af6519657b79 +U mistachkin +Z 6fd91dd1e10822b3b57f7222d98e1fa9 diff --git a/manifest.uuid b/manifest.uuid index b182c8c316..482b361956 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83913515830aa850f9e38406f9422d7e88dcab66 \ No newline at end of file +d026f0c944ce812732d3595eaa3c5d432a86c7dd \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 1a1e441cd4..34eb1d42c5 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1558,6 +1558,7 @@ case OP_Function: { ctx.pVdbe = p; MemSetTypeFlag(ctx.pOut, MEM_Null); ctx.fErrorOrAux = 0; + assert( db->lastRowid==lastRowid ); (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ lastRowid = db->lastRowid; /* Remember rowid changes made by xFunc */ From 588400b861b2fa82a591b78a85cf3f460435ba8a Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 27 Sep 2014 05:00:25 +0000 Subject: [PATCH 397/710] Reduce the amount of memcpy() required by defragmentPage(). FossilOrigin-Name: 3edab9957cc7bb90b52fd40b02613c2cb03fc166 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/btree.c | 25 ++++++++++++++++--------- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 131d77b3a9..574bfe2c8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s"PRAGMA\sintegrity_check"\scommand\sso\sthat\sit\savoids\sformatting\serror\nmessage\scontext\smessages\suntil\sit\sactually\sneeds\sto\sgenerate\san\serror\smessage.\nThis\savoids\smuch\sformatting,\sand\shence\sgreatly\simproves\sthe\sperformance\sof\n"PRAGMA\sintegrity_check"\sin\sthe\scommon\scase\swhen\sthere\sare\sno\serrors.\s\sIt\salso\nmakes\sthe\scode\sa\slittle\ssmaller. -D 2014-09-26T02:41:05.726 +C Reduce\sthe\samount\sof\smemcpy()\srequired\sby\sdefragmentPage(). +D 2014-09-27T05:00:25.096 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 59f03e421dad3cb6e27cc7d2393d3a7459be4b5e +F src/btree.c 95a942a6ebdb23eb2a5d925526d35169aa6742f6 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e93aecc090c2a1d3c231bb2bde044886eff0bdf7 -R 32be19747fff5e8f2465eed0f224b45d +P 83913515830aa850f9e38406f9422d7e88dcab66 +R 66a1e1f00a844450677737824735607d +T *branch * defrag-opt +T *sym-defrag-opt * +T -sym-trunk * U drh -Z f56b9000203c19d0f3a8172e8374b279 +Z f60a1f2e4650c574e91ad245c0c67dee diff --git a/manifest.uuid b/manifest.uuid index b182c8c316..bf8d17f152 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83913515830aa850f9e38406f9422d7e88dcab66 \ No newline at end of file +3edab9957cc7bb90b52fd40b02613c2cb03fc166 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 135b40139f..c4832b4ee9 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1151,6 +1151,7 @@ static int defragmentPage(MemPage *pPage){ int nCell; /* Number of cells on the page */ unsigned char *data; /* The page data */ unsigned char *temp; /* Temp area for cell content */ + unsigned char *src; /* Source of content */ int iCellFirst; /* First allowable cell index */ int iCellLast; /* Last possible cell index */ @@ -1160,15 +1161,13 @@ static int defragmentPage(MemPage *pPage){ assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - data = pPage->aData; + temp = 0; + src = data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; assert( nCell==get2byte(&data[hdr+3]) ); usableSize = pPage->pBt->usableSize; - cbrk = get2byte(&data[hdr+5]); - memcpy(&temp[cbrk], &data[cbrk], usableSize - cbrk); cbrk = usableSize; iCellFirst = cellOffset + 2*nCell; iCellLast = usableSize - 4; @@ -1187,7 +1186,7 @@ static int defragmentPage(MemPage *pPage){ } #endif assert( pc>=iCellFirst && pc<=iCellLast ); - size = cellSizePtr(pPage, &temp[pc]); + size = cellSizePtr(pPage, &src[pc]); cbrk -= size; #if defined(SQLITE_ENABLE_OVERSIZE_CELL_CHECK) if( cbrk=iCellFirst ); testcase( cbrk+size==usableSize ); testcase( pc+size==usableSize ); - memcpy(&data[cbrk], &temp[pc], size); put2byte(pAddr, cbrk); + if( temp==0 ){ + int x; + if( cbrk==pc ) continue; + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + x = get2byte(&data[hdr+5]); + memcpy(&temp[x], &data[x], (cbrk+size) - x); + src = temp; + } + memcpy(&data[cbrk], &src[pc], size); } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); @@ -5955,13 +5962,13 @@ static void assemblePage( assert( pPage->nCell==0 ); assert( get2byteNotZero(&data[hdr+5])==nUsable ); - pCellptr = &pPage->aCellIdx[nCell*2]; + pCellptr = pPage->aCellIdx; cellbody = nUsable; - for(i=nCell-1; i>=0; i--){ + for(i=0; i Date: Mon, 29 Sep 2014 15:00:28 +0000 Subject: [PATCH 398/710] Ensure that the OP_Prev opcode verifies that content has not been deleted out from under the cursor. Fix for ticket [209d31e3161b9e9ff]. FossilOrigin-Name: 414f0d6a647a4d040b5463c73c5e15e699d85b4c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 3 +-- test/eval.test | 12 ++++++++++++ 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 63e97cbf9f..68c67e07c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sassert()\sto\sverify\sthe\slast-row-id\sfor\sthe\sdatabase\sjust\sprior\sto\scalling\sa\sSQL\sfunction. -D 2014-09-26T18:30:11.093 +C Ensure\sthat\sthe\sOP_Prev\sopcode\sverifies\sthat\scontent\shas\snot\sbeen\sdeleted\nout\sfrom\sunder\sthe\scursor.\s\sFix\sfor\sticket\s[209d31e3161b9e9ff]. +D 2014-09-29T15:00:28.761 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 59f03e421dad3cb6e27cc7d2393d3a7459be4b5e +F src/btree.c ede8348a7d623257ee6c06ca4796ceaee13b8657 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -469,7 +469,7 @@ F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020 F test/eqp.test 85873fa5816c48915c82c4e74cb5c35a5b48160f F test/errmsg.test f31592a594b44ee121371d25ddd5d63497bb3401 -F test/eval.test bc269c365ba877554948441e91ad5373f9f91be3 +F test/eval.test a64c9105d6ff163df7cf09d6ac29cdad5922078c F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 83913515830aa850f9e38406f9422d7e88dcab66 -R dc4da31df9102dea4f18af6519657b79 -U mistachkin -Z 6fd91dd1e10822b3b57f7222d98e1fa9 +P d026f0c944ce812732d3595eaa3c5d432a86c7dd +R 4bbd3babbb639de348c82ac1e98901ef +U drh +Z e609ec38acb2ceba655fca4c2f22f4af diff --git a/manifest.uuid b/manifest.uuid index 482b361956..909bc1e4db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d026f0c944ce812732d3595eaa3c5d432a86c7dd \ No newline at end of file +414f0d6a647a4d040b5463c73c5e15e699d85b4c \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 135b40139f..503a2fb5d0 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4985,8 +4985,7 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur, int *pRes){ assert( (pCur->curFlags & (BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey))==0 ); assert( pCur->info.nSize==0 ); if( pCur->eState!=CURSOR_VALID ){ - assert( pCur->eState>=CURSOR_REQUIRESEEK ); - rc = btreeRestoreCursorPosition(pCur); + rc = restoreCursorPosition(pCur); if( rc!=SQLITE_OK ){ return rc; } diff --git a/test/eval.test b/test/eval.test index 912dc8215b..360d158f3c 100644 --- a/test/eval.test +++ b/test/eval.test @@ -58,6 +58,18 @@ do_test eval-2.2 { SELECT * FROM t2 } } {} +do_test eval-2.3 { + execsql { + INSERT INTO t2 SELECT x, x+1 FROM t1 WHERE x<5; + SELECT x, test_eval('DELETE FROM t2 WHERE x='||x), y FROM t2 + ORDER BY rowid DESC; + } +} {4 {} {} 3 {} {} 2 {} {} 1 {} {}} +do_test eval-2.4 { + execsql { + SELECT * FROM t2 + } +} {} # Modify a row while it is being read. # From 39c4b82b5a82977db329f688ebf4bf0a2d901ec1 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 29 Sep 2014 15:42:01 +0000 Subject: [PATCH 399/710] Fix the header comment in sqlite3VdbeDeletePriorOpcode(). No changes to code. FossilOrigin-Name: 7fb1626866c2f8dad84c7e6184824be3efd71ca2 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 68c67e07c1..c39ace7ac0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\sOP_Prev\sopcode\sverifies\sthat\scontent\shas\snot\sbeen\sdeleted\nout\sfrom\sunder\sthe\scursor.\s\sFix\sfor\sticket\s[209d31e3161b9e9ff]. -D 2014-09-29T15:00:28.761 +C Fix\sthe\sheader\scomment\sin\ssqlite3VdbeDeletePriorOpcode().\s\sNo\schanges\sto\ncode. +D 2014-09-29T15:42:01.115 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,7 +293,7 @@ F src/vdbe.c 93eeb6f9c3a3084133225a196f220454d71cca10 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h bb7f7ecfdead1a2ae0251b59f86f5724838d975c F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d -F src/vdbeaux.c a05adc3c96abdaf3db14768ddd63132fc9678060 +F src/vdbeaux.c 8e016c6051c013a394f8e8679be1ca60723707bd F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 1e105dacf5190fc85a8ec2107c0dcc1884e75099 F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d026f0c944ce812732d3595eaa3c5d432a86c7dd -R 4bbd3babbb639de348c82ac1e98901ef +P 414f0d6a647a4d040b5463c73c5e15e699d85b4c +R f6298f1a1c0cd9e9dc9c3dc07f24407e U drh -Z e609ec38acb2ceba655fca4c2f22f4af +Z 0f82639c06e932d5523a9215bf58cd9c diff --git a/manifest.uuid b/manifest.uuid index 909bc1e4db..b5ea57e647 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -414f0d6a647a4d040b5463c73c5e15e699d85b4c \ No newline at end of file +7fb1626866c2f8dad84c7e6184824be3efd71ca2 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 87b14a6d87..2562c63b62 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -752,7 +752,8 @@ void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ } /* -** Remove the last opcode inserted +** If the last opcode is "op" and it is not a jump destination, +** then remove it. Return true if and only if an opcode was removed. */ int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){ From 4fa4a54f7eb58c465186cac6fe9ab4f5cbddbc12 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Sep 2014 12:33:33 +0000 Subject: [PATCH 400/710] Remove the SQLITE_ENABLE_TREE_EXPLAIN compile-time option. Add alternative debugging display routines: sqlite3TreeViewExpr(), sqlite3TreeViewExprList(), and sqlite3TreeViewSelect(). FossilOrigin-Name: 4ff51325d6b41d0c59e303b573700ec80c51d216 --- manifest | 34 +++++----- manifest.uuid | 2 +- src/expr.c | 167 ++++++++++++++++++++++-------------------------- src/main.c | 16 ----- src/parse.y | 3 - src/printf.c | 64 +++++++++++++++++++ src/select.c | 124 +++++++++++++++++------------------ src/shell.c | 9 --- src/sqlite.h.in | 2 +- src/sqliteInt.h | 45 +++++++------ src/vdbeInt.h | 4 -- src/vdbeaux.c | 4 -- src/vdbetrace.c | 115 --------------------------------- src/where.c | 59 ----------------- 14 files changed, 245 insertions(+), 403 deletions(-) diff --git a/manifest b/manifest index c39ace7ac0..642f78d5ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sheader\scomment\sin\ssqlite3VdbeDeletePriorOpcode().\s\sNo\schanges\sto\ncode. -D 2014-09-29T15:42:01.115 +C Remove\sthe\sSQLITE_ENABLE_TREE_EXPLAIN\scompile-time\soption.\s\sAdd\salternative\ndebugging\sdisplay\sroutines:\ssqlite3TreeViewExpr(),\ssqlite3TreeViewExprList(),\nand\ssqlite3TreeViewSelect(). +D 2014-09-30T12:33:33.546 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f -F src/expr.c f32119248996680aa73c5c37bfdd42820804dc17 +F src/expr.c 46a8ca93361d09f2ec6d9b7d524751510569d737 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c d15621461fb0c52675eba2b650492ed1beef69ab +F src/main.c 4a507a467cc20979579e4320ca6466b8ed0be268 F src/malloc.c 5bb99ee1e08ad58e457063cf79ce521db0e24195 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -217,22 +217,22 @@ F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c caab007743821d96752597c9cfd7351654697b06 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y b98772da2bb5415970085b707203f92569400aa8 +F src/parse.y ce1494308578d2f10a68cd8debc9fc156dda1094 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 3a47f526b173813d9a7f4e7044007771ba68cde1 +F src/printf.c 0db94d24f97b4e562e9da9d2ce85e8a69531daf6 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c a83ed8bc2a31c131e3addb6f0488b68334085e7b -F src/shell.c dad23987c34faddb061a339da3e92e05ccc6935e -F src/sqlite.h.in 8b018219ce988913e5977d5de9ab4beb33be23b6 +F src/select.c 373da54c2bd7e38993bc926a284bc05a53b01b8b +F src/shell.c 38f627b0885191357f55902a3ac199de90d79715 +F src/sqlite.h.in 159f2cb9eef74b6c99aeeb4c071e7745835f04f6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 5e09fe04f999223680801ddf8fbae6b60751d613 +F src/sqliteInt.h 8242d04760ec2a943a152eba2a006ea121a3eafd F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -291,18 +291,18 @@ F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 93eeb6f9c3a3084133225a196f220454d71cca10 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h bb7f7ecfdead1a2ae0251b59f86f5724838d975c +F src/vdbeInt.h 0b97a3190f8fbf460655985a9183019f5a702754 F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d -F src/vdbeaux.c 8e016c6051c013a394f8e8679be1ca60723707bd +F src/vdbeaux.c 5b687d7b5beaaa5b97189edf25cf08c311834933 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 1e105dacf5190fc85a8ec2107c0dcc1884e75099 F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef -F src/vdbetrace.c 4f29b04edb0cec3d5fcd9b566d9f0e75c8984362 +F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c a14d3d8042adeb51f81731c1b47b3e481d1cc23a +F src/where.c 5a2c700f6f29da91ac633015d1378b91dcf237b6 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 414f0d6a647a4d040b5463c73c5e15e699d85b4c -R f6298f1a1c0cd9e9dc9c3dc07f24407e +P 7fb1626866c2f8dad84c7e6184824be3efd71ca2 +R f57511c6ad57aee6db1d51feabde515b U drh -Z 0f82639c06e932d5523a9215bf58cd9c +Z d2ed6dec33496c4977b9e22861714f01 diff --git a/manifest.uuid b/manifest.uuid index b5ea57e647..246f7552d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fb1626866c2f8dad84c7e6184824be3efd71ca2 \ No newline at end of file +4ff51325d6b41d0c59e303b573700ec80c51d216 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 57e462ea6e..31ff98ea4f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3236,90 +3236,86 @@ void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ exprToRegister(pExpr, iMem); } -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) +#ifdef SQLITE_DEBUG /* ** Generate a human-readable explanation of an expression tree. */ -void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ - int op; /* The opcode being coded */ +void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ const char *zBinOp = 0; /* Binary operator */ const char *zUniOp = 0; /* Unary operator */ + pView = sqlite3TreeViewPush(pView, moreToFollow); if( pExpr==0 ){ - op = TK_NULL; - }else{ - op = pExpr->op; + sqlite3TreeViewLine(pView, "nil"); + sqlite3TreeViewPop(pView); + return; } - switch( op ){ + switch( pExpr->op ){ case TK_AGG_COLUMN: { - sqlite3ExplainPrintf(pOut, "AGG{%d:%d}", + sqlite3TreeViewLine(pView, "AGG{%d:%d}", pExpr->iTable, pExpr->iColumn); break; } case TK_COLUMN: { if( pExpr->iTable<0 ){ /* This only happens when coding check constraints */ - sqlite3ExplainPrintf(pOut, "COLUMN(%d)", pExpr->iColumn); + sqlite3TreeViewLine(pView, "COLUMN(%d)", pExpr->iColumn); }else{ - sqlite3ExplainPrintf(pOut, "{%d:%d}", + sqlite3TreeViewLine(pView, "{%d:%d}", pExpr->iTable, pExpr->iColumn); } break; } case TK_INTEGER: { if( pExpr->flags & EP_IntValue ){ - sqlite3ExplainPrintf(pOut, "%d", pExpr->u.iValue); + sqlite3TreeViewLine(pView, "%d", pExpr->u.iValue); }else{ - sqlite3ExplainPrintf(pOut, "%s", pExpr->u.zToken); + sqlite3TreeViewLine(pView, "%s", pExpr->u.zToken); } break; } #ifndef SQLITE_OMIT_FLOATING_POINT case TK_FLOAT: { - sqlite3ExplainPrintf(pOut,"%s", pExpr->u.zToken); + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); break; } #endif case TK_STRING: { - sqlite3ExplainPrintf(pOut,"%Q", pExpr->u.zToken); + sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken); break; } case TK_NULL: { - sqlite3ExplainPrintf(pOut,"NULL"); + sqlite3TreeViewLine(pView,"NULL"); break; } #ifndef SQLITE_OMIT_BLOB_LITERAL case TK_BLOB: { - sqlite3ExplainPrintf(pOut,"%s", pExpr->u.zToken); + sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); break; } #endif case TK_VARIABLE: { - sqlite3ExplainPrintf(pOut,"VARIABLE(%s,%d)", - pExpr->u.zToken, pExpr->iColumn); + sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)", + pExpr->u.zToken, pExpr->iColumn); break; } case TK_REGISTER: { - sqlite3ExplainPrintf(pOut,"REGISTER(%d)", pExpr->iTable); + sqlite3TreeViewLine(pView,"REGISTER(%d)", pExpr->iTable); break; } case TK_AS: { - sqlite3ExplainExpr(pOut, pExpr->pLeft); + sqlite3TreeViewLine(pView,"AS %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + case TK_ID: { + sqlite3TreeViewLine(pView,"ID %Q", pExpr->u.zToken); break; } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ - const char *zAff = "unk"; - switch( sqlite3AffinityType(pExpr->u.zToken, 0) ){ - case SQLITE_AFF_TEXT: zAff = "TEXT"; break; - case SQLITE_AFF_NONE: zAff = "NONE"; break; - case SQLITE_AFF_NUMERIC: zAff = "NUMERIC"; break; - case SQLITE_AFF_INTEGER: zAff = "INTEGER"; break; - case SQLITE_AFF_REAL: zAff = "REAL"; break; - } - sqlite3ExplainPrintf(pOut, "CAST-%s(", zAff); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut, ")"); + sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); break; } #endif /* SQLITE_OMIT_CAST */ @@ -3352,8 +3348,8 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ case TK_NOTNULL: zUniOp = "NOTNULL"; break; case TK_COLLATE: { - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut,".COLLATE(%s)",pExpr->u.zToken); + sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); break; } @@ -3365,41 +3361,36 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ }else{ pFarg = pExpr->x.pList; } - if( op==TK_AGG_FUNCTION ){ - sqlite3ExplainPrintf(pOut, "AGG_FUNCTION%d:%s(", + if( pExpr->op==TK_AGG_FUNCTION ){ + sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", pExpr->op2, pExpr->u.zToken); }else{ - sqlite3ExplainPrintf(pOut, "FUNCTION:%s(", pExpr->u.zToken); + sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); } if( pFarg ){ - sqlite3ExplainExprList(pOut, pFarg); + sqlite3TreeViewExprList(pView, pFarg, 0, 0); } - sqlite3ExplainPrintf(pOut, ")"); break; } #ifndef SQLITE_OMIT_SUBQUERY case TK_EXISTS: { - sqlite3ExplainPrintf(pOut, "EXISTS("); - sqlite3ExplainSelect(pOut, pExpr->x.pSelect); - sqlite3ExplainPrintf(pOut,")"); + sqlite3TreeViewLine(pView, "EXISTS-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); break; } case TK_SELECT: { - sqlite3ExplainPrintf(pOut, "("); - sqlite3ExplainSelect(pOut, pExpr->x.pSelect); - sqlite3ExplainPrintf(pOut, ")"); + sqlite3TreeViewLine(pView, "SELECT-expr"); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); break; } case TK_IN: { - sqlite3ExplainPrintf(pOut, "IN("); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut, ","); + sqlite3TreeViewLine(pView, "IN"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - sqlite3ExplainSelect(pOut, pExpr->x.pSelect); + sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); }else{ - sqlite3ExplainExprList(pOut, pExpr->x.pList); + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); } - sqlite3ExplainPrintf(pOut, ")"); break; } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -3419,13 +3410,10 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ Expr *pX = pExpr->pLeft; Expr *pY = pExpr->x.pList->a[0].pExpr; Expr *pZ = pExpr->x.pList->a[1].pExpr; - sqlite3ExplainPrintf(pOut, "BETWEEN("); - sqlite3ExplainExpr(pOut, pX); - sqlite3ExplainPrintf(pOut, ","); - sqlite3ExplainExpr(pOut, pY); - sqlite3ExplainPrintf(pOut, ","); - sqlite3ExplainExpr(pOut, pZ); - sqlite3ExplainPrintf(pOut, ")"); + sqlite3TreeViewLine(pView, "BETWEEN"); + sqlite3TreeViewExpr(pView, pX, 1); + sqlite3TreeViewExpr(pView, pY, 1); + sqlite3TreeViewExpr(pView, pZ, 0); break; } case TK_TRIGGER: { @@ -3436,15 +3424,14 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ ** is set to the column of the pseudo-table to read, or to -1 to ** read the rowid field. */ - sqlite3ExplainPrintf(pOut, "%s(%d)", + sqlite3TreeViewLine(pView, "%s(%d)", pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); break; } case TK_CASE: { - sqlite3ExplainPrintf(pOut, "CASE("); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut, ","); - sqlite3ExplainExprList(pOut, pExpr->x.pList); + sqlite3TreeViewLine(pView, "CASE"); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); break; } #ifndef SQLITE_OMIT_TRIGGER @@ -3456,55 +3443,57 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){ case OE_Fail: zType = "fail"; break; case OE_Ignore: zType = "ignore"; break; } - sqlite3ExplainPrintf(pOut, "RAISE-%s(%s)", zType, pExpr->u.zToken); + sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); break; } #endif + default: { + sqlite3TreeViewLine(pView, "op=%d", pExpr->op); + break; + } } if( zBinOp ){ - sqlite3ExplainPrintf(pOut,"%s(", zBinOp); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut,","); - sqlite3ExplainExpr(pOut, pExpr->pRight); - sqlite3ExplainPrintf(pOut,")"); + sqlite3TreeViewLine(pView, "%s", zBinOp); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + sqlite3TreeViewExpr(pView, pExpr->pRight, 0); }else if( zUniOp ){ - sqlite3ExplainPrintf(pOut,"%s(", zUniOp); - sqlite3ExplainExpr(pOut, pExpr->pLeft); - sqlite3ExplainPrintf(pOut,")"); + sqlite3TreeViewLine(pView, "%s", zUniOp); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); } + sqlite3TreeViewPop(pView); } -#endif /* defined(SQLITE_ENABLE_TREE_EXPLAIN) */ +#endif /* SQLITE_DEBUG */ -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) +#ifdef SQLITE_DEBUG /* ** Generate a human-readable explanation of an expression list. */ -void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){ +void sqlite3TreeViewExprList( + TreeView *pView, + const ExprList *pList, + u8 moreToFollow, + const char *zLabel +){ int i; - if( pList==0 || pList->nExpr==0 ){ - sqlite3ExplainPrintf(pOut, "(empty-list)"); - return; - }else if( pList->nExpr==1 ){ - sqlite3ExplainExpr(pOut, pList->a[0].pExpr); + pView = sqlite3TreeViewPush(pView, moreToFollow); + if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; + if( pList==0 ){ + sqlite3TreeViewLine(pView, "%s (empty)", zLabel); }else{ - sqlite3ExplainPush(pOut); + sqlite3TreeViewLine(pView, "%s", zLabel); for(i=0; inExpr; i++){ - sqlite3ExplainPrintf(pOut, "item[%d] = ", i); - sqlite3ExplainPush(pOut); - sqlite3ExplainExpr(pOut, pList->a[i].pExpr); - sqlite3ExplainPop(pOut); - if( pList->a[i].zName ){ + sqlite3TreeViewExpr(pView, pList->a[i].pExpr, inExpr-1); +#if 0 + if( pList->a[i].zName ){ sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); } if( pList->a[i].bSpanIsTab ){ sqlite3ExplainPrintf(pOut, " (%s)", pList->a[i].zSpan); } - if( inExpr-1 ){ - sqlite3ExplainNL(pOut); - } +#endif } - sqlite3ExplainPop(pOut); } + sqlite3TreeViewPop(pView); } #endif /* SQLITE_DEBUG */ diff --git a/src/main.c b/src/main.c index 231890de4b..dc17917796 100644 --- a/src/main.c +++ b/src/main.c @@ -3327,22 +3327,6 @@ int sqlite3_test_control(int op, ...){ break; } -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) - /* sqlite3_test_control(SQLITE_TESTCTRL_EXPLAIN_STMT, - ** sqlite3_stmt*,const char**); - ** - ** If compiled with SQLITE_ENABLE_TREE_EXPLAIN, each sqlite3_stmt holds - ** a string that describes the optimized parse tree. This test-control - ** returns a pointer to that string. - */ - case SQLITE_TESTCTRL_EXPLAIN_STMT: { - sqlite3_stmt *pStmt = va_arg(ap, sqlite3_stmt*); - const char **pzRet = va_arg(ap, const char**); - *pzRet = sqlite3VdbeExplanation((Vdbe*)pStmt); - break; - } -#endif - /* sqlite3_test_control(SQLITE_TESTCTRL_NEVER_CORRUPT, int); ** ** Set or clear a flag that indicates that the database file is always well- diff --git a/src/parse.y b/src/parse.y index 30a6dc5ff4..b47f531ee3 100644 --- a/src/parse.y +++ b/src/parse.y @@ -399,9 +399,6 @@ cmd ::= DROP VIEW ifexists(E) fullname(X). { cmd ::= select(X). { SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; sqlite3Select(pParse, X, &dest); - sqlite3ExplainBegin(pParse->pVdbe); - sqlite3ExplainSelect(pParse->pVdbe, X); - sqlite3ExplainFinish(pParse->pVdbe); sqlite3SelectDelete(pParse->db, X); } diff --git a/src/printf.c b/src/printf.c index 6d4b1b4ac7..92e8e100de 100644 --- a/src/printf.c +++ b/src/printf.c @@ -1056,6 +1056,70 @@ void sqlite3DebugPrintf(const char *zFormat, ...){ } #endif +#ifdef SQLITE_DEBUG +/************************************************************************* +** Routines for implementing the "TreeView" display of hierarchical +** data structures for debugging. +** +** The main entry points (coded elsewhere) are: +** sqlite3TreeViewExpr(0, pExpr, 0); +** sqlite3TreeViewExprList(0, pList, 0, 0); +** sqlite3TreeViewSelect(0, pSelect, 0); +** Insert calls to those routines while debugging in order to display +** a diagram of Expr, ExprList, and Select objects. +** +*/ +/* Add a new subitem to the tree. The moreToFollow flag indicates that this +** is not the last item in the tree. */ +TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ + if( p==0 ){ + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ) return 0; + memset(p, 0, sizeof(*p)); + }else{ + p->iLevel++; + } + assert( moreToFollow==0 || moreToFollow==1 ); + p->mLine &= ~(1<iLevel); + p->mLine |= moreToFollow << p->iLevel; + return p; +} +/* Finished with one layer of the tree */ +void sqlite3TreeViewPop(TreeView *p){ + if( p==0 ) return; + p->iLevel--; + if( p->iLevel<0 ) sqlite3_free(p); +} +/* Generate a single line of output for the tree, with a prefix that contains +** all the appropriate tree lines */ +void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ + va_list ap; + int i; + StrAccum acc; + char zBuf[500]; + sqlite3StrAccumInit(&acc, zBuf, sizeof(zBuf), 0); + acc.useMalloc = 0; + if( p ){ + for(i=0; iiLevel; i++){ + sqlite3StrAccumAppend(&acc, (p->mLine & (1<mLine & (1<selFlags & (SF_Distinct|SF_Aggregate) ){ - if( p->selFlags & SF_Distinct ){ - sqlite3ExplainPrintf(pVdbe, "DISTINCT "); - } - if( p->selFlags & SF_Aggregate ){ - sqlite3ExplainPrintf(pVdbe, "agg_flag "); - } - sqlite3ExplainNL(pVdbe); - sqlite3ExplainPrintf(pVdbe, " "); - } - sqlite3ExplainExprList(pVdbe, p->pEList); - sqlite3ExplainNL(pVdbe); +void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ + pView = sqlite3TreeViewPush(pView, moreToFollow); + sqlite3TreeViewLine(pView, "SELECT%s%s", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : "") + ); + sqlite3TreeViewExprList(pView, p->pEList, 1, "result-set"); if( p->pSrc && p->pSrc->nSrc ){ int i; - sqlite3ExplainPrintf(pVdbe, "FROM "); - sqlite3ExplainPush(pVdbe); + pView = sqlite3TreeViewPush(pView, 1); + sqlite3TreeViewLine(pView, "FROM"); for(i=0; ipSrc->nSrc; i++){ struct SrcList_item *pItem = &p->pSrc->a[i]; - sqlite3ExplainPrintf(pVdbe, "{%d,*} = ", pItem->iCursor); - if( pItem->pSelect ){ - sqlite3ExplainSelect(pVdbe, pItem->pSelect); - if( pItem->pTab ){ - sqlite3ExplainPrintf(pVdbe, " (tabname=%s)", pItem->pTab->zName); - } + StrAccum x; + char zLine[100]; + sqlite3StrAccumInit(&x, zLine, sizeof(zLine), 0); + sqlite3XPrintf(&x, 0, "{%d,*}", pItem->iCursor); + if( pItem->zDatabase ){ + sqlite3XPrintf(&x, 0, " %s.%s", pItem->zDatabase, pItem->zName); }else if( pItem->zName ){ - sqlite3ExplainPrintf(pVdbe, "%s", pItem->zName); + sqlite3XPrintf(&x, 0, " %s", pItem->zName); + } + if( pItem->pTab ){ + sqlite3XPrintf(&x, 0, " tabname=%Q", pItem->pTab->zName); } if( pItem->zAlias ){ - sqlite3ExplainPrintf(pVdbe, " (AS %s)", pItem->zAlias); + sqlite3XPrintf(&x, 0, " (AS %s)", pItem->zAlias); } if( pItem->jointype & JT_LEFT ){ - sqlite3ExplainPrintf(pVdbe, " LEFT-JOIN"); + sqlite3XPrintf(&x, 0, " LEFT-JOIN"); } - sqlite3ExplainNL(pVdbe); + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); + if( pItem->pSelect ){ + sqlite3TreeViewSelect(pView, pItem->pSelect, 0); + } + sqlite3TreeViewPop(pView); } - sqlite3ExplainPop(pVdbe); + sqlite3TreeViewPop(pView); } if( p->pWhere ){ - sqlite3ExplainPrintf(pVdbe, "WHERE "); - sqlite3ExplainExpr(pVdbe, p->pWhere); - sqlite3ExplainNL(pVdbe); + sqlite3TreeViewItem(pView, "WHERE", 1); + sqlite3TreeViewExpr(pView, p->pWhere, 0); + sqlite3TreeViewPop(pView); } if( p->pGroupBy ){ - sqlite3ExplainPrintf(pVdbe, "GROUPBY "); - sqlite3ExplainExprList(pVdbe, p->pGroupBy); - sqlite3ExplainNL(pVdbe); + sqlite3TreeViewExprList(pView, p->pGroupBy, 1, "GROUPBY"); } if( p->pHaving ){ - sqlite3ExplainPrintf(pVdbe, "HAVING "); - sqlite3ExplainExpr(pVdbe, p->pHaving); - sqlite3ExplainNL(pVdbe); + sqlite3TreeViewItem(pView, "HAVING", 1); + sqlite3TreeViewExpr(pView, p->pHaving, 0); + sqlite3TreeViewPop(pView); } if( p->pOrderBy ){ - sqlite3ExplainPrintf(pVdbe, "ORDERBY "); - sqlite3ExplainExprList(pVdbe, p->pOrderBy); - sqlite3ExplainNL(pVdbe); + sqlite3TreeViewExprList(pView, p->pOrderBy, 1, "ORDERBY"); } if( p->pLimit ){ - sqlite3ExplainPrintf(pVdbe, "LIMIT "); - sqlite3ExplainExpr(pVdbe, p->pLimit); - sqlite3ExplainNL(pVdbe); + sqlite3TreeViewItem(pView, "LIMIT", 1); + sqlite3TreeViewExpr(pView, p->pLimit, 0); + sqlite3TreeViewPop(pView); } if( p->pOffset ){ - sqlite3ExplainPrintf(pVdbe, "OFFSET "); - sqlite3ExplainExpr(pVdbe, p->pOffset); - sqlite3ExplainNL(pVdbe); + sqlite3TreeViewItem(pView, "OFFSET", 1); + sqlite3TreeViewExpr(pView, p->pOffset, 0); + sqlite3TreeViewPop(pView); } + if( p->pPrior ){ + const char *zOp = "UNION"; + switch( p->op ){ + case TK_ALL: zOp = "UNION ALL"; break; + case TK_INTERSECT: zOp = "INTERSECT"; break; + case TK_EXCEPT: zOp = "EXCEPT"; break; + } + sqlite3TreeViewItem(pView, zOp, 1); + sqlite3TreeViewSelect(pView, p->pPrior, 1); + sqlite3TreeViewPop(pView); + } + sqlite3TreeViewItem(pView, "END-SELECT", 0); + sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(pView); } -void sqlite3ExplainSelect(Vdbe *pVdbe, Select *p){ - if( p==0 ){ - sqlite3ExplainPrintf(pVdbe, "(null-select)"); - return; - } - sqlite3ExplainPush(pVdbe); - while( p ){ - explainOneSelect(pVdbe, p); - p = p->pNext; - if( p==0 ) break; - sqlite3ExplainNL(pVdbe); - sqlite3ExplainPrintf(pVdbe, "%s\n", selectOpName(p->op)); - } - sqlite3ExplainPrintf(pVdbe, "END"); - sqlite3ExplainPop(pVdbe); -} - -/* End of the structure debug printing code -*****************************************************************************/ -#endif /* defined(SQLITE_ENABLE_TREE_EXPLAIN) */ +#endif /* SQLITE_DEBUG */ diff --git a/src/shell.c b/src/shell.c index 9745e0ff25..cd4dadb08c 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1353,15 +1353,6 @@ static int shell_exec( sqlite3_free(zEQP); } - /* Output TESTCTRL_EXPLAIN text of requested */ - if( pArg && pArg->mode==MODE_Explain ){ - const char *zExplain = 0; - sqlite3_test_control(SQLITE_TESTCTRL_EXPLAIN_STMT, pStmt, &zExplain); - if( zExplain && zExplain[0] ){ - fprintf(pArg->out, "%s", zExplain); - } - } - /* If the shell is currently in ".explain" mode, gather the extra ** data required to add indents to the output.*/ if( pArg && pArg->mode==MODE_Explain ){ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 8d7ac79ee7..ffb020058f 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6205,7 +6205,7 @@ int sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_ISKEYWORD 16 #define SQLITE_TESTCTRL_SCRATCHMALLOC 17 #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 -#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 +#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 #define SQLITE_TESTCTRL_VDBE_COVERAGE 21 #define SQLITE_TESTCTRL_BYTEORDER 22 diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b7e4d071ac..d637df782a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -469,6 +469,11 @@ #define MIN(A,B) ((A)<(B)?(A):(B)) #define MAX(A,B) ((A)>(B)?(A):(B)) +/* +** Swap two objects of type TYPE. +*/ +#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} + /* ** Check to see if this machine uses EBCDIC. (Yes, believe it or ** not, there are still machines out there that use EBCDIC.) @@ -855,6 +860,7 @@ typedef struct StrAccum StrAccum; typedef struct Table Table; typedef struct TableLock TableLock; typedef struct Token Token; +typedef struct TreeView TreeView; typedef struct Trigger Trigger; typedef struct TriggerPrg TriggerPrg; typedef struct TriggerStep TriggerStep; @@ -2923,6 +2929,17 @@ struct With { } a[1]; }; +#ifdef SQLITE_DEBUG +/* +** An instance of the TreeView object is used for printing the content of +** data structures on sqlite3DebugPrintf() using a tree-like view. +*/ +struct TreeView { + int iLevel; /* Which level of the tree we are on */ + u64 mLine; /* Mask of continuation lines to be drawn */ +}; +#endif /* SQLITE_DEBUG */ + /* ** Assuming zIn points to the first byte of a UTF-8 character, ** advance zIn to point to the first byte of the next UTF-8 character. @@ -3087,25 +3104,14 @@ char *sqlite3MAppendf(sqlite3*,char*,const char*,...); void *sqlite3TestTextToPtr(const char*); #endif -/* Output formatting for SQLITE_TESTCTRL_EXPLAIN */ -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) - void sqlite3ExplainBegin(Vdbe*); - void sqlite3ExplainPrintf(Vdbe*, const char*, ...); - void sqlite3ExplainNL(Vdbe*); - void sqlite3ExplainPush(Vdbe*); - void sqlite3ExplainPop(Vdbe*); - void sqlite3ExplainFinish(Vdbe*); - void sqlite3ExplainSelect(Vdbe*, Select*); - void sqlite3ExplainExpr(Vdbe*, Expr*); - void sqlite3ExplainExprList(Vdbe*, ExprList*); - const char *sqlite3VdbeExplanation(Vdbe*); -#else -# define sqlite3ExplainBegin(X) -# define sqlite3ExplainSelect(A,B) -# define sqlite3ExplainExpr(A,B) -# define sqlite3ExplainExprList(A,B) -# define sqlite3ExplainFinish(X) -# define sqlite3VdbeExplanation(X) 0 +#if defined(SQLITE_DEBUG) + TreeView *sqlite3TreeViewPush(TreeView*,u8); + void sqlite3TreeViewPop(TreeView*); + void sqlite3TreeViewLine(TreeView*, const char*, ...); + void sqlite3TreeViewItem(TreeView*, const char*, u8); + void sqlite3TreeViewExpr(TreeView*, const Expr*, u8); + void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); + void sqlite3TreeViewSelect(TreeView*, const Select*, u8); #endif @@ -3128,6 +3134,7 @@ Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*); Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); void sqlite3ExprAssignVarNumber(Parse*, Expr*); void sqlite3ExprDelete(sqlite3*, Expr*); +void sqlite3ExprFactor(sqlite3*, Expr*, u8); ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index f54c9c6d3b..aa1132e31c 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -360,10 +360,6 @@ struct Vdbe { i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ char *zSql; /* Text of the SQL statement that generated this */ void *pFree; /* Free this when deleting the vdbe */ -#ifdef SQLITE_ENABLE_TREE_EXPLAIN - Explain *pExplain; /* The explainer */ - char *zExplain; /* Explanation of data structures */ -#endif VdbeFrame *pFrame; /* Parent frame */ VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */ int nFrame; /* Number of frames in pFrame list */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 2562c63b62..33fa055369 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2678,10 +2678,6 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); sqlite3DbFree(db, p->pFree); -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) - sqlite3DbFree(db, p->zExplain); - sqlite3DbFree(db, p->pExplain); -#endif } /* diff --git a/src/vdbetrace.c b/src/vdbetrace.c index d27693450e..507c2f12fc 100644 --- a/src/vdbetrace.c +++ b/src/vdbetrace.c @@ -183,118 +183,3 @@ char *sqlite3VdbeExpandSql( } #endif /* #ifndef SQLITE_OMIT_TRACE */ - -/***************************************************************************** -** The following code implements the data-structure explaining logic -** for the Vdbe. -*/ - -#if defined(SQLITE_ENABLE_TREE_EXPLAIN) - -/* -** Allocate a new Explain object -*/ -void sqlite3ExplainBegin(Vdbe *pVdbe){ - if( pVdbe ){ - Explain *p; - sqlite3BeginBenignMalloc(); - p = (Explain *)sqlite3MallocZero( sizeof(Explain) ); - if( p ){ - p->pVdbe = pVdbe; - sqlite3_free(pVdbe->pExplain); - pVdbe->pExplain = p; - sqlite3StrAccumInit(&p->str, p->zBase, sizeof(p->zBase), - SQLITE_MAX_LENGTH); - p->str.useMalloc = 2; - }else{ - sqlite3EndBenignMalloc(); - } - } -} - -/* -** Return true if the Explain ends with a new-line. -*/ -static int endsWithNL(Explain *p){ - return p && p->str.zText && p->str.nChar - && p->str.zText[p->str.nChar-1]=='\n'; -} - -/* -** Append text to the indentation -*/ -void sqlite3ExplainPrintf(Vdbe *pVdbe, const char *zFormat, ...){ - Explain *p; - if( pVdbe && (p = pVdbe->pExplain)!=0 ){ - va_list ap; - if( p->nIndent && endsWithNL(p) ){ - int n = p->nIndent; - if( n>ArraySize(p->aIndent) ) n = ArraySize(p->aIndent); - sqlite3AppendSpace(&p->str, p->aIndent[n-1]); - } - va_start(ap, zFormat); - sqlite3VXPrintf(&p->str, SQLITE_PRINTF_INTERNAL, zFormat, ap); - va_end(ap); - } -} - -/* -** Append a '\n' if there is not already one. -*/ -void sqlite3ExplainNL(Vdbe *pVdbe){ - Explain *p; - if( pVdbe && (p = pVdbe->pExplain)!=0 && !endsWithNL(p) ){ - sqlite3StrAccumAppend(&p->str, "\n", 1); - } -} - -/* -** Push a new indentation level. Subsequent lines will be indented -** so that they begin at the current cursor position. -*/ -void sqlite3ExplainPush(Vdbe *pVdbe){ - Explain *p; - if( pVdbe && (p = pVdbe->pExplain)!=0 ){ - if( p->str.zText && p->nIndentaIndent) ){ - const char *z = p->str.zText; - int i = p->str.nChar-1; - int x; - while( i>=0 && z[i]!='\n' ){ i--; } - x = (p->str.nChar - 1) - i; - if( p->nIndent && xaIndent[p->nIndent-1] ){ - x = p->aIndent[p->nIndent-1]; - } - p->aIndent[p->nIndent] = x; - } - p->nIndent++; - } -} - -/* -** Pop the indentation stack by one level. -*/ -void sqlite3ExplainPop(Vdbe *p){ - if( p && p->pExplain ) p->pExplain->nIndent--; -} - -/* -** Free the indentation structure -*/ -void sqlite3ExplainFinish(Vdbe *pVdbe){ - if( pVdbe && pVdbe->pExplain ){ - sqlite3_free(pVdbe->zExplain); - sqlite3ExplainNL(pVdbe); - pVdbe->zExplain = sqlite3StrAccumFinish(&pVdbe->pExplain->str); - sqlite3_free(pVdbe->pExplain); - pVdbe->pExplain = 0; - sqlite3EndBenignMalloc(); - } -} - -/* -** Return the explanation of a virtual machine. -*/ -const char *sqlite3VdbeExplanation(Vdbe *pVdbe){ - return (pVdbe && pVdbe->zExplain) ? pVdbe->zExplain : 0; -} -#endif /* defined(SQLITE_DEBUG) */ diff --git a/src/where.c b/src/where.c index 40f95acc49..9ece59f939 100644 --- a/src/where.c +++ b/src/where.c @@ -364,11 +364,6 @@ static int allowedOp(int op){ return op==TK_IN || (op>=TK_EQ && op<=TK_GE) || op==TK_ISNULL; } -/* -** Swap two objects of type TYPE. -*/ -#define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} - /* ** Commute a comparison operator. Expressions of the form "X op Y" ** are converted into "Y op X". @@ -3761,22 +3756,6 @@ static Bitmask codeOneLoopStart( return pLevel->notReady; } -#if defined(WHERETRACE_ENABLED) && defined(SQLITE_ENABLE_TREE_EXPLAIN) -/* -** Generate "Explanation" text for a WhereTerm. -*/ -static void whereExplainTerm(Vdbe *v, WhereTerm *pTerm){ - char zType[4]; - memcpy(zType, "...", 4); - if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; - if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; - if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; - sqlite3ExplainPrintf(v, "%s ", zType); - sqlite3ExplainExpr(v, pTerm->pExpr); -} -#endif /* WHERETRACE_ENABLED && SQLITE_ENABLE_TREE_EXPLAIN */ - - #ifdef WHERETRACE_ENABLED /* ** Print a WhereLoop object for debugging purposes @@ -3819,27 +3798,6 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); } sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); -#ifdef SQLITE_ENABLE_TREE_EXPLAIN - /* If the 0x100 bit of wheretracing is set, then show all of the constraint - ** expressions in the WhereLoop.aLTerm[] array. - */ - if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ /* WHERETRACE 0x100 */ - int i; - Vdbe *v = pWInfo->pParse->pVdbe; - sqlite3ExplainBegin(v); - for(i=0; inLTerm; i++){ - WhereTerm *pTerm = p->aLTerm[i]; - if( pTerm==0 ) continue; - sqlite3ExplainPrintf(v, " (%d) #%-2d ", i+1, (int)(pTerm-pWC->a)); - sqlite3ExplainPush(v); - whereExplainTerm(v, pTerm); - sqlite3ExplainPop(v); - sqlite3ExplainNL(v); - } - sqlite3ExplainFinish(v); - sqlite3DebugPrintf("%s", sqlite3VdbeExplanation(v)); - } -#endif } #endif @@ -6172,23 +6130,6 @@ WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ WHERETRACE(0xffff,("*** Optimizer Start ***\n")); - /* Display all terms of the WHERE clause */ -#if defined(WHERETRACE_ENABLED) && defined(SQLITE_ENABLE_TREE_EXPLAIN) - if( sqlite3WhereTrace & 0x100 ){ - int i; - Vdbe *v = pParse->pVdbe; - sqlite3ExplainBegin(v); - for(i=0; inTerm; i++){ - sqlite3ExplainPrintf(v, "#%-2d ", i); - sqlite3ExplainPush(v); - whereExplainTerm(v, &sWLB.pWC->a[i]); - sqlite3ExplainPop(v); - sqlite3ExplainNL(v); - } - sqlite3ExplainFinish(v); - sqlite3DebugPrintf("%s", sqlite3VdbeExplanation(v)); - } -#endif if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ rc = whereLoopAddAll(&sWLB); if( rc ) goto whereBeginError; From c90713d3d2feb19ca1d4f78e679a0dcc0b0e8948 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Sep 2014 13:46:49 +0000 Subject: [PATCH 401/710] Show tree diagrams of data structures in the debugging output when the 0x100 bit is set on sqlite3WhereTrace or sqlite3SelectTrace. FossilOrigin-Name: 92e0b4bd4d75e8b000586e51a07b3e181d9af20b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 12 +++++++++++- src/where.c | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 642f78d5ff..4ef89a6d0a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sSQLITE_ENABLE_TREE_EXPLAIN\scompile-time\soption.\s\sAdd\salternative\ndebugging\sdisplay\sroutines:\ssqlite3TreeViewExpr(),\ssqlite3TreeViewExprList(),\nand\ssqlite3TreeViewSelect(). -D 2014-09-30T12:33:33.546 +C Show\stree\sdiagrams\sof\sdata\sstructures\sin\sthe\sdebugging\soutput\swhen\sthe\s0x100\nbit\sis\sset\son\ssqlite3WhereTrace\sor\ssqlite3SelectTrace. +D 2014-09-30T13:46:49.195 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -227,7 +227,7 @@ F src/printf.c 0db94d24f97b4e562e9da9d2ce85e8a69531daf6 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 373da54c2bd7e38993bc926a284bc05a53b01b8b +F src/select.c b5304314d9456850e755a106d64b378d60c62644 F src/shell.c 38f627b0885191357f55902a3ac199de90d79715 F src/sqlite.h.in 159f2cb9eef74b6c99aeeb4c071e7745835f04f6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 5a2c700f6f29da91ac633015d1378b91dcf237b6 +F src/where.c 5924c54986ec694e0b9e90eca506a930cfc71f6f F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7fb1626866c2f8dad84c7e6184824be3efd71ca2 -R f57511c6ad57aee6db1d51feabde515b +P 4ff51325d6b41d0c59e303b573700ec80c51d216 +R 1c3af6544b324d9bf4b577de17e6b173 U drh -Z d2ed6dec33496c4977b9e22861714f01 +Z e2f1f26ff32680c40b90d9645bff7468 diff --git a/manifest.uuid b/manifest.uuid index 246f7552d4..1357a3ed6c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4ff51325d6b41d0c59e303b573700ec80c51d216 \ No newline at end of file +92e0b4bd4d75e8b000586e51a07b3e181d9af20b \ No newline at end of file diff --git a/src/select.c b/src/select.c index 595926fa1a..4fb3860fb3 100644 --- a/src/select.c +++ b/src/select.c @@ -3642,6 +3642,13 @@ static int flattenSubquery( */ sqlite3SelectDelete(db, pSub1); +#if SELECTTRACE_ENABLED + if( sqlite3SelectTrace & 0x100 ){ + sqlite3DebugPrintf("After flattening:\n"); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + return 1; } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ @@ -4649,7 +4656,10 @@ int sqlite3Select( memset(&sAggInfo, 0, sizeof(sAggInfo)); #if SELECTTRACE_ENABLED pParse->nSelectIndent++; - SELECTTRACE(1,pParse,p, ("begin processing\n")); + SELECTTRACE(1,pParse,p, ("begin processing:\n")); + if( sqlite3SelectTrace & 0x100 ){ + sqlite3TreeViewSelect(0, p, 0); + } #endif assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); diff --git a/src/where.c b/src/where.c index 9ece59f939..e2020ca06e 100644 --- a/src/where.c +++ b/src/where.c @@ -3756,6 +3756,23 @@ static Bitmask codeOneLoopStart( return pLevel->notReady; } +#ifdef WHERETRACE_ENABLED +/* +** Print the content of a WhereTerm object +*/ +static void whereTermPrint(WhereTerm *pTerm, int iTerm){ + char zType[4]; + memcpy(zType, "...", 4); + if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; + if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + sqlite3DebugPrintf("TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x\n", iTerm, + pTerm, zType, pTerm->leftCursor, pTerm->truthProb, + pTerm->eOperator); + sqlite3TreeViewExpr(0, pTerm->pExpr, 0); +} +#endif + #ifdef WHERETRACE_ENABLED /* ** Print a WhereLoop object for debugging purposes @@ -3798,6 +3815,14 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); } sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); + if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ + int i; + for(i=0; inLTerm; i++){ + WhereTerm *pTerm = p->aLTerm[i]; + if( pTerm==0 ) continue; + whereTermPrint(pTerm, i); + } + } } #endif @@ -6130,6 +6155,16 @@ WhereInfo *sqlite3WhereBegin( /* Construct the WhereLoop objects */ WHERETRACE(0xffff,("*** Optimizer Start ***\n")); +#if defined(WHERETRACE_ENABLED) + /* Display all terms of the WHERE clause */ + if( sqlite3WhereTrace & 0x100 ){ + int i; + for(i=0; inTerm; i++){ + whereTermPrint(&sWLB.pWC->a[i], i); + } + } +#endif + if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ rc = whereLoopAddAll(&sWLB); if( rc ) goto whereBeginError; From 5265149c0d5eed40f7a4858aeaf73776da69f78b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Sep 2014 14:14:19 +0000 Subject: [PATCH 402/710] Enhanced debug output for OR-logic in the query loop optimizer. FossilOrigin-Name: 2e375eae473e4a9f2e7870d59e22ba39051ecbce --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 ++++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 4ef89a6d0a..1c128cf92e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Show\stree\sdiagrams\sof\sdata\sstructures\sin\sthe\sdebugging\soutput\swhen\sthe\s0x100\nbit\sis\sset\son\ssqlite3WhereTrace\sor\ssqlite3SelectTrace. -D 2014-09-30T13:46:49.195 +C Enhanced\sdebug\soutput\sfor\sOR-logic\sin\sthe\squery\sloop\soptimizer. +D 2014-09-30T14:14:19.732 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 5924c54986ec694e0b9e90eca506a930cfc71f6f +F src/where.c 1d22623f4cc01ad34f4660daae77826550033c58 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4ff51325d6b41d0c59e303b573700ec80c51d216 -R 1c3af6544b324d9bf4b577de17e6b173 +P 92e0b4bd4d75e8b000586e51a07b3e181d9af20b +R ad11511f8b3c0903ec6511fb5cd0e86b U drh -Z e2f1f26ff32680c40b90d9645bff7468 +Z 55bafa322513796a5af056ecffd49293 diff --git a/manifest.uuid b/manifest.uuid index 1357a3ed6c..e80a6f2549 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -92e0b4bd4d75e8b000586e51a07b3e181d9af20b \ No newline at end of file +2e375eae473e4a9f2e7870d59e22ba39051ecbce \ No newline at end of file diff --git a/src/where.c b/src/where.c index e2020ca06e..3cde23be49 100644 --- a/src/where.c +++ b/src/where.c @@ -5037,6 +5037,12 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ continue; } sCur.n = 0; +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x200 ){ + sqlite3DebugPrintf("OR-term %d:\n",(int)(pOrTerm-pOrWC->a)); + sqlite3TreeViewExpr(0, pOrTerm->pExpr, 0); + } +#endif #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pItem->pTab) ){ rc = whereLoopAddVirtual(&sSubBuild, mExtra); From 0a99ba3bc86ceecde75b8b3b8ed8a689183d24b8 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Sep 2014 17:03:35 +0000 Subject: [PATCH 403/710] Further enhancements to the "wheretrace" debugging output. FossilOrigin-Name: 670993eb8113f386cb2cb8b1507917f6da3b4d98 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 38 +++++++++++++++++++++++--------------- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 1c128cf92e..2a2a4b68cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhanced\sdebug\soutput\sfor\sOR-logic\sin\sthe\squery\sloop\soptimizer. -D 2014-09-30T14:14:19.732 +C Further\senhancements\sto\sthe\s"wheretrace"\sdebugging\soutput. +D 2014-09-30T17:03:35.868 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 1d22623f4cc01ad34f4660daae77826550033c58 +F src/where.c a459332dc671138a6997904850ead36d83bfb8e0 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 92e0b4bd4d75e8b000586e51a07b3e181d9af20b -R ad11511f8b3c0903ec6511fb5cd0e86b +P 2e375eae473e4a9f2e7870d59e22ba39051ecbce +R c96a5d9652d91d70c0e54df21653ce3e U drh -Z 55bafa322513796a5af056ecffd49293 +Z 7116ce524f0c5c15e04b43bf011b61b4 diff --git a/manifest.uuid b/manifest.uuid index e80a6f2549..42170fe6be 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e375eae473e4a9f2e7870d59e22ba39051ecbce \ No newline at end of file +670993eb8113f386cb2cb8b1507917f6da3b4d98 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 3cde23be49..bc7e58a7fe 100644 --- a/src/where.c +++ b/src/where.c @@ -3537,6 +3537,7 @@ static Bitmask codeOneLoopStart( pOrExpr = pAndExpr; } /* Loop through table entries that match term pOrTerm. */ + WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, wctrlFlags, iCovCur); assert( pSubWInfo || pParse->nErr || db->mallocFailed ); @@ -3761,15 +3762,19 @@ static Bitmask codeOneLoopStart( ** Print the content of a WhereTerm object */ static void whereTermPrint(WhereTerm *pTerm, int iTerm){ - char zType[4]; - memcpy(zType, "...", 4); - if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; - if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; - if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; - sqlite3DebugPrintf("TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x\n", iTerm, - pTerm, zType, pTerm->leftCursor, pTerm->truthProb, - pTerm->eOperator); - sqlite3TreeViewExpr(0, pTerm->pExpr, 0); + if( pTerm==0 ){ + sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); + }else{ + char zType[4]; + memcpy(zType, "...", 4); + if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; + if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; + if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + sqlite3DebugPrintf("TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x\n", + iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb, + pTerm->eOperator); + sqlite3TreeViewExpr(0, pTerm->pExpr, 0); + } } #endif @@ -3818,9 +3823,7 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ int i; for(i=0; inLTerm; i++){ - WhereTerm *pTerm = p->aLTerm[i]; - if( pTerm==0 ) continue; - whereTermPrint(pTerm, i); + whereTermPrint(p->aLTerm[i], i); } } } @@ -5023,6 +5026,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ sSubBuild.pOrderBy = 0; sSubBuild.pOrSet = &sCur; + WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; @@ -5038,9 +5042,12 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ } sCur.n = 0; #ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace & 0x200 ){ - sqlite3DebugPrintf("OR-term %d:\n",(int)(pOrTerm-pOrWC->a)); - sqlite3TreeViewExpr(0, pOrTerm->pExpr, 0); + WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", + (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); + if( sqlite3WhereTrace & 0x400 ){ + for(i=0; inTerm; i++){ + whereTermPrint(&sSubBuild.pWC->a[i], i); + } } #endif #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -5095,6 +5102,7 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); } + WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm)); } } return rc; From 36be4c49e42951cc38894244dd8b591803c6bd3f Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Sep 2014 17:31:23 +0000 Subject: [PATCH 404/710] Enable the query planner to deal with WHERE clauses that have OR terms nested within AND terms that are nested within OR terms. Also remove an unused function declaration. FossilOrigin-Name: b6b289182f6590288ebc7b9efbcb29b6b4480538 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 3 +-- src/where.c | 9 ++++++--- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 2a2a4b68cc..84f200ceb8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\senhancements\sto\sthe\s"wheretrace"\sdebugging\soutput. -D 2014-09-30T17:03:35.868 +C Enable\sthe\squery\splanner\sto\sdeal\swith\sWHERE\sclauses\sthat\shave\sOR\sterms\nnested\swithin\sAND\sterms\sthat\sare\snested\swithin\sOR\sterms.\s\sAlso\sremove\san\nunused\sfunction\sdeclaration. +D 2014-09-30T17:31:23.408 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -232,7 +232,7 @@ F src/shell.c 38f627b0885191357f55902a3ac199de90d79715 F src/sqlite.h.in 159f2cb9eef74b6c99aeeb4c071e7745835f04f6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 8242d04760ec2a943a152eba2a006ea121a3eafd +F src/sqliteInt.h 254797e62264c53184172d98a491aa2b8cd4ad88 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c a459332dc671138a6997904850ead36d83bfb8e0 +F src/where.c 2f42fe0d19303e0f5ce29aff3afbd3e43cbd6efb F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2e375eae473e4a9f2e7870d59e22ba39051ecbce -R c96a5d9652d91d70c0e54df21653ce3e +P 670993eb8113f386cb2cb8b1507917f6da3b4d98 +R 1a06ccbab92129ac2bab7054741ffba8 U drh -Z 7116ce524f0c5c15e04b43bf011b61b4 +Z 36668dfa98188ac65878a88086fca9b9 diff --git a/manifest.uuid b/manifest.uuid index 42170fe6be..9e6ff3561c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -670993eb8113f386cb2cb8b1507917f6da3b4d98 \ No newline at end of file +b6b289182f6590288ebc7b9efbcb29b6b4480538 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d637df782a..35cadcdbc0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2231,7 +2231,7 @@ struct SrcList { #define WHERE_OMIT_OPEN_CLOSE 0x0010 /* Table cursors are already open */ #define WHERE_FORCE_TABLE 0x0020 /* Do not use an index-only search */ #define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ -#define WHERE_AND_ONLY 0x0080 /* Don't use indices for OR terms */ + /* 0x0080 // not currently used */ #define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ #define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ #define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ @@ -3134,7 +3134,6 @@ Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*); Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); void sqlite3ExprAssignVarNumber(Parse*, Expr*); void sqlite3ExprDelete(sqlite3*, Expr*); -void sqlite3ExprFactor(sqlite3*, Expr*, u8); ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*); diff --git a/src/where.c b/src/where.c index bc7e58a7fe..8974895c45 100644 --- a/src/where.c +++ b/src/where.c @@ -3524,8 +3524,9 @@ static Bitmask codeOneLoopStart( ** eliminating duplicates from other WHERE clauses, the action for each ** sub-WHERE clause is to to invoke the main loop body as a subroutine. */ - wctrlFlags = WHERE_OMIT_OPEN_CLOSE | WHERE_AND_ONLY | - WHERE_FORCE_TABLE | WHERE_ONETABLE_ONLY; + wctrlFlags = WHERE_OMIT_OPEN_CLOSE + | WHERE_FORCE_TABLE + | WHERE_ONETABLE_ONLY; for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ @@ -5005,7 +5006,6 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ struct SrcList_item *pItem; pWC = pBuilder->pWC; - if( pWInfo->wctrlFlags & WHERE_AND_ONLY ) return SQLITE_OK; pWCEnd = pWC->a + pWC->nTerm; pNew = pBuilder->pNew; memset(&sSum, 0, sizeof(sSum)); @@ -5058,6 +5058,9 @@ static int whereLoopAddOr(WhereLoopBuilder *pBuilder, Bitmask mExtra){ { rc = whereLoopAddBtree(&sSubBuild, mExtra); } + if( rc==SQLITE_OK ){ + rc = whereLoopAddOr(&sSubBuild, mExtra); + } assert( rc==SQLITE_OK || sCur.n==0 ); if( sCur.n==0 ){ sSum.n = 0; From b08cd3f34517bbb7a7cd8c745b2c51fe5bd09082 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Sep 2014 19:04:41 +0000 Subject: [PATCH 405/710] Improvements to the new syntax-tree output routines: Omit the "END SELECT" mark and instead terminate the graph at the last item. Increase the maximum tree depth to 100. FossilOrigin-Name: 5ce05757aac80b99c3b2141cd301809f8e28e661 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/printf.c | 9 ++++----- src/select.c | 31 +++++++++++++++++++------------ src/sqliteInt.h | 2 +- 5 files changed, 33 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 84f200ceb8..4f1b85c92d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sthe\squery\splanner\sto\sdeal\swith\sWHERE\sclauses\sthat\shave\sOR\sterms\nnested\swithin\sAND\sterms\sthat\sare\snested\swithin\sOR\sterms.\s\sAlso\sremove\san\nunused\sfunction\sdeclaration. -D 2014-09-30T17:31:23.408 +C Improvements\sto\sthe\snew\ssyntax-tree\soutput\sroutines:\s\sOmit\sthe\s"END\sSELECT"\nmark\sand\sinstead\sterminate\sthe\sgraph\sat\sthe\slast\sitem.\s\sIncrease\sthe\smaximum\ntree\sdepth\sto\s100. +D 2014-09-30T19:04:41.396 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,16 +223,16 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 0db94d24f97b4e562e9da9d2ce85e8a69531daf6 +F src/printf.c 6b79bbd063dcbadca4cf617a4cde255bcc13ea64 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c b5304314d9456850e755a106d64b378d60c62644 +F src/select.c f11533162b57ed5ed37f549add34cbcdf51f6712 F src/shell.c 38f627b0885191357f55902a3ac199de90d79715 F src/sqlite.h.in 159f2cb9eef74b6c99aeeb4c071e7745835f04f6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 254797e62264c53184172d98a491aa2b8cd4ad88 +F src/sqliteInt.h 5a430c5443717d7c5e2c224f9dcc2534348dc3f6 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 670993eb8113f386cb2cb8b1507917f6da3b4d98 -R 1a06ccbab92129ac2bab7054741ffba8 +P b6b289182f6590288ebc7b9efbcb29b6b4480538 +R cfd4c6e5c7836f29218c39baf2122e42 U drh -Z 36668dfa98188ac65878a88086fca9b9 +Z 3bfcd52f8fd5ecba827fd0c1ccf2615c diff --git a/manifest.uuid b/manifest.uuid index 9e6ff3561c..f78de65f58 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b6b289182f6590288ebc7b9efbcb29b6b4480538 \ No newline at end of file +5ce05757aac80b99c3b2141cd301809f8e28e661 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 92e8e100de..c0b3c70f6b 100644 --- a/src/printf.c +++ b/src/printf.c @@ -1080,8 +1080,7 @@ TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ p->iLevel++; } assert( moreToFollow==0 || moreToFollow==1 ); - p->mLine &= ~(1<iLevel); - p->mLine |= moreToFollow << p->iLevel; + if( p->iLevelbLine) ) p->bLine[p->iLevel] = moreToFollow; return p; } /* Finished with one layer of the tree */ @@ -1100,10 +1099,10 @@ void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ sqlite3StrAccumInit(&acc, zBuf, sizeof(zBuf), 0); acc.useMalloc = 0; if( p ){ - for(i=0; iiLevel; i++){ - sqlite3StrAccumAppend(&acc, (p->mLine & (1<iLevel && ibLine)-1; i++){ + sqlite3StrAccumAppend(&acc, p->bLine[i] ? "| " : " ", 4); } - sqlite3StrAccumAppend(&acc, (p->mLine & (1<bLine[i] ? "|-- " : "'-- ", 4); } va_start(ap, zFormat); sqlite3VXPrintf(&acc, 0, zFormat, ap); diff --git a/src/select.c b/src/select.c index 4fb3860fb3..411bca0df4 100644 --- a/src/select.c +++ b/src/select.c @@ -5432,15 +5432,24 @@ select_end: ** Generate a human-readable description of a the Select object. */ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ + int n = 0; pView = sqlite3TreeViewPush(pView, moreToFollow); sqlite3TreeViewLine(pView, "SELECT%s%s", ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), ((p->selFlags & SF_Aggregate) ? " agg_flag" : "") ); - sqlite3TreeViewExprList(pView, p->pEList, 1, "result-set"); + if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pWhere ) n++; + if( p->pGroupBy ) n++; + if( p->pHaving ) n++; + if( p->pOrderBy ) n++; + if( p->pLimit ) n++; + if( p->pOffset ) n++; + if( p->pPrior ) n++; + sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); if( p->pSrc && p->pSrc->nSrc ){ int i; - pView = sqlite3TreeViewPush(pView, 1); + pView = sqlite3TreeViewPush(pView, (n--)>0); sqlite3TreeViewLine(pView, "FROM"); for(i=0; ipSrc->nSrc; i++){ struct SrcList_item *pItem = &p->pSrc->a[i]; @@ -5472,28 +5481,28 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ sqlite3TreeViewPop(pView); } if( p->pWhere ){ - sqlite3TreeViewItem(pView, "WHERE", 1); + sqlite3TreeViewItem(pView, "WHERE", (n--)>0); sqlite3TreeViewExpr(pView, p->pWhere, 0); sqlite3TreeViewPop(pView); } if( p->pGroupBy ){ - sqlite3TreeViewExprList(pView, p->pGroupBy, 1, "GROUPBY"); + sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); } if( p->pHaving ){ - sqlite3TreeViewItem(pView, "HAVING", 1); + sqlite3TreeViewItem(pView, "HAVING", (n--)>0); sqlite3TreeViewExpr(pView, p->pHaving, 0); sqlite3TreeViewPop(pView); } if( p->pOrderBy ){ - sqlite3TreeViewExprList(pView, p->pOrderBy, 1, "ORDERBY"); + sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); } if( p->pLimit ){ - sqlite3TreeViewItem(pView, "LIMIT", 1); + sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); sqlite3TreeViewExpr(pView, p->pLimit, 0); sqlite3TreeViewPop(pView); } if( p->pOffset ){ - sqlite3TreeViewItem(pView, "OFFSET", 1); + sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); sqlite3TreeViewExpr(pView, p->pOffset, 0); sqlite3TreeViewPop(pView); } @@ -5504,12 +5513,10 @@ void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ case TK_INTERSECT: zOp = "INTERSECT"; break; case TK_EXCEPT: zOp = "EXCEPT"; break; } - sqlite3TreeViewItem(pView, zOp, 1); - sqlite3TreeViewSelect(pView, p->pPrior, 1); + sqlite3TreeViewItem(pView, zOp, (n--)>0); + sqlite3TreeViewSelect(pView, p->pPrior, 0); sqlite3TreeViewPop(pView); } - sqlite3TreeViewItem(pView, "END-SELECT", 0); - sqlite3TreeViewPop(pView); sqlite3TreeViewPop(pView); } #endif /* SQLITE_DEBUG */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 35cadcdbc0..695b63d753 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2936,7 +2936,7 @@ struct With { */ struct TreeView { int iLevel; /* Which level of the tree we are on */ - u64 mLine; /* Mask of continuation lines to be drawn */ + u8 bLine[100]; /* Draw vertical in column i if bLine[i] is true */ }; #endif /* SQLITE_DEBUG */ From 9501a6451650d7dd19b616ce71a4b1fec45fdf64 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 1 Oct 2014 12:01:10 +0000 Subject: [PATCH 406/710] Avoid ever writing before the start of an allocated buffer in the DIRECT_OVERFLOW_READ code. Fix for [e3a290961a6]. FossilOrigin-Name: c3c15d20c6913811956a5041c959a56ca4eeb5eb --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/btree.c | 4 ++++ test/ovfl.test | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 8 deletions(-) create mode 100644 test/ovfl.test diff --git a/manifest b/manifest index 4f1b85c92d..22b9dc1ea8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\snew\ssyntax-tree\soutput\sroutines:\s\sOmit\sthe\s"END\sSELECT"\nmark\sand\sinstead\sterminate\sthe\sgraph\sat\sthe\slast\sitem.\s\sIncrease\sthe\smaximum\ntree\sdepth\sto\s100. -D 2014-09-30T19:04:41.396 +C Avoid\sever\swriting\sbefore\sthe\sstart\sof\san\sallocated\sbuffer\sin\sthe\sDIRECT_OVERFLOW_READ\scode.\sFix\sfor\s[e3a290961a6]. +D 2014-10-01T12:01:10.959 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c ede8348a7d623257ee6c06ca4796ceaee13b8657 +F src/btree.c fa00618117fb6bb46c243452c56997c0d22d4fc9 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -749,6 +749,7 @@ F test/orderby5.test 8f08a54836d21fb7c70245360751aedd1c2286fb F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/orderby7.test 3d1383d52ade5b9eb3a173b3147fdd296f0202da F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 +F test/ovfl.test 4f7ca651cba5c059a12d8c67dddd49bec5747799 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f @@ -1200,7 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b6b289182f6590288ebc7b9efbcb29b6b4480538 -R cfd4c6e5c7836f29218c39baf2122e42 -U drh -Z 3bfcd52f8fd5ecba827fd0c1ccf2615c +P 5ce05757aac80b99c3b2141cd301809f8e28e661 +R 8b86b2d12e4b9100e4b861428290f6cc +U dan +Z 9b09f2a5bed05af5296fa69f0721cad2 diff --git a/manifest.uuid b/manifest.uuid index f78de65f58..8b1c98cc6b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5ce05757aac80b99c3b2141cd301809f8e28e661 \ No newline at end of file +c3c15d20c6913811956a5041c959a56ca4eeb5eb \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 503a2fb5d0..12dcb44cba 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4022,6 +4022,7 @@ static int accessPayload( MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ #ifdef SQLITE_DIRECT_OVERFLOW_READ + unsigned char * const pBufStart = pBuf; int bEnd; /* True if reading to end of data */ #endif @@ -4149,6 +4150,7 @@ static int accessPayload( ** 4) there is no open write-transaction, and ** 5) the database is not a WAL database, ** 6) all data from the page is being read. + ** 7) at least 4 bytes have already been read into the output buffer ** ** then data can be read directly from the database file into the ** output buffer, bypassing the page-cache altogether. This speeds @@ -4160,9 +4162,11 @@ static int accessPayload( && pBt->inTransaction==TRANS_READ /* (4) */ && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ && pBt->pPage1->aData[19]==0x01 /* (5) */ + && &pBuf[-4]>=pBufStart /* (7) */ ){ u8 aSave[4]; u8 *aWrite = &pBuf[-4]; + assert( aWrite>=pBufStart ); /* hence (7) */ memcpy(aSave, aWrite, 4); rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); nextPage = get4byte(aWrite); diff --git a/test/ovfl.test b/test/ovfl.test new file mode 100644 index 0000000000..075b1e43dd --- /dev/null +++ b/test/ovfl.test @@ -0,0 +1,49 @@ +# 2014 October 01 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the SQLITE_DIRECT_OVERFLOW_READ logic. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix ovfl + +# Populate table t2: +# +# CREATE TABLE t1(c1 TEXT, c2 TEXT); +# +# with 2000 rows. In each row, c2 spans multiple overflow pages. The text +# value of c1 ranges in size from 1 to 2000 bytes. The idea is to create +# at least one row where the first byte of c2 is also the first byte of +# an overflow page. This was at one point exposing an obscure bug in the +# SQLITE_DIRECT_OVERFLOW_READ logic. +# +do_test 1.1 { + set c2 [string repeat abcdefghij 200] + execsql { + PRAGMA cache_size = 10; + CREATE TABLE t1(c1 TEXT, c2 TEXT); + BEGIN; + } + for {set i 1} {$i <= 2000} {incr i} { + set c1 [string repeat . $i] + execsql { INSERT INTO t1 VALUES($c1, $c2) } + } + execsql COMMIT +} {} + +do_execsql_test 1.2 { + SELECT sum(length(c2)) FROM t1; +} [expr 2000 * 2000] + +finish_test + + From ccaba81e26a04e0b36c29cf6e9b3dcf44b50953e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Oct 2014 13:17:34 +0000 Subject: [PATCH 407/710] Show the TK_DOT operator in the TreeView debugging output. No changes to production code. FossilOrigin-Name: 07c89940c49a5dca3205a4b6fa8290f23bcb6e10 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 22b9dc1ea8..06559afc44 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sever\swriting\sbefore\sthe\sstart\sof\san\sallocated\sbuffer\sin\sthe\sDIRECT_OVERFLOW_READ\scode.\sFix\sfor\s[e3a290961a6]. -D 2014-10-01T12:01:10.959 +C Show\sthe\sTK_DOT\soperator\sin\sthe\sTreeView\sdebugging\soutput.\nNo\schanges\sto\sproduction\scode. +D 2014-10-01T13:17:34.666 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f -F src/expr.c 46a8ca93361d09f2ec6d9b7d524751510569d737 +F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee @@ -1201,7 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5ce05757aac80b99c3b2141cd301809f8e28e661 -R 8b86b2d12e4b9100e4b861428290f6cc -U dan -Z 9b09f2a5bed05af5296fa69f0721cad2 +P c3c15d20c6913811956a5041c959a56ca4eeb5eb +R edd47e8e1fee70147349c2e68dee2e83 +U drh +Z 9e63ea12305cf84dbf6e1aed1e91c8fa diff --git a/manifest.uuid b/manifest.uuid index 8b1c98cc6b..b9538ddd77 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3c15d20c6913811956a5041c959a56ca4eeb5eb \ No newline at end of file +07c89940c49a5dca3205a4b6fa8290f23bcb6e10 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 31ff98ea4f..1ad9a879a3 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3339,6 +3339,7 @@ void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ case TK_LSHIFT: zBinOp = "LSHIFT"; break; case TK_RSHIFT: zBinOp = "RSHIFT"; break; case TK_CONCAT: zBinOp = "CONCAT"; break; + case TK_DOT: zBinOp = "DOT"; break; case TK_UMINUS: zUniOp = "UMINUS"; break; case TK_UPLUS: zUniOp = "UPLUS"; break; From 2a3d1d17fd6007bb2ed86ed86fe38df4f9cb4804 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Oct 2014 21:52:35 +0000 Subject: [PATCH 408/710] Avoid a NULL pointer deference when processing the IS operator if the right-hand side is an illegal "#ID" style variable. Fix for ticket [8c32a33a53092c85a15b] FossilOrigin-Name: ffe7573636c8057614b02f0a85559e1857fd04e4 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/parse.y | 2 +- test/expr.test | 4 ++++ 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 06559afc44..fe8f3f84fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Show\sthe\sTK_DOT\soperator\sin\sthe\sTreeView\sdebugging\soutput.\nNo\schanges\sto\sproduction\scode. -D 2014-10-01T13:17:34.666 +C Avoid\sa\sNULL\spointer\sdeference\swhen\sprocessing\sthe\sIS\soperator\sif\sthe\nright-hand\sside\sis\san\sillegal\s"#ID"\sstyle\svariable.\nFix\sfor\sticket\s[8c32a33a53092c85a15b] +D 2014-10-02T21:52:35.759 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c caab007743821d96752597c9cfd7351654697b06 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y ce1494308578d2f10a68cd8debc9fc156dda1094 +F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa @@ -474,7 +474,7 @@ F test/exclusive.test c7ebbc756eacf544c108b15eed64d7d4e5f86b75 F test/exclusive2.test 32798111aae78a5deec980eee383213f189df308 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7 F test/exists.test 8f7b27b61c2fbe5822f0a1f899c715d14e416e30 -F test/expr.test 67c9fd6f8f829e239dc8b0f4a08a73c08b09196d +F test/expr.test c4b9bf0cc60b26862475e19999fbd2609ca8259c F test/extension01.test 00d13cec817f331a687a243e0e5a2d87b0e358c9 F test/fallocate.test 3e979af17dfa7e5e9dda5eba1a696c04fa9d47f7 F test/filectrl.test 14fa712e42c4cb791e09dfd58a6a03efb47ef13a @@ -1201,7 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c3c15d20c6913811956a5041c959a56ca4eeb5eb -R edd47e8e1fee70147349c2e68dee2e83 +P 07c89940c49a5dca3205a4b6fa8290f23bcb6e10 +R 3ff78086c12aaca80a7d944710785692 U drh -Z 9e63ea12305cf84dbf6e1aed1e91c8fa +Z 3bce8c89e00f85ac67bad3c4bc80b65e diff --git a/manifest.uuid b/manifest.uuid index b9538ddd77..839ebd889d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -07c89940c49a5dca3205a4b6fa8290f23bcb6e10 \ No newline at end of file +ffe7573636c8057614b02f0a85559e1857fd04e4 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index b47f531ee3..877827e68d 100644 --- a/src/parse.y +++ b/src/parse.y @@ -961,7 +961,7 @@ expr(A) ::= expr(X) NOT NULL(E). {spanUnaryPostfix(&A,pParse,TK_NOTNULL,&X,&E);} ** unary TK_ISNULL or TK_NOTNULL expression. */ static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ sqlite3 *db = pParse->db; - if( db->mallocFailed==0 && pY->op==TK_NULL ){ + if( pY && pA && pY->op==TK_NULL ){ pA->op = (u8)op; sqlite3ExprDelete(db, pA->pRight); pA->pRight = 0; diff --git a/test/expr.test b/test/expr.test index cc4c9c67f1..8d913d2a1a 100644 --- a/test/expr.test +++ b/test/expr.test @@ -205,6 +205,10 @@ test_expr expr-1.125 {i1=6, i2=NULL} \ test_expr expr-1.126 {i1=8, i2=8} \ {CASE WHEN i1 IS NOT i2 THEN 'yes' ELSE 'no' END} no +do_catchsql_test expr-1.127 { + SELECT 1 IS #1; +} {1 {near "#1": syntax error}} + ifcapable floatingpoint {if {[working_64bit_int]} { test_expr expr-1.200\ {i1=9223372036854775806, i2=1} {i1+i2} 9223372036854775807 From 8da47419ddf4b06ac31ad7e72e4c236d82a47765 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Oct 2014 14:54:47 +0000 Subject: [PATCH 409/710] Update to requirements marks related to changes in the memory allocation interface and enhancement of the documentation regarding DEFAULT clauses in CREATE TABLE. FossilOrigin-Name: 440705b98a3429b830ea85e71cc1e414bc6d8058 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/malloc.c | 10 +++++----- test/e_createtable.test | 15 ++++++++------- 4 files changed, 21 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index fe8f3f84fe..3abf1e8c7b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\sNULL\spointer\sdeference\swhen\sprocessing\sthe\sIS\soperator\sif\sthe\nright-hand\sside\sis\san\sillegal\s"#ID"\sstyle\svariable.\nFix\sfor\sticket\s[8c32a33a53092c85a15b] -D 2014-10-02T21:52:35.759 +C Update\sto\srequirements\smarks\srelated\sto\schanges\sin\sthe\smemory\sallocation\ninterface\sand\senhancement\sof\sthe\sdocumentation\sregarding\sDEFAULT\sclauses\nin\sCREATE\sTABLE. +D 2014-10-03T14:54:47.347 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 F src/main.c 4a507a467cc20979579e4320ca6466b8ed0be268 -F src/malloc.c 5bb99ee1e08ad58e457063cf79ce521db0e24195 +F src/malloc.c 7cf86b4f2310898675d8a962342e5650779dfb3f F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f @@ -448,7 +448,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 -F test/e_createtable.test 181653f6f45e3adde73f8686600ce5ad7515466b +F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 F test/e_dropview.test 0c9f7f60989164a70a67a9d9c26d1083bc808306 @@ -1201,7 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 07c89940c49a5dca3205a4b6fa8290f23bcb6e10 -R 3ff78086c12aaca80a7d944710785692 +P ffe7573636c8057614b02f0a85559e1857fd04e4 +R aa87a5e90d62a634fb6995b4f5b80a7f U drh -Z 3bce8c89e00f85ac67bad3c4bc80b65e +Z fba6f65828fcf9ec8fdba2a5c0c0fdb8 diff --git a/manifest.uuid b/manifest.uuid index 839ebd889d..4fde6cabd2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffe7573636c8057614b02f0a85559e1857fd04e4 \ No newline at end of file +440705b98a3429b830ea85e71cc1e414bc6d8058 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 8ba5fa0a84..7562ce2c41 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -310,7 +310,7 @@ void *sqlite3Malloc(u64 n){ }else{ p = sqlite3GlobalConfig.m.xMalloc((int)n); } - assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-04675-44850 */ + assert( EIGHT_BYTE_ALIGNMENT(p) ); /* IMP: R-11148-40995 */ return p; } @@ -533,10 +533,10 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){ int nOld, nNew, nDiff; void *pNew; if( pOld==0 ){ - return sqlite3Malloc(nBytes); /* IMP: R-28354-25769 */ + return sqlite3Malloc(nBytes); /* IMP: R-04300-56712 */ } if( nBytes==0 ){ - sqlite3_free(pOld); /* IMP: R-31593-10574 */ + sqlite3_free(pOld); /* IMP: R-26507-47431 */ return 0; } if( nBytes>=0x7fffff00 ){ @@ -573,7 +573,7 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){ }else{ pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } - assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-04675-44850 */ + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); /* IMP: R-11148-40995 */ return pNew; } @@ -585,7 +585,7 @@ void *sqlite3_realloc(void *pOld, int n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif - if( n<0 ) n = 0; + if( n<0 ) n = 0; /* IMP: R-26507-47431 */ return sqlite3Realloc(pOld, n); } void *sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ diff --git a/test/e_createtable.test b/test/e_createtable.test index 08f606f65b..2921d86c6f 100644 --- a/test/e_createtable.test +++ b/test/e_createtable.test @@ -862,11 +862,11 @@ do_createtable_tests 3.2.3 -query { 3 "INSERT INTO t1 DEFAULT VALUES" {NULL NULL NULL} } -# EVIDENCE-OF: R-62940-43005 An explicit DEFAULT clause may specify that +# EVIDENCE-OF: R-07343-35026 An explicit DEFAULT clause may specify that # the default value is NULL, a string constant, a blob constant, a -# signed-number, or any constant expression enclosed in parentheses. An -# explicit default value may also be one of the special case-independent -# keywords CURRENT_TIME, CURRENT_DATE or CURRENT_TIMESTAMP. +# signed-number, or any constant expression enclosed in parentheses. A +# default value may also be one of the special case-independent keywords +# CURRENT_TIME, CURRENT_DATE or CURRENT_TIMESTAMP. # do_execsql_test e_createtable-3.3.1 { CREATE TABLE t4( @@ -884,9 +884,9 @@ do_execsql_test e_createtable-3.3.1 { ); } {} -# EVIDENCE-OF: R-36381-62919 For the purposes of the DEFAULT clause, an -# expression is considered constant provided that it does not contain -# any sub-queries, column or table references, or string literals +# EVIDENCE-OF: R-18415-27776 For the purposes of the DEFAULT clause, an +# expression is considered constant if it does contains no sub-queries, +# column or table references, bound parameters, or string literals # enclosed in double-quotes instead of single-quotes. # do_createtable_tests 3.4.1 -error { @@ -896,6 +896,7 @@ do_createtable_tests 3.4.1 -error { 2 {CREATE TABLE t5(x DEFAULT ( "abc" ))} {} 3 {CREATE TABLE t5(x DEFAULT ( 1 IN (SELECT 1) ))} {} 4 {CREATE TABLE t5(x DEFAULT ( EXISTS (SELECT 1) ))} {} + 5 {CREATE TABLE t5(x DEFAULT ( x!=?1 ))} {} } do_createtable_tests 3.4.2 -repair { catchsql { DROP TABLE t5 } From 79f7af9a9e16c65c5ab3947bb0358b28a7240519 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Oct 2014 16:00:51 +0000 Subject: [PATCH 410/710] Add requirements marks on the sqlite3_db_status() interface implementation. Fix a typo in the documentation. Fix the new sqlite3_result_text64() routine so that it works correctly with an encoding parameter of SQLITE_UTF16. FossilOrigin-Name: d2fc322728331ae2d147c8496129df5e3c655eb5 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqlite.h.in | 2 +- src/status.c | 8 +++++--- src/vdbeapi.c | 1 + 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 3abf1e8c7b..1c0365b036 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sto\srequirements\smarks\srelated\sto\schanges\sin\sthe\smemory\sallocation\ninterface\sand\senhancement\sof\sthe\sdocumentation\sregarding\sDEFAULT\sclauses\nin\sCREATE\sTABLE. -D 2014-10-03T14:54:47.347 +C Add\srequirements\smarks\son\sthe\ssqlite3_db_status()\sinterface\simplementation.\nFix\sa\stypo\sin\sthe\sdocumentation.\s\sFix\sthe\snew\ssqlite3_result_text64()\sroutine\nso\sthat\sit\sworks\scorrectly\swith\san\sencoding\sparameter\sof\sSQLITE_UTF16. +D 2014-10-03T16:00:51.115 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,12 +229,12 @@ F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c f11533162b57ed5ed37f549add34cbcdf51f6712 F src/shell.c 38f627b0885191357f55902a3ac199de90d79715 -F src/sqlite.h.in 159f2cb9eef74b6c99aeeb4c071e7745835f04f6 +F src/sqlite.h.in a0b09ea5f73f3629c20b9788e0cde2a70f1703f5 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 5a430c5443717d7c5e2c224f9dcc2534348dc3f6 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d -F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 +F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 F src/test1.c 523cd70ded28db71af9a30ec184cbe0957de9575 @@ -292,7 +292,7 @@ F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 93eeb6f9c3a3084133225a196f220454d71cca10 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h 0b97a3190f8fbf460655985a9183019f5a702754 -F src/vdbeapi.c e9e33b59834e3edc8790209765e069874c269d9d +F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 F src/vdbeaux.c 5b687d7b5beaaa5b97189edf25cf08c311834933 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 1e105dacf5190fc85a8ec2107c0dcc1884e75099 @@ -1201,7 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ffe7573636c8057614b02f0a85559e1857fd04e4 -R aa87a5e90d62a634fb6995b4f5b80a7f +P 440705b98a3429b830ea85e71cc1e414bc6d8058 +R 8a35df4f3d17b4f437ae0dc791626bd3 U drh -Z fba6f65828fcf9ec8fdba2a5c0c0fdb8 +Z 3c9270c816943439d7ab9b669a742153 diff --git a/manifest.uuid b/manifest.uuid index 4fde6cabd2..4d17ba50dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -440705b98a3429b830ea85e71cc1e414bc6d8058 \ No newline at end of file +d2fc322728331ae2d147c8496129df5e3c655eb5 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ffb020058f..cc7bcd620e 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4445,7 +4445,7 @@ typedef void (*sqlite3_destructor_type)(void*); ** of the application-defined function to be NULL. ** ** ^The sqlite3_result_text(), sqlite3_result_text16(), -** sqlite3_result_text16le(), and sqlite3_result_text16be() +** sqlite3_result_text16le(), and sqlite3_result_text16be() interfaces ** set the return value of the application-defined function to be ** a text string which is represented as UTF-8, UTF-16 native byte order, ** UTF-16 little endian, or UTF-16 big endian, respectively. diff --git a/src/status.c b/src/status.c index 5fcb68ddc3..79a8001b8a 100644 --- a/src/status.c +++ b/src/status.c @@ -213,7 +213,7 @@ int sqlite3_db_status( } db->pnBytesFreed = 0; - *pHighwater = 0; + *pHighwater = 0; /* IMP: R-64479-57858 */ *pCurrent = nByte; break; @@ -238,7 +238,9 @@ int sqlite3_db_status( sqlite3PagerCacheStat(pPager, op, resetFlag, &nRet); } } - *pHighwater = 0; + *pHighwater = 0; /* IMP: R-42420-56072 */ + /* IMP: R-54100-20147 */ + /* IMP: R-29431-39229 */ *pCurrent = nRet; break; } @@ -248,7 +250,7 @@ int sqlite3_db_status( ** have been satisfied. The *pHighwater is always set to zero. */ case SQLITE_DBSTATUS_DEFERRED_FKS: { - *pHighwater = 0; + *pHighwater = 0; /* IMP: R-11967-56545 */ *pCurrent = db->nDeferredImmCons>0 || db->nDeferredCons>0; break; } diff --git a/src/vdbeapi.c b/src/vdbeapi.c index dc38132382..0ab76e0784 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -318,6 +318,7 @@ void sqlite3_result_text64( ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; if( n>0x7fffffff ){ (void)invokeValueDestructor(z, xDel, pCtx); }else{ From 43085d742548e19d3a61c26c547d4dc3cf42bf36 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 3 Oct 2014 19:16:53 +0000 Subject: [PATCH 411/710] Improve the accuracy of the estimates used when searching an index for values not present in any stat4 samples under some circumstances. FossilOrigin-Name: e6f7f97dbc677c9f01b23142928c3fa7307c2fba --- manifest | 19 +++++++++-------- manifest.uuid | 2 +- src/analyze.c | 54 ++++++++++++++++++++++++++++++++++++------------- src/sqliteInt.h | 1 + 4 files changed, 53 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 1c0365b036..f180b21817 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\srequirements\smarks\son\sthe\ssqlite3_db_status()\sinterface\simplementation.\nFix\sa\stypo\sin\sthe\sdocumentation.\s\sFix\sthe\snew\ssqlite3_result_text64()\sroutine\nso\sthat\sit\sworks\scorrectly\swith\san\sencoding\sparameter\sof\sSQLITE_UTF16. -D 2014-10-03T16:00:51.115 +C Improve\sthe\saccuracy\sof\sthe\sestimates\sused\swhen\ssearching\san\sindex\sfor\svalues\snot\spresent\sin\sany\sstat4\ssamples\sunder\ssome\scircumstances. +D 2014-10-03T19:16:53.018 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 6290a109be876daaa242cd7216f97240f5401776 +F src/analyze.c 418c2fc20cd36f1acc82456b5bd9baae77dbda78 F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -232,7 +232,7 @@ F src/shell.c 38f627b0885191357f55902a3ac199de90d79715 F src/sqlite.h.in a0b09ea5f73f3629c20b9788e0cde2a70f1703f5 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 5a430c5443717d7c5e2c224f9dcc2534348dc3f6 +F src/sqliteInt.h 3e4bd1b2288528b6a7f2d52709618572b422ab7e F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -1201,7 +1201,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 440705b98a3429b830ea85e71cc1e414bc6d8058 -R 8a35df4f3d17b4f437ae0dc791626bd3 -U drh -Z 3c9270c816943439d7ab9b669a742153 +P d2fc322728331ae2d147c8496129df5e3c655eb5 +R 57c99795ca217ceed92eda9a33e89730 +T *branch * stat4-avgeq +T *sym-stat4-avgeq * +T -sym-trunk * +U dan +Z 343ac493fd77cb5d6f7bcecf2a2a97e2 diff --git a/manifest.uuid b/manifest.uuid index 4d17ba50dd..ea230c321e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2fc322728331ae2d147c8496129df5e3c655eb5 \ No newline at end of file +e6f7f97dbc677c9f01b23142928c3fa7307c2fba \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index aec1f021ea..154033d067 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1448,12 +1448,12 @@ static void decodeIntArray( #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( aOut ){ aOut[i] = v; - }else + } #else assert( aOut==0 ); UNUSED_PARAMETER(aOut); #endif - { + if( aLog ){ aLog[i] = sqlite3LogEst(v); } if( *z==' ' ) z++; @@ -1516,8 +1516,16 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ z = argv[2]; if( pIndex ){ + int nCol = pIndex->nKeyCol+1; +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + tRowcnt * const aiRowEst = pIndex->aiRowEst = (tRowcnt*)sqlite3DbMallocZero( + pInfo->db, sizeof(tRowcnt) * nCol + ); +#else + tRowcnt * const aiRowEst = 0; +#endif pIndex->bUnordered = 0; - decodeIntArray((char*)z, pIndex->nKeyCol+1, 0, pIndex->aiRowLogEst, pIndex); + decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; }else{ Index fakeIdx; @@ -1576,25 +1584,38 @@ static void initAvgEq(Index *pIdx){ pIdx->aAvgEq[nCol] = 1; } for(iCol=0; iColnSample; int i; /* Used to iterate through samples */ tRowcnt sumEq = 0; /* Sum of the nEq values */ - tRowcnt nSum = 0; /* Number of terms contributing to sumEq */ tRowcnt avgEq = 0; - tRowcnt nDLt = pFinal->anDLt[iCol]; + tRowcnt nRow; /* Number of rows in index */ + i64 nSum100 = 0; /* Number of terms contributing to sumEq */ + i64 nDist100; /* Number of distinct values in index */ + + if( pIdx->aiRowEst==0 ){ + nRow = pFinal->anLt[iCol]; + nDist100 = (i64)100 * pFinal->anDLt[iCol]; + nSample--; + }else{ + nRow = pIdx->aiRowEst[0]; + nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1]; + } /* Set nSum to the number of distinct (iCol+1) field prefixes that - ** occur in the stat4 table for this index before pFinal. Set - ** sumEq to the sum of the nEq values for column iCol for the same - ** set (adding the value only once where there exist duplicate - ** prefixes). */ - for(i=0; i<(pIdx->nSample-1); i++){ - if( aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] ){ + ** occur in the stat4 table for this index. Set sumEq to the sum of + ** the nEq values for column iCol for the same set (adding the value + ** only once where there exist duplicate prefixes). */ + for(i=0; inSample-1) + || aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] + ){ sumEq += aSample[i].anEq[iCol]; - nSum++; + nSum100 += 100; } } - if( nDLt>nSum ){ - avgEq = (pFinal->anLt[iCol] - sumEq)/(nDLt - nSum); + + if( nDist100>nSum100 ){ + avgEq = ((i64)100 * (nRow - sumEq))/(nDist100 - nSum100); } if( avgEq==0 ) avgEq = 1; pIdx->aAvgEq[iCol] = avgEq; @@ -1846,6 +1867,11 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ rc = loadStat4(db, sInfo.zDatabase); db->lookaside.bEnabled = lookasideEnabled; } + for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + sqlite3DbFree(db, pIdx->aiRowEst); + pIdx->aiRowEst = 0; + } #endif if( rc==SQLITE_NOMEM ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 695b63d753..74c36c8db5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1801,6 +1801,7 @@ struct Index { int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ + tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this table */ #endif }; From 0c1a18b2944ef0a3dedfb192a90927dc69b32068 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 3 Oct 2014 19:29:39 +0000 Subject: [PATCH 412/710] Fix a division-by-zero error that might occur if the sqlite_stat1 table is corrupt. FossilOrigin-Name: f9c053b23ece877a7fdbe82204a10592f2d24a2d --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/analyze.c | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f180b21817..db1dca9628 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\saccuracy\sof\sthe\sestimates\sused\swhen\ssearching\san\sindex\sfor\svalues\snot\spresent\sin\sany\sstat4\ssamples\sunder\ssome\scircumstances. -D 2014-10-03T19:16:53.018 +C Fix\sa\sdivision-by-zero\serror\sthat\smight\soccur\sif\sthe\ssqlite_stat1\stable\sis\scorrupt. +D 2014-10-03T19:29:39.807 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 418c2fc20cd36f1acc82456b5bd9baae77dbda78 +F src/analyze.c 8d5a138936dab3436e67ca3a0f6466ad2f18d86b F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -1201,10 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d2fc322728331ae2d147c8496129df5e3c655eb5 -R 57c99795ca217ceed92eda9a33e89730 -T *branch * stat4-avgeq -T *sym-stat4-avgeq * -T -sym-trunk * +P e6f7f97dbc677c9f01b23142928c3fa7307c2fba +R d767aa2120b870307998a73a73f29d86 U dan -Z 343ac493fd77cb5d6f7bcecf2a2a97e2 +Z 9bc979da9e7cfcf210fc63943ac10f56 diff --git a/manifest.uuid b/manifest.uuid index ea230c321e..662bc404af 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6f7f97dbc677c9f01b23142928c3fa7307c2fba \ No newline at end of file +f9c053b23ece877a7fdbe82204a10592f2d24a2d \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 154033d067..2f65fe3d3e 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1592,7 +1592,7 @@ static void initAvgEq(Index *pIdx){ i64 nSum100 = 0; /* Number of terms contributing to sumEq */ i64 nDist100; /* Number of distinct values in index */ - if( pIdx->aiRowEst==0 ){ + if( pIdx->aiRowEst==0 || pIdx->aiRowEst[iCol+1]==0 ){ nRow = pFinal->anLt[iCol]; nDist100 = (i64)100 * pFinal->anDLt[iCol]; nSample--; From 75b170b16431b1b38aaf1bf64e29b8de5aec6325 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 4 Oct 2014 00:07:44 +0000 Subject: [PATCH 413/710] Avoid leaking Index.aiRowEst memory if an OOM causes a rollback which deletes the index before the aiRowEst deletion code in sqlite3AnalysisLoad() routine has a chance to run. Since the aiRowEst now might be deleted from freeIndex() which does not always have a db pointer, make sure the aiRowEst memory is not held in lookaside. FossilOrigin-Name: efd87ba142723ba131fcc985db6eb45c5a3c637b --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 7 ++++--- src/build.c | 3 +++ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index db1dca9628..2b01d7bbcc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sdivision-by-zero\serror\sthat\smight\soccur\sif\sthe\ssqlite_stat1\stable\sis\scorrupt. -D 2014-10-03T19:29:39.807 +C Avoid\sleaking\sIndex.aiRowEst\smemory\sif\san\sOOM\scauses\sa\srollback\swhich\sdeletes\nthe\sindex\sbefore\sthe\saiRowEst\sdeletion\scode\sin\ssqlite3AnalysisLoad()\sroutine\nhas\sa\schance\sto\srun.\s\sSince\sthe\saiRowEst\snow\smight\sbe\sdeleted\sfrom\sfreeIndex()\nwhich\sdoes\snot\salways\shave\sa\sdb\spointer,\smake\ssure\sthe\saiRowEst\smemory\sis\nnot\sheld\sin\slookaside. +D 2014-10-04T00:07:44.206 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 8d5a138936dab3436e67ca3a0f6466ad2f18d86b +F src/analyze.c ee85c504829aea05489ed0c67cbcd68d6a1ea7dd F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -175,7 +175,7 @@ F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c fa00618117fb6bb46c243452c56997c0d22d4fc9 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d -F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 +F src/build.c 9e5205db9a0c8a1a4ce7379d60a2a34cb0b7339c F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 @@ -1201,7 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e6f7f97dbc677c9f01b23142928c3fa7307c2fba -R d767aa2120b870307998a73a73f29d86 -U dan -Z 9bc979da9e7cfcf210fc63943ac10f56 +P f9c053b23ece877a7fdbe82204a10592f2d24a2d +R f53222c51c5cd542b15f6fc746109d5c +U drh +Z 12ba20164b8e53ca4d39b40be557570a diff --git a/manifest.uuid b/manifest.uuid index 662bc404af..e923963045 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f9c053b23ece877a7fdbe82204a10592f2d24a2d \ No newline at end of file +efd87ba142723ba131fcc985db6eb45c5a3c637b \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 2f65fe3d3e..6b244dd9a7 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1518,9 +1518,10 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pIndex ){ int nCol = pIndex->nKeyCol+1; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - tRowcnt * const aiRowEst = pIndex->aiRowEst = (tRowcnt*)sqlite3DbMallocZero( - pInfo->db, sizeof(tRowcnt) * nCol + tRowcnt * const aiRowEst = pIndex->aiRowEst = (tRowcnt*)sqlite3MallocZero( + sizeof(tRowcnt) * nCol ); + if( aiRowEst==0 ) pInfo->db->mallocFailed = 1; #else tRowcnt * const aiRowEst = 0; #endif @@ -1869,7 +1870,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ } for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - sqlite3DbFree(db, pIdx->aiRowEst); + sqlite3_free(pIdx->aiRowEst); pIdx->aiRowEst = 0; } #endif diff --git a/src/build.c b/src/build.c index 777831aab5..14d8aab587 100644 --- a/src/build.c +++ b/src/build.c @@ -435,6 +435,9 @@ static void freeIndex(sqlite3 *db, Index *p){ sqlite3ExprDelete(db, p->pPartIdxWhere); sqlite3DbFree(db, p->zColAff); if( p->isResized ) sqlite3DbFree(db, p->azColl); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + sqlite3_free(p->aiRowEst); +#endif sqlite3DbFree(db, p); } From 4ee3eb0ad482c160527260af9210f410581e6436 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Oct 2014 10:22:01 +0000 Subject: [PATCH 414/710] Add a test to show that the change on this branch is effective. FossilOrigin-Name: fc619be057975b8be6d0958024c5d436edbdf084 --- manifest | 13 ++--- manifest.uuid | 2 +- test/analyzeD.test | 117 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 test/analyzeD.test diff --git a/manifest b/manifest index 2b01d7bbcc..7ac24e1526 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sleaking\sIndex.aiRowEst\smemory\sif\san\sOOM\scauses\sa\srollback\swhich\sdeletes\nthe\sindex\sbefore\sthe\saiRowEst\sdeletion\scode\sin\ssqlite3AnalysisLoad()\sroutine\nhas\sa\schance\sto\srun.\s\sSince\sthe\saiRowEst\snow\smight\sbe\sdeleted\sfrom\sfreeIndex()\nwhich\sdoes\snot\salways\shave\sa\sdb\spointer,\smake\ssure\sthe\saiRowEst\smemory\sis\nnot\sheld\sin\slookaside. -D 2014-10-04T00:07:44.206 +C Add\sa\stest\sto\sshow\sthat\sthe\schange\son\sthis\sbranch\sis\seffective. +D 2014-10-04T10:22:01.856 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -326,6 +326,7 @@ F test/analyze9.test 72795c8113604b5dcd47a1498a61d6d7fb5d041a F test/analyzeA.test 3335697f6700c7052295cfd0067fc5b2aacddf9a F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/analyzeC.test 555a6cc388b9818b6eda6df816f01ce0a75d3a93 +F test/analyzeD.test 08f9d0bee4e118a66fff3a32d02dbe0ee0a2b594 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -1201,7 +1202,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f9c053b23ece877a7fdbe82204a10592f2d24a2d -R f53222c51c5cd542b15f6fc746109d5c -U drh -Z 12ba20164b8e53ca4d39b40be557570a +P efd87ba142723ba131fcc985db6eb45c5a3c637b +R 4172251d18757975548e0803ec5466b1 +U dan +Z d36e743f41577a93de7b943cc5387e20 diff --git a/manifest.uuid b/manifest.uuid index e923963045..cfe8277192 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -efd87ba142723ba131fcc985db6eb45c5a3c637b \ No newline at end of file +fc619be057975b8be6d0958024c5d436edbdf084 \ No newline at end of file diff --git a/test/analyzeD.test b/test/analyzeD.test new file mode 100644 index 0000000000..4d46be6c64 --- /dev/null +++ b/test/analyzeD.test @@ -0,0 +1,117 @@ +# 2005 July 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# This file implements tests for the ANALYZE command. +# +# $Id: analyze.test,v 1.9 2008/08/11 18:44:58 drh Exp $ + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix analyzeD + +ifcapable {!stat4} { + finish_test + return +} + + +# Set up a table with the following properties: +# +# * Contains 1000 rows. +# * Column a contains even integers between 0 and 18, inclusive (so that +# a=? for any such integer matches 100 rows). +# * Column b contains integers between 0 and 9, inclusive. +# * Column c contains integers between 0 and 199, inclusive (so that +# for any such integer, c=? matches 5 rows). +# * Then add 7 rows with a new value for "a" - 3001. The stat4 table will +# not contain any samples with a=3001. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c); +} +do_test 1.1 { + for {set i 1} {$i < 1000} {incr i} { + set c [expr $i % 200] + execsql { INSERT INTO t1(a, b, c) VALUES( 2*($i/100), $i%10, $c ) } + } + + execsql { + INSERT INTO t1 VALUES(3001, 3001, 3001); + INSERT INTO t1 VALUES(3001, 3001, 3002); + INSERT INTO t1 VALUES(3001, 3001, 3003); + INSERT INTO t1 VALUES(3001, 3001, 3004); + INSERT INTO t1 VALUES(3001, 3001, 3005); + INSERT INTO t1 VALUES(3001, 3001, 3006); + INSERT INTO t1 VALUES(3001, 3001, 3007); + + CREATE INDEX t1_ab ON t1(a, b); + CREATE INDEX t1_c ON t1(c); + + ANALYZE; + } +} {} + +# With full ANALYZE data, SQLite sees that c=150 (5 rows) is better than +# a=3001 (7 rows). +# +do_eqp_test 1.2 { + SELECT * FROM t1 WHERE a=3001 AND c=150; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)} +} + +do_test 1.3 { + execsql { DELETE FROM sqlite_stat1 } + db close + sqlite3 db test.db +} {} + +# Without stat1, because 3001 is larger than all samples in the stat4 +# table, SQLite things that a=3001 matches just 1 row. So it (incorrectly) +# chooses it over the c=150 index (5 rows). Even with stat1 data, things +# worked this way before commit [e6f7f97dbc]. +# +do_eqp_test 1.4 { + SELECT * FROM t1 WHERE a=3001 AND c=150; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1_ab (a=?)} +} + +do_test 1.5 { + execsql { + UPDATE t1 SET a=13 WHERE a = 3001; + ANALYZE; + } +} {} + +do_eqp_test 1.6 { + SELECT * FROM t1 WHERE a=13 AND c=150; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)} +} + +do_test 1.7 { + execsql { DELETE FROM sqlite_stat1 } + db close + sqlite3 db test.db +} {} + +# Same test as 1.4, except this time the 7 rows that match the a=? condition +# do not feature larger values than all rows in the stat4 table. So SQLite +# gets this right, even without stat1 data. +do_eqp_test 1.8 { + SELECT * FROM t1 WHERE a=13 AND c=150; +} { + 0 0 0 {SEARCH TABLE t1 USING INDEX t1_c (c=?)} +} + +finish_test + From 00729cba46c0b51d5001027f139b596ab81497da Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 4 Oct 2014 11:59:33 +0000 Subject: [PATCH 415/710] Updates to documentation and requirements marks. No code changes. FossilOrigin-Name: 0f8102d71a0ee828629f037775ad86fe2a544120 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/global.c | 7 +++++++ src/main.c | 11 +++++++++-- src/sqlite.h.in | 13 ++++++------- test/e_uri.test | 3 +++ 6 files changed, 35 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 1c0365b036..ec8a762863 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\srequirements\smarks\son\sthe\ssqlite3_db_status()\sinterface\simplementation.\nFix\sa\stypo\sin\sthe\sdocumentation.\s\sFix\sthe\snew\ssqlite3_result_text64()\sroutine\nso\sthat\sit\sworks\scorrectly\swith\san\sencoding\sparameter\sof\sSQLITE_UTF16. -D 2014-10-03T16:00:51.115 +C Updates\sto\sdocumentation\sand\srequirements\smarks.\s\sNo\scode\schanges. +D 2014-10-04T11:59:33.912 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -185,7 +185,7 @@ F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee -F src/global.c 5110fa12e09729b84eee0191c984ec4008e21937 +F src/global.c 01c1f36ecfcf10770db648422a8852c222308bb9 F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 4a507a467cc20979579e4320ca6466b8ed0be268 +F src/main.c bbe872b0ac0007bed0ebe1936fc493b039ad4f51 F src/malloc.c 7cf86b4f2310898675d8a962342e5650779dfb3f F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -229,7 +229,7 @@ F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c f11533162b57ed5ed37f549add34cbcdf51f6712 F src/shell.c 38f627b0885191357f55902a3ac199de90d79715 -F src/sqlite.h.in a0b09ea5f73f3629c20b9788e0cde2a70f1703f5 +F src/sqlite.h.in 7e5a6b5cce31f0b4844376566b9374fabc4985f9 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 5a430c5443717d7c5e2c224f9dcc2534348dc3f6 @@ -461,7 +461,7 @@ F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 F test/e_select.test 52692ff3849541e828ad4661fe3773a9b8711763 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 -F test/e_uri.test a2c92d80093a7efdcfbb11093651cbea87097b6b +F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 @@ -1201,7 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 440705b98a3429b830ea85e71cc1e414bc6d8058 -R 8a35df4f3d17b4f437ae0dc791626bd3 +P d2fc322728331ae2d147c8496129df5e3c655eb5 +R 48a7506351fd9c89c269926766fa27eb U drh -Z 3c9270c816943439d7ab9b669a742153 +Z 0614c4bed4c215b7b30b213ff48ceb06 diff --git a/manifest.uuid b/manifest.uuid index 4d17ba50dd..de0530ba8a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d2fc322728331ae2d147c8496129df5e3c655eb5 \ No newline at end of file +0f8102d71a0ee828629f037775ad86fe2a544120 \ No newline at end of file diff --git a/src/global.c b/src/global.c index 22b990699b..e769eb425f 100644 --- a/src/global.c +++ b/src/global.c @@ -129,6 +129,13 @@ const unsigned char sqlite3CtypeMap[256] = { }; #endif +/* EVIDENCE-OF: R-02982-34736 In order to maintain full backwards +** compatibility for legacy applications, the URI filename capability is +** disabled by default. +** +** EVIDENCE-OF: R-38799-08373 URI filenames can be enabled or disabled +** using the SQLITE_USE_URI=1 or SQLITE_USE_URI=0 compile-time options. +*/ #ifndef SQLITE_USE_URI # define SQLITE_USE_URI 0 #endif diff --git a/src/main.c b/src/main.c index dc17917796..ea03f2639f 100644 --- a/src/main.c +++ b/src/main.c @@ -476,6 +476,11 @@ int sqlite3_config(int op, ...){ break; } + /* EVIDENCE-OF: R-55548-33817 The compile-time setting for URI filenames + ** can be changed at start-time using the + ** sqlite3_config(SQLITE_CONFIG_URI,1) or + ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. + */ case SQLITE_CONFIG_URI: { sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); break; @@ -2213,7 +2218,7 @@ int sqlite3ParseUri( assert( *pzErrMsg==0 ); if( ((flags & SQLITE_OPEN_URI) || sqlite3GlobalConfig.bOpenUri) - && nUri>=5 && memcmp(zUri, "file:", 5)==0 + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ ){ char *zOpt; int eState; /* Parser state when parsing URI */ @@ -2443,7 +2448,9 @@ static int openDatabase( testcase( (1<<(flags&7))==0x02 ); /* READONLY */ testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ - if( ((1<<(flags&7)) & 0x46)==0 ) return SQLITE_MISUSE_BKPT; + if( ((1<<(flags&7)) & 0x46)==0 ){ + return SQLITE_MISUSE_BKPT; /* IMP: R-65497-44594 */ + } if( sqlite3GlobalConfig.bCoreMutex==0 ){ isThreadsafe = 0; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index cc7bcd620e..d8e99c13e1 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2754,13 +2754,14 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** then it is interpreted as an absolute path. ^If the path does not begin ** with a '/' (meaning that the authority section is omitted from the URI) ** then the path is interpreted as a relative path. -** ^On windows, the first component of an absolute path -** is a drive specification (e.g. "C:"). +** ^(On windows, the first component of an absolute path +** is a drive specification (e.g. "C:").)^ ** ** [[core URI query parameters]] ** The query component of a URI may contain parameters that are interpreted ** either by SQLite itself, or by a [VFS | custom VFS implementation]. -** SQLite interprets the following three query parameters: +** SQLite and its built-in [VFSes] interpret the +** following query parameters: ** **
      **
    • vfs: ^The "vfs" parameter may be used to specify the name of @@ -2795,11 +2796,9 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** a URI filename, its value overrides any behavior requested by setting ** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. ** -**
    • psow: ^The psow parameter may be "true" (or "on" or "yes" or -** "1") or "false" (or "off" or "no" or "0") to indicate that the +**
    • psow: ^The psow parameter indicates whether or not the ** [powersafe overwrite] property does or does not apply to the -** storage media on which the database file resides. ^The psow query -** parameter only works for the built-in unix and Windows VFSes. +** storage media on which the database file resides. ** **
    • nolock: ^The nolock parameter is a boolean query parameter ** which if set disables file locking in rollback journal modes. This diff --git a/test/e_uri.test b/test/e_uri.test index a8865cad28..d1590e4108 100644 --- a/test/e_uri.test +++ b/test/e_uri.test @@ -125,6 +125,9 @@ if {$tcl_platform(platform) == "unix"} { sqlite3_shutdown sqlite3_config_uri 1 +# EVIDENCE-OF: R-06842-00595 If the URI contains an authority, then it +# must be either an empty string or the string "localhost". +# # EVIDENCE-OF: R-17482-00398 If the authority is not an empty string or # "localhost", an error is returned to the caller. # From df868a4fbf0a0055ff3416c121c4c1ad5e675a15 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 4 Oct 2014 19:31:53 +0000 Subject: [PATCH 416/710] Tweaks to documentation on sqlite3_open() and sqlite3_bind(). No code changes. FossilOrigin-Name: b8f7f19dc06c59de2e194d83e6c052fb7d28c71d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 15 +++++++-------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index ec8a762863..5ed0c93fb2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sdocumentation\sand\srequirements\smarks.\s\sNo\scode\schanges. -D 2014-10-04T11:59:33.912 +C Tweaks\sto\sdocumentation\son\ssqlite3_open()\sand\ssqlite3_bind().\s\sNo\scode\schanges. +D 2014-10-04T19:31:53.403 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c f11533162b57ed5ed37f549add34cbcdf51f6712 F src/shell.c 38f627b0885191357f55902a3ac199de90d79715 -F src/sqlite.h.in 7e5a6b5cce31f0b4844376566b9374fabc4985f9 +F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 5a430c5443717d7c5e2c224f9dcc2534348dc3f6 @@ -1201,7 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d2fc322728331ae2d147c8496129df5e3c655eb5 -R 48a7506351fd9c89c269926766fa27eb +P 0f8102d71a0ee828629f037775ad86fe2a544120 +R 9fc77b39bf813ff4a1732fefd3521b90 U drh -Z 0614c4bed4c215b7b30b213ff48ceb06 +Z 7cb27b268889a7c66aa5ab31d2f83fa7 diff --git a/manifest.uuid b/manifest.uuid index de0530ba8a..4a5b1c0284 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f8102d71a0ee828629f037775ad86fe2a544120 \ No newline at end of file +b8f7f19dc06c59de2e194d83e6c052fb7d28c71d \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d8e99c13e1..f1d4e406e8 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2664,9 +2664,9 @@ void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); ** an English language description of the error following a failure of any ** of the sqlite3_open() routines. ** -** ^The default encoding for the database will be UTF-8 if -** sqlite3_open() or sqlite3_open_v2() is called and -** UTF-16 in the native byte order if sqlite3_open16() is used. +** ^The default encoding will be UTF-8 for databases created using +** sqlite3_open() or sqlite3_open_v2(). ^The default encoding for databases +** created using sqlite3_open16() will be UTF-16 in the native byte order. ** ** Whether or not an error occurs when it is opened, resources ** associated with the [database connection] handle should be released by @@ -3393,11 +3393,10 @@ typedef struct sqlite3_context sqlite3_context; ** contain embedded NULs. The result of expressions involving strings ** with embedded NULs is undefined. ** -** ^The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and -** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or +** ^The fifth argument to the BLOB and string binding interfaces +** is a destructor used to dispose of the BLOB or ** string after SQLite has finished with it. ^The destructor is called -** to dispose of the BLOB or string even if the call to sqlite3_bind_blob(), -** sqlite3_bind_text(), or sqlite3_bind_text16() fails. +** to dispose of the BLOB or string even if the call to bind API fails. ** ^If the fifth argument is ** the special value [SQLITE_STATIC], then SQLite assumes that the ** information is in static, unmanaged space and does not need to be freed. @@ -3408,7 +3407,7 @@ typedef struct sqlite3_context sqlite3_context; ** ^The sixth argument to sqlite3_bind_text64() must be one of ** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] ** to specify the encoding of the text in the third parameter. If -** the sixth argument to sqlite3_bind_text64() is not how of the +** the sixth argument to sqlite3_bind_text64() is not one of the ** allowed values shown above, or if the text encoding is different ** from the encoding specified by the sixth parameter, then the behavior ** is undefined. From b8e8d5055a19500a5e38e9b95fe967cb06605206 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 6 Oct 2014 12:41:57 +0000 Subject: [PATCH 417/710] Fix a harmless compiler warning inside an assert() in FTS4. FossilOrigin-Name: 418f3c9ad28672e5fe38d772d34e7cf8d26bc0e1 --- ext/fts3/fts3.c | 2 +- manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 4f4b667430..582b7e27a1 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -4426,7 +4426,7 @@ static int fts3EvalIncrPhraseNext( bMaxSet = 1; } } - assert( rc!=SQLITE_OK || a[p->nToken-1].bIgnore==0 ); + assert( rc!=SQLITE_OK || (p->nToken>=1 && a[p->nToken-1].bIgnore==0) ); assert( rc!=SQLITE_OK || bMaxSet ); /* Keep advancing iterators until they all point to the same document */ diff --git a/manifest b/manifest index 5ed0c93fb2..e86e7242aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tweaks\sto\sdocumentation\son\ssqlite3_open()\sand\ssqlite3_bind().\s\sNo\scode\schanges. -D 2014-10-04T19:31:53.403 +C Fix\sa\sharmless\scompiler\swarning\sinside\san\sassert()\sin\sFTS4. +D 2014-10-06T12:41:57.462 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 2f5e925bdb9d6d3e488c5a981af60cad4f9cdfe7 +F ext/fts3/fts3.c 66f39c425fa834b939d06caeeb14f49e038d443b F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 53d4eca1fb23eab00681fb028fb82eb5705c1e21 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 @@ -1201,7 +1201,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0f8102d71a0ee828629f037775ad86fe2a544120 -R 9fc77b39bf813ff4a1732fefd3521b90 +P b8f7f19dc06c59de2e194d83e6c052fb7d28c71d +R 4c5bc018ecacdd56e0a0b436c7d98214 U drh -Z 7cb27b268889a7c66aa5ab31d2f83fa7 +Z bf48e2ee6f6966054d7eb786534348de diff --git a/manifest.uuid b/manifest.uuid index 4a5b1c0284..3d35e96c0c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b8f7f19dc06c59de2e194d83e6c052fb7d28c71d \ No newline at end of file +418f3c9ad28672e5fe38d772d34e7cf8d26bc0e1 \ No newline at end of file From 85d117bc56a04688971a4e550717cea554939492 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 6 Oct 2014 18:33:49 +0000 Subject: [PATCH 418/710] Remove unreachable branches in decodeIntArray() when compiling without STAT3 or STAT4. FossilOrigin-Name: 80e1baa5c225c78902e08dbea9d577ff5757847f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 12 +++++------- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 8a54593787..8f581e16e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\saccuracy\sof\sthe\sestimates\sused\swhen\ssearching\san\sindex\sfor\svalues\snot\spresent\sin\sany\sstat4\ssamples. -D 2014-10-06T14:37:48.824 +C Remove\sunreachable\sbranches\sin\sdecodeIntArray()\swhen\scompiling\swithout\nSTAT3\sor\sSTAT4. +D 2014-10-06T18:33:49.122 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c ee85c504829aea05489ed0c67cbcd68d6a1ea7dd +F src/analyze.c 8c322e1ecc08909526dbd5ab4421889d05f2263d F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -1202,7 +1202,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 418f3c9ad28672e5fe38d772d34e7cf8d26bc0e1 fc619be057975b8be6d0958024c5d436edbdf084 -R 08d6dd1c11aa3044fd1c3c851cac54a0 -U dan -Z 0fcae61ad3d53701c4c43039d043043c +P 3aff9a9cac7aa994dfdaa0ab5c23ae73a1e820f0 +R 2a9f3a3ac6efdd67ef1666d9c29f8a31 +U drh +Z 0a50a0d9b770dc9e76964bc37ef2d3c3 diff --git a/manifest.uuid b/manifest.uuid index 3fb60d9ba8..3d0480e133 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3aff9a9cac7aa994dfdaa0ab5c23ae73a1e820f0 \ No newline at end of file +80e1baa5c225c78902e08dbea9d577ff5757847f \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 6b244dd9a7..7d36f01318 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1437,7 +1437,7 @@ static void decodeIntArray( #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( z==0 ) z = ""; #else - if( NEVER(z==0) ) z = ""; + assert( z!=0 ); #endif for(i=0; *z && i Date: Tue, 7 Oct 2014 15:46:54 +0000 Subject: [PATCH 419/710] Enhance (and fix) the MEMTYPE tags associated with heap memory allocations when SQLITE_MEMDEBUG is used. FossilOrigin-Name: ca5b789e33c4e5ce366d8f5372d086442f84e230 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/malloc.c | 31 ++++++++++++++++--------------- src/mem2.c | 4 ++-- src/sqliteInt.h | 3 +-- 5 files changed, 28 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 8f581e16e1..6b9f0fa8b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunreachable\sbranches\sin\sdecodeIntArray()\swhen\scompiling\swithout\nSTAT3\sor\sSTAT4. -D 2014-10-06T18:33:49.122 +C Enhance\s(and\sfix)\sthe\sMEMTYPE\stags\sassociated\swith\sheap\smemory\sallocations\nwhen\sSQLITE_MEMDEBUG\sis\sused. +D 2014-10-07T15:46:54.844 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,10 +195,10 @@ F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 F src/main.c bbe872b0ac0007bed0ebe1936fc493b039ad4f51 -F src/malloc.c 7cf86b4f2310898675d8a962342e5650779dfb3f +F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f -F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f +F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c 61eeb90134f9a5be6c2e68d8daae7628b25953fb F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 @@ -232,7 +232,7 @@ F src/shell.c 38f627b0885191357f55902a3ac199de90d79715 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 3e4bd1b2288528b6a7f2d52709618572b422ab7e +F src/sqliteInt.h 6ac5e34a590ad7ea22af91d190bdb212b12107be F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -1202,7 +1202,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3aff9a9cac7aa994dfdaa0ab5c23ae73a1e820f0 -R 2a9f3a3ac6efdd67ef1666d9c29f8a31 +P 80e1baa5c225c78902e08dbea9d577ff5757847f +R feb5d8a0d69f5af350e2ea3e74be0e6b U drh -Z 0a50a0d9b770dc9e76964bc37ef2d3c3 +Z 1e9604671a765b8eca12ebcbe81357fc diff --git a/manifest.uuid b/manifest.uuid index 3d0480e133..579a427501 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -80e1baa5c225c78902e08dbea9d577ff5757847f \ No newline at end of file +ca5b789e33c4e5ce366d8f5372d086442f84e230 \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index 7562ce2c41..6fb9d53d1b 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -447,25 +447,27 @@ static int isLookaside(sqlite3 *db, void *p){ */ int sqlite3MallocSize(void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); return sqlite3GlobalConfig.m.xSize(p); } int sqlite3DbMallocSize(sqlite3 *db, void *p){ if( db==0 ){ + assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); return sqlite3MallocSize(p); }else{ assert( sqlite3_mutex_held(db->mutex) ); if( isLookaside(db, p) ){ return db->lookaside.sz; }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); - assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, ~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); return sqlite3GlobalConfig.m.xSize(p); } } } sqlite3_uint64 sqlite3_msize(void *p){ + assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); return (sqlite3_uint64)sqlite3GlobalConfig.m.xSize(p); } @@ -474,8 +476,8 @@ sqlite3_uint64 sqlite3_msize(void *p){ */ void sqlite3_free(void *p){ if( p==0 ) return; /* IMP: R-49053-54554 */ - assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); sqlite3StatusAdd(SQLITE_STATUS_MEMORY_USED, -sqlite3MallocSize(p)); @@ -519,8 +521,8 @@ void sqlite3DbFree(sqlite3 *db, void *p){ return; } } - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, ~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( db!=0 || sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); sqlite3_free(p); @@ -532,6 +534,8 @@ void sqlite3DbFree(sqlite3 *db, void *p){ void *sqlite3Realloc(void *pOld, u64 nBytes){ int nOld, nNew, nDiff; void *pNew; + assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); + assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); if( pOld==0 ){ return sqlite3Malloc(nBytes); /* IMP: R-04300-56712 */ } @@ -558,8 +562,6 @@ void *sqlite3Realloc(void *pOld, u64 nBytes){ mem0.alarmThreshold-nDiff ){ sqlite3MallocAlarm(nDiff); } - assert( sqlite3MemdebugHasType(pOld, MEMTYPE_HEAP) ); - assert( sqlite3MemdebugNoType(pOld, ~MEMTYPE_HEAP) ); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); if( pNew==0 && mem0.alarmCallback ){ sqlite3MallocAlarm((int)nBytes); @@ -672,8 +674,8 @@ void *sqlite3DbMallocRaw(sqlite3 *db, u64 n){ if( !p && db ){ db->mallocFailed = 1; } - sqlite3MemdebugSetType(p, MEMTYPE_DB | - ((db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); + sqlite3MemdebugSetType(p, + (db && db->lookaside.bEnabled) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); return p; } @@ -699,15 +701,14 @@ void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ sqlite3DbFree(db, p); } }else{ - assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_LOOKASIDE|MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, ~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); pNew = sqlite3_realloc64(p, n); if( !pNew ){ - sqlite3MemdebugSetType(p, MEMTYPE_DB|MEMTYPE_HEAP); db->mallocFailed = 1; } - sqlite3MemdebugSetType(pNew, MEMTYPE_DB | + sqlite3MemdebugSetType(pNew, (db->lookaside.bEnabled ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP)); } } diff --git a/src/mem2.c b/src/mem2.c index 99ea42517e..51ea297c6a 100644 --- a/src/mem2.c +++ b/src/mem2.c @@ -394,7 +394,7 @@ void sqlite3MemdebugSetType(void *p, u8 eType){ ** This routine is designed for use within an assert() statement, to ** verify the type of an allocation. For example: ** -** assert( sqlite3MemdebugHasType(p, MEMTYPE_DB) ); +** assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); */ int sqlite3MemdebugHasType(void *p, u8 eType){ int rc = 1; @@ -416,7 +416,7 @@ int sqlite3MemdebugHasType(void *p, u8 eType){ ** This routine is designed for use within an assert() statement, to ** verify the type of an allocation. For example: ** -** assert( sqlite3MemdebugNoType(p, MEMTYPE_DB) ); +** assert( sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); */ int sqlite3MemdebugNoType(void *p, u8 eType){ int rc = 1; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 74c36c8db5..7998638c7f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3800,10 +3800,9 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...); # define sqlite3MemdebugNoType(X,Y) 1 #endif #define MEMTYPE_HEAP 0x01 /* General heap allocations */ -#define MEMTYPE_LOOKASIDE 0x02 /* Might have been lookaside memory */ +#define MEMTYPE_LOOKASIDE 0x02 /* Heap that might have been lookaside */ #define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ #define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ -#define MEMTYPE_DB 0x10 /* Uses sqlite3DbMalloc, not sqlite_malloc */ /* ** Threading interface From 3b335fce5cf01719274718986b785fa60500b667 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Oct 2014 16:59:22 +0000 Subject: [PATCH 420/710] Restrict the scope of the valueToText() routine. FossilOrigin-Name: 13c962b33df411a0d9ead0bb1969596faa286f79 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6b9f0fa8b5..907be1a736 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\s(and\sfix)\sthe\sMEMTYPE\stags\sassociated\swith\sheap\smemory\sallocations\nwhen\sSQLITE_MEMDEBUG\sis\sused. -D 2014-10-07T15:46:54.844 +C Restrict\sthe\sscope\sof\sthe\svalueToText()\sroutine. +D 2014-10-07T16:59:22.085 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -295,7 +295,7 @@ F src/vdbeInt.h 0b97a3190f8fbf460655985a9183019f5a702754 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 F src/vdbeaux.c 5b687d7b5beaaa5b97189edf25cf08c311834933 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 1e105dacf5190fc85a8ec2107c0dcc1884e75099 +F src/vdbemem.c ee0c60af8c0f5535c6a06c49a624d87cf70b0573 F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1202,7 +1202,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 80e1baa5c225c78902e08dbea9d577ff5757847f -R feb5d8a0d69f5af350e2ea3e74be0e6b +P ca5b789e33c4e5ce366d8f5372d086442f84e230 +R 0b0659acf5a7c9ad242fd7056973fd11 U drh -Z 1e9604671a765b8eca12ebcbe81357fc +Z 23204866f7082d1eef3090df8e45b47e diff --git a/manifest.uuid b/manifest.uuid index 579a427501..43aa93cb2a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca5b789e33c4e5ce366d8f5372d086442f84e230 \ No newline at end of file +13c962b33df411a0d9ead0bb1969596faa286f79 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 36db80fa18..a4caf51759 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -994,7 +994,7 @@ int sqlite3VdbeMemFromBtree( ** Convert it into a string with encoding enc and return a pointer ** to a zero-terminated version of that string. */ -SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ +static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ assert( pVal!=0 ); assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); From 89a5833cb9a6c9be11b244d4ed6d3fe89b6f47d0 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Oct 2014 20:09:27 +0000 Subject: [PATCH 421/710] Fix the corruptI.test script so that it works with SQLITE_ENABLE_OVERSIZE_CELL_CHECK and with SQLITE_DEFAULT_AUTOVACUUM=1. FossilOrigin-Name: e405b9e4a9ef322d84b20e902234b4f6aa196b1b --- manifest | 12 +++++----- manifest.uuid | 2 +- test/corruptI.test | 57 ++++++++++++++++++++++++---------------------- 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 907be1a736..433a723055 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Restrict\sthe\sscope\sof\sthe\svalueToText()\sroutine. -D 2014-10-07T16:59:22.085 +C Fix\sthe\scorruptI.test\sscript\sso\sthat\sit\sworks\swith\nSQLITE_ENABLE_OVERSIZE_CELL_CHECK\sand\swith\sSQLITE_DEFAULT_AUTOVACUUM=1. +D 2014-10-07T20:09:27.561 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -420,7 +420,7 @@ F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb -F test/corruptI.test 0afbba50bfae006094cc548b4605f521c1179502 +F test/corruptI.test 221ad8b7f0a9ac6b80fc577e73b5ad8cdea31243 F test/cost.test 19d314526616ce4473eb4e4e450fcb94499ce318 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 @@ -1202,7 +1202,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ca5b789e33c4e5ce366d8f5372d086442f84e230 -R 0b0659acf5a7c9ad242fd7056973fd11 +P 13c962b33df411a0d9ead0bb1969596faa286f79 +R bc98a778a72f51d6ea08f3f2bdc76392 U drh -Z 23204866f7082d1eef3090df8e45b47e +Z 156ceab5010f208eb433cfa9f4253059 diff --git a/manifest.uuid b/manifest.uuid index 43aa93cb2a..d79d2fd138 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13c962b33df411a0d9ead0bb1969596faa286f79 \ No newline at end of file +e405b9e4a9ef322d84b20e902234b4f6aa196b1b \ No newline at end of file diff --git a/test/corruptI.test b/test/corruptI.test index 41200c5409..c8d0176236 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -75,31 +75,34 @@ do_test 2.2 { catchsql { SELECT * FROM r WHERE x >= 10 } } {1 {database disk image is malformed}} -reset_db - -do_execsql_test 3.1 { - PRAGMA page_size = 512; - CREATE TABLE t1(a INTEGER PRIMARY KEY, b); - WITH s(a, b) AS ( - SELECT 2, 'abcdefghij' - UNION ALL - SELECT a+2, b FROM s WHERe a < 40 - ) - INSERT INTO t1 SELECT * FROM s; -} {} - -do_test 3.2 { - hexio_write test.db [expr 512+3] 0054 - db close - sqlite3 db test.db - execsql { INSERT INTO t1 VALUES(5, 'klmnopqrst') } - execsql { INSERT INTO t1 VALUES(7, 'klmnopqrst') } -} {} - -db close -sqlite3 db test.db -do_catchsql_test 3.2 { - INSERT INTO t1 VALUES(9, 'klmnopqrst'); -} {1 {database disk image is malformed}} - +if {[db one {SELECT sqlite_compileoption_used('ENABLE_OVERSIZE_CELL_CHECK')}]} { + # The following tests only work if OVERSIZE_CELL_CHECK is disabled +} else { + reset_db + do_execsql_test 3.1 { + PRAGMA auto_vacuum=0; + PRAGMA page_size = 512; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + WITH s(a, b) AS ( + SELECT 2, 'abcdefghij' + UNION ALL + SELECT a+2, b FROM s WHERe a < 40 + ) + INSERT INTO t1 SELECT * FROM s; + } {} + + do_test 3.2 { + hexio_write test.db [expr 512+3] 0054 + db close + sqlite3 db test.db + execsql { INSERT INTO t1 VALUES(5, 'klmnopqrst') } + execsql { INSERT INTO t1 VALUES(7, 'klmnopqrst') } + } {} + + db close + sqlite3 db test.db + do_catchsql_test 3.3 { + INSERT INTO t1 VALUES(9, 'klmnopqrst'); + } {1 {database disk image is malformed}} +} ;# end-if !defined(ENABLE_OVERSIZE_CELL_CHECK) finish_test From 722246e801299ea9042db1278ab6365384d72e22 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 7 Oct 2014 23:02:24 +0000 Subject: [PATCH 422/710] Make sure the sqlite3VdbeMemClearAndResize() routine is never called with a zero size parameter, since a size of zero could lead to either a memory leak or an assertion fault. FossilOrigin-Name: f672a380e2e52bede95ff11a533fd9f7d412d494 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 3 ++- src/vdbemem.c | 13 ++++++++++--- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 433a723055..8b6acbd4de 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\scorruptI.test\sscript\sso\sthat\sit\sworks\swith\nSQLITE_ENABLE_OVERSIZE_CELL_CHECK\sand\swith\sSQLITE_DEFAULT_AUTOVACUUM=1. -D 2014-10-07T20:09:27.561 +C Make\ssure\sthe\ssqlite3VdbeMemClearAndResize()\sroutine\sis\snever\scalled\swith\sa\nzero\ssize\sparameter,\ssince\sa\ssize\sof\szero\scould\slead\sto\seither\sa\smemory\sleak\nor\san\sassertion\sfault. +D 2014-10-07T23:02:24.724 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,13 +289,13 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 93eeb6f9c3a3084133225a196f220454d71cca10 +F src/vdbe.c e6c964101382d6fb144853b1d5b288158a9aba0e F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h 0b97a3190f8fbf460655985a9183019f5a702754 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 F src/vdbeaux.c 5b687d7b5beaaa5b97189edf25cf08c311834933 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c ee0c60af8c0f5535c6a06c49a624d87cf70b0573 +F src/vdbemem.c 481327f50d9da330053aa7456702ce46d0a4e70f F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f @@ -1202,7 +1202,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 13c962b33df411a0d9ead0bb1969596faa286f79 -R bc98a778a72f51d6ea08f3f2bdc76392 +P e405b9e4a9ef322d84b20e902234b4f6aa196b1b +R 59a3e15d0b7c925a5c413304ec5e314b U drh -Z 156ceab5010f208eb433cfa9f4253059 +Z 1de6834c7b78df261ad24af6ce5dedbe diff --git a/manifest.uuid b/manifest.uuid index d79d2fd138..169d9b9887 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e405b9e4a9ef322d84b20e902234b4f6aa196b1b \ No newline at end of file +f672a380e2e52bede95ff11a533fd9f7d412d494 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 34eb1d42c5..18e1cd83a9 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4333,7 +4333,8 @@ case OP_RowData: { goto too_big; } } - if( sqlite3VdbeMemClearAndResize(pOut, n) ){ + testcase( n==0 ); + if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){ goto no_mem; } pOut->n = n; diff --git a/src/vdbemem.c b/src/vdbemem.c index a4caf51759..0c62db0720 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -31,7 +31,10 @@ int sqlite3VdbeCheckMemInvariants(Mem *p){ */ assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 ); - /* MEM_Dyn may only be set if Mem.szMalloc==0 */ + /* MEM_Dyn may only be set if Mem.szMalloc==0. In this way we + ** ensure that if Mem.szMalloc>0 then it is safe to do + ** Mem.z = Mem.zMalloc without having to check Mem.flags&MEM_Dyn. + ** That saves a few cycles in inner loops. */ assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 ); /* Cannot be both MEM_Int and MEM_Real at the same time */ @@ -167,7 +170,8 @@ SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ ** if unable to complete the resizing. */ int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ - assert( szNew>=0 ); + assert( szNew>0 ); + assert( (pMem->flags & MEM_Dyn)==0 || pMem->szMalloc==0 ); if( pMem->szMallociLimit ){ return SQLITE_TOOBIG; } - if( sqlite3VdbeMemClearAndResize(pMem, nAlloc) ){ + testcase( nAlloc==0 ); + testcase( nAlloc==31 ); + testcase( nAlloc==32 ); + if( sqlite3VdbeMemClearAndResize(pMem, MAX(nAlloc,32)) ){ return SQLITE_NOMEM; } memcpy(pMem->z, z, nAlloc); From 9a7b41d74a384431c53bc986e78852cf5d6e9575 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Oct 2014 00:08:08 +0000 Subject: [PATCH 423/710] More intuitive labels on ".wheretrace" output. FossilOrigin-Name: adcb3fed489b580221c7bf2692a60e24248b23a0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 8b6acbd4de..7b23da26a8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\ssqlite3VdbeMemClearAndResize()\sroutine\sis\snever\scalled\swith\sa\nzero\ssize\sparameter,\ssince\sa\ssize\sof\szero\scould\slead\sto\seither\sa\smemory\sleak\nor\san\sassertion\sfault. -D 2014-10-07T23:02:24.724 +C More\sintuitive\slabels\son\s".wheretrace"\soutput. +D 2014-10-08T00:08:08.905 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2f42fe0d19303e0f5ce29aff3afbd3e43cbd6efb +F src/where.c 74e1f7e136bfb52c9c65a55909f8a24873b1edb5 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1202,7 +1202,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e405b9e4a9ef322d84b20e902234b4f6aa196b1b -R 59a3e15d0b7c925a5c413304ec5e314b +P f672a380e2e52bede95ff11a533fd9f7d412d494 +R b98bb7ee2a60e85b3c67deaa06ee1000 U drh -Z 1de6834c7b78df261ad24af6ce5dedbe +Z 71a8e8f52b1016a5adbdd28eb6eb38bb diff --git a/manifest.uuid b/manifest.uuid index 169d9b9887..bfe9547db3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f672a380e2e52bede95ff11a533fd9f7d412d494 \ No newline at end of file +adcb3fed489b580221c7bf2692a60e24248b23a0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8974895c45..23481bb05e 100644 --- a/src/where.c +++ b/src/where.c @@ -4139,7 +4139,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** than pTemplate, so just ignore pTemplate */ #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf("ins-noop: "); + sqlite3DebugPrintf(" skip: "); whereLoopPrint(pTemplate, pBuilder->pWC); } #endif @@ -4155,10 +4155,10 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ if( p!=0 ){ - sqlite3DebugPrintf("ins-del: "); + sqlite3DebugPrintf("replace: "); whereLoopPrint(p, pBuilder->pWC); } - sqlite3DebugPrintf("ins-new: "); + sqlite3DebugPrintf(" add: "); whereLoopPrint(pTemplate, pBuilder->pWC); } #endif @@ -4182,7 +4182,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ *ppTail = pToDel->pNextLoop; #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ - sqlite3DebugPrintf("ins-del: "); + sqlite3DebugPrintf(" delete: "); whereLoopPrint(pToDel, pBuilder->pWC); } #endif From 69afd9980efa75371340a16bee238721f8b27cb2 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Oct 2014 02:53:25 +0000 Subject: [PATCH 424/710] Fix the STAT4 range scan estimates for DESC indexes. FossilOrigin-Name: e3fe84005259ef9a6027d25793514cebb2d4e7e0 --- manifest | 13 +-- manifest.uuid | 2 +- src/where.c | 13 ++- test/analyzeE.test | 242 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 259 insertions(+), 11 deletions(-) create mode 100644 test/analyzeE.test diff --git a/manifest b/manifest index 7b23da26a8..36cf57e18d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\sintuitive\slabels\son\s".wheretrace"\soutput. -D 2014-10-08T00:08:08.905 +C Fix\sthe\sSTAT4\srange\sscan\sestimates\sfor\sDESC\sindexes. +D 2014-10-08T02:53:25.568 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 74e1f7e136bfb52c9c65a55909f8a24873b1edb5 +F src/where.c 982f1ce21355452f2e5cd284359ab141c1eff547 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -327,6 +327,7 @@ F test/analyzeA.test 3335697f6700c7052295cfd0067fc5b2aacddf9a F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d F test/analyzeC.test 555a6cc388b9818b6eda6df816f01ce0a75d3a93 F test/analyzeD.test 08f9d0bee4e118a66fff3a32d02dbe0ee0a2b594 +F test/analyzeE.test 8684e8ac5722fb97c251887ad97e5d496a98af1d F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -1202,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f672a380e2e52bede95ff11a533fd9f7d412d494 -R b98bb7ee2a60e85b3c67deaa06ee1000 +P adcb3fed489b580221c7bf2692a60e24248b23a0 +R 50f3751b824cfe9cc6f7ad401beb29e1 U drh -Z 71a8e8f52b1016a5adbdd28eb6eb38bb +Z 07946a58b794ebe5e4cb12afd1dbd645 diff --git a/manifest.uuid b/manifest.uuid index bfe9547db3..ee71c7f98a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adcb3fed489b580221c7bf2692a60e24248b23a0 \ No newline at end of file +e3fe84005259ef9a6027d25793514cebb2d4e7e0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 23481bb05e..236337a405 100644 --- a/src/where.c +++ b/src/where.c @@ -2207,16 +2207,22 @@ static int whereRangeScanEst( iUpper = a[0] + a[1]; } + assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); + assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); + if( p->pKeyInfo && p->pKeyInfo->aSortOrder[nEq] ){ + /* The roles of pLower and pUpper are swapped for a DESC index */ + SWAP(WhereTerm*, pLower, pUpper); + } + /* If possible, improve on the iLower estimate using ($P:$L). */ if( pLower ){ int bOk; /* True if value is extracted from pExpr */ Expr *pExpr = pLower->pExpr->pRight; - assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); if( rc==SQLITE_OK && bOk ){ tRowcnt iNew; whereKeyStats(pParse, p, pRec, 0, a); - iNew = a[0] + ((pLower->eOperator & WO_GT) ? a[1] : 0); + iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); if( iNew>iLower ) iLower = iNew; nOut--; pLower = 0; @@ -2227,12 +2233,11 @@ static int whereRangeScanEst( if( pUpper ){ int bOk; /* True if value is extracted from pExpr */ Expr *pExpr = pUpper->pExpr->pRight; - assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); if( rc==SQLITE_OK && bOk ){ tRowcnt iNew; whereKeyStats(pParse, p, pRec, 1, a); - iNew = a[0] + ((pUpper->eOperator & WO_LE) ? a[1] : 0); + iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); if( iNew2500 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-1.8 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1900 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-1.9 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1100 +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-1.10 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1100 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-1.11 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1900 +} {/SCAN TABLE t1/} + +# Verify that everything works the same on a DESCENDING index. +# +do_execsql_test analyzeE-2.0 { + DROP INDEX t1a; + CREATE INDEX t1a ON t1(a DESC); + ANALYZE; +} {} +do_execsql_test analyzeE-2.1 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 500 AND 2500; +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-2.2 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 2900 AND 3000; +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.3 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1700 AND 1750; +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.4 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1 AND 500 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.5 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 3000 AND 3000000 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.6 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<500 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.7 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>2500 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.8 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1900 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.9 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1100 +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-2.10 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1100 +} {/SEARCH TABLE t1 USING INDEX t1a/} +do_execsql_test analyzeE-2.11 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1900 +} {/SCAN TABLE t1/} + +# Now do a range query on the second term of an ASCENDING index +# where the first term is constrained by equality. +# +do_execsql_test analyzeE-3.0 { + DROP TABLE t1; + CREATE TABLE t1(a,b,c); + WITH RECURSIVE + cnt(x) AS (VALUES(1000) UNION ALL SELECT x+1 FROM cnt WHERE x<2000) + INSERT INTO t1(a,b,c) SELECT x, x, 123 FROM cnt; + CREATE INDEX t1ca ON t1(c,a); + ANALYZE; +} {} +do_execsql_test analyzeE-3.1 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 500 AND 2500 AND c=123; +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-3.2 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 2900 AND 3000 AND c=123; +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.3 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1700 AND 1750 AND c=123; +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.4 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1 AND 500 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.5 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 3000 AND 3000000 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.6 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<500 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.7 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>2500 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.8 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1900 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.9 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1100 AND c=123 +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-3.10 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1100 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-3.11 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1900 AND c=123 +} {/SCAN TABLE t1/} + +# Repeat the 3.x tests using a DESCENDING index +# +do_execsql_test analyzeE-4.0 { + DROP INDEX t1ca; + CREATE INDEX t1ca ON t1(c ASC,a DESC); + ANALYZE; +} {} +do_execsql_test analyzeE-4.1 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 500 AND 2500 AND c=123; +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-4.2 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 2900 AND 3000 AND c=123; +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.3 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1700 AND 1750 AND c=123; +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.4 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 1 AND 500 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.5 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a BETWEEN 3000 AND 3000000 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.6 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<500 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.7 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>2500 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.8 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1900 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.9 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a>1100 AND c=123 +} {/SCAN TABLE t1/} +do_execsql_test analyzeE-4.10 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1100 AND c=123 +} {/SEARCH TABLE t1 USING INDEX t1ca/} +do_execsql_test analyzeE-4.11 { + EXPLAIN QUERY PLAN + SELECT * FROM t1 WHERE a<1900 AND c=123 +} {/SCAN TABLE t1/} + +finish_test From 923c0b53beffb478a8f279170fdb980d93c7d951 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 8 Oct 2014 11:11:24 +0000 Subject: [PATCH 425/710] Remove some temporary code in mallocA.test that was accidentally checked in. FossilOrigin-Name: dedd15f7cd13868f3be37646dd30ab7ceac5dea7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/mallocA.test | 2 -- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 36cf57e18d..61a2a23950 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sSTAT4\srange\sscan\sestimates\sfor\sDESC\sindexes. -D 2014-10-08T02:53:25.568 +C Remove\ssome\stemporary\scode\sin\smallocA.test\sthat\swas\saccidentally\schecked\sin. +D 2014-10-08T11:11:24.999 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -692,7 +692,7 @@ F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151 F test/malloc7.test 7c68a32942858bc715284856c5507446bba88c3a F test/malloc8.test 9b7a3f8cb9cf0b12fff566e80a980b1767bd961d F test/malloc9.test 2307c6ee3703b0a21391f3ea92388b4b73f9105e -F test/mallocA.test c049224adeb0244b8f6eb770c1fa6ac40f9b3518 +F test/mallocA.test 672cd7dedb63771bade3a6f557f851a4ad161d4a F test/mallocAll.test 98f1be74bc9f49a858bc4f361fc58e26486798be F test/mallocB.test bc475ab850cda896142ab935bbfbc74c24e51ed6 F test/mallocC.test 3dffe16532f109293ce1ccecd0c31dca55ef08c4 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P adcb3fed489b580221c7bf2692a60e24248b23a0 -R 50f3751b824cfe9cc6f7ad401beb29e1 -U drh -Z 07946a58b794ebe5e4cb12afd1dbd645 +P e3fe84005259ef9a6027d25793514cebb2d4e7e0 +R 041f735fd7729856664b7e587c89b3dc +U dan +Z 7e47868b81dfec753c3dcd2a3c6c6ccd diff --git a/manifest.uuid b/manifest.uuid index ee71c7f98a..0fb51d4d96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e3fe84005259ef9a6027d25793514cebb2d4e7e0 \ No newline at end of file +dedd15f7cd13868f3be37646dd30ab7ceac5dea7 \ No newline at end of file diff --git a/test/mallocA.test b/test/mallocA.test index d6d6de8222..a78073d833 100644 --- a/test/mallocA.test +++ b/test/mallocA.test @@ -119,8 +119,6 @@ do_execsql_test 7.0 { PRAGMA cache_size = 5; } do_faultsim_test 7 -faults oom-trans* -prep { - if {$iFail < 500} { set iFail 2000 } - if {$iFail > 1215} { set iFail 2000 } } -body { execsql { WITH r(x,y) AS ( From 6a15440378465848343def4f398861d04a6c5e5f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Oct 2014 13:34:21 +0000 Subject: [PATCH 426/710] Ensure that the Pager.pTmpSpace allocation is correct even if an OOM error occurs while reducing the page size. FossilOrigin-Name: e4b43967fd9a0b4944be9ab5575bff3678be8ed5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 6 ++++-- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 61a2a23950..2d5dcf21ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\stemporary\scode\sin\smallocA.test\sthat\swas\saccidentally\schecked\sin. -D 2014-10-08T11:11:24.999 +C Ensure\sthat\sthe\sPager.pTmpSpace\sallocation\sis\scorrect\seven\sif\san\sOOM\serror\noccurs\swhile\sreducing\sthe\spage\ssize. +D 2014-10-08T13:34:21.522 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -215,7 +215,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c caab007743821d96752597c9cfd7351654697b06 +F src/pager.c a171cf9dd09c6cb162b262c328d4dfd198e04f80 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e3fe84005259ef9a6027d25793514cebb2d4e7e0 -R 041f735fd7729856664b7e587c89b3dc -U dan -Z 7e47868b81dfec753c3dcd2a3c6c6ccd +P dedd15f7cd13868f3be37646dd30ab7ceac5dea7 +R 4cf08d774ef596973588c752956e991d +U drh +Z e3bcbea265a57269fe808fe62b242441 diff --git a/manifest.uuid b/manifest.uuid index 0fb51d4d96..b199078c31 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dedd15f7cd13868f3be37646dd30ab7ceac5dea7 \ No newline at end of file +e4b43967fd9a0b4944be9ab5575bff3678be8ed5 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 79bfe15f10..d3a36ef484 100644 --- a/src/pager.c +++ b/src/pager.c @@ -3618,13 +3618,15 @@ int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ if( rc==SQLITE_OK ){ pager_reset(pPager); - sqlite3PageFree(pPager->pTmpSpace); - pPager->pTmpSpace = pNew; rc = sqlite3PcacheSetPageSize(pPager->pPCache, pageSize); } if( rc==SQLITE_OK ){ + sqlite3PageFree(pPager->pTmpSpace); + pPager->pTmpSpace = pNew; pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); pPager->pageSize = pageSize; + }else{ + sqlite3PageFree(pNew); } } From f6aff8052551c6818d8077cf18ae50482c3c729e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Oct 2014 14:28:31 +0000 Subject: [PATCH 427/710] Set the connection-specific lastRowid value before calling any SQL function. FossilOrigin-Name: dff0f6422e60a7e2e4efb658aab202a119cfa702 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2d5dcf21ef..4860d29d3d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\sPager.pTmpSpace\sallocation\sis\scorrect\seven\sif\san\sOOM\serror\noccurs\swhile\sreducing\sthe\spage\ssize. -D 2014-10-08T13:34:21.522 +C Set\sthe\sconnection-specific\slastRowid\svalue\sbefore\scalling\sany\sSQL\sfunction. +D 2014-10-08T14:28:31.133 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c e6c964101382d6fb144853b1d5b288158a9aba0e +F src/vdbe.c fee8286ff026bb9cf96ce87971b60aba53863b78 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h 0b97a3190f8fbf460655985a9183019f5a702754 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dedd15f7cd13868f3be37646dd30ab7ceac5dea7 -R 4cf08d774ef596973588c752956e991d +P e4b43967fd9a0b4944be9ab5575bff3678be8ed5 +R 4b3fd8ad675c779547c8afe5951393e2 U drh -Z e3bcbea265a57269fe808fe62b242441 +Z c9285a151f73aa9d039cef676eec6a31 diff --git a/manifest.uuid b/manifest.uuid index b199078c31..27c74f4c9b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4b43967fd9a0b4944be9ab5575bff3678be8ed5 \ No newline at end of file +dff0f6422e60a7e2e4efb658aab202a119cfa702 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 18e1cd83a9..c039dcc862 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1558,7 +1558,7 @@ case OP_Function: { ctx.pVdbe = p; MemSetTypeFlag(ctx.pOut, MEM_Null); ctx.fErrorOrAux = 0; - assert( db->lastRowid==lastRowid ); + db->lastRowid = lastRowid; (*ctx.pFunc->xFunc)(&ctx, n, apVal); /* IMP: R-24505-23230 */ lastRowid = db->lastRowid; /* Remember rowid changes made by xFunc */ From 3705ef6a736d656e96d6de2ba5d069d5b0a6e7ae Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Oct 2014 15:53:21 +0000 Subject: [PATCH 428/710] Fix up test cases to account for the new SQLITE_LIMIT_WORKER_THREADS limit. FossilOrigin-Name: 6483d426c4c5c772cd49412ea37e0fa7a0378904 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/test1.c | 3 ++- src/test_config.c | 1 + test/sqllimits1.test | 7 +++++++ 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 4860d29d3d..91746f0685 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Set\sthe\sconnection-specific\slastRowid\svalue\sbefore\scalling\sany\sSQL\sfunction. -D 2014-10-08T14:28:31.133 +C Fix\sup\stest\scases\sto\saccount\sfor\sthe\snew\sSQLITE_LIMIT_WORKER_THREADS\slimit. +D 2014-10-08T15:53:21.620 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -237,7 +237,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 -F src/test1.c 523cd70ded28db71af9a30ec184cbe0957de9575 +F src/test1.c 518db4305d76b29dd9da3f022ca899c8fcdf9fc7 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -250,7 +250,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c 6f721f0337b96d58e81ff69bba101113c8168c2b +F src/test_config.c a4cdebe093474c02eecc5e4008b1a22198edf975 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -862,7 +862,7 @@ F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b F test/speedtest1.c 83f6b3318f7ee60e52b978b5a5e5dd7e83dfb7ee F test/spellfix.test 24f676831acddd2f4056a598fd731a72c6311f49 -F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 +F test/sqllimits1.test 9014524e7ab16e2a4976b13397db4c29cc29c6d9 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 F test/subquery.test 666fdecceac258f5fd84bed09a64e49d9f37edd9 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e4b43967fd9a0b4944be9ab5575bff3678be8ed5 -R 4b3fd8ad675c779547c8afe5951393e2 +P dff0f6422e60a7e2e4efb658aab202a119cfa702 +R 7321f912fc295ac13d3e667e8ae8fde4 U drh -Z c9285a151f73aa9d039cef676eec6a31 +Z a0d7563bd184d766bb67ad4df881d191 diff --git a/manifest.uuid b/manifest.uuid index 27c74f4c9b..44dfb50593 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dff0f6422e60a7e2e4efb658aab202a119cfa702 \ No newline at end of file +6483d426c4c5c772cd49412ea37e0fa7a0378904 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 62b575989d..85a16488ba 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5513,10 +5513,11 @@ static int test_limit( { "SQLITE_LIMIT_LIKE_PATTERN_LENGTH", SQLITE_LIMIT_LIKE_PATTERN_LENGTH }, { "SQLITE_LIMIT_VARIABLE_NUMBER", SQLITE_LIMIT_VARIABLE_NUMBER }, { "SQLITE_LIMIT_TRIGGER_DEPTH", SQLITE_LIMIT_TRIGGER_DEPTH }, + { "SQLITE_LIMIT_WORKER_THREADS", SQLITE_LIMIT_WORKER_THREADS }, /* Out of range test cases */ { "SQLITE_LIMIT_TOOSMALL", -1, }, - { "SQLITE_LIMIT_TOOBIG", SQLITE_LIMIT_TRIGGER_DEPTH+1 }, + { "SQLITE_LIMIT_TOOBIG", SQLITE_LIMIT_WORKER_THREADS+1 }, }; int i, id; int val; diff --git a/src/test_config.c b/src/test_config.c index 074faf2116..2c11f713fb 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -644,6 +644,7 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); LINKVAR( DEFAULT_FILE_FORMAT ); LINKVAR( MAX_ATTACHED ); LINKVAR( MAX_DEFAULT_PAGE_SIZE ); + LINKVAR( MAX_WORKER_THREADS ); { static const int cv_TEMP_STORE = SQLITE_TEMP_STORE; diff --git a/test/sqllimits1.test b/test/sqllimits1.test index 2cbad3ffb8..57fc931f7c 100644 --- a/test/sqllimits1.test +++ b/test/sqllimits1.test @@ -51,6 +51,13 @@ do_test sqllimits1-1.9 { do_test sqllimits1-1.10 { sqlite3_limit db SQLITE_LIMIT_VARIABLE_NUMBER -1 } $SQLITE_MAX_VARIABLE_NUMBER +do_test sqllimits1-1.11 { + sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH -1 +} $SQLITE_MAX_TRIGGER_DEPTH +do_test sqllimits1-1.12 { + sqlite3_limit db SQLITE_LIMIT_WORKER_THREADS 99999 + sqlite3_limit db SQLITE_LIMIT_WORKER_THREADS -1 +} $SQLITE_MAX_WORKER_THREADS # Limit parameters out of range. # From 8e0a8f681ab3a7f5be1d0f7d080c8064f5b27127 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 8 Oct 2014 19:33:54 +0000 Subject: [PATCH 429/710] Remove an always-true branch in whereRangeScanEst(). Replace it with an assert(). FossilOrigin-Name: 42e48fd3a6a6219d9bd6135d821b38c5157922ba --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 91746f0685..d52803d558 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sup\stest\scases\sto\saccount\sfor\sthe\snew\sSQLITE_LIMIT_WORKER_THREADS\slimit. -D 2014-10-08T15:53:21.620 +C Remove\san\salways-true\sbranch\sin\swhereRangeScanEst().\s\sReplace\sit\swith\san\nassert(). +D 2014-10-08T19:33:54.518 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 982f1ce21355452f2e5cd284359ab141c1eff547 +F src/where.c 6fe21e0f60a449af5d75d00e6d480370464a9a48 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dff0f6422e60a7e2e4efb658aab202a119cfa702 -R 7321f912fc295ac13d3e667e8ae8fde4 +P 6483d426c4c5c772cd49412ea37e0fa7a0378904 +R df1d18db7990f8a7d78a79273193f180 U drh -Z a0d7563bd184d766bb67ad4df881d191 +Z 70427f180e63fb554c9a4909a2862788 diff --git a/manifest.uuid b/manifest.uuid index 44dfb50593..ba92b776aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6483d426c4c5c772cd49412ea37e0fa7a0378904 \ No newline at end of file +42e48fd3a6a6219d9bd6135d821b38c5157922ba \ No newline at end of file diff --git a/src/where.c b/src/where.c index 236337a405..011ad66c00 100644 --- a/src/where.c +++ b/src/where.c @@ -2209,7 +2209,8 @@ static int whereRangeScanEst( assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); - if( p->pKeyInfo && p->pKeyInfo->aSortOrder[nEq] ){ + assert( p->pKeyInfo!=0 && p->pKeyInfo->aSortOrder!=0 ); + if( p->pKeyInfo->aSortOrder[nEq] ){ /* The roles of pLower and pUpper are swapped for a DESC index */ SWAP(WhereTerm*, pLower, pUpper); } From a8950d5038d443a0014eb7ca6c14544b4d48ab9e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 9 Oct 2014 14:00:49 +0000 Subject: [PATCH 430/710] Fix a memory leak associated with the FTS4 matchinfo() function. FossilOrigin-Name: fb8da82411b80a234c6a5481622027815450996a --- ext/fts3/fts3.c | 1 + manifest | 12 ++++++------ manifest.uuid | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 582b7e27a1..2b93c62715 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -3116,6 +3116,7 @@ static int fts3FilterMethod( /* In case the cursor has been used before, clear it now. */ sqlite3_finalize(pCsr->pStmt); sqlite3_free(pCsr->aDoclist); + sqlite3_free(pCsr->aMatchinfo); sqlite3Fts3ExprFree(pCsr->pExpr); memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); diff --git a/manifest b/manifest index d52803d558..213759fab2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\salways-true\sbranch\sin\swhereRangeScanEst().\s\sReplace\sit\swith\san\nassert(). -D 2014-10-08T19:33:54.518 +C Fix\sa\smemory\sleak\sassociated\swith\sthe\sFTS4\smatchinfo()\sfunction. +D 2014-10-09T14:00:49.227 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 66f39c425fa834b939d06caeeb14f49e038d443b +F ext/fts3/fts3.c 8b6cceb3e0be22da26d83a3cec0e0e337e6b8ec6 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h 53d4eca1fb23eab00681fb028fb82eb5705c1e21 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6483d426c4c5c772cd49412ea37e0fa7a0378904 -R df1d18db7990f8a7d78a79273193f180 +P 42e48fd3a6a6219d9bd6135d821b38c5157922ba +R b5e5fdd1e3ce299c36e7187cabf29d23 U drh -Z 70427f180e63fb554c9a4909a2862788 +Z 0cd30ecf2cac304fdb077fd52d8a3789 diff --git a/manifest.uuid b/manifest.uuid index ba92b776aa..261ccbda05 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42e48fd3a6a6219d9bd6135d821b38c5157922ba \ No newline at end of file +fb8da82411b80a234c6a5481622027815450996a \ No newline at end of file From 622d4f8bb1f3c530cf5886242a19fbc931585801 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 9 Oct 2014 14:10:38 +0000 Subject: [PATCH 431/710] Add a test case for the memory leak fixed by the previous check-in. FossilOrigin-Name: bae36d544676c90e337381a83f4513b4d925ab05 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fts3matchinfo.test | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 213759fab2..96904c6bca 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\smemory\sleak\sassociated\swith\sthe\sFTS4\smatchinfo()\sfunction. -D 2014-10-09T14:00:49.227 +C Add\sa\stest\scase\sfor\sthe\smemory\sleak\sfixed\sby\sthe\sprevious\scheck-in. +D 2014-10-09T14:10:38.803 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -568,7 +568,7 @@ F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 F test/fts3join.test 53e66a0c21eb568580674a43b21c059acb26f499 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6 -F test/fts3matchinfo.test ff423e73faab8fc6d7adeefedf74dd8e2b0b14e0 +F test/fts3matchinfo.test 58544fa4d254000fa4e7f494b0a832f7ba61d45e F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905 F test/fts3prefix.test b36d4f00b128a51e7b386cc013a874246d9d7dc1 F test/fts3prefix2.test e1f0a822ca661dced7f12ce392e14eaf65609dce @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 42e48fd3a6a6219d9bd6135d821b38c5157922ba -R b5e5fdd1e3ce299c36e7187cabf29d23 +P fb8da82411b80a234c6a5481622027815450996a +R 3f68f640c5da00a278a8bfe99729cf27 U drh -Z 0cd30ecf2cac304fdb077fd52d8a3789 +Z d27fedea65c2ff594a69f62bdfd5cd22 diff --git a/manifest.uuid b/manifest.uuid index 261ccbda05..66a5fa9d03 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fb8da82411b80a234c6a5481622027815450996a \ No newline at end of file +bae36d544676c90e337381a83f4513b4d925ab05 \ No newline at end of file diff --git a/test/fts3matchinfo.test b/test/fts3matchinfo.test index fd475af2e4..36c9121118 100644 --- a/test/fts3matchinfo.test +++ b/test/fts3matchinfo.test @@ -433,4 +433,21 @@ do_execsql_test 9.1 { SELECT snippet(ft2, '[', ']', '', -1, 1) FROM ft2 WHERE ft2 MATCH 'c'; } {{[c]} {[c]}} +#--------------------------------------------------------------------------- +# Test for a memory leak +# +do_execsql_test 10.1 { + DROP TABLE t10; + CREATE VIRTUAL TABLE t10 USING fts4(idx, value); + INSERT INTO t10 values (1, 'one'),(2, 'two'),(3, 'three'); + SELECT docId, t10.* + FROM t10 + JOIN (SELECT 1 AS idx UNION SELECT 2 UNION SELECT 3) AS x + WHERE t10 MATCH x.idx + AND matchinfo(t10) not null + GROUP BY docId + ORDER BY 1; +} {1 1 one 2 2 two 3 3 three} + + finish_test From 6e1a03735790e1ccf2f243d26197090e7954b9fc Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 9 Oct 2014 15:08:17 +0000 Subject: [PATCH 432/710] Allow FTS tokenizers to choose whether or not to consider the "*" character part of tokens or not. This restores the pre-[e21bf7a2ad] behaviour. Also fix a problem causing FTS to interpret tokens beginning with "*" characters as EOF. FossilOrigin-Name: 49dfee7cd1c9ab2901b8a871a6cd00b2ead76801 --- ext/fts3/fts3_expr.c | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/fts3expr4.test | 31 ++++++++++++++++++++++++++++--- 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/ext/fts3/fts3_expr.c b/ext/fts3/fts3_expr.c index f5d28cbfcc..2ba786ce80 100644 --- a/ext/fts3/fts3_expr.c +++ b/ext/fts3/fts3_expr.c @@ -190,7 +190,7 @@ static int getNextToken( /* Set variable i to the maximum number of bytes of input to tokenize. */ for(i=0; i Date: Thu, 9 Oct 2014 19:35:37 +0000 Subject: [PATCH 433/710] Change the balance_nonroot() routine to reduce the amount of memcpy work that takes place. This is a work in progress. FossilOrigin-Name: 29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 --- manifest | 21 +-- manifest.uuid | 2 +- src/btree.c | 494 +++++++++++++++++++++++++++----------------------- src/pager.c | 9 + src/pager.h | 2 + 5 files changed, 292 insertions(+), 236 deletions(-) diff --git a/manifest b/manifest index 574bfe2c8d..aeefa53238 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\samount\sof\smemcpy()\srequired\sby\sdefragmentPage(). -D 2014-09-27T05:00:25.096 +C Change\sthe\sbalance_nonroot()\sroutine\sto\sreduce\sthe\samount\sof\smemcpy\swork\sthat\stakes\splace.\sThis\sis\sa\swork\sin\sprogress. +D 2014-10-09T19:35:37.452 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 95a942a6ebdb23eb2a5d925526d35169aa6742f6 +F src/btree.c 7b89fde3bffa5b7300e94c4aeb69ccff926ef513 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -215,8 +215,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c caab007743821d96752597c9cfd7351654697b06 -F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 +F src/pager.c 0abcb0904a78d68b96357f360c6b160bcfc2a3e0 +F src/pager.h 8b6707cb32c788cf36bfc3d63f6d4b4fa689e7c2 F src/parse.y b98772da2bb5415970085b707203f92569400aa8 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a @@ -1200,10 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 83913515830aa850f9e38406f9422d7e88dcab66 -R 66a1e1f00a844450677737824735607d -T *branch * defrag-opt -T *sym-defrag-opt * -T -sym-trunk * -U drh -Z f60a1f2e4650c574e91ad245c0c67dee +P 3edab9957cc7bb90b52fd40b02613c2cb03fc166 +R 5fca1836b5a4d862df24682ecb47d048 +U dan +Z 8527330c8f275358262176fc962502e2 diff --git a/manifest.uuid b/manifest.uuid index bf8d17f152..bb422fd7dd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3edab9957cc7bb90b52fd40b02613c2cb03fc166 \ No newline at end of file +29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index c4832b4ee9..dbf002ecdb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5977,6 +5977,49 @@ static void assemblePage( pPage->nCell = (u16)nCell; } + +static void rebuildPage( + MemPage *pPg, /* Edit this page */ + int nRemove, /* Cells to remove from start of page */ + int nCell, /* Final number of cells on page */ + u8 **apCell, /* Array of nCell final cells */ + u16 *szCell /* Array of nCell cell sizes */ +){ + const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ + u8 * const aData = pPg->aData; /* Pointer to data for pPg */ + const int usableSize = pPg->pBt->usableSize; + u8 * const pEnd = &aData[usableSize]; + int i; + u8 *pCellptr = pPg->aCellIdx; + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + u8 *pData; + + i = get2byte(&aData[hdr+5]); + memcpy(&pTmp[i], &aData[i], usableSize - i); + pData = &aData[usableSize]; + + for(i=0; iaData && pCellnFree = (pData - pCellptr); + pPg->nCell = nCell; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+1], 0); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + aData[hdr+7] = 0x00; +} + /* ** The following parameters determine how many adjacent pages get involved ** in a balancing operation. NN is the number of neighbors on either side @@ -6098,7 +6141,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ } #endif /* SQLITE_OMIT_QUICKBALANCE */ -#if 0 +#if 1 /* ** This function does not contribute anything to the operation of SQLite. ** it is sometimes activated temporarily while debugging code responsible @@ -6265,7 +6308,6 @@ static int balance_nonroot( int iOvflSpace = 0; /* First unused byte of aOvflSpace[] */ int szScratch; /* Size of scratch memory requested */ MemPage *apOld[NB]; /* pPage and up to two siblings */ - MemPage *apCopy[NB]; /* Private copies of apOld[] pages */ MemPage *apNew[NB+2]; /* pPage and up to NB siblings after balancing */ u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ @@ -6276,6 +6318,13 @@ static int balance_nonroot( u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ + int aShiftLeft[NB+2]; + int aShiftRight[NB+2]; + u8 abDone[NB+2]; + Pgno aPgno[NB+2]; + u16 aPgFlags[NB+2]; + + memset(abDone, 0, sizeof(abDone)); pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); @@ -6384,12 +6433,10 @@ static int balance_nonroot( /* ** Allocate space for memory structures */ - k = pBt->pageSize + ROUND8(sizeof(MemPage)); szScratch = nMaxCells*sizeof(u8*) /* apCell */ + nMaxCells*sizeof(u16) /* szCell */ - + pBt->pageSize /* aSpace1 */ - + k*nOld; /* Page copies (apCopy) */ + + pBt->pageSize; /* aSpace1 */ apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; @@ -6402,8 +6449,8 @@ static int balance_nonroot( /* ** Load pointers to all cells on sibling pages and the divider cells ** into the local apCell[] array. Make copies of the divider cells - ** into space obtained from aSpace1[] and remove the divider cells - ** from pParent. + ** into space obtained from aSpace1[]. The divider cells have already + ** been removed from pParent. ** ** If the siblings are on leaf pages, then the child pointers of the ** divider cells are stripped from the cells before they are copied @@ -6419,15 +6466,7 @@ static int balance_nonroot( leafData = apOld[0]->intKeyLeaf; for(i=0; ipageSize + k*i]; - memcpy(pOld, apOld[i], sizeof(MemPage)); - pOld->aData = (void*)&pOld[1]; - memcpy(pOld->aData, apOld[i]->aData, pBt->pageSize); + MemPage *pOld = apOld[i]; limit = pOld->nCell+pOld->nOverflow; if( pOld->nOverflow>0 ){ @@ -6556,10 +6595,10 @@ static int balance_nonroot( assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); #endif - TRACE(("BALANCE: old: %d %d %d ", - apOld[0]->pgno, - nOld>=2 ? apOld[1]->pgno : 0, - nOld>=3 ? apOld[2]->pgno : 0 + TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + apOld[0]->pgno, apOld[0]->nCell, + nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, + nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 )); /* @@ -6582,6 +6621,7 @@ static int balance_nonroot( assert( i>0 ); rc = allocateBtreePage(pBt, &pNew, &pgno, (bBulk ? 1 : pgno), 0); if( rc ) goto balance_cleanup; + zeroPage(pNew, pageFlags); apNew[i] = pNew; nNew++; @@ -6595,135 +6635,223 @@ static int balance_nonroot( } } - /* Free any old pages that were not reused as new pages. - */ - while( ipgno; - int minI = i; - for(j=i+1; jpgno<(unsigned)minV ){ - minI = j; - minV = apNew[j]->pgno; + for(i=0; ipgno; + aPgFlags[i] = apNew[i]->pDbPage->flags; + } + for(i=0; ipgno); + Pgno iMin = 0; + u16 flags = 0; + for(j=0; jiGt && (iMin==0 || iPgnoi ){ - MemPage *pT; - pT = apNew[i]; - apNew[i] = apNew[minI]; - apNew[minI] = pT; + if( apNew[i]->pgno!=iMin ){ + apNew[i]->pDbPage->flags = flags; + sqlite3PagerRekey(apNew[i]->pDbPage, iMin); + apNew[i]->pgno = iMin; } } - TRACE(("new: %d(%d) %d(%d) %d(%d) %d(%d) %d(%d)\n", - apNew[0]->pgno, szNew[0], + + TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " + "%d(%d nc=%d) %d(%d nc=%d)\n", + apNew[0]->pgno, szNew[0], cntNew[0], nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, + nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, nNew>=3 ? apNew[2]->pgno : 0, nNew>=3 ? szNew[2] : 0, + nNew>=3 ? cntNew[2] - cntNew[1] - !leafData : 0, nNew>=4 ? apNew[3]->pgno : 0, nNew>=4 ? szNew[3] : 0, - nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0)); + nNew>=4 ? cntNew[3] - cntNew[2] - !leafData : 0, + nNew>=5 ? apNew[4]->pgno : 0, nNew>=5 ? szNew[4] : 0, + nNew>=5 ? cntNew[4] - cntNew[3] - !leafData : 0 + )); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); put4byte(pRight, apNew[nNew-1]->pgno); - /* - ** Evenly distribute the data in apCell[] across the new pages. - ** Insert divider cells into pParent as necessary. - */ j = 0; for(i=0; inCell>0 || (nNew==1 && cntNew[0]==0) ); - assert( pNew->nOverflow==0 ); + /* At this point, "j" is the apCell[] index of the first cell currently + ** stored on page apNew[i]. Or, if apNew[i] was not one of the original + ** sibling pages, "j" should be set to nCell. Variable iFirst is set + ** to the apCell[] index of the first cell that will appear on the + ** page following this balancing operation. */ + int iFirst = (i==0 ? 0 : cntNew[i-1] + !leafData); /* new first cell */ + assert( inCell + apNew[i]->nOverflow; + aShiftRight[i] = cntNew[i] - j; + assert( i!=nOld-1 || j==nCell ); + if( jnOld ? apNew : apOld)[nOld-1]; + memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); + } - /* If the sibling page assembled above was not the right-most sibling, - ** insert a divider cell into the parent page. - */ - assert( iaData; + int cntOldNext = pNew->nCell + pNew->nOverflow; + int usableSize = pBt->usableSize; + int iNew = 0; + int iOld = 0; - assert( jleaf ){ - memcpy(&pNew->aData[8], pCell, 4); - }else if( leafData ){ - /* If the tree is a leaf-data tree, and the siblings are leaves, - ** then there is no divider cell in apCell[]. Instead, the divider - ** cell consists of the integer key for the right-most cell of - ** the sibling-page assembled above only. - */ - CellInfo info; - j--; - btreeParseCellPtr(pNew, apCell[j], &info); - pCell = pTemp; - sz = 4 + putVarint(&pCell[4], info.nKey); - pTemp = 0; - }else{ - pCell -= 4; - /* Obscure case for non-leaf-data trees: If the cell at pCell was - ** previously stored on a leaf node, and its reported size was 4 - ** bytes, then it may actually be smaller than this - ** (see btreeParseCellPtr(), 4 bytes is the minimum size of - ** any cell). But it is important to pass the correct size to - ** insertCell(), so reparse the cell now. - ** - ** Note that this can never happen in an SQLite data file, as all - ** cells are at least 4 bytes. It only happens in b-trees used - ** to evaluate "IN (SELECT ...)" and similar clauses. - */ - if( szCell[j]==4 ){ - assert(leafCorrection==4); - sz = cellSizePtr(pParent, pCell); + for(i=0; inCell + pOld->nOverflow + !leafData; + aOld = pOld->aData; + } + if( i==cntNew[iNew] ){ + pNew = apNew[++iNew]; + if( !leafData ) continue; + } + + /* Cell pCell is destined for new sibling page pNew. Originally, it + ** was either part of sibling page iOld (possibly an overflow page), + ** or else the divider cell to the left of sibling page iOld. So, + ** if sibling page iOld had the same page number as pNew, and if + ** pCell really was a part of sibling page iOld (not a divider or + ** overflow cell), we can skip updating the pointer map entries. */ + if( pNew->pgno!=aPgno[iOld] || pCell=&aOld[usableSize] ){ + if( !leafCorrection ){ + ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); + } + if( szCell[i]>pNew->minLocal ){ + ptrmapPutOvflPtr(pNew, pCell, &rc); } } - iOvflSpace += sz; - assert( sz<=pBt->maxLocal+23 ); - assert( iOvflSpace <= (int)pBt->pageSize ); - insertCell(pParent, nxDiv, pCell, sz, pTemp, pNew->pgno, &rc); - if( rc!=SQLITE_OK ) goto balance_cleanup; - assert( sqlite3PagerIswriteable(pParent->pDbPage) ); - - j++; - nxDiv++; } } - assert( j==nCell ); + + /* Insert new divider cells into pParent. */ + for(i=0; ileaf ){ + memcpy(&pNew->aData[8], pCell, 4); + }else if( leafData ){ + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of + ** the sibling-page assembled above only. + */ + CellInfo info; + j--; + btreeParseCellPtr(pNew, apCell[j], &info); + pCell = pTemp; + sz = 4 + putVarint(&pCell[4], info.nKey); + pTemp = 0; + }else{ + pCell -= 4; + /* Obscure case for non-leaf-data trees: If the cell at pCell was + ** previously stored on a leaf node, and its reported size was 4 + ** bytes, then it may actually be smaller than this + ** (see btreeParseCellPtr(), 4 bytes is the minimum size of + ** any cell). But it is important to pass the correct size to + ** insertCell(), so reparse the cell now. + ** + ** Note that this can never happen in an SQLite data file, as all + ** cells are at least 4 bytes. It only happens in b-trees used + ** to evaluate "IN (SELECT ...)" and similar clauses. + */ + if( szCell[j]==4 ){ + assert(leafCorrection==4); + sz = cellSizePtr(pParent, pCell); + } + } + iOvflSpace += sz; + assert( sz<=pBt->maxLocal+23 ); + assert( iOvflSpace <= (int)pBt->pageSize ); + insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc); + if( rc!=SQLITE_OK ) goto balance_cleanup; + assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + } + + /* Now update the actual sibling pages. The order in which they are updated + ** is important, as this code needs to avoid disrupting any page from which + ** cells may still to be read. In practice, this means: + ** + ** 1) If the aShiftLeft[] entry is less than 0, it is not safe to + ** update the page until the page to the left of the current page + ** (apNew[i-1]) has already been updated. + ** + ** 2) If the aShiftRight[] entry is less than 0, it is not safe to + ** update the page until the page to the right of the current page + ** (apNew[i+1]) has already been updated. + ** + ** If neither of the above apply, the page is safe to update. + */ + assert( aShiftRight[nNew-1]>=0 && aShiftLeft[0]==0 ); + for(i=0; i=nNew ? i-nNew : nNew-1-i); + if( abDone[iPg]==0 + && (aShiftLeft[iPg]>=0 || abDone[iPg-1]) + && (aShiftRight[iPg]>=0 || abDone[iPg+1]) + ){ + MemPage *pNew = apNew[iPg]; + int iLeft = ((iPg==0) ? 0 : cntNew[iPg-1] + !leafData); + rebuildPage(pNew, + aShiftLeft[iPg] < 0 ? (aShiftLeft[iPg]*-1) : 0, + cntNew[iPg] - iLeft, + &apCell[iLeft], + &szCell[iLeft] + ); + abDone[iPg] = 1; + assert( pNew->nOverflow==0 ); + assert( pNew->nCell==(cntNew[iPg] - (iPg==0?0:cntNew[iPg-1]+!leafData)) ); + } + } + assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); + assert( nOld>0 ); assert( nNew>0 ); - if( (pageFlags & PTF_LEAF)==0 ){ - u8 *zChild = &apCopy[nOld-1]->aData[8]; - memcpy(&apNew[nNew-1]->aData[8], zChild, 4); - } if( isRoot && pParent->nCell==0 && pParent->hdrOffset<=apNew[0]->nFree ){ /* The root page of the b-tree now contains no cells. The only sibling @@ -6746,116 +6874,36 @@ static int balance_nonroot( ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM ){ - /* Fix the pointer-map entries for all the cells that were shifted around. - ** There are several different types of pointer-map entries that need to - ** be dealt with by this routine. Some of these have been set already, but - ** many have not. The following is a summary: - ** - ** 1) The entries associated with new sibling pages that were not - ** siblings when this function was called. These have already - ** been set. We don't need to worry about old siblings that were - ** moved to the free-list - the freePage() code has taken care - ** of those. - ** - ** 2) The pointer-map entries associated with the first overflow - ** page in any overflow chains used by new divider cells. These - ** have also already been taken care of by the insertCell() code. - ** - ** 3) If the sibling pages are not leaves, then the child pages of - ** cells stored on the sibling pages may need to be updated. - ** - ** 4) If the sibling pages are not internal intkey nodes, then any - ** overflow pages used by these cells may need to be updated - ** (internal intkey nodes never contain pointers to overflow pages). - ** - ** 5) If the sibling pages are not leaves, then the pointer-map - ** entries for the right-child pages of each sibling may need - ** to be updated. - ** - ** Cases 1 and 2 are dealt with above by other code. The next - ** block deals with cases 3 and 4 and the one after that, case 5. Since - ** setting a pointer map entry is a relatively expensive operation, this - ** code only sets pointer map entries for child or overflow pages that have - ** actually moved between pages. */ - MemPage *pNew = apNew[0]; - MemPage *pOld = apCopy[0]; - int nOverflow = pOld->nOverflow; - int iNextOld = pOld->nCell + nOverflow; - int iOverflow = (nOverflow ? pOld->aiOvfl[0] : -1); - j = 0; /* Current 'old' sibling page */ - k = 0; /* Current 'new' sibling page */ - for(i=0; inCell + pOld->nOverflow; - if( pOld->nOverflow ){ - nOverflow = pOld->nOverflow; - iOverflow = i + !leafData + pOld->aiOvfl[0]; - } - isDivider = !leafData; - } - - assert(nOverflow>0 || iOverflowaiOvfl[0]==pOld->aiOvfl[1]-1); - assert(nOverflow<3 || pOld->aiOvfl[1]==pOld->aiOvfl[2]-1); - if( i==iOverflow ){ - isDivider = 1; - if( (--nOverflow)>0 ){ - iOverflow++; - } - } - - if( i==cntNew[k] ){ - /* Cell i is the cell immediately following the last cell on new - ** sibling page k. If the siblings are not leaf pages of an - ** intkey b-tree, then cell i is a divider cell. */ - pNew = apNew[++k]; - if( !leafData ) continue; - } - assert( jpgno!=pNew->pgno ){ - if( !leafCorrection ){ - ptrmapPut(pBt, get4byte(apCell[i]), PTRMAP_BTREE, pNew->pgno, &rc); - } - if( szCell[i]>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, apCell[i], &rc); - } - } + }else if( ISAUTOVACUUM && !leafCorrection ){ + /* Fix the pointer map entries associated with the right-child of each + ** sibling page. All other pointer map entries have already been taken + ** care of. */ + for(i=0; iaData[8]); + ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); } + } - if( !leafCorrection ){ - for(i=0; iaData[8]); - ptrmapPut(pBt, key, PTRMAP_BTREE, apNew[i]->pgno, &rc); - } - } + assert( pParent->isInit ); + TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + nOld, nNew, nCell)); -#if 0 + /* Free any old pages that were not reused as new pages. + */ + for(i=nNew; iisInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may ** cause an assert() statement to fail. */ ptrmapCheckPages(apNew, nNew); ptrmapCheckPages(&pParent, 1); -#endif } - - assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", - nOld, nNew, nCell)); +#endif /* ** Cleanup before returning. diff --git a/src/pager.c b/src/pager.c index 79bfe15f10..e68d147de5 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6835,6 +6835,14 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ return SQLITE_OK; } + +void sqlite3PagerRekey(DbPage *pPage, Pgno iNew){ + PgHdr *pPg = (PgHdr*)pPage; + assert( pPg->flags & PGHDR_DIRTY ); + assert( !subjRequiresPage(pPg) ); + sqlite3PcacheMove(pPg, iNew); +} + #endif /* @@ -7235,4 +7243,5 @@ int sqlite3PagerWalFramesize(Pager *pPager){ } #endif + #endif /* SQLITE_OMIT_DISKIO */ diff --git a/src/pager.h b/src/pager.h index c9ca8553b9..f3a04bbca6 100644 --- a/src/pager.h +++ b/src/pager.h @@ -188,6 +188,8 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); +void sqlite3PagerRekey(DbPage*, Pgno); + #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); #endif From bf59bf94b288a9be1fc6537b65ce5fa15742fda6 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Oct 2014 13:08:33 +0000 Subject: [PATCH 434/710] Cause the command-line shell to return non-zero if the final SQL statement is incomplete. FossilOrigin-Name: 177fff3b98b101b98f1e7d334b6a80530b645565 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 993d89e658..1d67001ce9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sFTS\stokenizers\sto\schoose\swhether\sor\snot\sto\sconsider\sthe\s"*"\scharacter\spart\sof\stokens\sor\snot.\sThis\srestores\sthe\spre-[e21bf7a2ad]\sbehaviour.\sAlso\sfix\sa\sproblem\scausing\sFTS\sto\sinterpret\stokens\sbeginning\swith\s"*"\scharacters\sas\sEOF. -D 2014-10-09T15:08:17.615 +C Cause\sthe\scommand-line\sshell\sto\sreturn\snon-zero\sif\sthe\sfinal\sSQL\sstatement\nis\sincomplete. +D 2014-10-10T13:08:33.966 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c f11533162b57ed5ed37f549add34cbcdf51f6712 -F src/shell.c 38f627b0885191357f55902a3ac199de90d79715 +F src/shell.c 18ee8bbe9502d8848072dc2eddd1ea09254ba494 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P bae36d544676c90e337381a83f4513b4d925ab05 -R f3cda043ab8d3408ba2f57353f2b5a88 -U dan -Z 374021995a45280d12595d646a4004c7 +P 49dfee7cd1c9ab2901b8a871a6cd00b2ead76801 +R cb2ddf412f0df2d20c53503ae15115d8 +U drh +Z 473a503ca2f65e4944005f1668696768 diff --git a/manifest.uuid b/manifest.uuid index f52ad1a7ce..20fdb6bf5a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49dfee7cd1c9ab2901b8a871a6cd00b2ead76801 \ No newline at end of file +177fff3b98b101b98f1e7d334b6a80530b645565 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index cd4dadb08c..3ca4b094bc 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3725,6 +3725,7 @@ static int process_input(ShellState *p, FILE *in){ if( nSql ){ if( !_all_whitespace(zSql) ){ fprintf(stderr, "Error: incomplete SQL: %s\n", zSql); + errCnt++; } free(zSql); } From 681fca0018cd931953350475bfe63e46d6ef9374 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Oct 2014 15:01:46 +0000 Subject: [PATCH 435/710] Fix a potential problem in the whereRangeScanEst() routine when STAT4 is active. The problem was introduced by recent enhancements. FossilOrigin-Name: 68e1b4de700b5291f79249a03e1a750c6b2c9ae4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1d67001ce9..a49120ba19 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Cause\sthe\scommand-line\sshell\sto\sreturn\snon-zero\sif\sthe\sfinal\sSQL\sstatement\nis\sincomplete. -D 2014-10-10T13:08:33.966 +C Fix\sa\spotential\sproblem\sin\sthe\swhereRangeScanEst()\sroutine\swhen\sSTAT4\sis\nactive.\s\sThe\sproblem\swas\sintroduced\sby\srecent\senhancements. +D 2014-10-10T15:01:46.922 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 6fe21e0f60a449af5d75d00e6d480370464a9a48 +F src/where.c d1e315e338f51e4e7ff83ac6231a81d1c0cd30f9 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 49dfee7cd1c9ab2901b8a871a6cd00b2ead76801 -R cb2ddf412f0df2d20c53503ae15115d8 +P 177fff3b98b101b98f1e7d334b6a80530b645565 +R 07357edada657f8492081fa01ca31a82 U drh -Z 473a503ca2f65e4944005f1668696768 +Z a44558a0f7d62314eeda3bb13a5723b9 diff --git a/manifest.uuid b/manifest.uuid index 20fdb6bf5a..dc8efe9532 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -177fff3b98b101b98f1e7d334b6a80530b645565 \ No newline at end of file +68e1b4de700b5291f79249a03e1a750c6b2c9ae4 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 011ad66c00..e3ab542525 100644 --- a/src/where.c +++ b/src/where.c @@ -2209,8 +2209,8 @@ static int whereRangeScanEst( assert( pLower==0 || (pLower->eOperator & (WO_GT|WO_GE))!=0 ); assert( pUpper==0 || (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); - assert( p->pKeyInfo!=0 && p->pKeyInfo->aSortOrder!=0 ); - if( p->pKeyInfo->aSortOrder[nEq] ){ + assert( p->aSortOrder!=0 ); + if( p->aSortOrder[nEq] ){ /* The roles of pLower and pUpper are swapped for a DESC index */ SWAP(WhereTerm*, pLower, pUpper); } From 6c97789d2eb2df436506132cc3a4f211a68c6aa5 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Oct 2014 15:47:46 +0000 Subject: [PATCH 436/710] Simplify the code used to generate the text for EXPLAIN QUERY PLAN. FossilOrigin-Name: beea1efc3a49cad08087fcbb18dbce71c873fe57 --- manifest | 12 +++--- manifest.uuid | 2 +- src/where.c | 100 +++++++++++++++++++++++++------------------------- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/manifest b/manifest index a49120ba19..d11aa809a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\sproblem\sin\sthe\swhereRangeScanEst()\sroutine\swhen\sSTAT4\sis\nactive.\s\sThe\sproblem\swas\sintroduced\sby\srecent\senhancements. -D 2014-10-10T15:01:46.922 +C Simplify\sthe\scode\sused\sto\sgenerate\sthe\stext\sfor\sEXPLAIN\sQUERY\sPLAN. +D 2014-10-10T15:47:46.266 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c d1e315e338f51e4e7ff83ac6231a81d1c0cd30f9 +F src/where.c 23b9e5dd96a51657fb7d81091c28cd84f54dc8a0 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 177fff3b98b101b98f1e7d334b6a80530b645565 -R 07357edada657f8492081fa01ca31a82 +P 68e1b4de700b5291f79249a03e1a750c6b2c9ae4 +R 1d64962e74f9e7c1f94f06e344d62c75 U drh -Z a44558a0f7d62314eeda3bb13a5723b9 +Z 5a7fbeb924ed3728c1bc69e5188e226a diff --git a/manifest.uuid b/manifest.uuid index dc8efe9532..948793a5c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -68e1b4de700b5291f79249a03e1a750c6b2c9ae4 \ No newline at end of file +beea1efc3a49cad08087fcbb18dbce71c873fe57 \ No newline at end of file diff --git a/src/where.c b/src/where.c index e3ab542525..d1f379b6c3 100644 --- a/src/where.c +++ b/src/where.c @@ -2737,9 +2737,8 @@ static void explainAppendTerm( /* ** Argument pLevel describes a strategy for scanning table pTab. This -** function returns a pointer to a string buffer containing a description -** of the subset of table rows scanned by the strategy in the form of an -** SQL expression. Or, if all rows are scanned, NULL is returned. +** function appends text to pStr that describes the subset of table +** rows scanned by the strategy in the form of an SQL expression. ** ** For example, if the query: ** @@ -2749,49 +2748,37 @@ static void explainAppendTerm( ** string similar to: ** ** "a=? AND b>?" -** -** The returned pointer points to memory obtained from sqlite3DbMalloc(). -** It is the responsibility of the caller to free the buffer when it is -** no longer required. */ -static char *explainIndexRange(sqlite3 *db, WhereLoop *pLoop, Table *pTab){ +void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ Index *pIndex = pLoop->u.btree.pIndex; u16 nEq = pLoop->u.btree.nEq; u16 nSkip = pLoop->u.btree.nSkip; int i, j; Column *aCol = pTab->aCol; i16 *aiColumn = pIndex->aiColumn; - StrAccum txt; - if( nEq==0 && (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ){ - return 0; - } - sqlite3StrAccumInit(&txt, 0, 0, SQLITE_MAX_LENGTH); - txt.db = db; - sqlite3StrAccumAppend(&txt, " (", 2); + if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; + sqlite3StrAccumAppend(pStr, " (", 2); for(i=0; i=nSkip ){ - explainAppendTerm(&txt, i, z, "="); + explainAppendTerm(pStr, i, z, "="); }else{ - if( i ) sqlite3StrAccumAppend(&txt, " AND ", 5); - sqlite3StrAccumAppend(&txt, "ANY(", 4); - sqlite3StrAccumAppendAll(&txt, z); - sqlite3StrAccumAppend(&txt, ")", 1); + if( i ) sqlite3StrAccumAppend(pStr, " AND ", 5); + sqlite3XPrintf(pStr, 0, "ANY(%s)", z); } } j = i; if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; - explainAppendTerm(&txt, i++, z, ">"); + explainAppendTerm(pStr, i++, z, ">"); } if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ char *z = aiColumn[j] < 0 ? "rowid" : aCol[aiColumn[j]].zName; - explainAppendTerm(&txt, i, z, "<"); + explainAppendTerm(pStr, i, z, "<"); } - sqlite3StrAccumAppend(&txt, ")", 1); - return sqlite3StrAccumFinish(&txt); + sqlite3StrAccumAppend(pStr, ")", 1); } /* @@ -2815,11 +2802,13 @@ static void explainOneScan( struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite3 *db = pParse->db; /* Database handle */ - char *zMsg; /* Text to add to EQP output */ int iId = pParse->iSelectId; /* Select id (left-most output column) */ int isSearch; /* True for a SEARCH. False for SCAN. */ WhereLoop *pLoop; /* The controlling WhereLoop object */ u32 flags; /* Flags that describe this loop */ + char *zMsg; /* Text to add to EQP output */ + StrAccum str; /* EQP output string */ + char zBuf[100]; /* Initial space for EQP output string */ pLoop = pLevel->pWLoop; flags = pLoop->wsFlags; @@ -2829,54 +2818,65 @@ static void explainOneScan( || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); - zMsg = sqlite3MPrintf(db, "%s", isSearch?"SEARCH":"SCAN"); + sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + str.db = db; + sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s SUBQUERY %d", zMsg,pItem->iSelectId); + sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); }else{ - zMsg = sqlite3MAppendf(db, zMsg, "%s TABLE %s", zMsg, pItem->zName); + sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); } if( pItem->zAlias ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s AS %s", zMsg, pItem->zAlias); + sqlite3XPrintf(&str, 0, " AS %s", pItem->zAlias); } - if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 - && ALWAYS(pLoop->u.btree.pIndex!=0) - ){ - const char *zFmt; - Index *pIdx = pLoop->u.btree.pIndex; - char *zWhere = explainIndexRange(db, pLoop, pItem->pTab); + if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + const char *zFmt = 0; + Index *pIdx; + + assert( pLoop->u.btree.pIndex!=0 ); + pIdx = pLoop->u.btree.pIndex; assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ - zFmt = zWhere ? "%s USING PRIMARY KEY%.0s%s" : "%s%.0s%s"; + if( pLoop->u.btree.nEq>0 + || (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 + ){ + zFmt = "PRIMARY KEY"; + } }else if( flags & WHERE_AUTO_INDEX ){ - zFmt = "%s USING AUTOMATIC COVERING INDEX%.0s%s"; + zFmt = "AUTOMATIC COVERING INDEX"; }else if( flags & WHERE_IDX_ONLY ){ - zFmt = "%s USING COVERING INDEX %s%s"; + zFmt = "COVERING INDEX %s"; }else{ - zFmt = "%s USING INDEX %s%s"; + zFmt = "INDEX %s"; + } + if( zFmt ){ + sqlite3StrAccumAppend(&str, " USING ", 7); + sqlite3XPrintf(&str, 0, zFmt, pIdx->zName); + explainIndexRange(&str, pLoop, pItem->pTab); } - zMsg = sqlite3MAppendf(db, zMsg, zFmt, zMsg, pIdx->zName, zWhere); - sqlite3DbFree(db, zWhere); }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s USING INTEGER PRIMARY KEY", zMsg); - + const char *zRange; if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid=?)", zMsg); + zRange = "(rowid=?)"; }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid>? AND rowid? AND rowid?)", zMsg); - }else if( ALWAYS(flags&WHERE_TOP_LIMIT) ){ - zMsg = sqlite3MAppendf(db, zMsg, "%s (rowid?)"; + }else{ + assert( flags&WHERE_TOP_LIMIT); + zRange = "(rowidu.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif - zMsg = sqlite3MAppendf(db, zMsg, "%s", zMsg); + zMsg = sqlite3StrAccumFinish(&str); sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); } } From 98545bb2edf182c7b29a14710fd1e80979b78151 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Oct 2014 17:20:39 +0000 Subject: [PATCH 437/710] Add the estimated number of output rows to the EXPLAIN QUERY PLAN output if compiled with SQLITE_EXPLAIN_ESTIMATED_ROWS. This feature is off by default for the time being. FossilOrigin-Name: daa8314fba9dc3c4f5e7fbda42c97604fbfc4392 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 7 +++++++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d11aa809a0..52085777df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\scode\sused\sto\sgenerate\sthe\stext\sfor\sEXPLAIN\sQUERY\sPLAN. -D 2014-10-10T15:47:46.266 +C Add\sthe\sestimated\snumber\sof\soutput\srows\sto\sthe\sEXPLAIN\sQUERY\sPLAN\soutput\nif\scompiled\swith\sSQLITE_EXPLAIN_ESTIMATED_ROWS.\s\sThis\sfeature\sis\soff\sby\ndefault\sfor\sthe\stime\sbeing. +D 2014-10-10T17:20:39.349 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 23b9e5dd96a51657fb7d81091c28cd84f54dc8a0 +F src/where.c b511252533ca9f70e6adfc0ffd5c82594df137a2 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 68e1b4de700b5291f79249a03e1a750c6b2c9ae4 -R 1d64962e74f9e7c1f94f06e344d62c75 +P beea1efc3a49cad08087fcbb18dbce71c873fe57 +R a0896f7681cce645a858136a6e4bda47 U drh -Z 5a7fbeb924ed3728c1bc69e5188e226a +Z 8fddc2c9d3b88cabdf0b2692b84338ef diff --git a/manifest.uuid b/manifest.uuid index 948793a5c8..10d95ef678 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -beea1efc3a49cad08087fcbb18dbce71c873fe57 \ No newline at end of file +daa8314fba9dc3c4f5e7fbda42c97604fbfc4392 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d1f379b6c3..5b639946cf 100644 --- a/src/where.c +++ b/src/where.c @@ -2875,6 +2875,13 @@ static void explainOneScan( sqlite3XPrintf(&str, 0, " VIRTUAL TABLE INDEX %d:%s", pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } +#endif +#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS + if( pLoop->nOut>=10 ){ + sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); + }else{ + sqlite3StrAccumAppend(&str, " (~1 row)", 9); + } #endif zMsg = sqlite3StrAccumFinish(&str); sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); From 2eeb7ae437b080ae73789bf7957effc77cc0bc90 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Oct 2014 17:44:03 +0000 Subject: [PATCH 438/710] Provide the new "-config CONFIG" option to the test/releasetest.tcl script, which allows one to run a single test configuration selected by name. FossilOrigin-Name: d479e32be205f6cd0474f002282eae6eec613f36 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/releasetest.tcl | 13 ++++++++++++- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 52085777df..0af18811ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sestimated\snumber\sof\soutput\srows\sto\sthe\sEXPLAIN\sQUERY\sPLAN\soutput\nif\scompiled\swith\sSQLITE_EXPLAIN_ESTIMATED_ROWS.\s\sThis\sfeature\sis\soff\sby\ndefault\sfor\sthe\stime\sbeing. -D 2014-10-10T17:20:39.349 +C Provide\sthe\snew\s"-config\sCONFIG"\soption\sto\sthe\stest/releasetest.tcl\sscript,\nwhich\sallows\sone\sto\srun\sa\ssingle\stest\sconfiguration\sselected\sby\sname. +D 2014-10-10T17:44:03.334 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -783,7 +783,7 @@ F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a -F test/releasetest.tcl a0df0dfc5e3ee83ade87b9cc96db41b52d590b9e +F test/releasetest.tcl aa7aea9bacd0d76b3d9d5754bb1846d10103af57 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P beea1efc3a49cad08087fcbb18dbce71c873fe57 -R a0896f7681cce645a858136a6e4bda47 +P daa8314fba9dc3c4f5e7fbda42c97604fbfc4392 +R d16c504828c83cfa33bdf3d0eaba9ed0 U drh -Z 8fddc2c9d3b88cabdf0b2692b84338ef +Z b39929cd4032458ed60510f729a1a0e8 diff --git a/manifest.uuid b/manifest.uuid index 10d95ef678..fd80ff2ddf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -daa8314fba9dc3c4f5e7fbda42c97604fbfc4392 \ No newline at end of file +d479e32be205f6cd0474f002282eae6eec613f36 \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index eb2e440013..99790d3248 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -13,6 +13,7 @@ optional) are: -makefile PATH-TO-MAKEFILE (default "releasetest.mk") -platform PLATFORM (see below) -quick BOOLEAN (default "0") + -config CONFIGNAME (Run only CONFIGNAME) The default value for -makefile is "./releasetest.mk". @@ -292,6 +293,7 @@ proc run_test_suite {name testtarget config} { proc process_options {argv} { set ::MAKEFILE releasetest.mk ;# Default value set ::QUICK 0 ;# Default value + set config {} set platform $::tcl_platform(os)-$::tcl_platform(machine) for {set i 0} {$i < [llength $argv]} {incr i} { @@ -310,6 +312,11 @@ proc process_options {argv} { incr i set ::QUICK [lindex $argv $i] } + + -config { + incr i + set config [lindex $argv $i] + } default { puts stderr "" @@ -333,7 +340,11 @@ proc process_options {argv} { exit } - set ::CONFIGLIST $::Platforms($platform) + if {$config!=""} { + set ::CONFIGLIST $config + } else { + set ::CONFIGLIST $::Platforms($platform) + } puts "Running the following configurations for $platform:" puts " [string trim $::CONFIGLIST]" } From e35626fabdb0183eafd481930e2a487a973cee3d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Oct 2014 17:47:00 +0000 Subject: [PATCH 439/710] When using the -config option to releasetest.tcl, default the testing type to "fulltest" if it is not specified on the command line. FossilOrigin-Name: cf291cbe9f49396f03cfca39c2e892f27f750107 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/releasetest.tcl | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 0af18811ed..258e0ade2b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Provide\sthe\snew\s"-config\sCONFIG"\soption\sto\sthe\stest/releasetest.tcl\sscript,\nwhich\sallows\sone\sto\srun\sa\ssingle\stest\sconfiguration\sselected\sby\sname. -D 2014-10-10T17:44:03.334 +C When\susing\sthe\s-config\soption\sto\sreleasetest.tcl,\sdefault\sthe\stesting\stype\nto\s"fulltest"\sif\sit\sis\snot\sspecified\son\sthe\scommand\sline. +D 2014-10-10T17:47:00.230 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -783,7 +783,7 @@ F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a -F test/releasetest.tcl aa7aea9bacd0d76b3d9d5754bb1846d10103af57 +F test/releasetest.tcl 4296b9adbc5992bcd0b0b2876b7651f57c1494f2 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P daa8314fba9dc3c4f5e7fbda42c97604fbfc4392 -R d16c504828c83cfa33bdf3d0eaba9ed0 +P d479e32be205f6cd0474f002282eae6eec613f36 +R 931ed67b9f38d30c31761dd7af84265b U drh -Z b39929cd4032458ed60510f729a1a0e8 +Z 3cea3f1ee45c98cf88d87a6691b7a96c diff --git a/manifest.uuid b/manifest.uuid index fd80ff2ddf..29909b01ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d479e32be205f6cd0474f002282eae6eec613f36 \ No newline at end of file +cf291cbe9f49396f03cfca39c2e892f27f750107 \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index 99790d3248..589319d680 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -341,6 +341,7 @@ proc process_options {argv} { } if {$config!=""} { + if {[llength $config]==1} {lappend config fulltest} set ::CONFIGLIST $config } else { set ::CONFIGLIST $::Platforms($platform) From ab993380a27226883381d21d5401d76df7fccaee Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Oct 2014 18:09:52 +0000 Subject: [PATCH 440/710] Changes to enable compiling using VC6. FossilOrigin-Name: 9ee5686ab3b0bca8cabdf455c75bd9410cdc5378 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 2 +- src/threads.c | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 258e0ade2b..fd3281f6c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\susing\sthe\s-config\soption\sto\sreleasetest.tcl,\sdefault\sthe\stesting\stype\nto\s"fulltest"\sif\sit\sis\snot\sspecified\son\sthe\scommand\sline. -D 2014-10-10T17:47:00.230 +C Changes\sto\senable\scompiling\susing\sVC6. +D 2014-10-10T18:09:52.113 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -232,7 +232,7 @@ F src/shell.c 18ee8bbe9502d8848072dc2eddd1ea09254ba494 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 6ac5e34a590ad7ea22af91d190bdb212b12107be +F src/sqliteInt.h c417a25e2369f705b651897a2f1cc8da0e6aa1c4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -282,7 +282,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 22dded4283dc4b25422f6444cdcb8d6b1ea0b5ff +F src/threads.c 60c9d400abf17ccdc8767cdc6af90b9c5acf58bd F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d479e32be205f6cd0474f002282eae6eec613f36 -R 931ed67b9f38d30c31761dd7af84265b +P cf291cbe9f49396f03cfca39c2e892f27f750107 +R 9d22f3b27c328e6d3cb49e4853606989 U drh -Z 3cea3f1ee45c98cf88d87a6691b7a96c +Z 2221fbb0fe2956ed5002647d0de83cc0 diff --git a/manifest.uuid b/manifest.uuid index 29909b01ca..8d12ba50f3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cf291cbe9f49396f03cfca39c2e892f27f750107 \ No newline at end of file +9ee5686ab3b0bca8cabdf455c75bd9410cdc5378 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 7998638c7f..e648430353 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -159,7 +159,7 @@ */ #if defined(__GNUC__) # define SQLITE_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) +#elif defined(_MSC_VER) && _MSC_VER>=1310 # define SQLITE_NOINLINE __declspec(noinline) #else # define SQLITE_NOINLINE diff --git a/src/threads.c b/src/threads.c index 213a129c99..6d39042fd9 100644 --- a/src/threads.c +++ b/src/threads.c @@ -105,7 +105,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ /* A running thread */ struct SQLiteThread { - uintptr_t tid; /* The thread handle */ + void *tid; /* The thread handle */ unsigned id; /* The thread identifier */ void *(*xTask)(void*); /* The routine to run as a thread */ void *pIn; /* Argument to xTask */ @@ -153,7 +153,7 @@ int sqlite3ThreadCreate( }else{ p->xTask = xTask; p->pIn = pIn; - p->tid = _beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id); + p->tid = (void*)_beginthreadex(0, 0, sqlite3ThreadProc, p, 0, &p->id); if( p->tid==0 ){ memset(p, 0, sizeof(*p)); } From 5bd8af7c6b9b0b3947d1da11311af25dee1875b4 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 10 Oct 2014 19:10:59 +0000 Subject: [PATCH 441/710] Fix a failing test case in index5.test. Also tweak the way cache memory is divided between read-only and dirty pages when using SQLITE_CONFIG_PAGECACHE to reduce IO in some cases. FossilOrigin-Name: 8541dfb3bbdf63dc9ab304d8a0ab8b290cdc9d96 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pcache1.c | 2 +- test/index5.test | 5 ++++- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index fd3281f6c5..1b392cbba7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\senable\scompiling\susing\sVC6. -D 2014-10-10T18:09:52.113 +C Fix\sa\sfailing\stest\scase\sin\sindex5.test.\sAlso\stweak\sthe\sway\scache\smemory\sis\sdivided\sbetween\sread-only\sand\sdirty\spages\swhen\susing\sSQLITE_CONFIG_PAGECACHE\sto\sreduce\sIO\sin\ssome\scases. +D 2014-10-10T19:10:59.145 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -220,7 +220,7 @@ F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a -F src/pcache1.c dab8ab930d4a73b99768d881185994f34b80ecaa +F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c 6b79bbd063dcbadca4cf617a4cde255bcc13ea64 @@ -631,7 +631,7 @@ F test/index.test 4d990005a67a36984e4f1a5f1bdccea8d08da4ee F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 55a90cff99834305e8141df7afaef39674b57062 F test/index4.test ab92e736d5946840236cd61ac3191f91a7856bf6 -F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33 +F test/index5.test 25b0b451aceed4ac5f7d49f856f6de7257470b3e F test/index6.test fb370966ac3cd0989053dd5385757b5c3e24ab6a F test/index7.test 917cf1e1c7439bb155abbeabec511b28945e157b F test/indexedby.test b2f22f3e693a53813aa3f50b812eb609ba6df1ec @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cf291cbe9f49396f03cfca39c2e892f27f750107 -R 9d22f3b27c328e6d3cb49e4853606989 -U drh -Z 2221fbb0fe2956ed5002647d0de83cc0 +P 9ee5686ab3b0bca8cabdf455c75bd9410cdc5378 +R a867243fa8e0f0e41c6d9028f2d48296 +U dan +Z 113af8b0b8de7f680d672bce9ecfd1d0 diff --git a/manifest.uuid b/manifest.uuid index 8d12ba50f3..14bf033b25 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ee5686ab3b0bca8cabdf455c75bd9410cdc5378 \ No newline at end of file +8541dfb3bbdf63dc9ab304d8a0ab8b290cdc9d96 \ No newline at end of file diff --git a/src/pcache1.c b/src/pcache1.c index 9d15e8514c..a8c3217382 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -688,7 +688,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( if( createFlag==1 && ( nPinned>=pGroup->mxPinned || nPinned>=pCache->n90pct - || pcache1UnderMemoryPressure(pCache) + || (pcache1UnderMemoryPressure(pCache) && pCache->nRecyclable Date: Fri, 10 Oct 2014 19:15:35 +0000 Subject: [PATCH 442/710] Restrict the scope of the explainIndexRange() function in where.c. FossilOrigin-Name: c30124520027f0f860223bf842e2f09db3dafb5f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1b392cbba7..d84631e220 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfailing\stest\scase\sin\sindex5.test.\sAlso\stweak\sthe\sway\scache\smemory\sis\sdivided\sbetween\sread-only\sand\sdirty\spages\swhen\susing\sSQLITE_CONFIG_PAGECACHE\sto\sreduce\sIO\sin\ssome\scases. -D 2014-10-10T19:10:59.145 +C Restrict\sthe\sscope\sof\sthe\sexplainIndexRange()\sfunction\sin\swhere.c. +D 2014-10-10T19:15:35.023 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c b511252533ca9f70e6adfc0ffd5c82594df137a2 +F src/where.c 75af78b4a4125e891f62d997d6d93b006a8c7d68 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9ee5686ab3b0bca8cabdf455c75bd9410cdc5378 -R a867243fa8e0f0e41c6d9028f2d48296 -U dan -Z 113af8b0b8de7f680d672bce9ecfd1d0 +P 8541dfb3bbdf63dc9ab304d8a0ab8b290cdc9d96 +R 56927b83c6bc9f8ed8f382263f0b3a19 +U drh +Z 9069bb5e423e6471fadcdc96cb083909 diff --git a/manifest.uuid b/manifest.uuid index 14bf033b25..ced410d581 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8541dfb3bbdf63dc9ab304d8a0ab8b290cdc9d96 \ No newline at end of file +c30124520027f0f860223bf842e2f09db3dafb5f \ No newline at end of file diff --git a/src/where.c b/src/where.c index 5b639946cf..bd59055b4a 100644 --- a/src/where.c +++ b/src/where.c @@ -2749,7 +2749,7 @@ static void explainAppendTerm( ** ** "a=? AND b>?" */ -void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ +static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ Index *pIndex = pLoop->u.btree.pIndex; u16 nEq = pLoop->u.btree.nEq; u16 nSkip = pLoop->u.btree.nSkip; From b6453201d2ec13d7c71da62e2d11cae427d3bf04 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 10 Oct 2014 20:52:53 +0000 Subject: [PATCH 443/710] Account for the ASC/DESC properties of ORDER BY expressions when using the same index for GROUP BY and ORDER BY. Candidate fix for [ba7cbfaedc]. FossilOrigin-Name: 2a9573962b837973c4959465d8a5f2641d109a5a --- manifest | 18 ++++++----- manifest.uuid | 2 +- src/where.c | 17 +++++++---- test/tkt-ba7cbfaedc.test | 65 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 13 deletions(-) create mode 100644 test/tkt-ba7cbfaedc.test diff --git a/manifest b/manifest index d84631e220..ced935d9a0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Restrict\sthe\sscope\sof\sthe\sexplainIndexRange()\sfunction\sin\swhere.c. -D 2014-10-10T19:15:35.023 +C Account\sfor\sthe\sASC/DESC\sproperties\sof\sORDER\sBY\sexpressions\swhen\susing\sthe\ssame\sindex\sfor\sGROUP\sBY\sand\sORDER\sBY.\sCandidate\sfix\sfor\s[ba7cbfaedc]. +D 2014-10-10T20:52:53.529 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 75af78b4a4125e891f62d997d6d93b006a8c7d68 +F src/where.c 7137da023fa550b44f98251ab6fcf132ce4e371e F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -933,6 +933,7 @@ F test/tkt-b1d3a2e531.test 8f7576e41ca179289ee1a8fee28386fd8e4b0550 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 F test/tkt-b75a9ca6b0.test 97cc2d5eeaf82799eb42138c0a1ff64370238ce4 +F test/tkt-ba7cbfaedc.test e76d88e572e489ee0d64fe4caf4af18b3d1dc688 F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 F test/tkt-bdc6bbbb38.test fc38bb09bdd440e3513a1f5f98fc60a075182d7d F test/tkt-c48d99d690.test ba61977d62ab612fc515b3c488a6fbd6464a2447 @@ -1203,7 +1204,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8541dfb3bbdf63dc9ab304d8a0ab8b290cdc9d96 -R 56927b83c6bc9f8ed8f382263f0b3a19 -U drh -Z 9069bb5e423e6471fadcdc96cb083909 +P c30124520027f0f860223bf842e2f09db3dafb5f +R 8de40b978c4dd5c8456f5f796f63e003 +T *branch * experimental +T *sym-experimental * +T -sym-trunk * +U dan +Z 060424e4e85bb8595890bea0de65ac0e diff --git a/manifest.uuid b/manifest.uuid index ced410d581..604705047d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c30124520027f0f860223bf842e2f09db3dafb5f \ No newline at end of file +2a9573962b837973c4959465d8a5f2641d109a5a \ No newline at end of file diff --git a/src/where.c b/src/where.c index bd59055b4a..6a5e918669 100644 --- a/src/where.c +++ b/src/where.c @@ -5827,12 +5827,19 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr ){ - Bitmask notUsed = 0; - int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, - pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used - ); + Bitmask revMask = 0; + int nOrder; + assert( pWInfo->wctrlFlags & WHERE_GROUPBY ); assert( pWInfo->sorted==0 ); - pWInfo->sorted = (nOrder==pWInfo->pOrderBy->nExpr); + pWInfo->wctrlFlags &= ~WHERE_GROUPBY; + nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, + pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask + ); + pWInfo->wctrlFlags |= WHERE_GROUPBY; + if( nOrder==pWInfo->pOrderBy->nExpr ){ + pWInfo->sorted = 1; + pWInfo->revMask = revMask; + } } } diff --git a/test/tkt-ba7cbfaedc.test b/test/tkt-ba7cbfaedc.test new file mode 100644 index 0000000000..a558d4cec3 --- /dev/null +++ b/test/tkt-ba7cbfaedc.test @@ -0,0 +1,65 @@ +# 2014-10-11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# Test that ticket [ba7cbfaedc] has been fixed. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix tkt-ba7cbfaedc + +do_execsql_test 1 { + CREATE TABLE t1 (x, y); + INSERT INTO t1 VALUES (3, 'a'); + INSERT INTO t1 VALUES (1, 'a'); + INSERT INTO t1 VALUES (2, 'b'); + INSERT INTO t1 VALUES (2, 'a'); + INSERT INTO t1 VALUES (3, 'b'); + INSERT INTO t1 VALUES (1, 'b'); +} + +do_execsql_test 1.1 { + CREATE INDEX i1 ON t1(x, y); +} + +foreach {n idx} { + 1 { CREATE INDEX i1 ON t1(x, y) } + 2 { CREATE INDEX i1 ON t1(x DESC, y) } + 3 { CREATE INDEX i1 ON t1(x, y DESC) } + 4 { CREATE INDEX i1 ON t1(x DESC, y DESC) } +} { + catchsql { DROP INDEX i1 } + execsql $idx + foreach {tn q res} { + 1 "GROUP BY x, y ORDER BY x, y" {1 a 1 b 2 a 2 b 3 a 3 b} + 2 "GROUP BY x, y ORDER BY x DESC, y" {3 a 3 b 2 a 2 b 1 a 1 b} + 3 "GROUP BY x, y ORDER BY x, y DESC" {1 b 1 a 2 b 2 a 3 b 3 a} + 4 "GROUP BY x, y ORDER BY x DESC, y DESC" {3 b 3 a 2 b 2 a 1 b 1 a} + } { + do_execsql_test 1.$n.$tn "SELECT * FROM t1 $q" $res + } +} + +do_execsql_test 2.0 { + drop table if exists t1; + create table t1(id int); + insert into t1(id) values(1),(2),(3),(4),(5); + create index t1_idx_id on t1(id asc); + select * from t1 group by id order by id; + select * from t1 group by id order by id asc; + select * from t1 group by id order by id desc; +} { + 1 2 3 4 5 1 2 3 4 5 5 4 3 2 1 +} + +finish_test + + From c631faa922b129adac37c2e26dd38e77897fcb7a Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Oct 2014 01:22:16 +0000 Subject: [PATCH 444/710] Remove an unnecessary conditional from the EXPLAIN QUERY PLAN logic. FossilOrigin-Name: c5dc83ebded914f07286b7f98d0a50c28c16f609 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 4 +--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index d84631e220..b8991f9317 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Restrict\sthe\sscope\sof\sthe\sexplainIndexRange()\sfunction\sin\swhere.c. -D 2014-10-10T19:15:35.023 +C Remove\san\sunnecessary\sconditional\sfrom\sthe\sEXPLAIN\sQUERY\sPLAN\slogic. +D 2014-10-11T01:22:16.169 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 75af78b4a4125e891f62d997d6d93b006a8c7d68 +F src/where.c 45202a9a2848848a17b64316fe87bdb5a75f5910 F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1203,7 +1203,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8541dfb3bbdf63dc9ab304d8a0ab8b290cdc9d96 -R 56927b83c6bc9f8ed8f382263f0b3a19 +P c30124520027f0f860223bf842e2f09db3dafb5f +R 76a395c9de317da89295170f9596fa71 U drh -Z 9069bb5e423e6471fadcdc96cb083909 +Z efd3e7a68804083aa9d3740b204464db diff --git a/manifest.uuid b/manifest.uuid index ced410d581..e90d20a635 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c30124520027f0f860223bf842e2f09db3dafb5f \ No newline at end of file +c5dc83ebded914f07286b7f98d0a50c28c16f609 \ No newline at end of file diff --git a/src/where.c b/src/where.c index bd59055b4a..4aa208d2ee 100644 --- a/src/where.c +++ b/src/where.c @@ -2838,9 +2838,7 @@ static void explainOneScan( pIdx = pLoop->u.btree.pIndex; assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ - if( pLoop->u.btree.nEq>0 - || (pLoop->wsFlags & (WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 - ){ + if( isSearch ){ zFmt = "PRIMARY KEY"; } }else if( flags & WHERE_AUTO_INDEX ){ From 8683e08676d96538bc2cb05d04cbff4db110f890 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Oct 2014 10:52:54 +0000 Subject: [PATCH 445/710] Require the SQLITE_ENABLE_RTREE compile-time option in speedtest1.c in order to enable the R-Tree tests. FossilOrigin-Name: 5d29a033b0f17b0fd74656b28a8367a9a9067f81 --- manifest | 13 ++++++------- manifest.uuid | 2 +- test/speedtest1.c | 11 +++++++++++ 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0395f4e8a7..f11096d9d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthat\sa\sGROUP\sBY\sthat\salso\simplements\san\sORDER\sBY\sscans\sthe\stable\nin\sthe\scorrect\sorder.\s\sFix\sfor\sticket\s[ba7cbfaedc7e6]. -D 2014-10-11T02:12:58.777 +C Require\sthe\sSQLITE_ENABLE_RTREE\scompile-time\soption\sin\sspeedtest1.c\sin\sorder\nto\senable\sthe\sR-Tree\stests. +D 2014-10-11T10:52:54.590 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -860,7 +860,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 83f6b3318f7ee60e52b978b5a5e5dd7e83dfb7ee +F test/speedtest1.c e4e2aa37ff66bad9f414a50a8cb9edfaac65c9e5 F test/spellfix.test 24f676831acddd2f4056a598fd731a72c6311f49 F test/sqllimits1.test 9014524e7ab16e2a4976b13397db4c29cc29c6d9 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1204,8 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c5dc83ebded914f07286b7f98d0a50c28c16f609 2a9573962b837973c4959465d8a5f2641d109a5a -R 6f36f8a6a865ae8ec418dbbf6437f4b9 -T +closed 2a9573962b837973c4959465d8a5f2641d109a5a +P 7a32fdfd4be2138c0ab00f3dc6f54a70e4e07be4 +R 4281e286fbeeaa18c9716d9bb0cdd63c U drh -Z b85f219ca12b10b385d8fb433d0ee994 +Z 50870d5e316ccf74380338fbaf696334 diff --git a/manifest.uuid b/manifest.uuid index c394cc6a6e..44905c9e54 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7a32fdfd4be2138c0ab00f3dc6f54a70e4e07be4 \ No newline at end of file +5d29a033b0f17b0fd74656b28a8367a9a9067f81 \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 8589b16333..8e5b74c56e 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -934,6 +934,7 @@ void testset_cte(void){ } +#ifdef SQLITE_ENABLE_RTREE /* Generate two numbers between 1 and mx. The first number is less than ** the second. Usually the numbers are near each other but can sometimes ** be far apart. @@ -954,7 +955,9 @@ static void twoCoords( *pX0 = x0; *pX1 = x1; } +#endif +#ifdef SQLITE_ENABLE_RTREE /* The following routine is an R-Tree geometry callback. It returns ** true if the object overlaps a slice on the Y coordinate between the ** two values given as arguments. In other words @@ -974,7 +977,9 @@ static int xsliceGeometryCallback( *pRes = aCoord[3]>=p->aParam[0] && aCoord[2]<=p->aParam[1]; return SQLITE_OK; } +#endif /* SQLITE_ENABLE_RTREE */ +#ifdef SQLITE_ENABLE_RTREE /* ** A testset for the R-Tree virtual table */ @@ -1110,6 +1115,7 @@ void testset_rtree(int p1, int p2){ } speedtest1_end_test(); } +#endif /* SQLITE_ENABLE_RTREE */ /* ** A testset used for debugging speedtest1 itself. @@ -1329,7 +1335,12 @@ int main(int argc, char **argv){ }else if( strcmp(zTSet,"cte")==0 ){ testset_cte(); }else if( strcmp(zTSet,"rtree")==0 ){ +#ifdef SQLITE_ENABLE_RTREE testset_rtree(6, 147); +#else + fatal_error("compile with -DSQLITE_ENABLE_RTREE to enable " + "the R-Tree tests\n"); +#endif }else{ fatal_error("unknown testset: \"%s\"\nChoices: main debug1 cte rtree\n", zTSet); From d6176c413164d7ea4240af9be56889a0461715be Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Oct 2014 17:22:55 +0000 Subject: [PATCH 446/710] Simplification to the insertCell() routine in btree.c, resulting in a performance boost and a very small size decrease. It turns out that the extra work involved in sometimes avoiding an memcpy() of the first four bytes of a record takes more time than just unconditionally copying those four bytes. FossilOrigin-Name: 66de15580d3c289601e67debfe1edee286f4db5f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 10 ++-------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index f11096d9d2..517771d6a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Require\sthe\sSQLITE_ENABLE_RTREE\scompile-time\soption\sin\sspeedtest1.c\sin\sorder\nto\senable\sthe\sR-Tree\stests. -D 2014-10-11T10:52:54.590 +C Simplification\sto\sthe\sinsertCell()\sroutine\sin\sbtree.c,\sresulting\sin\sa\nperformance\sboost\sand\sa\svery\ssmall\ssize\sdecrease.\s\sIt\sturns\sout\sthat\sthe\nextra\swork\sinvolved\sin\ssometimes\savoiding\san\smemcpy()\sof\sthe\sfirst\sfour\sbytes\nof\sa\srecord\stakes\smore\stime\sthan\sjust\sunconditionally\scopying\sthose\nfour\sbytes. +D 2014-10-11T17:22:55.486 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c fa00618117fb6bb46c243452c56997c0d22d4fc9 +F src/btree.c 5c0b78c49d00da49a0e8e3d2738900a431df7fca F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c 9e5205db9a0c8a1a4ce7379d60a2a34cb0b7339c @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7a32fdfd4be2138c0ab00f3dc6f54a70e4e07be4 -R 4281e286fbeeaa18c9716d9bb0cdd63c +P 5d29a033b0f17b0fd74656b28a8367a9a9067f81 +R 8d2ebada6cfcd93628f86065b0703bad U drh -Z 50870d5e316ccf74380338fbaf696334 +Z b915e33ee9cbdc5e832f0d58216a79da diff --git a/manifest.uuid b/manifest.uuid index 44905c9e54..e1067663de 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5d29a033b0f17b0fd74656b28a8367a9a9067f81 \ No newline at end of file +66de15580d3c289601e67debfe1edee286f4db5f \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 12dcb44cba..cfe623342a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5845,11 +5845,6 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** in pTemp or the original pCell) and also record its index. ** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. -** -** If nSkip is non-zero, then do not copy the first nSkip bytes of the -** cell. The caller will overwrite them after this function returns. If -** nSkip is non-zero, then pCell may not point to an invalid memory location -** (but pCell+nSkip is always valid). */ static void insertCell( MemPage *pPage, /* Page into which we are copying */ @@ -5866,7 +5861,6 @@ static void insertCell( int ins; /* Index in data[] where new cell pointer is inserted */ int cellOffset; /* Address of first cell pointer in data[] */ u8 *data; /* The content of the whole page */ - int nSkip = (iChild ? 4 : 0); if( *pRC ) return; @@ -5884,7 +5878,7 @@ static void insertCell( assert( sz==cellSizePtr(pPage, pCell) || (sz==8 && iChild>0) ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ - memcpy(pTemp+nSkip, pCell+nSkip, sz-nSkip); + memcpy(pTemp, pCell, sz); pCell = pTemp; } if( iChild ){ @@ -5913,7 +5907,7 @@ static void insertCell( assert( idx+sz <= (int)pPage->pBt->usableSize ); pPage->nCell++; pPage->nFree -= (u16)(2 + sz); - memcpy(&data[idx+nSkip], pCell+nSkip, sz-nSkip); + memcpy(&data[idx], pCell, sz); if( iChild ){ put4byte(&data[idx], iChild); } From 09c6840153697ea6679650b65dfa50bc15cef536 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 11 Oct 2014 20:00:24 +0000 Subject: [PATCH 447/710] Attempt to further reduce memcpy() in balance_nonroot(). FossilOrigin-Name: fec849dcca3aead2bc2d4ecffeda750684d32fb0 --- manifest | 12 ++-- manifest.uuid | 2 +- src/btree.c | 185 ++++++++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 179 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index aeefa53238..7e782aa82c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sbalance_nonroot()\sroutine\sto\sreduce\sthe\samount\sof\smemcpy\swork\sthat\stakes\splace.\sThis\sis\sa\swork\sin\sprogress. -D 2014-10-09T19:35:37.452 +C Attempt\sto\sfurther\sreduce\smemcpy()\sin\sbalance_nonroot(). +D 2014-10-11T20:00:24.552 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 7b89fde3bffa5b7300e94c4aeb69ccff926ef513 +F src/btree.c d5f4f74e309f79ace4b4025c433874ead635bed2 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3edab9957cc7bb90b52fd40b02613c2cb03fc166 -R 5fca1836b5a4d862df24682ecb47d048 +P 29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 +R 3235ac37769ff059d7c714947e8a11dd U dan -Z 8527330c8f275358262176fc962502e2 +Z 17fdfd7faf788bbb79d2ec9941c17025 diff --git a/manifest.uuid b/manifest.uuid index bb422fd7dd..006b3c0008 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 \ No newline at end of file +fec849dcca3aead2bc2d4ecffeda750684d32fb0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index dbf002ecdb..13ec912748 100644 --- a/src/btree.c +++ b/src/btree.c @@ -5980,10 +5980,9 @@ static void assemblePage( static void rebuildPage( MemPage *pPg, /* Edit this page */ - int nRemove, /* Cells to remove from start of page */ int nCell, /* Final number of cells on page */ - u8 **apCell, /* Array of nCell final cells */ - u16 *szCell /* Array of nCell cell sizes */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ ){ const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ u8 * const aData = pPg->aData; /* Pointer to data for pPg */ @@ -6020,6 +6019,129 @@ static void rebuildPage( aData[hdr+7] = 0x00; } +static void editPage( + MemPage *pPg, /* Edit this page */ + int iOld, /* Index of first cell currently on page */ + int iNew, /* Index of new first cell on page */ + int nNew, /* Final number of cells on page */ + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ +){ + + if( 1 ){ + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nFree = pPg->nFree; /* Free bytes on pPg */ + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + +#ifdef SQLITE_DEBUG + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); +#endif + + /* Remove cells from the start and end of the page */ + if( iOldaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ + freeSpace(pPg, apCell[i] - aData, szCell[i]); + nFree += szCell[i] + 2; + nShift++; + } + } + nCell -= nShift; + memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + } + for(i=iNew+nNew; iaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ + freeSpace(pPg, apCell[i] - aData, szCell[i]); + nFree += szCell[i] + 2; + nCell--; + } + } + pData = &aData[get2byte(&aData[hdr+5])]; + if( pDataaCellIdx; + memmove(&pCellptr[(iOld-iNew)*2], pCellptr, nCell*2); + for(i=iNew; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + int sz = szCell[iCell+iNew]; + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + pData -= sz; + if( pDataaCellIdx[nCell*2]; + for(i=iNew+nCell; i<(iNew+nNew); i++){ + pData -= szCell[i]; + if( pDatanFree = nFree; + pPg->nCell = nNew; + pPg->nOverflow = 0; + + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); + +#ifdef SQLITE_DEBUG + for(i=0; iaCellIdx[i*2]); + if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ + pCell = &pTmp[pCell - aData]; + } + assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + } +#endif + +#if 0 + printf("EDIT\n"); +#endif + + return; + } + + editpage_fail: +#if 0 + printf("REBUILD\n"); +#endif + /* Unable to edit this page. Rebuild it from scratch instead. */ + rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]); +} + /* ** The following parameters determine how many adjacent pages get involved ** in a balancing operation. NN is the number of neighbors on either side @@ -6312,6 +6434,7 @@ static int balance_nonroot( u8 *pRight; /* Location in parent of right-sibling pointer */ u8 *apDiv[NB-1]; /* Divider cells in pParent */ int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ + int cntOld[NB+2]; /* Old index in aCell[] after i-th page */ int szNew[NB+2]; /* Combined size of cells place on i-th page */ u8 **apCell = 0; /* All cells begin balanced */ u16 *szCell; /* Local size of all cells in apCell[] */ @@ -6487,6 +6610,7 @@ static int balance_nonroot( nCell++; } } + cntOld[i] = nCell; if( i %d@%d", apNew[i]->pgno, + pNew->nCell+pNew->nOverflow, j, + cntNew[i] - iFirst, iFirst + ); + for(iCell=iFirst; iCellaData + || apCell[iCell]>=&pNew->aData[pBt->usableSize] + ){ + nCta += szCell[iCell]; + } + } + nFree = get2byte(&pNew->aData[pNew->hdrOffset+5]); + nFree -= (pNew->cellOffset + (cntNew[i] - iFirst) * 2); + printf(" cta=%d free=%d\n", nCta, nFree); + if( i==(nNew-1) ){ + printf("-----\n"); + fflush(stdout); + } +#endif + assert( inCell + apNew[i]->nOverflow; @@ -6835,17 +6987,23 @@ static int balance_nonroot( && (aShiftLeft[iPg]>=0 || abDone[iPg-1]) && (aShiftRight[iPg]>=0 || abDone[iPg+1]) ){ - MemPage *pNew = apNew[iPg]; - int iLeft = ((iPg==0) ? 0 : cntNew[iPg-1] + !leafData); - rebuildPage(pNew, - aShiftLeft[iPg] < 0 ? (aShiftLeft[iPg]*-1) : 0, - cntNew[iPg] - iLeft, - &apCell[iLeft], - &szCell[iLeft] - ); + int iNew; + int iOld; + int nNewCell; + + if( iPg==0 ){ + iNew = iOld = 0; + nNewCell = cntNew[0]; + }else{ + iOld = iPgnOverflow==0 ); - assert( pNew->nCell==(cntNew[iPg] - (iPg==0?0:cntNew[iPg-1]+!leafData)) ); + assert( apNew[iPg]->nOverflow==0 ); + assert( apNew[iPg]->nCell==nNewCell ); } } assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); @@ -6869,6 +7027,7 @@ static int balance_nonroot( ** is important if the parent page happens to be page 1 of the database ** image. */ assert( nNew==1 ); + defragmentPage(apNew[0]); assert( apNew[0]->nFree == (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) ); From c81aa2e1203c4e3aa389f69e715c019f50fe1420 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Oct 2014 23:31:52 +0000 Subject: [PATCH 448/710] Performance optimization and very slight size reduction for OP_Column. FossilOrigin-Name: 869c30e45cc87063be423c650f16b99e8adb3df0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 25 +++++++++++++++++-------- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 517771d6a6..62b5d0c870 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplification\sto\sthe\sinsertCell()\sroutine\sin\sbtree.c,\sresulting\sin\sa\nperformance\sboost\sand\sa\svery\ssmall\ssize\sdecrease.\s\sIt\sturns\sout\sthat\sthe\nextra\swork\sinvolved\sin\ssometimes\savoiding\san\smemcpy()\sof\sthe\sfirst\sfour\sbytes\nof\sa\srecord\stakes\smore\stime\sthan\sjust\sunconditionally\scopying\sthose\nfour\sbytes. -D 2014-10-11T17:22:55.486 +C Performance\soptimization\sand\svery\sslight\ssize\sreduction\sfor\sOP_Column. +D 2014-10-11T23:31:52.159 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c fee8286ff026bb9cf96ce87971b60aba53863b78 +F src/vdbe.c 58c19340f009d29b63d3701bd1871aad03e4c134 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h 0b97a3190f8fbf460655985a9183019f5a702754 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5d29a033b0f17b0fd74656b28a8367a9a9067f81 -R 8d2ebada6cfcd93628f86065b0703bad +P 66de15580d3c289601e67debfe1edee286f4db5f +R 512122d0d240ac7f0b4d899edd214ecb U drh -Z b915e33ee9cbdc5e832f0d58216a79da +Z 7a9b66d8d53ebbf7b4743296d025a097 diff --git a/manifest.uuid b/manifest.uuid index e1067663de..2c31982904 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -66de15580d3c289601e67debfe1edee286f4db5f \ No newline at end of file +869c30e45cc87063be423c650f16b99e8adb3df0 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index c039dcc862..8223d21611 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2332,14 +2332,6 @@ case OP_Column: { pC->iHdrOffset = getVarint32(pC->aRow, offset); pC->nHdrParsed = 0; aOffset[0] = offset; - if( availaRow does not have to hold the entire row, but it does at least - ** need to cover the header of the record. If pC->aRow does not contain - ** the complete header, then set it to zero, forcing the header to be - ** dynamically allocated. */ - pC->aRow = 0; - pC->szRow = 0; - } /* Make sure a corrupt database has not given us an oversize header. ** Do this now to avoid an oversize memory allocation. @@ -2354,6 +2346,22 @@ case OP_Column: { rc = SQLITE_CORRUPT_BKPT; goto op_column_error; } + + if( availaRow does not have to hold the entire row, but it does at least + ** need to cover the header of the record. If pC->aRow does not contain + ** the complete header, then set it to zero, forcing the header to be + ** dynamically allocated. */ + pC->aRow = 0; + pC->szRow = 0; + } + + /* The following goto is an optimization. It can be omitted and + ** everything will still work. But OP_Column is measurably faster + ** by skipping the subsequent conditional, which is always true. + */ + assert( pC->nHdrParsed<=p2 ); /* Conditional skipped */ + goto op_column_read_header; } /* Make sure at least the first p2+1 entries of the header have been @@ -2363,6 +2371,7 @@ case OP_Column: { /* If there is more header available for parsing in the record, try ** to extract additional fields up through the p2+1-th field */ + op_column_read_header: if( pC->iHdrOffsetaRow==0 ){ From b53a5a9e50d376a6cda897ce629f4af55b96c193 Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 12 Oct 2014 22:37:22 +0000 Subject: [PATCH 449/710] Remove the VdbeCursor.lastRowid cache of the current rowid, since maintaining the correct cache value uses more CPU cycles than just recomputing the rowid on the occasions when it is actually needed. Replace it with the VdbeCursor.aOffset field which used to be computed from VdbeCursor.aType when needed. Saves 100 bytes of code space and runs 0.2% faster. FossilOrigin-Name: 91384a7d727ef0f285cd430e829ba9f3852db50e --- manifest | 16 +++++++-------- manifest.uuid | 2 +- src/vdbe.c | 56 +++++++++++++++++---------------------------------- src/vdbeInt.h | 3 +-- src/vdbeaux.c | 2 -- 5 files changed, 28 insertions(+), 51 deletions(-) diff --git a/manifest b/manifest index 62b5d0c870..0d0dc6d249 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\soptimization\sand\svery\sslight\ssize\sreduction\sfor\sOP_Column. -D 2014-10-11T23:31:52.159 +C Remove\sthe\sVdbeCursor.lastRowid\scache\sof\sthe\scurrent\srowid,\ssince\smaintaining\nthe\scorrect\scache\svalue\suses\smore\sCPU\scycles\sthan\sjust\srecomputing\sthe\srowid\non\sthe\soccasions\swhen\sit\sis\sactually\sneeded.\s\sReplace\sit\swith\sthe\nVdbeCursor.aOffset\sfield\swhich\sused\sto\sbe\scomputed\sfrom\sVdbeCursor.aType\nwhen\sneeded.\sSaves\s100\sbytes\sof\scode\sspace\sand\sruns\s0.2%\sfaster. +D 2014-10-12T22:37:22.384 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,11 +289,11 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 58c19340f009d29b63d3701bd1871aad03e4c134 +F src/vdbe.c 97c6c50e272ed531bc3af308d5f156cfca0ce4f4 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h 0b97a3190f8fbf460655985a9183019f5a702754 +F src/vdbeInt.h e1173bd72b282633c2ec8f3a2f78b5117229f268 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 -F src/vdbeaux.c 5b687d7b5beaaa5b97189edf25cf08c311834933 +F src/vdbeaux.c e223d15ab39c844c04ad19931e8c71966c20c68d F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 481327f50d9da330053aa7456702ce46d0a4e70f F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 66de15580d3c289601e67debfe1edee286f4db5f -R 512122d0d240ac7f0b4d899edd214ecb +P 869c30e45cc87063be423c650f16b99e8adb3df0 +R d569318d3e6a6b025c49ffecae9bf7d4 U drh -Z 7a9b66d8d53ebbf7b4743296d025a097 +Z e6cf351e034f82bffef92232dfc65ffe diff --git a/manifest.uuid b/manifest.uuid index 2c31982904..a04b930976 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -869c30e45cc87063be423c650f16b99e8adb3df0 \ No newline at end of file +91384a7d727ef0f285cd430e829ba9f3852db50e \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 8223d21611..347b604bf4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -214,6 +214,7 @@ static VdbeCursor *allocateCursor( memset(pCx, 0, sizeof(VdbeCursor)); pCx->iDb = iDb; pCx->nField = nField; + pCx->aOffset = &pCx->aType[nField]; if( isBtreeCursor ){ pCx->pCursor = (BtCursor*) &pMem->z[ROUND8(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; @@ -2276,7 +2277,7 @@ case OP_Column: { pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( p2nField ); - aOffset = pC->aType + pC->nField; + aOffset = pC->aOffset; #ifndef SQLITE_OMIT_VIRTUALTABLE assert( pC->pVtabCursor==0 ); /* OP_Column never called on virtual table */ #endif @@ -3563,7 +3564,6 @@ case OP_SeekGT: { /* jump, in3 */ applyNumericAffinity(pIn3, 0); } iKey = sqlite3VdbeIntValue(pIn3); - pC->rowidIsValid = 0; /* If the P3 value could not be converted into an integer without ** loss of information, then special processing is required... */ @@ -3599,13 +3599,10 @@ case OP_SeekGT: { /* jump, in3 */ } } rc = sqlite3BtreeMovetoUnpacked(pC->pCursor, 0, (u64)iKey, 0, &res); + pC->movetoTarget = iKey; /* Used by OP_Delete */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - if( res==0 ){ - pC->rowidIsValid = 1; - pC->lastRowid = iKey; - } }else{ nField = pOp->p4.i; assert( pOp->p4type==P4_INT32 ); @@ -3635,7 +3632,6 @@ case OP_SeekGT: { /* jump, in3 */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - pC->rowidIsValid = 0; } pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -3647,7 +3643,6 @@ case OP_SeekGT: { /* jump, in3 */ res = 0; rc = sqlite3BtreeNext(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; - pC->rowidIsValid = 0; }else{ res = 0; } @@ -3657,7 +3652,6 @@ case OP_SeekGT: { /* jump, in3 */ res = 0; rc = sqlite3BtreePrevious(pC->pCursor, &res); if( rc!=SQLITE_OK ) goto abort_due_to_error; - pC->rowidIsValid = 0; }else{ /* res might be negative because the table is empty. Check to ** see if this is the case. @@ -3694,7 +3688,6 @@ case OP_Seek: { /* in2 */ pC->nullRow = 0; pIn2 = &aMem[pOp->p2]; pC->movetoTarget = sqlite3VdbeIntValue(pIn2); - pC->rowidIsValid = 0; pC->deferredMoveto = 1; break; } @@ -3880,15 +3873,13 @@ case OP_NotExists: { /* jump, in3 */ res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); - pC->lastRowid = pIn3->u.i; - pC->rowidIsValid = res==0 ?1:0; + pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; pC->deferredMoveto = 0; VdbeBranchTaken(res!=0,2); if( res!=0 ){ pc = pOp->p2 - 1; - assert( pC->rowidIsValid==0 ); } pC->seekResult = res; break; @@ -4036,7 +4027,6 @@ case OP_NewRowid: { /* out2-prerelease */ } assert( v>0 ); /* EV: R-40812-03570 */ } - pC->rowidIsValid = 0; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } @@ -4141,7 +4131,6 @@ case OP_InsertInt: { pData->z, pData->n, nZero, (pOp->p5 & OPFLAG_APPEND)!=0, seekResult ); - pC->rowidIsValid = 0; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -4178,33 +4167,32 @@ case OP_InsertInt: { ** using OP_NotFound prior to invoking this opcode. */ case OP_Delete: { - i64 iKey; VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->pCursor!=0 ); /* Only valid for real tables, no pseudotables */ - iKey = pC->lastRowid; /* Only used for the update hook */ - - /* The OP_Delete opcode always follows an OP_NotExists or OP_Last or - ** OP_Column on the same table without any intervening operations that - ** might move or invalidate the cursor. Hence cursor pC is always pointing - ** to the row to be deleted and the sqlite3VdbeCursorMoveto() operation - ** below is always a no-op and cannot fail. We will run it anyhow, though, - ** to guard against future changes to the code generator. - **/ assert( pC->deferredMoveto==0 ); - rc = sqlite3VdbeCursorMoveto(pC); - if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; +#ifdef SQLITE_DEBUG + /* The seek operation that positioned the cursor prior to OP_Delete will + ** have also set the pC->movetoTarget field to the rowid of the row that + ** is being deleted */ + if( pOp->p4.z && pC->isTable ){ + i64 iKey = 0; + sqlite3BtreeKeySize(pC->pCursor, &iKey); + assert( pC->movetoTarget==iKey ); + } +#endif + rc = sqlite3BtreeDelete(pC->pCursor); pC->cacheStatus = CACHE_STALE; /* Invoke the update-hook if required. */ if( rc==SQLITE_OK && db->xUpdateCallback && pOp->p4.z && pC->isTable ){ db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, - db->aDb[pC->iDb].zName, pOp->p4.z, iKey); + db->aDb[pC->iDb].zName, pOp->p4.z, pC->movetoTarget); assert( pC->iDb>=0 ); } if( pOp->p2 & OPFLAG_NCHANGE ) p->nChange++; @@ -4396,12 +4384,8 @@ case OP_Rowid: { /* out2-prerelease */ assert( pC->pCursor!=0 ); rc = sqlite3VdbeCursorMoveto(pC); if( rc ) goto abort_due_to_error; - if( pC->rowidIsValid ){ - v = pC->lastRowid; - }else{ - rc = sqlite3BtreeKeySize(pC->pCursor, &v); - assert( rc==SQLITE_OK ); /* Always so because of CursorMoveto() above */ - } + rc = sqlite3BtreeKeySize(pC->pCursor, &v); + assert( rc==SQLITE_OK ); /* Always so because of CursorMoveto() above */ } pOut->u.i = v; break; @@ -4420,7 +4404,6 @@ case OP_NullRow: { pC = p->apCsr[pOp->p1]; assert( pC!=0 ); pC->nullRow = 1; - pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; if( pC->pCursor ){ sqlite3BtreeClearCursor(pC->pCursor); @@ -4454,7 +4437,6 @@ case OP_Last: { /* jump */ rc = sqlite3BtreeLast(pCrsr, &res); pC->nullRow = (u8)res; pC->deferredMoveto = 0; - pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; #ifdef SQLITE_DEBUG pC->seekOp = OP_Last; @@ -4521,7 +4503,6 @@ case OP_Rewind: { /* jump */ rc = sqlite3BtreeFirst(pCrsr, &res); pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; - pC->rowidIsValid = 0; } pC->nullRow = (u8)res; assert( pOp->p2>0 && pOp->p2nOp ); @@ -4647,7 +4628,6 @@ next_tail: }else{ pC->nullRow = 1; } - pC->rowidIsValid = 0; goto check_for_interrupt; } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index aa1132e31c..1c7bebc9f6 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -73,7 +73,6 @@ struct VdbeCursor { #endif i8 iDb; /* Index of cursor database in db->aDb[] (or -1) */ u8 nullRow; /* True if pointing to a row with no data */ - u8 rowidIsValid; /* True if lastRowid is valid */ u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ Bool isEphemeral:1; /* True for an ephemeral table */ Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */ @@ -83,7 +82,6 @@ struct VdbeCursor { sqlite3_vtab_cursor *pVtabCursor; /* The cursor for a virtual table */ i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ - i64 lastRowid; /* Rowid being deleted by OP_Delete */ VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ /* Cached information about the header for the data record that the @@ -100,6 +98,7 @@ struct VdbeCursor { u32 szRow; /* Byte available in aRow */ u32 iHdrOffset; /* Offset to next unparsed byte of the header */ const u8 *aRow; /* Data for the current row, if all on one page */ + u32 *aOffset; /* Pointer to aType[nField] */ u32 aType[1]; /* Type values for all entries in the record */ /* 2*nField extra array elements allocated for aType[], beyond the one ** static element declared in the structure. nField total array slots for diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 33fa055369..fefa0ef337 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2718,9 +2718,7 @@ static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ assert( p->isTable ); rc = sqlite3BtreeMovetoUnpacked(p->pCursor, 0, p->movetoTarget, 0, &res); if( rc ) return rc; - p->lastRowid = p->movetoTarget; if( res!=0 ) return SQLITE_CORRUPT_BKPT; - p->rowidIsValid = 1; #ifdef SQLITE_TEST sqlite3_search_count++; #endif From 6cf4a7dfa65fb5c4d16aca12d1065bd04257c831 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Oct 2014 13:00:58 +0000 Subject: [PATCH 450/710] Remove the OPFLAG_CLEARCACHE flag from OP_Column. In its place, change the P3 parameter of OP_SorterData to be the index of the pseudo-table cursor whose record header cache is to be cleared. This gives a small size reduction and performance increase. FossilOrigin-Name: 20062f49428a2349a2dd705af570c60b499a3eef --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/build.c | 2 +- src/select.c | 9 ++------- src/sqliteInt.h | 1 - src/vdbe.c | 13 +++++++++++-- 6 files changed, 24 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 0d0dc6d249..ea3a0d9c70 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sVdbeCursor.lastRowid\scache\sof\sthe\scurrent\srowid,\ssince\smaintaining\nthe\scorrect\scache\svalue\suses\smore\sCPU\scycles\sthan\sjust\srecomputing\sthe\srowid\non\sthe\soccasions\swhen\sit\sis\sactually\sneeded.\s\sReplace\sit\swith\sthe\nVdbeCursor.aOffset\sfield\swhich\sused\sto\sbe\scomputed\sfrom\sVdbeCursor.aType\nwhen\sneeded.\sSaves\s100\sbytes\sof\scode\sspace\sand\sruns\s0.2%\sfaster. -D 2014-10-12T22:37:22.384 +C Remove\sthe\sOPFLAG_CLEARCACHE\sflag\sfrom\sOP_Column.\s\sIn\sits\splace,\schange\sthe\nP3\sparameter\sof\sOP_SorterData\sto\sbe\sthe\sindex\sof\sthe\spseudo-table\scursor\swhose\nrecord\sheader\scache\sis\sto\sbe\scleared.\s\sThis\sgives\sa\ssmall\ssize\sreduction\nand\sperformance\sincrease. +D 2014-10-13T13:00:58.198 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c 5c0b78c49d00da49a0e8e3d2738900a431df7fca F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d -F src/build.c 9e5205db9a0c8a1a4ce7379d60a2a34cb0b7339c +F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 @@ -227,12 +227,12 @@ F src/printf.c 6b79bbd063dcbadca4cf617a4cde255bcc13ea64 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c f11533162b57ed5ed37f549add34cbcdf51f6712 +F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 18ee8bbe9502d8848072dc2eddd1ea09254ba494 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h c417a25e2369f705b651897a2f1cc8da0e6aa1c4 +F src/sqliteInt.h f7812f74f2d0c6041ef6b91a99c5a45f775dd408 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 97c6c50e272ed531bc3af308d5f156cfca0ce4f4 +F src/vdbe.c 6e07ea132d94b464fa9d28dd7ea85a977524de31 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e1173bd72b282633c2ec8f3a2f78b5117229f268 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 869c30e45cc87063be423c650f16b99e8adb3df0 -R d569318d3e6a6b025c49ffecae9bf7d4 +P 91384a7d727ef0f285cd430e829ba9f3852db50e +R 398856b8d4cf81a7bc85ff63a35b9f22 U drh -Z e6cf351e034f82bffef92232dfc65ffe +Z cd7cdfd82fa519c77c73eb77000032e4 diff --git a/manifest.uuid b/manifest.uuid index a04b930976..9e8c25c8d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91384a7d727ef0f285cd430e829ba9f3852db50e \ No newline at end of file +20062f49428a2349a2dd705af570c60b499a3eef \ No newline at end of file diff --git a/src/build.c b/src/build.c index 14d8aab587..b897494db3 100644 --- a/src/build.c +++ b/src/build.c @@ -2747,7 +2747,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ }else{ addr2 = sqlite3VdbeCurrentAddr(v); } - sqlite3VdbeAddOp2(v, OP_SorterData, iSorter, regRecord); + sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 1); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); diff --git a/src/select.c b/src/select.c index 411bca0df4..3b422f1100 100644 --- a/src/select.c +++ b/src/select.c @@ -1181,7 +1181,6 @@ static void generateSortTail( int nKey; int iSortTab; /* Sorter cursor to read from */ int nSortData; /* Trailing values to read from sorter */ - u8 p5; /* p5 parameter for 1st OP_Column */ int i; int bSeq; /* True if sorter record includes seq. no. */ #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS @@ -1215,19 +1214,16 @@ static void generateSortTail( addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); - sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); - p5 = OPFLAG_CLEARCACHE; + sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); bSeq = 0; }else{ addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); iSortTab = iTab; - p5 = 0; bSeq = 1; } for(i=0; inExpr; j++){ if( groupBySort ){ sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); - if( j==0 ) sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ sAggInfo.directMode = 1; sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e648430353..cba89b03e7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2666,7 +2666,6 @@ struct AuthContext { #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ -#define OPFLAG_CLEARCACHE 0x20 /* Clear pseudo-table cache in OP_Column */ #define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ diff --git a/src/vdbe.c b/src/vdbe.c index 347b604bf4..19cf117ce5 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2288,7 +2288,7 @@ case OP_Column: { /* If the cursor cache is stale, bring it up-to-date */ rc = sqlite3VdbeCursorMoveto(pC); if( rc ) goto abort_due_to_error; - if( pC->cacheStatus!=p->cacheCtr || (pOp->p5&OPFLAG_CLEARCACHE)!=0 ){ + if( pC->cacheStatus!=p->cacheCtr ){ if( pC->nullRow ){ if( pCrsr==0 ){ assert( pC->pseudoTableReg>0 ); @@ -4245,10 +4245,17 @@ case OP_SorterCompare: { break; }; -/* Opcode: SorterData P1 P2 * * * +/* Opcode: SorterData P1 P2 P3 * * ** Synopsis: r[P2]=data ** ** Write into register P2 the current sorter data for sorter cursor P1. +** Then clear the column header cache on cursor P3. +** +** This opcode is normally use to move a record out of the sorter and into +** a register that is the source for a pseudo-table cursor created using +** OpenPseudo. That pseudo-table cursor is the one that is identified by +** parameter P3. Clearing the P3 column cache as part of this opcode saves +** us from having to issue a separate NullRow instruction to clear that cache. */ case OP_SorterData: { VdbeCursor *pC; @@ -4258,6 +4265,8 @@ case OP_SorterData: { assert( isSorter(pC) ); rc = sqlite3VdbeSorterRowkey(pC, pOut); assert( rc!=SQLITE_OK || (pOut->flags & MEM_Blob) ); + assert( pOp->p1>=0 && pOp->p1nCursor ); + p->apCsr[pOp->p3]->cacheStatus = CACHE_STALE; break; } From c22284f4b3939c1a09c0df2759bd5cca2dc34bad Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Oct 2014 16:02:20 +0000 Subject: [PATCH 451/710] Reduce the CPU load imposed by sqlit3VdbeCursorMoveto() by factoring out some of its functions and by avoiding unnecessary calls. FossilOrigin-Name: c2799aece17d347c64217a0e407bb10e50c184a3 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/btree.c | 2 +- src/vdbe.c | 28 +++++++++++++++++++--------- src/vdbeInt.h | 1 + src/vdbeaux.c | 13 ++++++++++++- 6 files changed, 43 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index ea3a0d9c70..72398a68f7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sOPFLAG_CLEARCACHE\sflag\sfrom\sOP_Column.\s\sIn\sits\splace,\schange\sthe\nP3\sparameter\sof\sOP_SorterData\sto\sbe\sthe\sindex\sof\sthe\spseudo-table\scursor\swhose\nrecord\sheader\scache\sis\sto\sbe\scleared.\s\sThis\sgives\sa\ssmall\ssize\sreduction\nand\sperformance\sincrease. -D 2014-10-13T13:00:58.198 +C Reduce\sthe\sCPU\sload\simposed\sby\ssqlit3VdbeCursorMoveto()\sby\sfactoring\sout\nsome\sof\sits\sfunctions\sand\sby\savoiding\sunnecessary\scalls. +D 2014-10-13T16:02:20.134 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 5c0b78c49d00da49a0e8e3d2738900a431df7fca +F src/btree.c c9fcae8145436f728c61272cba72b1469c07f30d F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -289,11 +289,11 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 6e07ea132d94b464fa9d28dd7ea85a977524de31 +F src/vdbe.c cf7c24e531aad047689e4235a66b22526e765e46 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h e1173bd72b282633c2ec8f3a2f78b5117229f268 +F src/vdbeInt.h 90c0ae53382d16a4f77ad5a9c4b3e35278e9e061 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 -F src/vdbeaux.c e223d15ab39c844c04ad19931e8c71966c20c68d +F src/vdbeaux.c 8d101333912e33ed36201898d839cd560d8e87f4 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 481327f50d9da330053aa7456702ce46d0a4e70f F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 91384a7d727ef0f285cd430e829ba9f3852db50e -R 398856b8d4cf81a7bc85ff63a35b9f22 +P 20062f49428a2349a2dd705af570c60b499a3eef +R a5158d9f69a4de7fce20f1a1e7deba5e U drh -Z cd7cdfd82fa519c77c73eb77000032e4 +Z 9da431167b91f4974349edc450eddb7a diff --git a/manifest.uuid b/manifest.uuid index 9e8c25c8d6..5724d7dce8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -20062f49428a2349a2dd705af570c60b499a3eef \ No newline at end of file +c2799aece17d347c64217a0e407bb10e50c184a3 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index cfe623342a..3553924c0f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -776,7 +776,7 @@ static int btreeRestoreCursorPosition(BtCursor *pCur){ ** back to where it ought to be if this routine returns true. */ int sqlite3BtreeCursorHasMoved(BtCursor *pCur){ - return pCur && pCur->eState!=CURSOR_VALID; + return pCur->eState!=CURSOR_VALID; } /* diff --git a/src/vdbe.c b/src/vdbe.c index 19cf117ce5..7e0e35fb0e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4313,16 +4313,20 @@ case OP_RowData: { assert( pC->pseudoTableReg==0 ); assert( pC->pCursor!=0 ); pCrsr = pC->pCursor; - assert( sqlite3BtreeCursorIsValid(pCrsr) ); /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or ** OP_Rewind/Op_Next with no intervening instructions that might invalidate - ** the cursor. Hence the following sqlite3VdbeCursorMoveto() call is always - ** a no-op and can never fail. But we leave it in place as a safety. + ** the cursor. If this where not the case, on of the following assert()s + ** would fail. Should this ever change (because of changes in the code + ** generator) then the fix would be to insert a call to + ** sqlite3VdbeCursorMoveto(). */ assert( pC->deferredMoveto==0 ); + assert( sqlite3BtreeCursorIsValid(pCrsr) ); +#if 0 /* Not required due to the previous to assert() statements */ rc = sqlite3VdbeCursorMoveto(pC); - if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + if( rc!=SQLITE_OK ) goto abort_due_to_error; +#endif if( pC->isTable==0 ){ assert( !pC->isTable ); @@ -4391,10 +4395,10 @@ case OP_Rowid: { /* out2-prerelease */ #endif /* SQLITE_OMIT_VIRTUALTABLE */ }else{ assert( pC->pCursor!=0 ); - rc = sqlite3VdbeCursorMoveto(pC); + rc = sqlite3VdbeCursorRestore(pC); if( rc ) goto abort_due_to_error; rc = sqlite3BtreeKeySize(pC->pCursor, &v); - assert( rc==SQLITE_OK ); /* Always so because of CursorMoveto() above */ + assert( rc==SQLITE_OK ); /* Always so because of CursorRestore() above */ } pOut->u.i = v; break; @@ -4752,10 +4756,16 @@ case OP_IdxRowid: { /* out2-prerelease */ pCrsr = pC->pCursor; assert( pCrsr!=0 ); pOut->flags = MEM_Null; - rc = sqlite3VdbeCursorMoveto(pC); - if( NEVER(rc) ) goto abort_due_to_error; - assert( pC->deferredMoveto==0 ); assert( pC->isTable==0 ); + assert( pC->deferredMoveto==0 ); + + /* sqlite3VbeCursorRestore() can only fail if the record has been deleted + ** out from under the cursor. That will never happend for an IdxRowid + ** opcode, hence the NEVER() arround the check of the return value. + */ + rc = sqlite3VdbeCursorRestore(pC); + if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + if( !pC->nullRow ){ rowid = 0; /* Not needed. Only used to silence a warning. */ rc = sqlite3VdbeIdxRowid(db, pCrsr, &rowid); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 1c7bebc9f6..1b27e88e72 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -383,6 +383,7 @@ struct Vdbe { void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); void sqliteVdbePopStack(Vdbe*,int); int sqlite3VdbeCursorMoveto(VdbeCursor*); +int sqlite3VdbeCursorRestore(VdbeCursor*); #if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) void sqlite3VdbePrintOp(FILE*, int, Op*); #endif diff --git a/src/vdbeaux.c b/src/vdbeaux.c index fefa0ef337..88bf2c3dd8 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2744,6 +2744,17 @@ static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ return rc; } +/* +** Check to ensure that the cursor is valid. Restore the cursor +** if need be. Return any I/O error from the restore operation. +*/ +int sqlite3VdbeCursorRestore(VdbeCursor *p){ + if( sqlite3BtreeCursorHasMoved(p->pCursor) ){ + return handleMovedCursor(p); + } + return SQLITE_OK; +} + /* ** Make sure the cursor p is ready to read or write the row to which it ** was last positioned. Return an error code if an OOM fault or I/O error @@ -2761,7 +2772,7 @@ int sqlite3VdbeCursorMoveto(VdbeCursor *p){ if( p->deferredMoveto ){ return handleDeferredMoveto(p); } - if( sqlite3BtreeCursorHasMoved(p->pCursor) ){ + if( p->pCursor && sqlite3BtreeCursorHasMoved(p->pCursor) ){ return handleMovedCursor(p); } return SQLITE_OK; From f526dcad299bbb3d00817a84c9268ef9191bf07e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Oct 2014 17:42:05 +0000 Subject: [PATCH 452/710] Three small optimizations to vdbeaux.c. FossilOrigin-Name: 04892f8ba6c55cec4fe37bfe59b6349fd2a40698 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 9 +++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 72398a68f7..f76c05d5cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\sCPU\sload\simposed\sby\ssqlit3VdbeCursorMoveto()\sby\sfactoring\sout\nsome\sof\sits\sfunctions\sand\sby\savoiding\sunnecessary\scalls. -D 2014-10-13T16:02:20.134 +C Three\ssmall\soptimizations\sto\svdbeaux.c. +D 2014-10-13T17:42:05.891 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,7 +293,7 @@ F src/vdbe.c cf7c24e531aad047689e4235a66b22526e765e46 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h 90c0ae53382d16a4f77ad5a9c4b3e35278e9e061 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 -F src/vdbeaux.c 8d101333912e33ed36201898d839cd560d8e87f4 +F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 481327f50d9da330053aa7456702ce46d0a4e70f F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 20062f49428a2349a2dd705af570c60b499a3eef -R a5158d9f69a4de7fce20f1a1e7deba5e +P c2799aece17d347c64217a0e407bb10e50c184a3 +R f9a2ff50dcb19a8595444c5c4cad6f5f U drh -Z 9da431167b91f4974349edc450eddb7a +Z bf4e3c223808aa3476302aafe1aa8090 diff --git a/manifest.uuid b/manifest.uuid index 5724d7dce8..02b54f919a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c2799aece17d347c64217a0e407bb10e50c184a3 \ No newline at end of file +04892f8ba6c55cec4fe37bfe59b6349fd2a40698 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 88bf2c3dd8..c0018bb71c 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1745,7 +1745,7 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ sqlite3BtreeCloseCursor(pCx->pCursor); } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pCx->pVtabCursor ){ + else if( pCx->pVtabCursor ){ sqlite3_vtab_cursor *pVtabCursor = pCx->pVtabCursor; const sqlite3_module *pModule = pVtabCursor->pVtab->pModule; p->inVtabMethod = 1; @@ -1788,9 +1788,10 @@ static void closeAllCursors(Vdbe *p){ VdbeFrame *pFrame; for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); sqlite3VdbeFrameRestore(pFrame); + p->pFrame = 0; + p->nFrame = 0; } - p->pFrame = 0; - p->nFrame = 0; + assert( p->nFrame==0 ); if( p->apCsr ){ int i; @@ -1812,7 +1813,7 @@ static void closeAllCursors(Vdbe *p){ } /* Delete any auxdata allocations made by the VM */ - sqlite3VdbeDeleteAuxData(p, -1, 0); + if( p->pAuxData ) sqlite3VdbeDeleteAuxData(p, -1, 0); assert( p->pAuxData==0 ); } From d7b545bbcb2225a33e8c8c5192ab11416a855999 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Oct 2014 18:03:27 +0000 Subject: [PATCH 453/710] Further work on balance_nonroot(). FossilOrigin-Name: 6594f9b420e2fa642737722ff8521f756ecef227 --- manifest | 12 +-- manifest.uuid | 2 +- src/btree.c | 265 ++++++++++++++++++++++++++++++++------------------ 3 files changed, 180 insertions(+), 99 deletions(-) diff --git a/manifest b/manifest index 7e782aa82c..9f25f390d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Attempt\sto\sfurther\sreduce\smemcpy()\sin\sbalance_nonroot(). -D 2014-10-11T20:00:24.552 +C Further\swork\son\sbalance_nonroot(). +D 2014-10-13T18:03:27.743 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c d5f4f74e309f79ace4b4025c433874ead635bed2 +F src/btree.c 63211ca1d4ae867eede39a37901aec4746d904a7 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c bde83dd5cf812e310a7e5ad2846790a14745bef4 @@ -1200,7 +1200,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 29304499ea4b72dbb6701e10cc19b5d41f7e5ac9 -R 3235ac37769ff059d7c714947e8a11dd +P fec849dcca3aead2bc2d4ecffeda750684d32fb0 +R 79e5b5564c934fc68257d82dc897cefd U dan -Z 17fdfd7faf788bbb79d2ec9941c17025 +Z ae5b74eee28666050af6baaa639de56c diff --git a/manifest.uuid b/manifest.uuid index 006b3c0008..fcba485229 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fec849dcca3aead2bc2d4ecffeda750684d32fb0 \ No newline at end of file +6594f9b420e2fa642737722ff8521f756ecef227 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 13ec912748..7c9a7228de 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6009,7 +6009,7 @@ static void rebuildPage( assert( szCell[i]==cellSizePtr(pPg, pCell) ); } - pPg->nFree = (pData - pCellptr); + /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ pPg->nCell = nCell; pPg->nOverflow = 0; @@ -6019,6 +6019,106 @@ static void rebuildPage( aData[hdr+7] = 0x00; } +static u8 *pageFindSlot(MemPage *pPg, int nByte){ + const int hdr = pPg->hdrOffset; + u8 * const aData = pPg->aData; + int iAddr; + int pc; + int usableSize = pPg->pBt->usableSize; + + for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ + int size; /* Size of the free slot */ + if( pc>usableSize-4 || pc=nByte ){ + int x = size - nByte; + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + if( aData[hdr+7]>=60 ) return 0; + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else if( size+pc > usableSize ){ + return 0; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + } + + return 0; +} + +static int pageInsertArray( + MemPage *pPg, + u8 *pBegin, + u8 **ppData, + u8 *pCellptr, + int nCell, + u8 **apCell, /* Array of cells */ + u16 *szCell /* Array of cell sizes */ +){ + int i; + u8 *aData = pPg->aData; + u8 *pData = *ppData; + for(i=0; iaData; + u8 * const pEnd = &aData[pPg->pBt->usableSize]; + int nRet = 0; + int i; + u8 *pFree = 0; + int szFree = 0; + + for(i=0; iaData && pCellnFree field is invalid when this function returns. It is the +** responsibility of the caller to set it correctly. +*/ static void editPage( MemPage *pPg, /* Edit this page */ int iOld, /* Index of first cell currently on page */ @@ -6027,113 +6127,93 @@ static void editPage( u8 **apCell, /* Array of cells */ u16 *szCell /* Array of cell sizes */ ){ - - if( 1 ){ - u8 * const aData = pPg->aData; - const int hdr = pPg->hdrOffset; - u8 *pBegin = &pPg->aCellIdx[nNew * 2]; - int nFree = pPg->nFree; /* Free bytes on pPg */ - int nCell = pPg->nCell; /* Cells stored on pPg */ - u8 *pData; - u8 *pCellptr; - int i; - int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + u8 * const aData = pPg->aData; + const int hdr = pPg->hdrOffset; + u8 *pBegin = &pPg->aCellIdx[nNew * 2]; + int nCell = pPg->nCell; /* Cells stored on pPg */ + u8 *pData; + u8 *pCellptr; + int i; + int iOldEnd = iOld + pPg->nCell + pPg->nOverflow; + int iNewEnd = iNew + nNew; #ifdef SQLITE_DEBUG - u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); - memcpy(pTmp, aData, pPg->pBt->usableSize); + u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); + memcpy(pTmp, aData, pPg->pBt->usableSize); #endif - /* Remove cells from the start and end of the page */ - if( iOldaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ - freeSpace(pPg, apCell[i] - aData, szCell[i]); - nFree += szCell[i] + 2; - nShift++; - } - } - nCell -= nShift; - memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); - } - for(i=iNew+nNew; iaData && apCell[i]<&aData[pPg->pBt->usableSize] ){ - freeSpace(pPg, apCell[i] - aData, szCell[i]); - nFree += szCell[i] + 2; - nCell--; - } - } - pData = &aData[get2byte(&aData[hdr+5])]; - if( pDataaCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); + nCell -= nShift; + } + if( iNewEnd < iOldEnd ){ + nCell -= pageFreeArray( + pPg, iOldEnd-iNewEnd, &apCell[iNewEnd], &szCell[iNewEnd] + ); + } - /* Add cells to the start of the page */ - if( iNewaCellIdx; - memmove(&pCellptr[(iOld-iNew)*2], pCellptr, nCell*2); - for(i=iNew; iaCellIdx; + memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + nAdd, &apCell[iNew], &szCell[iNew] + ) ) goto editpage_fail; + nCell += nAdd; + } + + /* Add any overflow cells */ + for(i=0; inOverflow; i++){ + int iCell = (iOld + pPg->aiOvfl[i]) - iNew; + if( iCell>=0 && iCellaCellIdx[iCell * 2]; + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + nCell++; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + 1, &apCell[iCell + iNew], &szCell[iCell + iNew] + ) ) goto editpage_fail; } + } - /* Add any overflow cells */ - for(i=0; inOverflow; i++){ - int iCell = (iOld + pPg->aiOvfl[i]) - iNew; - if( iCell>=0 && iCellaCellIdx[iCell * 2]; - int sz = szCell[iCell+iNew]; - memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); - pData -= sz; - if( pDataaCellIdx[nCell*2]; + if( pageInsertArray( + pPg, pBegin, &pData, pCellptr, + nNew-nCell, &apCell[iNew+nCell], &szCell[iNew+nCell] + ) ) goto editpage_fail; - /* Append cells to the end of the page */ - pCellptr = &pPg->aCellIdx[nCell*2]; - for(i=iNew+nCell; i<(iNew+nNew); i++){ - pData -= szCell[i]; - if( pDatanCell = nNew; + pPg->nOverflow = 0; - pPg->nFree = nFree; - pPg->nCell = nNew; - pPg->nOverflow = 0; - - put2byte(&aData[hdr+3], pPg->nCell); - put2byte(&aData[hdr+5], pData - aData); + put2byte(&aData[hdr+3], pPg->nCell); + put2byte(&aData[hdr+5], pData - aData); #ifdef SQLITE_DEBUG - for(i=0; iaCellIdx[i*2]); - if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ - pCell = &pTmp[pCell - aData]; - } - assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + for(i=0; iaCellIdx[i*2]); + if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ + pCell = &pTmp[pCell - aData]; } + assert( 0==memcmp(pCell, &aData[iOff], szCell[i+iNew]) ); + } #endif #if 0 - printf("EDIT\n"); +printf("EDIT\n"); #endif - return; - } - + return; editpage_fail: #if 0 printf("REBUILD\n"); @@ -6662,7 +6742,7 @@ static int balance_nonroot( assert( i usableSpace ){ - szNew[k] = subtotal - szCell[i]; + szNew[k] = subtotal - szCell[i] - 2; cntNew[k] = i; if( leafData ){ i--; } subtotal = 0; @@ -7002,6 +7082,7 @@ static int balance_nonroot( editPage(apNew[iPg], iOld, iNew, nNewCell, apCell, szCell); abDone[iPg] = 1; + apNew[iPg]->nFree = usableSpace-szNew[iPg]; assert( apNew[iPg]->nOverflow==0 ); assert( apNew[iPg]->nCell==nNewCell ); } From facf47a891b114635ef5a5c2ea76ba1f8981e1c6 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Oct 2014 20:12:47 +0000 Subject: [PATCH 454/710] Use the padding word in the Mem object as temporary storage for serial_type value in OP_Record, and thus avoid a redundant computation of the serial_type for each column. FossilOrigin-Name: 4b3b65ee5ea61e9b9671ca027940bf02689cb890 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 4 ++-- src/vdbeInt.h | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f76c05d5cd..cdda42bff1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Three\ssmall\soptimizations\sto\svdbeaux.c. -D 2014-10-13T17:42:05.891 +C Use\sthe\spadding\sword\sin\sthe\sMem\sobject\sas\stemporary\sstorage\sfor\nserial_type\svalue\sin\sOP_Record,\sand\sthus\savoid\sa\sredundant\scomputation\sof\nthe\sserial_type\sfor\seach\scolumn. +D 2014-10-13T20:12:47.457 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,9 +289,9 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c cf7c24e531aad047689e4235a66b22526e765e46 +F src/vdbe.c 8755e3bb0d7d26b2b156c6f29ddd6b3d32b77df2 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h 90c0ae53382d16a4f77ad5a9c4b3e35278e9e061 +F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c2799aece17d347c64217a0e407bb10e50c184a3 -R f9a2ff50dcb19a8595444c5c4cad6f5f +P 04892f8ba6c55cec4fe37bfe59b6349fd2a40698 +R b0e17ef60b5aa0edd0fe18cf5fe3de58 U drh -Z bf4e3c223808aa3476302aafe1aa8090 +Z 6462e46ba313ee1bcf094c3a26197f24 diff --git a/manifest.uuid b/manifest.uuid index 02b54f919a..8d0a012b7d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -04892f8ba6c55cec4fe37bfe59b6349fd2a40698 \ No newline at end of file +4b3b65ee5ea61e9b9671ca027940bf02689cb890 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 7e0e35fb0e..049bf32ecd 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2616,7 +2616,7 @@ case OP_MakeRecord: { pRec = pLast; do{ assert( memIsValid(pRec) ); - serial_type = sqlite3VdbeSerialType(pRec, file_format); + pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format); len = sqlite3VdbeSerialTypeLen(serial_type); if( pRec->flags & MEM_Zero ){ if( nData ){ @@ -2665,7 +2665,7 @@ case OP_MakeRecord: { assert( pData0<=pLast ); pRec = pData0; do{ - serial_type = sqlite3VdbeSerialType(pRec, file_format); + serial_type = pRec->uTemp; i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ }while( (++pRec)<=pLast ); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 1b27e88e72..bb504d64a1 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -175,7 +175,7 @@ struct Mem { /* ShallowCopy only needs to copy the information above */ char *zMalloc; /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */ int szMalloc; /* Size of the zMalloc allocation */ - int iPadding1; /* Padding for 8-byte alignment */ + u32 uTemp; /* Transient storage for serial_type in OP_MakeRecord */ sqlite3 *db; /* The associated database connection */ void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */ #ifdef SQLITE_DEBUG From 8dd8362d6446e8d83fd621122987f048216197c2 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Oct 2014 23:39:02 +0000 Subject: [PATCH 455/710] Optimize a database corruption test inside of the OP_Column opcode. FossilOrigin-Name: 005e5b388a8a97bca6d1f0e06c40d68d92aa1212 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 15 ++++++++------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index cdda42bff1..83cd0aa0f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sthe\spadding\sword\sin\sthe\sMem\sobject\sas\stemporary\sstorage\sfor\nserial_type\svalue\sin\sOP_Record,\sand\sthus\savoid\sa\sredundant\scomputation\sof\nthe\sserial_type\sfor\seach\scolumn. -D 2014-10-13T20:12:47.457 +C Optimize\sa\sdatabase\scorruption\stest\sinside\sof\sthe\sOP_Column\sopcode. +D 2014-10-13T23:39:02.463 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 8755e3bb0d7d26b2b156c6f29ddd6b3d32b77df2 +F src/vdbe.c 5ee15a66ce07e0482b92aa29e4dd0c5827a22d79 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 04892f8ba6c55cec4fe37bfe59b6349fd2a40698 -R b0e17ef60b5aa0edd0fe18cf5fe3de58 +P 4b3b65ee5ea61e9b9671ca027940bf02689cb890 +R 42e26b1c48be06a100ad5530a5448c12 U drh -Z 6462e46ba313ee1bcf094c3a26197f24 +Z 6b074ae9a762ef61cbe684b68fc815ba diff --git a/manifest.uuid b/manifest.uuid index 8d0a012b7d..1929c02aaf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4b3b65ee5ea61e9b9671ca027940bf02689cb890 \ No newline at end of file +005e5b388a8a97bca6d1f0e06c40d68d92aa1212 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 049bf32ecd..0f9f45c456 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2417,15 +2417,16 @@ case OP_Column: { sMem.flags = MEM_Null; } - /* If we have read more header data than was contained in the header, - ** or if the end of the last field appears to be past the end of the - ** record, or if the end of the last field appears to be before the end - ** of the record (when all fields present), then we must be dealing - ** with a corrupt database. + /* The record is corrupt if any of the following are true: + ** (1) the bytes of the header extend past the declared header size + ** (zHdr>zEndHdr) + ** (2) the entire header was used but not all data was used + ** (zHdr==zEndHdr && offset!=pC->payloadSize) + ** (3) the end of the data extends beyond the end of the record. + ** (offset > pC->payloadSize) */ - if( (zHdr > zEndHdr) + if( (zHdr>=zEndHdr && (zHdr>zEndHdr || offset!=pC->payloadSize)) || (offset > pC->payloadSize) - || (zHdr==zEndHdr && offset!=pC->payloadSize) ){ rc = SQLITE_CORRUPT_BKPT; goto op_column_error; From 8e9ba0c335c5dc263f18f42f5c014233c7ad8976 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 14 Oct 2014 17:27:04 +0000 Subject: [PATCH 456/710] Fix some code duplication issues on this branch. Add minor optimizations to the new code. FossilOrigin-Name: 58d7793bd5d608ba9fc3a2cd44b9d9512e0332ba --- manifest | 12 +- manifest.uuid | 2 +- src/btree.c | 295 +++++++++++++++++++++----------------------------- 3 files changed, 128 insertions(+), 181 deletions(-) diff --git a/manifest b/manifest index f6d9403e16..347e58fbc2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges\sinto\sthis\sbranch. -D 2014-10-13T18:09:13.737 +C Fix\ssome\scode\sduplication\sissues\son\sthis\sbranch.\sAdd\sminor\soptimizations\sto\sthe\snew\scode. +D 2014-10-14T17:27:04.829 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 3d9d1db1d05724346876cf456a850628a9e75331 +F src/btree.c 3c72c5aa96b1a06ea15da24b82c526e6f14010ab F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6594f9b420e2fa642737722ff8521f756ecef227 04892f8ba6c55cec4fe37bfe59b6349fd2a40698 -R 76183a9b7ddeb938a48604dbc4dd8cd6 +P d5b7c5a88dd58de85b3060a1f28b6d96e6e21207 +R ff1ec1f8aba483920a0100e96e95dccf U dan -Z 504cabb1e1fb0c47450f76fec3d8495b +Z 9c750816a0b42cc370356f0c03af45ae diff --git a/manifest.uuid b/manifest.uuid index 32265a861c..065d26bf44 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5b7c5a88dd58de85b3060a1f28b6d96e6e21207 \ No newline at end of file +58d7793bd5d608ba9fc3a2cd44b9d9512e0332ba \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 7136b45b24..acf466304f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1224,6 +1224,57 @@ static int defragmentPage(MemPage *pPage){ return SQLITE_OK; } +/* +** Search the free-list on page pPg for space to store a cell nByte bytes in +** size. If one can be found, return a pointer to the space and remove it +** from the free-list. +** +** If no suitable space can be found on the free-list, return NULL. +** +** This function may detect corruption within pPg. If it does and argument +** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. +** Or, if corruption is detected by pRc is NULL, NULL is returned and the +** corruption goes unreported. +*/ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ + const int hdr = pPg->hdrOffset; + u8 * const aData = pPg->aData; + int iAddr; + int pc; + int usableSize = pPg->pBt->usableSize; + + for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ + int size; /* Size of the free slot */ + if( pc>usableSize-4 || pc=nByte ){ + int x = size - nByte; + testcase( x==4 ); + testcase( x==3 ); + if( x<4 ){ + if( aData[hdr+7]>=60 ) return 0; + /* Remove the slot from the free-list. Update the number of + ** fragmented bytes within the page. */ + memcpy(&aData[iAddr], &aData[pc], 2); + aData[hdr+7] += (u8)x; + }else if( size+pc > usableSize ){ + if( pRc ) *pRc = SQLITE_CORRUPT_BKPT; + return 0; + }else{ + /* The slot remains on the free-list. Reduce its size to account + ** for the portion used by the new allocation. */ + put2byte(&aData[pc+2], x); + } + return &aData[pc + x]; + } + } + + return 0; +} + /* ** Allocate nByte bytes of space from within the B-Tree page passed ** as the first argument. Write into *pIdx the index into pPage->aData[] @@ -1274,33 +1325,12 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+1==top ); testcase( gap==top ); if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ - int pc, addr; - for(addr=hdr+1; (pc = get2byte(&data[addr]))>0; addr=pc){ - int size; /* Size of the free slot */ - if( pc>usableSize-4 || pc=nByte ){ - int x = size - nByte; - testcase( x==4 ); - testcase( x==3 ); - if( x<4 ){ - if( data[hdr+7]>=60 ) goto defragment_page; - /* Remove the slot from the free-list. Update the number of - ** fragmented bytes within the page. */ - memcpy(&data[addr], &data[pc], 2); - data[hdr+7] += (u8)x; - }else if( size+pc > usableSize ){ - return SQLITE_CORRUPT_BKPT; - }else{ - /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ - put2byte(&data[pc+2], x); - } - *pIdx = pc + x; - return SQLITE_OK; - } + int rc = SQLITE_OK; + u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + if( rc ) return rc; + if( pSpace ){ + *pIdx = pSpace - data; + return SQLITE_OK; } } @@ -1309,7 +1339,6 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ -defragment_page: testcase( pPage->nCell==0 ); rc = defragmentPage(pPage); if( rc ) return rc; @@ -5933,48 +5962,18 @@ static void insertCell( } /* -** Add a list of cells to a page. The page should be initially empty. -** The cells are guaranteed to fit on the page. +** Array apCell[] contains pointers to nCell b-tree page cells. The +** szCell[] array contains the size in bytes of each cell. This function +** replaces the current contents of page pPg with the contents of the cell +** array. +** +** Some of the cells in apCell[] may currently be stored in pPg. This +** function works around problems caused by this by making a copy of any +** such cells before overwriting the page data. +** +** The MemPage.nFree field is invalidated by this function. It is the +** responsibility of the caller to set it correctly. */ -static void assemblePage( - MemPage *pPage, /* The page to be assembled */ - int nCell, /* The number of cells to add to this page */ - u8 **apCell, /* Pointers to cell bodies */ - u16 *aSize /* Sizes of the cells */ -){ - int i; /* Loop counter */ - u8 *pCellptr; /* Address of next cell pointer */ - int cellbody; /* Address of next cell body */ - u8 * const data = pPage->aData; /* Pointer to data for pPage */ - const int hdr = pPage->hdrOffset; /* Offset of header on pPage */ - const int nUsable = pPage->pBt->usableSize; /* Usable size of page */ - - assert( pPage->nOverflow==0 ); - assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - assert( nCell>=0 && nCell<=(int)MX_CELL(pPage->pBt) - && (int)MX_CELL(pPage->pBt)<=10921); - assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - - /* Check that the page has just been zeroed by zeroPage() */ - assert( pPage->nCell==0 ); - assert( get2byteNotZero(&data[hdr+5])==nUsable ); - - pCellptr = pPage->aCellIdx; - cellbody = nUsable; - for(i=0; inFree -= (nCell*2 + nUsable - cellbody); - pPage->nCell = (u16)nCell; -} - - static void rebuildPage( MemPage *pPg, /* Edit this page */ int nCell, /* Final number of cells on page */ @@ -5992,8 +5991,8 @@ static void rebuildPage( i = get2byte(&aData[hdr+5]); memcpy(&pTmp[i], &aData[i], usableSize - i); - pData = &aData[usableSize]; + pData = pEnd; for(i=0; iaData && pCellhdrOffset; - u8 * const aData = pPg->aData; - int iAddr; - int pc; - int usableSize = pPg->pBt->usableSize; - - for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ - int size; /* Size of the free slot */ - if( pc>usableSize-4 || pc=nByte ){ - int x = size - nByte; - testcase( x==4 ); - testcase( x==3 ); - if( x<4 ){ - if( aData[hdr+7]>=60 ) return 0; - /* Remove the slot from the free-list. Update the number of - ** fragmented bytes within the page. */ - memcpy(&aData[iAddr], &aData[pc], 2); - aData[hdr+7] += (u8)x; - }else if( size+pc > usableSize ){ - return 0; - }else{ - /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ - put2byte(&aData[pc+2], x); - } - return &aData[pc + x]; - } - } - - return 0; -} - +/* +** Array apCell[] contains nCell pointers to b-tree cells. Array szCell +** contains the size in bytes of each such cell. This function attempts to +** add the cells stored in the array to page pPg. If it cannot (because +** the page needs to be defragmented before the cells will fit), non-zero +** is returned. Otherwise, if the cells are added successfully, zero is +** returned. +** +** Argument pCellptr points to the first entry in the cell-pointer array +** (part of page pPg) to populate. After cell apCell[0] is written to the +** page body, a 16-bit offset is written to pCellptr. And so on, for each +** cell in the array. It is the responsibility of the caller to ensure +** that it is safe to overwrite this part of the cell-pointer array. +** +** When this function is called, *ppData points to the start of the +** content area on page pPg. If the size of the content area is extended, +** *ppData is updated to point to the new start of the content area +** before returning. +** +** Finally, argument pBegin points to the byte immediately following the +** end of the space required by this page for the cell-pointer area (for +** all cells - not just those inserted by the current call). If the content +** area must be extended to before this point in order to accomodate all +** cells in apCell[], then the cells do not fit and non-zero is returned. +*/ static int pageInsertArray( - MemPage *pPg, - u8 *pBegin, - u8 **ppData, - u8 *pCellptr, - int nCell, + MemPage *pPg, /* Page to add cells to */ + u8 *pBegin, /* End of cell-pointer array */ + u8 **ppData, /* IN/OUT: Page content -area pointer */ + u8 *pCellptr, /* Pointer to cell-pointer area */ + int nCell, /* Number of cells to add to pPg */ u8 **apCell, /* Array of cells */ u16 *szCell /* Array of cell sizes */ ){ int i; u8 *aData = pPg->aData; u8 *pData = *ppData; + const int bFreelist = aData[1] || aData[2]; + assert( pPg->hdrOffset==0 ); /* Never called on page 1 */ for(i=0; inFree field is invalid when this function returns. It is the ** responsibility of the caller to set it correctly. @@ -6206,15 +6205,8 @@ static void editPage( } #endif -#if 0 -printf("EDIT\n"); -#endif - return; editpage_fail: -#if 0 - printf("REBUILD\n"); -#endif /* Unable to edit this page. Rebuild it from scratch instead. */ rebuildPage(pPg, nNew, &apCell[iNew], &szCell[iNew]); } @@ -6288,7 +6280,8 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( sqlite3PagerIswriteable(pNew->pDbPage) ); assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - assemblePage(pNew, 1, &pCell, &szCell); + rebuildPage(pNew, 1, &pCell, &szCell); + pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map ** with entries for the new page, and any pointer from the @@ -6518,8 +6511,6 @@ static int balance_nonroot( u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ - int aShiftLeft[NB+2]; - int aShiftRight[NB+2]; u8 abDone[NB+2]; Pgno aPgno[NB+2]; u16 aPgFlags[NB+2]; @@ -6887,49 +6878,6 @@ static int balance_nonroot( assert( sqlite3PagerIswriteable(pParent->pDbPage) ); put4byte(pRight, apNew[nNew-1]->pgno); - j = 0; - for(i=0; i %d@%d", apNew[i]->pgno, - pNew->nCell+pNew->nOverflow, j, - cntNew[i] - iFirst, iFirst - ); - for(iCell=iFirst; iCellaData - || apCell[iCell]>=&pNew->aData[pBt->usableSize] - ){ - nCta += szCell[iCell]; - } - } - nFree = get2byte(&pNew->aData[pNew->hdrOffset+5]); - nFree -= (pNew->cellOffset + (cntNew[i] - iFirst) * 2); - printf(" cta=%d free=%d\n", nCta, nFree); - if( i==(nNew-1) ){ - printf("-----\n"); - fflush(stdout); - } -#endif - - assert( inCell + apNew[i]->nOverflow; - aShiftRight[i] = cntNew[i] - j; - assert( i!=nOld-1 || j==nCell ); - if( j=0 && aShiftLeft[0]==0 ); for(i=0; i=nNew ? i-nNew : nNew-1-i); if( abDone[iPg]==0 - && (aShiftLeft[iPg]>=0 || abDone[iPg-1]) - && (aShiftRight[iPg]>=0 || abDone[iPg+1]) + && (iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1]) + && (cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1]) ){ int iNew; int iOld; From 5604cc0d3b12e306cf7a601444e8d0f655e3bafc Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 14 Oct 2014 20:25:43 +0000 Subject: [PATCH 457/710] Make sure new sqlite3_vtab objects created by the xCreate() virtual table method are initialized by the system, in accordance with the documentation. FossilOrigin-Name: eab82330631187dcc3e5d2dddd23dbda5752904b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vtab.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 83cd0aa0f1..94c543c920 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimize\sa\sdatabase\scorruption\stest\sinside\sof\sthe\sOP_Column\sopcode. -D 2014-10-13T23:39:02.463 +C Make\ssure\snew\ssqlite3_vtab\sobjects\screated\sby\sthe\sxCreate()\svirtual\stable\nmethod\sare\sinitialized\sby\sthe\ssystem,\sin\saccordance\swith\sthe\sdocumentation. +D 2014-10-14T20:25:43.215 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -298,7 +298,7 @@ F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 481327f50d9da330053aa7456702ce46d0a4e70f F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 -F src/vtab.c 019dbfd0406a7447c990e1f7bd1dfcdb8895697f +F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4b3b65ee5ea61e9b9671ca027940bf02689cb890 -R 42e26b1c48be06a100ad5530a5448c12 +P 005e5b388a8a97bca6d1f0e06c40d68d92aa1212 +R 69c5ef77aff899ac36ba1309415e232b U drh -Z 6b074ae9a762ef61cbe684b68fc815ba +Z ba859354cbc6c4d678eb09a46ba9f63b diff --git a/manifest.uuid b/manifest.uuid index 1929c02aaf..83a00ae67f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -005e5b388a8a97bca6d1f0e06c40d68d92aa1212 \ No newline at end of file +eab82330631187dcc3e5d2dddd23dbda5752904b \ No newline at end of file diff --git a/src/vtab.c b/src/vtab.c index c7a8a5a33f..faee4ae478 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -519,6 +519,7 @@ static int vtabCallConstructor( }else if( ALWAYS(pVTable->pVtab) ){ /* Justification of ALWAYS(): A correct vtab constructor must allocate ** the sqlite3_vtab object if successful. */ + memset(pVTable->pVtab, 0, sizeof(pVTable->pVtab[0])); pVTable->pVtab->pModule = pMod->pModule; pVTable->nRef = 1; if( sCtx.pTab ){ From 74439167cb330f41e288270eba26b05a4f4af6a9 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Oct 2014 11:31:35 +0000 Subject: [PATCH 458/710] Rearrange an expression in vdbemem.c to avoid a (harmless) reference to a possibly unitialized variable. FossilOrigin-Name: 4a7b3fa049a9aa0668e318287edd4a78c0588bf8 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbemem.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 94c543c920..c9b3f13dde 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\snew\ssqlite3_vtab\sobjects\screated\sby\sthe\sxCreate()\svirtual\stable\nmethod\sare\sinitialized\sby\sthe\ssystem,\sin\saccordance\swith\sthe\sdocumentation. -D 2014-10-14T20:25:43.215 +C Rearrange\san\sexpression\sin\svdbemem.c\sto\savoid\sa\s(harmless)\sreference\sto\sa\spossibly\sunitialized\svariable. +D 2014-10-15T11:31:35.560 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -295,7 +295,7 @@ F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 -F src/vdbemem.c 481327f50d9da330053aa7456702ce46d0a4e70f +F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 005e5b388a8a97bca6d1f0e06c40d68d92aa1212 -R 69c5ef77aff899ac36ba1309415e232b -U drh -Z ba859354cbc6c4d678eb09a46ba9f63b +P eab82330631187dcc3e5d2dddd23dbda5752904b +R c46357701a0c6687b1cdb44cc5cea3cc +U dan +Z f35b20ba175f90e21204e2589425beed diff --git a/manifest.uuid b/manifest.uuid index 83a00ae67f..2ed282008d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eab82330631187dcc3e5d2dddd23dbda5752904b \ No newline at end of file +4a7b3fa049a9aa0668e318287edd4a78c0588bf8 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 0c62db0720..870fb5bd89 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -143,7 +143,7 @@ SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ } } - if( pMem->z && bPreserve && pMem->z!=pMem->zMalloc ){ + if( bPreserve && pMem->z && pMem->z!=pMem->zMalloc ){ memcpy(pMem->zMalloc, pMem->z, pMem->n); } if( (pMem->flags&MEM_Dyn)!=0 ){ From 92787cf0424ab04692843582c2ab9552c82b1c21 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Oct 2014 11:55:51 +0000 Subject: [PATCH 459/710] Add a four-byte prefix to the BtShared.pTmpSpace buffer to avoid reading before the beginning of an allocation. FossilOrigin-Name: 9386bfca128023583a24303e5f1d832987a49d43 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 22 +++++++++++++++++----- src/btreeInt.h | 2 +- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index c9b3f13dde..66f8f35d8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Rearrange\san\sexpression\sin\svdbemem.c\sto\savoid\sa\s(harmless)\sreference\sto\sa\spossibly\sunitialized\svariable. -D 2014-10-15T11:31:35.560 +C Add\sa\sfour-byte\sprefix\sto\sthe\sBtShared.pTmpSpace\sbuffer\sto\savoid\sreading\nbefore\sthe\sbeginning\sof\san\sallocation. +D 2014-10-15T11:55:51.434 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,9 +172,9 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c c9fcae8145436f728c61272cba72b1469c07f30d +F src/btree.c 1b1123cba0c65caa0baa51e71b8c089e3167c3ed F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 -F src/btreeInt.h 1bd7957161a1346a914f1f09231610e777a8e58d +F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eab82330631187dcc3e5d2dddd23dbda5752904b -R c46357701a0c6687b1cdb44cc5cea3cc -U dan -Z f35b20ba175f90e21204e2589425beed +P 4a7b3fa049a9aa0668e318287edd4a78c0588bf8 +R 0eaa9fa075886c1412c89d2319c66fd1 +U drh +Z fdb81bbd00d8de95e4a04ef38ee80f55 diff --git a/manifest.uuid b/manifest.uuid index 2ed282008d..864a483132 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a7b3fa049a9aa0668e318287edd4a78c0588bf8 \ No newline at end of file +9386bfca128023583a24303e5f1d832987a49d43 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 3553924c0f..758dfe6335 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2108,7 +2108,8 @@ static int removeFromSharingList(BtShared *pBt){ /* ** Make sure pBt->pTmpSpace points to an allocation of -** MX_CELL_SIZE(pBt) bytes. +** MX_CELL_SIZE(pBt) bytes with a 4-byte prefix for a left-child +** pointer. */ static void allocateTempSpace(BtShared *pBt){ if( !pBt->pTmpSpace ){ @@ -2123,8 +2124,16 @@ static void allocateTempSpace(BtShared *pBt){ ** it into a database page. This is not actually a problem, but it ** does cause a valgrind error when the 1 or 2 bytes of unitialized ** data is passed to system call write(). So to avoid this error, - ** zero the first 4 bytes of temp space here. */ - if( pBt->pTmpSpace ) memset(pBt->pTmpSpace, 0, 4); + ** zero the first 4 bytes of temp space here. + ** + ** Also: Provide four bytes of initialized space before the + ** beginning of pTmpSpace as an area available to prepend the + ** left-child pointer to the beginning of a cell. + */ + if( pBt->pTmpSpace ){ + memset(pBt->pTmpSpace, 0, 8); + pBt->pTmpSpace += 4; + } } } @@ -2132,8 +2141,11 @@ static void allocateTempSpace(BtShared *pBt){ ** Free the pBt->pTmpSpace allocation */ static void freeTempSpace(BtShared *pBt){ - sqlite3PageFree( pBt->pTmpSpace); - pBt->pTmpSpace = 0; + if( pBt->pTmpSpace ){ + pBt->pTmpSpace -= 4; + sqlite3PageFree(pBt->pTmpSpace); + pBt->pTmpSpace = 0; + } } /* diff --git a/src/btreeInt.h b/src/btreeInt.h index 9f648fceb0..2368e6c884 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -436,7 +436,7 @@ struct BtShared { BtLock *pLock; /* List of locks held on this shared-btree struct */ Btree *pWriter; /* Btree with currently open write transaction */ #endif - u8 *pTmpSpace; /* BtShared.pageSize bytes of space for tmp use */ + u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ }; /* From f41652419e75b83f7ebbcce40bffb5983087eb18 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Oct 2014 14:45:34 +0000 Subject: [PATCH 460/710] Fix a problem causing lock5.test to fail in mmap-mode. FossilOrigin-Name: b3e7b446bdb47cf9d7fe43dc37e3b4f8010ee09e --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/lock5.test | 7 ++++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 66f8f35d8a..479beb42cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sfour-byte\sprefix\sto\sthe\sBtShared.pTmpSpace\sbuffer\sto\savoid\sreading\nbefore\sthe\sbeginning\sof\san\sallocation. -D 2014-10-15T11:55:51.434 +C Fix\sa\sproblem\scausing\slock5.test\sto\sfail\sin\smmap-mode. +D 2014-10-15T14:45:34.650 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -677,7 +677,7 @@ F test/lock.test 87af515b0c4cf928576d0f89946d67d7c265dfb4 F test/lock2.test 5242d8ac4e2d59c403aebff606af449b455aceff F test/lock3.test f271375930711ae044080f4fe6d6eda930870d00 F test/lock4.test e175ae13865bc87680607563bafba21f31a26f12 -F test/lock5.test 5ad6a1f536036ff1be915cfdd41481aeafda3273 +F test/lock5.test c6c5e0ebcb21c61a572870cc86c0cb9f14cede38 F test/lock6.test ad5b387a3a8096afd3c68a55b9535056431b0cf5 F test/lock7.test 49f1eaff1cdc491cc5dee3669f3c671d9f172431 F test/lock_common.tcl 0c270b121d40959fa2f3add382200c27045b3d95 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4a7b3fa049a9aa0668e318287edd4a78c0588bf8 -R 0eaa9fa075886c1412c89d2319c66fd1 -U drh -Z fdb81bbd00d8de95e4a04ef38ee80f55 +P 9386bfca128023583a24303e5f1d832987a49d43 +R d767fb05b1c758bd73ecd7ed19e0f8f8 +U dan +Z deb18f7b499c6ed23800c00a8fad84b5 diff --git a/manifest.uuid b/manifest.uuid index 864a483132..ff3f963720 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9386bfca128023583a24303e5f1d832987a49d43 \ No newline at end of file +b3e7b446bdb47cf9d7fe43dc37e3b4f8010ee09e \ No newline at end of file diff --git a/test/lock5.test b/test/lock5.test index f0d495508a..99214afb19 100644 --- a/test/lock5.test +++ b/test/lock5.test @@ -154,6 +154,7 @@ do_test lock5-flock.8 { do_test lock5-none.1 { sqlite3 db test.db -vfs unix-none sqlite3 db2 test.db -vfs unix-none + execsql { PRAGMA mmap_size = 0 } db2 execsql { BEGIN; INSERT INTO t1 VALUES(3, 4); @@ -162,8 +163,8 @@ do_test lock5-none.1 { do_test lock5-none.2 { execsql { SELECT * FROM t1 } } {1 2 3 4} -do_test lock5-flock.3 { - execsql { SELECT * FROM t1 } db2 +do_test lock5-none.3 { + execsql { SELECT * FROM t1; } db2 } {1 2} do_test lock5-none.4 { execsql { @@ -183,7 +184,7 @@ ifcapable memorymanage { } {1 2 3 4} } -do_test lock5-flock.X { +do_test lock5-none.X { db close db2 close } {} From 5d510d4c412ec360cec5cd088fd8debff94e3354 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Oct 2014 15:28:27 +0000 Subject: [PATCH 461/710] Update releasetest.tcl so that x86-64 runs a superset of the x86 tests. FossilOrigin-Name: 3c1e70f4d55bc009ed9ed4cf6d756d7061985851 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/releasetest.tcl | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 479beb42cc..d416b82e23 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\scausing\slock5.test\sto\sfail\sin\smmap-mode. -D 2014-10-15T14:45:34.650 +C Update\sreleasetest.tcl\sso\sthat\sx86-64\sruns\sa\ssuperset\sof\sthe\sx86\stests. +D 2014-10-15T15:28:27.041 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -783,7 +783,7 @@ F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a -F test/releasetest.tcl 4296b9adbc5992bcd0b0b2876b7651f57c1494f2 +F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9386bfca128023583a24303e5f1d832987a49d43 -R d767fb05b1c758bd73ecd7ed19e0f8f8 +P b3e7b446bdb47cf9d7fe43dc37e3b4f8010ee09e +R d10aa2fd16b5624d36b7172d8913c506 U dan -Z deb18f7b499c6ed23800c00a8fad84b5 +Z 50710860f359ce0bbdbe754b3e261318 diff --git a/manifest.uuid b/manifest.uuid index ff3f963720..5726fd827b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b3e7b446bdb47cf9d7fe43dc37e3b4f8010ee09e \ No newline at end of file +3c1e70f4d55bc009ed9ed4cf6d756d7061985851 \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index 589319d680..d2a1bd2bb0 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -196,7 +196,8 @@ array set ::Platforms { "Device-Two" test "Ftrapv" test "No-lookaside" test - "Default" "threadtest test" + "Devkit" test + "Default" "threadtest fulltest" "Device-One" fulltest } Linux-i686 { From facd5fd6221a547a1fd92c0ee517d3978f34ce6d Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Oct 2014 11:45:14 +0000 Subject: [PATCH 462/710] Update the vdbe-compress.tcl script to account for expressions of the form (123>var), where "var" should be replaced with a reference to a union member. FossilOrigin-Name: 640345d880c6178f8434e3ce40329b7527588843 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/vdbe-compress.tcl | 5 +++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index d416b82e23..603373ecda 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sreleasetest.tcl\sso\sthat\sx86-64\sruns\sa\ssuperset\sof\sthe\sx86\stests. -D 2014-10-15T15:28:27.041 +C Update\sthe\svdbe-compress.tcl\sscript\sto\saccount\sfor\sexpressions\sof\sthe\sform\s(123>var),\swhere\s"var"\sshould\sbe\sreplaced\swith\sa\sreference\sto\sa\sunion\smember. +D 2014-10-16T11:45:14.341 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1199,12 +1199,12 @@ F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 -F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 +F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b3e7b446bdb47cf9d7fe43dc37e3b4f8010ee09e -R d10aa2fd16b5624d36b7172d8913c506 +P 3c1e70f4d55bc009ed9ed4cf6d756d7061985851 +R e0f61b3d5f668f3cfe77a35e5da93d93 U dan -Z 50710860f359ce0bbdbe754b3e261318 +Z d051f6e18c5411d016dc382904d39726 diff --git a/manifest.uuid b/manifest.uuid index 5726fd827b..be86e66b80 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c1e70f4d55bc009ed9ed4cf6d756d7061985851 \ No newline at end of file +640345d880c6178f8434e3ce40329b7527588843 \ No newline at end of file diff --git a/tool/vdbe-compress.tcl b/tool/vdbe-compress.tcl index a349830bcf..9477f4afe6 100644 --- a/tool/vdbe-compress.tcl +++ b/tool/vdbe-compress.tcl @@ -110,6 +110,11 @@ while {![eof stdin]} { foreach v $vlist { regsub -all "(\[^a-zA-Z0-9>.\])${v}(\\W)" $line "\\1u.$sname.$v\\2" line regsub -all "(\[^a-zA-Z0-9>.\])${v}(\\W)" $line "\\1u.$sname.$v\\2" line + + # The expressions above fail to catch instance of variable "abc" in + # expressions like (32>abc). The following expression makes those + # substitutions. + regsub -all "(\[^-\])>${v}(\\W)" $line "\\1>u.$sname.$v\\2" line } append afterUnion [string trimright $line]\n } elseif {$line=="" && [eof stdin]} { From 92c2e0da8f6fbec24e953f8b1acf26c6d33f55e0 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 16 Oct 2014 18:34:50 +0000 Subject: [PATCH 463/710] Changes to work around Win32 and MSVCRT APIs that are not present on Windows CE 200x. FossilOrigin-Name: 1418c006e377d7915a50577d4ccb21125b750bae --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/os_win.c | 7 ++++++- src/threads.c | 4 ++-- test/sort.test | 1 + 5 files changed, 19 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 603373ecda..fb97e6165f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\svdbe-compress.tcl\sscript\sto\saccount\sfor\sexpressions\sof\sthe\sform\s(123>var),\swhere\s"var"\sshould\sbe\sreplaced\swith\sa\sreference\sto\sa\sunion\smember. -D 2014-10-16T11:45:14.341 +C Changes\sto\swork\saround\sWin32\sand\sMSVCRT\sAPIs\sthat\sare\snot\spresent\son\sWindows\sCE\s200x. +D 2014-10-16T18:34:50.284 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 -F src/os_win.c 0a4042ef35f322e86fa01f6c8884c5e645b911e7 +F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c a171cf9dd09c6cb162b262c328d4dfd198e04f80 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -282,7 +282,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/threads.c 60c9d400abf17ccdc8767cdc6af90b9c5acf58bd +F src/threads.c 6de09362b657f19ba83e5fa521ee715787ce9fee F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 @@ -846,7 +846,7 @@ F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 -F test/sort.test 15e1d3014abc3f6d4357ed81b93b82117aefd235 +F test/sort.test c4400e7533748f6bd7413851ff148645e82b9e2d F test/sort2.test 269f4f50c6e468cc32b302ae7ff0add8338ec6de F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 F test/sort4.test 6c37d85f7cd28d50cce222fcab84ccd771e105cb @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3c1e70f4d55bc009ed9ed4cf6d756d7061985851 -R e0f61b3d5f668f3cfe77a35e5da93d93 -U dan -Z d051f6e18c5411d016dc382904d39726 +P 640345d880c6178f8434e3ce40329b7527588843 +R 46582f87dc5791f77f8a445cb47feb38 +U mistachkin +Z 04c8b48d78aac4913dca1bde0b860300 diff --git a/manifest.uuid b/manifest.uuid index be86e66b80..2cab3291b8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -640345d880c6178f8434e3ce40329b7527588843 \ No newline at end of file +1418c006e377d7915a50577d4ccb21125b750bae \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index e12ce4e532..8ca2107d90 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -943,7 +943,11 @@ static struct win_syscall { #define osWaitForSingleObject ((DWORD(WINAPI*)(HANDLE, \ DWORD))aSyscall[63].pCurrent) +#if !SQLITE_OS_WINCE { "WaitForSingleObjectEx", (SYSCALL)WaitForSingleObjectEx, 0 }, +#else + { "WaitForSingleObjectEx", (SYSCALL)0, 0 }, +#endif #define osWaitForSingleObjectEx ((DWORD(WINAPI*)(HANDLE,DWORD, \ BOOL))aSyscall[64].pCurrent) @@ -1286,7 +1290,8 @@ void sqlite3_win32_sleep(DWORD milliseconds){ #endif } -#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0 +#if SQLITE_MAX_WORKER_THREADS>0 && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && \ + SQLITE_THREADSAFE>0 DWORD sqlite3Win32Wait(HANDLE hObject){ DWORD rc; while( (rc = osWaitForSingleObjectEx(hObject, INFINITE, diff --git a/src/threads.c b/src/threads.c index 6d39042fd9..18d7320a12 100644 --- a/src/threads.c +++ b/src/threads.c @@ -98,7 +98,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ /********************************* Win32 Threads ****************************/ -#if SQLITE_OS_WIN && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0 +#if SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && SQLITE_THREADSAFE>0 #define SQLITE_THREADS_IMPLEMENTED 1 /* Prevent the single-thread code below */ #include @@ -191,7 +191,7 @@ int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ return (rc==WAIT_OBJECT_0) ? SQLITE_OK : SQLITE_ERROR; } -#endif /* SQLITE_OS_WIN && !SQLITE_OS_WINRT */ +#endif /* SQLITE_OS_WIN && !SQLITE_OS_WINCE && !SQLITE_OS_WINRT */ /******************************** End Win32 Threads *************************/ diff --git a/test/sort.test b/test/sort.test index 1c89552bb1..be2a6f531c 100644 --- a/test/sort.test +++ b/test/sort.test @@ -15,6 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix sort # Create a bunch of data to sort against # From 7bdc9749d5f3d34fe7ba6051673a58229f8fe91d Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 16 Oct 2014 21:39:17 +0000 Subject: [PATCH 464/710] Work around MSVC not being able to deduce that a local variable was initialized in a called function. FossilOrigin-Name: 06c576c152c4013080c255cbbeb45bf2e298be9f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fb97e6165f..6426f0698c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\swork\saround\sWin32\sand\sMSVCRT\sAPIs\sthat\sare\snot\spresent\son\sWindows\sCE\s200x. -D 2014-10-16T18:34:50.284 +C Work\saround\sMSVC\snot\sbeing\sable\sto\sdeduce\sthat\sa\slocal\svariable\swas\sinitialized\sin\sa\scalled\sfunction. +D 2014-10-16T21:39:17.263 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -296,7 +296,7 @@ F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f -F src/vdbesort.c 5c1bacf90578d22b630fbf6ed98ccf60d83435ef +F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 @@ -1204,7 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 640345d880c6178f8434e3ce40329b7527588843 -R 46582f87dc5791f77f8a445cb47feb38 +P 1418c006e377d7915a50577d4ccb21125b750bae +R 76bb09f8a8f8bdebeb708f5bb39aa31b U mistachkin -Z 04c8b48d78aac4913dca1bde0b860300 +Z baca4535ffaf8e732b4d2d60c4d0a012 diff --git a/manifest.uuid b/manifest.uuid index 2cab3291b8..5f80a79efb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1418c006e377d7915a50577d4ccb21125b750bae \ No newline at end of file +06c576c152c4013080c255cbbeb45bf2e298be9f \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index d9679caa06..46c9f3789d 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -2292,7 +2292,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ assert( pSorter->bUseThreads==0 || pSorter->nTask>1 ); if( pSorter->bUseThreads ){ int iTask; - PmaReader *pReadr; + PmaReader *pReadr = 0; SortSubtask *pLast = &pSorter->aTask[pSorter->nTask-1]; rc = vdbeSortAllocUnpacked(pLast); if( rc==SQLITE_OK ){ From 81b567a4c73c06069d72b95ee2b09a0aca46b3ec Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Oct 2014 11:24:17 +0000 Subject: [PATCH 465/710] Version 3.8.7 FossilOrigin-Name: e4ab094f8afce0817f4074e823fabe59fc29ebb4 --- manifest | 13 ++++++++----- manifest.uuid | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/manifest b/manifest index 6426f0698c..e210943d15 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\sMSVC\snot\sbeing\sable\sto\sdeduce\sthat\sa\slocal\svariable\swas\sinitialized\sin\sa\scalled\sfunction. -D 2014-10-16T21:39:17.263 +C Version\s3.8.7 +D 2014-10-17T11:24:17.839 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1204,7 +1204,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1418c006e377d7915a50577d4ccb21125b750bae +P 06c576c152c4013080c255cbbeb45bf2e298be9f R 76bb09f8a8f8bdebeb708f5bb39aa31b -U mistachkin -Z baca4535ffaf8e732b4d2d60c4d0a012 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.8.7 * +U drh +Z b2f516d1147acb0e1bf1c700327ee52e diff --git a/manifest.uuid b/manifest.uuid index 5f80a79efb..f4f611fe44 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -06c576c152c4013080c255cbbeb45bf2e298be9f \ No newline at end of file +e4ab094f8afce0817f4074e823fabe59fc29ebb4 \ No newline at end of file From 40253262e0b3f9bfd79b6a57c891eb0f0459a38f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Oct 2014 21:35:05 +0000 Subject: [PATCH 466/710] Fix a (probably harmless) but in the CSV output mode of the command-line shell. FossilOrigin-Name: 19fe4a0a475bd94f491031aea7a183f7c0515cf3 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/shell.c | 2 +- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e210943d15..894c7cd51a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.7 -D 2014-10-17T11:24:17.839 +C Fix\sa\s(probably\sharmless)\sbut\sin\sthe\sCSV\soutput\smode\sof\sthe\scommand-line\nshell. +D 2014-10-17T21:35:05.954 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 18ee8bbe9502d8848072dc2eddd1ea09254ba494 +F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1204,10 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 06c576c152c4013080c255cbbeb45bf2e298be9f -R 76bb09f8a8f8bdebeb708f5bb39aa31b -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.7 * +P e4ab094f8afce0817f4074e823fabe59fc29ebb4 +R bacb142e035c538b5014a902fee951f4 U drh -Z b2f516d1147acb0e1bf1c700327ee52e +Z 99accb7d1ae416e10fed302750dd8f99 diff --git a/manifest.uuid b/manifest.uuid index f4f611fe44..db419ccbec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4ab094f8afce0817f4074e823fabe59fc29ebb4 \ No newline at end of file +19fe4a0a475bd94f491031aea7a183f7c0515cf3 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 3ca4b094bc..59cd2011e7 100644 --- a/src/shell.c +++ b/src/shell.c @@ -882,7 +882,7 @@ static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int } fprintf(p->out,"%s",p->newline); } - if( azArg>0 ){ + if( nArg>0 ){ for(i=0; i Date: Tue, 21 Oct 2014 01:05:09 +0000 Subject: [PATCH 467/710] If a skip-scan is a proper subset of some other scan, then adjust the cost of the skip-scan upward so that it is more costly than the other scan. Such a cost imbalance can arise under STAT4 because of difficulties in getting an accurate estimate for skip-scans. FossilOrigin-Name: f4b22a2620a5dc48949048c2ecbd226755d4b2c3 --- manifest | 15 ++--- manifest.uuid | 2 +- src/where.c | 121 +++++++++++++++++------------------- src/whereInt.h | 4 +- test/skipscan6.test | 145 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 212 insertions(+), 75 deletions(-) create mode 100644 test/skipscan6.test diff --git a/manifest b/manifest index 894c7cd51a..3c0097dfe9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\s(probably\sharmless)\sbut\sin\sthe\sCSV\soutput\smode\sof\sthe\scommand-line\nshell. -D 2014-10-17T21:35:05.954 +C If\sa\sskip-scan\sis\sa\sproper\ssubset\sof\ssome\sother\sscan,\sthen\sadjust\sthe\ncost\sof\sthe\sskip-scan\supward\sso\sthat\sit\sis\smore\scostly\sthan\sthe\sother\sscan.\nSuch\sa\scost\simbalance\scan\sarise\sunder\sSTAT4\sbecause\sof\sdifficulties\sin\sgetting\nan\saccurate\sestimate\sfor\sskip-scans. +D 2014-10-21T01:05:09.795 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,8 +302,8 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2947912f1f3d6a7766fe087fd532a5d688d745b1 -F src/whereInt.h 124d970450955a6982e174b07c320ae6d62a595c +F src/where.c 2d1ff51eede0e4dcc87569dc8e3161237295162a +F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -844,6 +844,7 @@ F test/skipscan1.test 7e15e1cc524524e7b2c4595ec85c75501d22f4ff F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 +F test/skipscan6.test 3a891b45d6df266ced861a2ad9d03fca2bc7fcc5 F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test c4400e7533748f6bd7413851ff148645e82b9e2d @@ -1204,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e4ab094f8afce0817f4074e823fabe59fc29ebb4 -R bacb142e035c538b5014a902fee951f4 +P 19fe4a0a475bd94f491031aea7a183f7c0515cf3 +R 7ae877aeccb485915279368cd2f54415 U drh -Z 99accb7d1ae416e10fed302750dd8f99 +Z dece11d7f7de853e09f0182395012c8f diff --git a/manifest.uuid b/manifest.uuid index db419ccbec..ee9b76f602 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19fe4a0a475bd94f491031aea7a183f7c0515cf3 \ No newline at end of file +f4b22a2620a5dc48949048c2ecbd226755d4b2c3 \ No newline at end of file diff --git a/src/where.c b/src/where.c index bc0110779e..00b1880880 100644 --- a/src/where.c +++ b/src/where.c @@ -2638,7 +2638,7 @@ static int codeAllEqualityTerms( pLoop = pLevel->pWLoop; assert( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 ); nEq = pLoop->u.btree.nEq; - nSkip = pLoop->u.btree.nSkip; + nSkip = pLoop->nSkip; pIdx = pLoop->u.btree.pIndex; assert( pIdx!=0 ); @@ -2752,7 +2752,7 @@ static void explainAppendTerm( static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ Index *pIndex = pLoop->u.btree.pIndex; u16 nEq = pLoop->u.btree.nEq; - u16 nSkip = pLoop->u.btree.nSkip; + u16 nSkip = pLoop->nSkip; int i, j; Column *aCol = pTab->aCol; i16 *aiColumn = pIndex->aiColumn; @@ -3189,7 +3189,7 @@ static Bitmask codeOneLoopStart( pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; - assert( nEq>=pLoop->u.btree.nSkip ); + assert( nEq>=pLoop->nSkip ); /* If this loop satisfies a sort order (pOrderBy) request that ** was passed to this function to implement a "SELECT min(x) ..." @@ -3206,7 +3206,7 @@ static Bitmask codeOneLoopStart( && pWInfo->nOBSat>0 && (pIdx->nKeyCol>nEq) ){ - assert( pLoop->u.btree.nSkip==0 ); + assert( pLoop->nSkip==0 ); bSeekPastNull = 1; nExtraReg = 1; } @@ -3827,7 +3827,7 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3_free(z); } if( p->wsFlags & WHERE_SKIPSCAN ){ - sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->u.btree.nSkip); + sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); }else{ sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); } @@ -3956,12 +3956,15 @@ static int whereLoopCheaperProperSubset( const WhereLoop *pY /* Compare against this WhereLoop */ ){ int i, j; - if( pX->nLTerm >= pY->nLTerm ) return 0; /* X is not a subset of Y */ + if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ + return 0; /* X is not a subset of Y */ + } if( pX->rRun >= pY->rRun ){ if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ } for(i=pX->nLTerm-1; i>=0; i--){ + if( pX->aLTerm[i]==0 ) continue; for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } @@ -3983,25 +3986,12 @@ static int whereLoopCheaperProperSubset( ** To say "WhereLoop X is a proper subset of Y" means that X uses fewer ** WHERE clause terms than Y and that every WHERE clause term used by X is ** also used by Y. -** -** This adjustment is omitted for SKIPSCAN loops. In a SKIPSCAN loop, the -** WhereLoop.nLTerm field is not an accurate measure of the number of WHERE -** clause terms covered, since some of the first nLTerm entries in aLTerm[] -** will be NULL (because they are skipped). That makes it more difficult -** to compare the loops. We could add extra code to do the comparison, and -** perhaps we will someday. But SKIPSCAN is sufficiently uncommon, and this -** adjustment is sufficient minor, that it is very difficult to construct -** a test case where the extra code would improve the query plan. Better -** to avoid the added complexity and just omit cost adjustments to SKIPSCAN -** loops. */ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (pTemplate->wsFlags & WHERE_INDEXED)==0 ) return; - if( (pTemplate->wsFlags & WHERE_SKIPSCAN)!=0 ) return; for(; p; p=p->pNextLoop){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; - if( (p->wsFlags & WHERE_SKIPSCAN)!=0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ @@ -4295,7 +4285,7 @@ static int whereLoopAddBtreeIndex( Bitmask saved_prereq; /* Original value of pNew->prereq */ u16 saved_nLTerm; /* Original value of pNew->nLTerm */ u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ - u16 saved_nSkip; /* Original value of pNew->u.btree.nSkip */ + u16 saved_nSkip; /* Original value of pNew->nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ int iCol; /* Index of the column in the table */ @@ -4324,7 +4314,7 @@ static int whereLoopAddBtreeIndex( pTerm = whereScanInit(&scan, pBuilder->pWC, pSrc->iCursor, iCol, opMask, pProbe); saved_nEq = pNew->u.btree.nEq; - saved_nSkip = pNew->u.btree.nSkip; + saved_nSkip = pNew->nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; saved_prereq = pNew->prereq; @@ -4332,44 +4322,6 @@ static int whereLoopAddBtreeIndex( pNew->rSetup = 0; rSize = pProbe->aiRowLogEst[0]; rLogSize = estLog(rSize); - - /* Consider using a skip-scan if there are no WHERE clause constraints - ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. - ** - ** The magic number 18 is selected on the basis that scanning 17 rows - ** is almost always quicker than an index seek (even though if the index - ** contains fewer than 2^17 rows we assume otherwise in other parts of - ** the code). And, even if it is not, it should not be too much slower. - ** On the other hand, the extra seeks could end up being significantly - ** more expensive. */ - assert( 42==sqlite3LogEst(18) ); - if( saved_nEq==saved_nSkip - && saved_nEq+1nKeyCol - && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ - && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK - ){ - LogEst nIter; - pNew->u.btree.nEq++; - pNew->u.btree.nSkip++; - pNew->aLTerm[pNew->nLTerm++] = 0; - pNew->wsFlags |= WHERE_SKIPSCAN; - nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; - if( pTerm ){ - /* TUNING: When estimating skip-scan for a term that is also indexable, - ** multiply the cost of the skip-scan by 2.0, to make it a little less - ** desirable than the regular index lookup. */ - nIter += 10; assert( 10==sqlite3LogEst(2) ); - } - pNew->nOut -= nIter; - /* TUNING: Because uncertainties in the estimates for skip-scan queries, - ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ - nIter += 5; - whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); - pNew->nOut = saved_nOut; - pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nSkip = saved_nSkip; - } for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ u16 eOp = pTerm->eOperator; /* Shorthand for pTerm->eOperator */ LogEst rCostIdx; @@ -4532,10 +4484,50 @@ static int whereLoopAddBtreeIndex( } pNew->prereq = saved_prereq; pNew->u.btree.nEq = saved_nEq; - pNew->u.btree.nSkip = saved_nSkip; + pNew->nSkip = saved_nSkip; pNew->wsFlags = saved_wsFlags; pNew->nOut = saved_nOut; pNew->nLTerm = saved_nLTerm; + + /* Consider using a skip-scan if there are no WHERE clause constraints + ** available for the left-most terms of the index, and if the average + ** number of repeats in the left-most terms is at least 18. + ** + ** The magic number 18 is selected on the basis that scanning 17 rows + ** is almost always quicker than an index seek (even though if the index + ** contains fewer than 2^17 rows we assume otherwise in other parts of + ** the code). And, even if it is not, it should not be too much slower. + ** On the other hand, the extra seeks could end up being significantly + ** more expensive. */ + assert( 42==sqlite3LogEst(18) ); + if( saved_nEq==saved_nSkip + && saved_nEq+1nKeyCol + && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ + && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK + ){ + LogEst nIter; + pNew->u.btree.nEq++; + pNew->nSkip++; + pNew->aLTerm[pNew->nLTerm++] = 0; + pNew->wsFlags |= WHERE_SKIPSCAN; + nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; + if( pTerm ){ + /* TUNING: When estimating skip-scan for a term that is also indexable, + ** multiply the cost of the skip-scan by 2.0, to make it a little less + ** desirable than the regular index lookup. */ + nIter += 10; assert( 10==sqlite3LogEst(2) ); + } + pNew->nOut -= nIter; + /* TUNING: Because uncertainties in the estimates for skip-scan queries, + ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ + nIter += 5; + whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nIter + nInMul); + pNew->nOut = saved_nOut; + pNew->u.btree.nEq = saved_nEq; + pNew->nSkip = saved_nSkip; + pNew->wsFlags = saved_wsFlags; + } + return rc; } @@ -4714,7 +4706,7 @@ static int whereLoopAddBtree( if( pTerm->prereqRight & pNew->maskSelf ) continue; if( termCanDriveIndex(pTerm, pSrc, 0) ){ pNew->u.btree.nEq = 1; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->u.btree.pIndex = 0; pNew->nLTerm = 1; pNew->aLTerm[0] = pTerm; @@ -4755,7 +4747,7 @@ static int whereLoopAddBtree( } rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; - pNew->u.btree.nSkip = 0; + pNew->nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; pNew->rSetup = 0; @@ -5305,7 +5297,7 @@ static i8 wherePathSatisfiesOrderBy( /* Skip over == and IS NULL terms */ if( ju.btree.nEq - && pLoop->u.btree.nSkip==0 + && pLoop->nSkip==0 && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL))!=0 ){ if( i & WO_ISNULL ){ @@ -5878,7 +5870,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; pLoop->wsFlags = 0; - pLoop->u.btree.nSkip = 0; + pLoop->nSkip = 0; pTerm = findTerm(pWC, iCur, -1, 0, WO_EQ, 0); if( pTerm ){ pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; @@ -5890,7 +5882,6 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pLoop->aLTermSpace==pLoop->aLTerm ); - assert( ArraySize(pLoop->aLTermSpace)==4 ); if( !IsUniqueIndex(pIdx) || pIdx->pPartIdxWhere!=0 || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) diff --git a/src/whereInt.h b/src/whereInt.h index f17906e63a..e9eb8b7dd2 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -115,7 +115,6 @@ struct WhereLoop { union { struct { /* Information for internal btree tables */ u16 nEq; /* Number of equality constraints */ - u16 nSkip; /* Number of initial index columns to skip */ Index *pIndex; /* Index used, or NULL */ } btree; struct { /* Information for virtual tables */ @@ -128,12 +127,13 @@ struct WhereLoop { } u; u32 wsFlags; /* WHERE_* flags describing the plan */ u16 nLTerm; /* Number of entries in aLTerm[] */ + u16 nSkip; /* Number of NULL aLTerm[] entries */ /**** whereLoopXfer() copies fields above ***********************/ # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) u16 nLSlot; /* Number of slots allocated for aLTerm[] */ WhereTerm **aLTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ - WhereTerm *aLTermSpace[4]; /* Initial aLTerm[] space */ + WhereTerm *aLTermSpace[3]; /* Initial aLTerm[] space */ }; /* This object holds the prerequisites and the cost of running a diff --git a/test/skipscan6.test b/test/skipscan6.test new file mode 100644 index 0000000000..9eda9a66f3 --- /dev/null +++ b/test/skipscan6.test @@ -0,0 +1,145 @@ +# 2014-10-21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file implements tests of the "skip-scan" query strategy. In +# particular, this file verifies that use of all columns of an index +# is always preferred over the use of a skip-scan on some columns of +# the same index. Because of difficulties in scoring a skip-scan, +# the skip-scan can sometimes come out with a lower raw score when +# using STAT4. But the query planner should detect this and use the +# full index rather than the skip-scan. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix skipscan6 + +ifcapable !stat4 { + finish_test + return +} + +do_execsql_test 1.1 { + CREATE TABLE t1( + aa int, + bb int, + cc int, + dd int, + ee int + ); + CREATE INDEX ix on t1(aa, bb, cc, dd DESC); + ANALYZE sqlite_master; + INSERT INTO sqlite_stat1 VALUES('t1','ix','2695116 1347558 264 18 2'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5'); + INSERT INTO sqlite_stat4 VALUES('t1','ix','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35'); + ANALYZE sqlite_master; +} {} +do_execsql_test 1.2 { + EXPLAIN QUERY PLAN + SELECT COUNT(*) + FROM t1 + WHERE bb=21 + AND aa=1 + AND dd BETWEEN 1413833728 and 1413837331; +} {/INDEX ix .aa=. AND bb=../} + +do_execsql_test 2.1 { + DROP INDEX ix; + CREATE INDEX good on t1(bb, aa, dd DESC); + CREATE INDEX bad on t1(aa, bb, cc, dd DESC); + DELETE FROM sqlite_stat1; + DELETE FROM sqlite_stat4; + INSERT INTO sqlite_stat1 VALUES('t1','good','2695116 299 264 2'); + INSERT INTO sqlite_stat1 VALUES('t1','bad','2695116 1347558 264 18 2'); + INSERT INTO sqlite_stat4 VALUES('t1','good','197030 196859 32 1','15086 15086 92511 92536','19 25 81644 92536',X'05010904031552977BD725BD22'); + INSERT INTO sqlite_stat4 VALUES('t1','good','14972 14687 1 1','289878 289878 299457 299457','199 244 267460 299457',X'050209040301344F7E569402C419'); + INSERT INTO sqlite_stat4 VALUES('t1','good','19600 19313 22 1','327127 327127 346222 346243','261 319 306884 346243',X'0502090403018A49503BC01EC577'); + INSERT INTO sqlite_stat4 VALUES('t1','good','25666 25047 15 1','352087 352087 372692 372706','266 327 325601 372706',X'050209040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','good','42392 42327 26 1','378657 378657 382547 382572','268 331 333529 382572',X'05020904030193533B2FE326ED48'); + INSERT INTO sqlite_stat4 VALUES('t1','good','24619 24513 11 1','457872 457872 461748 461758','286 358 399322 461758',X'050209040301A752B1557825EA7C'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18969 18730 15 1','482491 482491 501105 501119','287 360 433605 501119',X'050209040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','good','119710 119603 1 1','576500 576500 598915 598915','404 505 519877 598915',X'05020904030247539A7A7912F617'); + INSERT INTO sqlite_stat4 VALUES('t1','good','11955 11946 1 1','889796 889796 898373 898373','938 1123 794694 898373',X'050209040304EF4DF9C4150BBB28'); + INSERT INTO sqlite_stat4 VALUES('t1','good','57197 57138 24 1','1129865 1129865 1151492 1151515','1967 2273 1027048 1151515',X'05020904030988533510BC26E20A'); + INSERT INTO sqlite_stat4 VALUES('t1','good','3609 3543 1 1','1196265 1196265 1197831 1197831','2002 2313 1070108 1197831',X'050209040309B050E95CD718D94D'); + INSERT INTO sqlite_stat4 VALUES('t1','good','25391 25365 13 1','1309378 1309378 1315567 1315579','2561 2936 1178358 1315579',X'05020904030C5F53DF9E13283570'); + INSERT INTO sqlite_stat4 VALUES('t1','good','45232 45180 17 1','1334769 1334769 1337946 1337962','2562 2938 1198998 1337962',X'05020904030C60541CACEE28BCAC'); + INSERT INTO sqlite_stat4 VALUES('t1','good','5496 5493 1 1','1495882 1495882 1497289 1497289','3043 3479 1348695 1497289',X'05020904030E99515C62AD0F0B34'); + INSERT INTO sqlite_stat4 VALUES('t1','good','26348 26139 17 1','1517381 1517381 1529990 1530006','3074 3519 1378320 1530006',X'05020904030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','good','102927 102894 10 1','1547088 1547088 1649950 1649959','3109 3559 1494260 1649959',X'05020904030EE34D309F671FFA47'); + INSERT INTO sqlite_stat4 VALUES('t1','good','3602 3576 1 1','1793873 1793873 1796747 1796747','3601 4128 1630783 1796747',X'050209040311294FE88B432219B9'); + INSERT INTO sqlite_stat4 VALUES('t1','good','154 154 1 1','2096059 2096059 2096205 2096205','5037 5779 1893039 2096205',X'050209040317994EFF05A016DCED'); + INSERT INTO sqlite_stat4 VALUES('t1','good','68153 66574 60 1','2244039 2244039 2268892 2268951','5899 6749 2027553 2268951',X'05020904031B8A532DBC5A26D2BA'); + INSERT INTO sqlite_stat4 VALUES('t1','good','321 321 1 1','2395618 2395618 2395663 2395663','6609 7528 2118435 2395663',X'05020904031EFA54078EEE1E2D65'); + INSERT INTO sqlite_stat4 VALUES('t1','good','19449 19440 22 1','2407769 2407769 2426049 2426070','6718 7651 2146904 2426070',X'05020904031F7450E6118C2336BD'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18383 18321 56 1','2539949 2539949 2551080 2551135','7838 8897 2245459 2551135',X'050209040324A752EA2E1E2642B2'); + INSERT INTO sqlite_stat4 VALUES('t1','good','22479 22384 60 1','2558332 2558332 2565233 2565292','7839 8899 2251202 2565292',X'050209040324A853926538279A5F'); + INSERT INTO sqlite_stat4 VALUES('t1','good','18771 18699 63 1','2580811 2580811 2596914 2596976','7840 8901 2263572 2596976',X'050209040324A9526C1DE9256E72'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 196859 196859 32 1','0 15043 15043 92468 92499','0 19 286 81846 92499',X'0609010804031552977BD725BD28'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 14687 161 1 1','0 289067 299306 299457 299457','0 199 6772 273984 299457',X'060902020403013406314D67456415B819'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19313 19308 22 1','0 325815 325815 343725 343746','0 261 9545 315009 343746',X'060902080403018A49B0A3AD1ED931'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25047 9051 15 1','0 350443 350443 356590 356604','0 266 9795 325519 356604',X'06090208040301914C2DD2E91F93CF'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 42327 9906 7 1','0 376381 376381 380291 380297','0 268 10100 344232 380297',X'06090208040301934BF672511F7ED3'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 24513 2237 1 1','0 455150 467779 470015 470015','0 286 10880 425401 470015',X'06090202040301A703464A28F2611EF1EE'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18730 18724 15 1','0 479663 479663 498271 498285','0 287 10998 450793 498285',X'06090208040301A8494AF3A41EC50C'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 119603 47125 1 1','0 572425 572425 598915 598915','0 404 14230 546497 598915',X'06090208040302474FD1929A03194F'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 1454 1454 1 1','0 898346 898346 898373 898373','0 952 31165 827562 898373',X'06090208040304FD53F6A2A2097F64'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 57138 7069 1 1','0 1122389 1122389 1129457 1129457','0 1967 46801 1045943 1129457',X'06090208040309884BC4C52F1F6EB7'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 285 11 1 1','0 1197683 1197824 1197831 1197831','0 2033 50990 1112280 1197831',X'06090202040309D80346503FE2A9038E4F'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 25365 9773 1 1','0 1301013 1301013 1310785 1310785','0 2561 58806 1217877 1310785',X'0609020804030C5F4C8F88AB0AF2A2'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 45180 7222 1 1','0 1326378 1326378 1333599 1333599','0 2562 59921 1240187 1333599',X'0609020804030C604CAB75490B0351'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 8537 41 1 1','0 1496959 1497288 1497289 1497289','0 3050 68246 1394126 1497289',X'0609020204030EA0057F527459B0257C4B'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 26139 26131 17 1','0 1507977 1507977 1520578 1520594','0 3074 69188 1416111 1520594',X'0609020804030EB95169453423D4EA'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 102894 29678 1 1','0 1537421 1550467 1564894 1564894','0 3109 69669 1459820 1564894',X'0609020204030EE3183652A6ED3006EBCB'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 319 3 1 1','0 1796728 1796746 1796747 1796747','0 3650 86468 1682243 1796747',X'0609020204031163033550D0C41018C28D'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 127 127 1 1','0 2096194 2096194 2096205 2096205','0 5145 106437 1951535 2096205',X'060902080403180F53BB1AF727EE50'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 66574 5252 1 1','0 2230524 2265961 2271212 2271212','0 5899 114976 2085829 2271212',X'0609020204031B8A05195009976D223B90'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 19440 19440 1 1','0 2391680 2391680 2395663 2395663','0 6718 123714 2184781 2395663',X'0609020804031F7452E00A7B07431A'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18321 2177 1 1','0 2522928 2523231 2525407 2525407','0 7838 139084 2299958 2525407',X'06090201040324A7475231103B1AA7B8'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 22384 1361 1 1','0 2541249 2544834 2546194 2546194','0 7839 139428 2308416 2546194',X'06090202040324A8011652323D4B1AA9EB'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','2677151 18699 855 1 1','0 2563633 2578178 2579032 2579032','0 7840 139947 2321671 2579032',X'06090202040324A9077452323D7D1052C5'); + INSERT INTO sqlite_stat4 VALUES('t1','bad','17965 1579 1579 1 1','2677151 2690666 2690666 2692244 2692244','1 9870 153959 2418294 2692244',X'060102080403021B8A4FE1AB84032B35'); + ANALYZE sqlite_master; +} {} +do_execsql_test 2.2 { + EXPLAIN QUERY PLAN + SELECT COUNT(*) + FROM t1 + WHERE bb=21 + AND aa=1 + AND dd BETWEEN 1413833728 and 1413837331; +} {/INDEX good .bb=. AND aa=. AND dd>. AND dd<../} + + + +finish_test From 1b131b7a7acc83c7bd1a5e7d0872c968d9e26489 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Oct 2014 16:01:40 +0000 Subject: [PATCH 468/710] Improvements to the WHERETRACE debugging logic. FossilOrigin-Name: ec1e942f08548695ff02645b3f3cd6bb2516bc9a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3c0097dfe9..d708dca4f3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\sskip-scan\sis\sa\sproper\ssubset\sof\ssome\sother\sscan,\sthen\sadjust\sthe\ncost\sof\sthe\sskip-scan\supward\sso\sthat\sit\sis\smore\scostly\sthan\sthe\sother\sscan.\nSuch\sa\scost\simbalance\scan\sarise\sunder\sSTAT4\sbecause\sof\sdifficulties\sin\sgetting\nan\saccurate\sestimate\sfor\sskip-scans. -D 2014-10-21T01:05:09.795 +C Improvements\sto\sthe\sWHERETRACE\sdebugging\slogic. +D 2014-10-21T16:01:40.774 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2d1ff51eede0e4dcc87569dc8e3161237295162a +F src/where.c 45cb63cb1422d7e5a9229c297e978d294ae51e16 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 19fe4a0a475bd94f491031aea7a183f7c0515cf3 -R 7ae877aeccb485915279368cd2f54415 +P f4b22a2620a5dc48949048c2ecbd226755d4b2c3 +R 59b7c6284af5e31186eaa6edb85f9e59 U drh -Z dece11d7f7de853e09f0182395012c8f +Z 3eaff9a0b8f4f97b5dd119bc1e1acc8b diff --git a/manifest.uuid b/manifest.uuid index ee9b76f602..838729f698 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4b22a2620a5dc48949048c2ecbd226755d4b2c3 \ No newline at end of file +ec1e942f08548695ff02645b3f3cd6bb2516bc9a \ No newline at end of file diff --git a/src/where.c b/src/where.c index 00b1880880..7dfe0f02bb 100644 --- a/src/where.c +++ b/src/where.c @@ -3995,11 +3995,15 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut - 1; }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ /* Adjust pTemplate cost upward so that it is costlier than p since ** pTemplate is a proper subset of p */ + WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", + pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1)); pTemplate->rRun = p->rRun; pTemplate->nOut = p->nOut + 1; } @@ -5751,7 +5755,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } #ifdef WHERETRACE_ENABLED /* >=2 */ - if( sqlite3WhereTrace>=2 ){ + if( sqlite3WhereTrace & 0x02 ){ sqlite3DebugPrintf("---- after round %d ----\n", iLoop); for(ii=0, pTo=aTo; ii Date: Tue, 21 Oct 2014 18:16:21 +0000 Subject: [PATCH 469/710] Further tuning of the cost estimates for skip-scan loops, especially for cases when skip-scan loops are in competition with regular loops. FossilOrigin-Name: a27861c28c4791e51d797aa37e9cca806cb58775 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 10 +++------- 3 files changed, 10 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index d708dca4f3..3daca49fdd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\sWHERETRACE\sdebugging\slogic. -D 2014-10-21T16:01:40.774 +C Further\stuning\sof\sthe\scost\sestimates\sfor\sskip-scan\sloops,\sespecially\sfor\scases\nwhen\sskip-scan\sloops\sare\sin\scompetition\swith\sregular\sloops. +D 2014-10-21T18:16:21.388 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 45cb63cb1422d7e5a9229c297e978d294ae51e16 +F src/where.c 994b38c8697aad095878ef1e4860902df457427f F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f4b22a2620a5dc48949048c2ecbd226755d4b2c3 -R 59b7c6284af5e31186eaa6edb85f9e59 +P ec1e942f08548695ff02645b3f3cd6bb2516bc9a +R 4f5d15d10f268e58d7c047d490bf8adc U drh -Z 3eaff9a0b8f4f97b5dd119bc1e1acc8b +Z 7d7f3f0271fe8b8df2863a2c22053442 diff --git a/manifest.uuid b/manifest.uuid index 838729f698..52d4aafd99 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec1e942f08548695ff02645b3f3cd6bb2516bc9a \ No newline at end of file +a27861c28c4791e51d797aa37e9cca806cb58775 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 7dfe0f02bb..7f51d00c87 100644 --- a/src/where.c +++ b/src/where.c @@ -3994,7 +3994,9 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its - ** subset p */ + ** subset p. Except, do not adjust the cost estimate downward for + ** a loop that skips more columns. */ + if( pTemplate->nSkip>p->nSkip ) continue; WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; @@ -4515,12 +4517,6 @@ static int whereLoopAddBtreeIndex( pNew->aLTerm[pNew->nLTerm++] = 0; pNew->wsFlags |= WHERE_SKIPSCAN; nIter = pProbe->aiRowLogEst[saved_nEq] - pProbe->aiRowLogEst[saved_nEq+1]; - if( pTerm ){ - /* TUNING: When estimating skip-scan for a term that is also indexable, - ** multiply the cost of the skip-scan by 2.0, to make it a little less - ** desirable than the regular index lookup. */ - nIter += 10; assert( 10==sqlite3LogEst(2) ); - } pNew->nOut -= nIter; /* TUNING: Because uncertainties in the estimates for skip-scan queries, ** add a 1.375 fudge factor to make skip-scan slightly less likely. */ From 442c5cd3cfc67d3e10aa64d9f180ef94fb3597cf Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Oct 2014 21:56:06 +0000 Subject: [PATCH 470/710] Call fsync() right after ftruncate() when in journal_mode=TRUNCATE and when synchronous=FULL in order to ensure that transactions are durable across a power loss that happens moments after the commit. Proposed fix for [https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. FossilOrigin-Name: 3e922208b68563489c7766abb9afb4885113e7b8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/pager.c | 8 ++++++++ 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3daca49fdd..f4a8a2a556 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\stuning\sof\sthe\scost\sestimates\sfor\sskip-scan\sloops,\sespecially\sfor\scases\nwhen\sskip-scan\sloops\sare\sin\scompetition\swith\sregular\sloops. -D 2014-10-21T18:16:21.388 +C Call\sfsync()\sright\safter\sftruncate()\swhen\sin\sjournal_mode=TRUNCATE\sand\nwhen\ssynchronous=FULL\sin\sorder\sto\sensure\sthat\stransactions\sare\sdurable\nacross\sa\spower\sloss\sthat\shappens\smoments\safter\sthe\scommit.\s\sProposed\nfix\sfor\s[https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. +D 2014-10-21T21:56:06.890 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -215,7 +215,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c a171cf9dd09c6cb162b262c328d4dfd198e04f80 +F src/pager.c a98547ad9b1b5dbbc5e7d1c520be041b5d2c0926 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ec1e942f08548695ff02645b3f3cd6bb2516bc9a -R 4f5d15d10f268e58d7c047d490bf8adc +P a27861c28c4791e51d797aa37e9cca806cb58775 +R 1375151e695773da263f40095fae22af U drh -Z 7d7f3f0271fe8b8df2863a2c22053442 +Z be5219411fed69e2fe0e0626925bcd5e diff --git a/manifest.uuid b/manifest.uuid index 52d4aafd99..db15d6664e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a27861c28c4791e51d797aa37e9cca806cb58775 \ No newline at end of file +3e922208b68563489c7766abb9afb4885113e7b8 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d3a36ef484..d840a39a15 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1941,6 +1941,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ rc = SQLITE_OK; }else{ rc = sqlite3OsTruncate(pPager->jfd, 0); + if( rc==SQLITE_OK && pPager->fullSync ){ + /* Make sure the new file size is written into the inode right away. + ** Otherwise the journal might resurrect following a power loss and + ** cause the last transaction to roll back. See + ** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773 + */ + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + } } pPager->journalOff = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST From 9f07cf7b2e43548f60bd7497dafba30856df3531 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Oct 2014 15:27:05 +0000 Subject: [PATCH 471/710] Take steps to avoid misestimating range query costs based on STAT4 data due to the roundoff error of converting from integers to LogEst and back to integers. FossilOrigin-Name: 3c933bf95f291f7957580d823dce92c981375a5c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 1 + src/sqliteInt.h | 3 ++- src/where.c | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f4a8a2a556..8290b14522 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Call\sfsync()\sright\safter\sftruncate()\swhen\sin\sjournal_mode=TRUNCATE\sand\nwhen\ssynchronous=FULL\sin\sorder\sto\sensure\sthat\stransactions\sare\sdurable\nacross\sa\spower\sloss\sthat\shappens\smoments\safter\sthe\scommit.\s\sProposed\nfix\sfor\s[https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. -D 2014-10-21T21:56:06.890 +C Take\ssteps\sto\savoid\smisestimating\srange\squery\scosts\sbased\son\sSTAT4\sdata\ndue\sto\sthe\sroundoff\serror\sof\sconverting\sfrom\sintegers\sto\sLogEst\sand\sback\nto\sintegers. +D 2014-10-22T15:27:05.734 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 8c322e1ecc08909526dbd5ab4421889d05f2263d +F src/analyze.c 567c94b763b67f7abda06dbf0ba34b0343ed9447 F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -232,7 +232,7 @@ F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h f7812f74f2d0c6041ef6b91a99c5a45f775dd408 +F src/sqliteInt.h d6d423b0f62846eb441236bc15417aeede2ebbdc F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 994b38c8697aad095878ef1e4860902df457427f +F src/where.c 5099c42e24c63969b3cf3b52e18c1a36cb841a34 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a27861c28c4791e51d797aa37e9cca806cb58775 -R 1375151e695773da263f40095fae22af +P 3e922208b68563489c7766abb9afb4885113e7b8 +R 5c508733aebac553de79291a7eabd709 U drh -Z be5219411fed69e2fe0e0626925bcd5e +Z a3d7104c7f173585d7e788d6453aa065 diff --git a/manifest.uuid b/manifest.uuid index db15d6664e..b84af28cee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3e922208b68563489c7766abb9afb4885113e7b8 \ No newline at end of file +3c933bf95f291f7957580d823dce92c981375a5c \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 7d36f01318..67bba1a28a 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1599,6 +1599,7 @@ static void initAvgEq(Index *pIdx){ nRow = pIdx->aiRowEst[0]; nDist100 = ((i64)100 * pIdx->aiRowEst[0]) / pIdx->aiRowEst[iCol+1]; } + pIdx->nRowEst0 = nRow; /* Set nSum to the number of distinct (iCol+1) field prefixes that ** occur in the stat4 table for this index. Set sumEq to the sum of diff --git a/src/sqliteInt.h b/src/sqliteInt.h index cba89b03e7..5409f7086f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1801,7 +1801,8 @@ struct Index { int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ - tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this table */ + tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ + tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif }; diff --git a/src/where.c b/src/where.c index 7f51d00c87..f708d7fad7 100644 --- a/src/where.c +++ b/src/where.c @@ -2198,7 +2198,7 @@ static int whereRangeScanEst( /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; - iUpper = sqlite3LogEstToInt(p->aiRowLogEst[0]); + iUpper = p->nRowEst0; }else{ /* Note: this call could be optimized away - since the same values must ** have been requested when testing key $P in whereEqualScanEst(). */ From d7d71470226bcc59799713b95ad9e10f9eff63cc Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Oct 2014 19:57:16 +0000 Subject: [PATCH 472/710] Change the 0x800 bit of SQLITE_TESTCTRL_OPTIMIZATIONS so that it disables the loading of STAT3 and STAT4 content, not just the using of that content. Change the internal name of that bit to SQLITE_Stat34. FossilOrigin-Name: ca3b00c44ec52d209642a5ba9ef82e085fac39db --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/analyze.c | 2 +- src/sqliteInt.h | 2 +- src/test1.c | 3 ++- src/where.c | 4 +--- 6 files changed, 15 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 8290b14522..0f41135502 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Take\ssteps\sto\savoid\smisestimating\srange\squery\scosts\sbased\son\sSTAT4\sdata\ndue\sto\sthe\sroundoff\serror\sof\sconverting\sfrom\sintegers\sto\sLogEst\sand\sback\nto\sintegers. -D 2014-10-22T15:27:05.734 +C Change\sthe\s0x800\sbit\sof\sSQLITE_TESTCTRL_OPTIMIZATIONS\sso\sthat\sit\sdisables\nthe\sloading\sof\sSTAT3\sand\sSTAT4\scontent,\snot\sjust\sthe\susing\sof\sthat\scontent.\nChange\sthe\sinternal\sname\sof\sthat\sbit\sto\sSQLITE_Stat34. +D 2014-10-22T19:57:16.520 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 567c94b763b67f7abda06dbf0ba34b0343ed9447 +F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e @@ -232,12 +232,12 @@ F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h d6d423b0f62846eb441236bc15417aeede2ebbdc +F src/sqliteInt.h 6e9e125698c1e5c78a51050ea61f179a281c766d F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 -F src/test1.c 518db4305d76b29dd9da3f022ca899c8fcdf9fc7 +F src/test1.c 63d4b1707c4052cf9c05c1cbb4a62666d70a0b48 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 5099c42e24c63969b3cf3b52e18c1a36cb841a34 +F src/where.c 2cd9e0af718d736459ae9d3b0f4532b4a80640d0 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3e922208b68563489c7766abb9afb4885113e7b8 -R 5c508733aebac553de79291a7eabd709 +P 3c933bf95f291f7957580d823dce92c981375a5c +R 3efbbff265eea57d45c364e4606463ea U drh -Z a3d7104c7f173585d7e788d6453aa065 +Z b4247f0ab9b19770a6354621b6169ee4 diff --git a/manifest.uuid b/manifest.uuid index b84af28cee..fff2e6a3fa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3c933bf95f291f7957580d823dce92c981375a5c \ No newline at end of file +ca3b00c44ec52d209642a5ba9ef82e085fac39db \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 67bba1a28a..597885237c 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1861,7 +1861,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat4 table. */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ int lookasideEnabled = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; rc = loadStat4(db, sInfo.zDatabase); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5409f7086f..1b3138be44 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1214,7 +1214,7 @@ struct sqlite3 { #define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ #define SQLITE_Transitive 0x0200 /* Transitive constraints */ #define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ -#define SQLITE_Stat3 0x0800 /* Use the SQLITE_STAT3 table */ +#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ #define SQLITE_AllOpts 0xffff /* All optimizations */ /* diff --git a/src/test1.c b/src/test1.c index 85a16488ba..802aa99b5b 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6293,7 +6293,8 @@ static int optimization_control( { "transitive", SQLITE_Transitive }, { "subquery-coroutine", SQLITE_SubqCoroutine }, { "omit-noop-join", SQLITE_OmitNoopJoin }, - { "stat3", SQLITE_Stat3 }, + { "stat3", SQLITE_Stat34 }, + { "stat4", SQLITE_Stat34 }, }; if( objc!=4 ){ diff --git a/src/where.c b/src/where.c index f708d7fad7..198a6c3fdd 100644 --- a/src/where.c +++ b/src/where.c @@ -1350,7 +1350,7 @@ static void exprAnalyze( if( pExpr->op==TK_NOTNULL && pExpr->pLeft->op==TK_COLUMN && pExpr->pLeft->iColumn>=0 - && OptimizationEnabled(db, SQLITE_Stat3) + && OptimizationEnabled(db, SQLITE_Stat34) ){ Expr *pNewExpr; Expr *pLeft = pExpr->pLeft; @@ -2159,7 +2159,6 @@ static int whereRangeScanEst( if( p->nSample>0 && nEqnSampleCol - && OptimizationEnabled(pParse->db, SQLITE_Stat3) ){ if( nEq==pBuilder->nRecValid ){ UnpackedRecord *pRec = pBuilder->pRec; @@ -4422,7 +4421,6 @@ static int whereLoopAddBtreeIndex( if( nInMul==0 && pProbe->nSample && pNew->u.btree.nEq<=pProbe->nSampleCol - && OptimizationEnabled(db, SQLITE_Stat3) && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) ){ Expr *pExpr = pTerm->pExpr; From b7288e28e48235e8f7671ae7b613f4c5c7251c44 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Oct 2014 20:07:19 +0000 Subject: [PATCH 473/710] Disable the use of strchrnul() unless specifically enabled by compile-time options. FossilOrigin-Name: e580470db77d6da970c755102790e603fb26b3c6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 6 +----- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 0f41135502..ae5d008c78 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\s0x800\sbit\sof\sSQLITE_TESTCTRL_OPTIMIZATIONS\sso\sthat\sit\sdisables\nthe\sloading\sof\sSTAT3\sand\sSTAT4\scontent,\snot\sjust\sthe\susing\sof\sthat\scontent.\nChange\sthe\sinternal\sname\sof\sthat\sbit\sto\sSQLITE_Stat34. -D 2014-10-22T19:57:16.520 +C Disable\sthe\suse\sof\sstrchrnul()\sunless\sspecifically\senabled\sby\scompile-time\noptions. +D 2014-10-22T20:07:19.558 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 6b79bbd063dcbadca4cf617a4cde255bcc13ea64 +F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3c933bf95f291f7957580d823dce92c981375a5c -R 3efbbff265eea57d45c364e4606463ea +P ca3b00c44ec52d209642a5ba9ef82e085fac39db +R 8aff7924cf3a3f93f4ee99fb545d1f5f U drh -Z b4247f0ab9b19770a6354621b6169ee4 +Z c6210337ac863b7dfe5cbe19c541a0aa diff --git a/manifest.uuid b/manifest.uuid index fff2e6a3fa..857c7b0e10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca3b00c44ec52d209642a5ba9ef82e085fac39db \ No newline at end of file +e580470db77d6da970c755102790e603fb26b3c6 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index c0b3c70f6b..1df287fbb6 100644 --- a/src/printf.c +++ b/src/printf.c @@ -21,11 +21,7 @@ ** the glibc version so the glibc version is definitely preferred. */ #if !defined(HAVE_STRCHRNUL) -# if defined(linux) -# define HAVE_STRCHRNUL 1 -# else -# define HAVE_STRCHRNUL 0 -# endif +# define HAVE_STRCHRNUL 0 #endif From 4f81bbb5289cdd248c21775b9e4cdb92e110e139 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 23 Oct 2014 01:01:26 +0000 Subject: [PATCH 474/710] Implement sqlite3_create_collation() by invoking sqlite3_create_collation_v2() with a NULL destructor argument. This saves a little space. FossilOrigin-Name: 9762ad0639cca2fc1ef0573113fb613ce9e7e83e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 8 +------- 3 files changed, 8 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index ae5d008c78..5b87a53662 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\suse\sof\sstrchrnul()\sunless\sspecifically\senabled\sby\scompile-time\noptions. -D 2014-10-22T20:07:19.558 +C Implement\ssqlite3_create_collation()\sby\sinvoking\s\nsqlite3_create_collation_v2()\swith\sa\sNULL\sdestructor\sargument.\s\sThis\ssaves\na\slittle\sspace. +D 2014-10-23T01:01:26.636 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c bbe872b0ac0007bed0ebe1936fc493b039ad4f51 +F src/main.c 8207ece1b4005b1efab55d1505d8ffb45bfced05 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ca3b00c44ec52d209642a5ba9ef82e085fac39db -R 8aff7924cf3a3f93f4ee99fb545d1f5f +P e580470db77d6da970c755102790e603fb26b3c6 +R f159389d59e4350424d5f438676d787b U drh -Z c6210337ac863b7dfe5cbe19c541a0aa +Z 177632e852e703fce9e831bfbe27f418 diff --git a/manifest.uuid b/manifest.uuid index 857c7b0e10..b0a7f1f074 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e580470db77d6da970c755102790e603fb26b3c6 \ No newline at end of file +9762ad0639cca2fc1ef0573113fb613ce9e7e83e \ No newline at end of file diff --git a/src/main.c b/src/main.c index ea03f2639f..e6cb4cd3df 100644 --- a/src/main.c +++ b/src/main.c @@ -2751,13 +2751,7 @@ int sqlite3_create_collation( void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*) ){ - int rc; - sqlite3_mutex_enter(db->mutex); - assert( !db->mallocFailed ); - rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, 0); - rc = sqlite3ApiExit(db, rc); - sqlite3_mutex_leave(db->mutex); - return rc; + return sqlite3_create_collation_v2(db, zName, enc, pCtx, xCompare, 0); } /* From 9ca95730e3a25bf5b335ec73a0a14f0112a421fb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 00:35:58 +0000 Subject: [PATCH 475/710] Add the SQLITE_ENABLE_API_ARMOR compile-time option. This is a work in progress and is not yet completely functional. FossilOrigin-Name: c297a84bc678f81ffc0aa9139ab73f0ca87c1971 --- manifest | 49 +++++++------- manifest.uuid | 2 +- src/auth.c | 3 + src/backup.c | 22 +++++++ src/build.c | 6 +- src/complete.c | 7 ++ src/ctime.c | 10 +++ src/main.c | 165 ++++++++++++++++++++++++++++++++++++++++++++++- src/malloc.c | 3 + src/mutex_unix.c | 8 ++- src/os.c | 4 ++ src/prepare.c | 13 ++-- src/printf.c | 21 ++++++ src/random.c | 11 ++-- src/status.c | 8 +++ src/table.c | 3 + src/util.c | 10 +++ src/vdbeapi.c | 31 +++++++-- src/vdbeblob.c | 5 ++ src/vtab.c | 16 ++++- 20 files changed, 351 insertions(+), 46 deletions(-) diff --git a/manifest b/manifest index 5b87a53662..0b02968a4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implement\ssqlite3_create_collation()\sby\sinvoking\s\nsqlite3_create_collation_v2()\swith\sa\sNULL\sdestructor\sargument.\s\sThis\ssaves\na\slittle\sspace. -D 2014-10-23T01:01:26.636 +C Add\sthe\sSQLITE_ENABLE_API_ARMOR\scompile-time\soption.\s\sThis\sis\sa\swork\sin\nprogress\sand\sis\snot\syet\scompletely\sfunctional. +D 2014-10-24T00:35:58.052 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,17 +168,17 @@ F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 -F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 -F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e +F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 +F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c 1b1123cba0c65caa0baa51e71b8c089e3167c3ed F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 -F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 +F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 -F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 -F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 +F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 +F src/ctime.c dfa83bfebb4201d07b16534acb8a0149592c3a25 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d @@ -194,8 +194,8 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 8207ece1b4005b1efab55d1505d8ffb45bfced05 -F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec +F src/main.c 2b882f64580ea72e2d972a5296f9eaa75a353161 +F src/malloc.c 5e02eab3e70cfcc265817f4de008f55e768dca51 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -205,10 +205,10 @@ F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 -F src/mutex_unix.c 1b10d5413dfc794364a8adf3eb3a192926b43fa3 +F src/mutex_unix.c 551e2f25f0fa0ee8fd7a43f50fc3d8be00e95dde F src/mutex_w32.c 06bfff9a3a83b53389a51a967643db3967032e1e F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 -F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace +F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa @@ -222,9 +222,9 @@ F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f -F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 -F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece +F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 +F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 +F src/random.c 689c95a50b90e7a8b3e6088e33d1d2d4212e6e88 F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b @@ -234,8 +234,8 @@ F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 6e9e125698c1e5c78a51050ea61f179a281c766d F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d -F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb -F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb +F src/status.c 81712116e826b0089bb221b018929536b2b5406f +F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 F src/test1.c 63d4b1707c4052cf9c05c1cbb4a62666d70a0b48 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 @@ -287,18 +287,18 @@ F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c -F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 +F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 5ee15a66ce07e0482b92aa29e4dd0c5827a22d79 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 -F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 +F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 -F src/vdbeblob.c 848238dc73e93e48432991bb5651bf87d865eca4 +F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 -F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de +F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 @@ -1205,7 +1205,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e580470db77d6da970c755102790e603fb26b3c6 -R f159389d59e4350424d5f438676d787b +P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e +R 98ebc598759882f163c17892cc0d9e3e +T *branch * api-armor +T *sym-api-armor * +T -sym-trunk * U drh -Z 177632e852e703fce9e831bfbe27f418 +Z 48418cab28475295c0204ad2ebe00aca diff --git a/manifest.uuid b/manifest.uuid index b0a7f1f074..64b7efe90b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9762ad0639cca2fc1ef0573113fb613ce9e7e83e \ No newline at end of file +c297a84bc678f81ffc0aa9139ab73f0ca87c1971 \ No newline at end of file diff --git a/src/auth.c b/src/auth.c index 1680c9a7c2..9768fc2fc0 100644 --- a/src/auth.c +++ b/src/auth.c @@ -72,6 +72,9 @@ int sqlite3_set_authorizer( int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; diff --git a/src/backup.c b/src/backup.c index 92c6334bde..da4303e5fd 100644 --- a/src/backup.c +++ b/src/backup.c @@ -138,6 +138,13 @@ sqlite3_backup *sqlite3_backup_init( ){ sqlite3_backup *p; /* Value to return */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pSrcDb)||!sqlite3SafetyCheckOk(pDestDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + /* Lock the source database handle. The destination database ** handle is not locked in this routine, but it is locked in ** sqlite3_backup_step(). The user is required to ensure that no @@ -334,6 +341,9 @@ int sqlite3_backup_step(sqlite3_backup *p, int nPage){ int pgszSrc = 0; /* Source page size */ int pgszDest = 0; /* Destination page size */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(p->pSrcDb->mutex); sqlite3BtreeEnter(p->pSrc); if( p->pDestDb ){ @@ -623,6 +633,12 @@ int sqlite3_backup_finish(sqlite3_backup *p){ ** call to sqlite3_backup_step(). */ int sqlite3_backup_remaining(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return p->nRemaining; } @@ -631,6 +647,12 @@ int sqlite3_backup_remaining(sqlite3_backup *p){ ** recent call to sqlite3_backup_step(). */ int sqlite3_backup_pagecount(sqlite3_backup *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return p->nPagecount; } diff --git a/src/build.c b/src/build.c index b897494db3..0b4affc664 100644 --- a/src/build.c +++ b/src/build.c @@ -307,7 +307,11 @@ int sqlite3UserAuthTable(const char *zTable){ Table *sqlite3FindTable(sqlite3 *db, const char *zName, const char *zDatabase){ Table *p = 0; int i; - assert( zName!=0 ); + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return 0; +#endif + /* All mutexes are required for schema access. Make sure we hold them. */ assert( zDatabase!=0 || sqlite3BtreeHoldsAllMutexes(db) ); #if SQLITE_USER_AUTHENTICATION diff --git a/src/complete.c b/src/complete.c index 6ab6f4a042..c439cfe181 100644 --- a/src/complete.c +++ b/src/complete.c @@ -105,6 +105,13 @@ int sqlite3_complete(const char *zSql){ u8 state = 0; /* Current state, using numbers defined in header comment */ u8 token; /* Value of the next token */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + #ifndef SQLITE_OMIT_TRIGGER /* A complex statement machine used to detect the end of a CREATE TRIGGER ** statement. This is the normal case. diff --git a/src/ctime.c b/src/ctime.c index 82a2f35204..36d0c266ee 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -63,6 +63,9 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_DISABLE_LFS "DISABLE_LFS", #endif +#ifdef SQLITE_ENABLE_API_ARMOR + "ENABLE_API_ARMOR", +#endif #ifdef SQLITE_ENABLE_ATOMIC_WRITE "ENABLE_ATOMIC_WRITE", #endif @@ -388,6 +391,13 @@ static const char * const azCompileOpt[] = { */ int sqlite3_compileoption_used(const char *zOptName){ int i, n; + +#ifdef SQLITE_ENABLE_API_ARMORE + if( zOptName==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; n = sqlite3Strlen30(zOptName); diff --git a/src/main.c b/src/main.c index e6cb4cd3df..5308698797 100644 --- a/src/main.c +++ b/src/main.c @@ -598,6 +598,12 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ ** Return the mutex associated with a database connection. */ sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->mutex; } @@ -607,6 +613,10 @@ sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ */ int sqlite3_db_release_memory(sqlite3 *db){ int i; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ @@ -737,6 +747,12 @@ static int nocaseCollatingFunc( ** Return the ROWID of the most recent insert */ sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->lastRowid; } @@ -744,6 +760,12 @@ sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ ** Return the number of changes in the most recent call to sqlite3_exec(). */ int sqlite3_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->nChange; } @@ -751,6 +773,12 @@ int sqlite3_changes(sqlite3 *db){ ** Return the number of changes since the database handle was opened. */ int sqlite3_total_changes(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->nTotalChange; } @@ -1296,6 +1324,9 @@ int sqlite3_busy_handler( int (*xBusy)(void*,int), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE; +#endif sqlite3_mutex_enter(db->mutex); db->busyHandler.xFunc = xBusy; db->busyHandler.pArg = pArg; @@ -1317,6 +1348,12 @@ void sqlite3_progress_handler( int (*xProgress)(void*), void *pArg ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif sqlite3_mutex_enter(db->mutex); if( nOps>0 ){ db->xProgress = xProgress; @@ -1337,6 +1374,9 @@ void sqlite3_progress_handler( ** specified number of milliseconds before returning 0. */ int sqlite3_busy_timeout(sqlite3 *db, int ms){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif if( ms>0 ){ sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); db->busyTimeout = ms; @@ -1350,6 +1390,12 @@ int sqlite3_busy_timeout(sqlite3 *db, int ms){ ** Cause any pending operation to stop at its earliest opportunity. */ void sqlite3_interrupt(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif db->u1.isInterrupted = 1; } @@ -1487,6 +1533,12 @@ int sqlite3_create_function_v2( ){ int rc = SQLITE_ERROR; FuncDestructor *pArg = 0; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); if( xDestroy ){ pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); @@ -1523,6 +1575,10 @@ int sqlite3_create_function16( ){ int rc; char *zFunc8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zFunctionName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); @@ -1554,6 +1610,12 @@ int sqlite3_overload_function( ){ int nName = sqlite3Strlen30(zName); int rc = SQLITE_OK; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); if( sqlite3FindFunction(db, zName, nName, nArg, SQLITE_UTF8, 0)==0 ){ rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, @@ -1575,6 +1637,13 @@ int sqlite3_overload_function( */ void *sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pTraceArg; db->xTrace = xTrace; @@ -1596,6 +1665,13 @@ void *sqlite3_profile( void *pArg ){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pProfileArg; db->xProfile = xProfile; @@ -1616,6 +1692,13 @@ void *sqlite3_commit_hook( void *pArg /* Argument to the function */ ){ void *pOld; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pOld = db->pCommitArg; db->xCommitCallback = xCallback; @@ -1634,6 +1717,13 @@ void *sqlite3_update_hook( void *pArg /* Argument to the function */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pUpdateArg; db->xUpdateCallback = xCallback; @@ -1652,6 +1742,13 @@ void *sqlite3_rollback_hook( void *pArg /* Argument to the function */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pRollbackArg; db->xRollbackCallback = xCallback; @@ -1698,6 +1795,9 @@ int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ UNUSED_PARAMETER(db); UNUSED_PARAMETER(nFrame); #else +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif if( nFrame>0 ){ sqlite3_wal_hook(db, sqlite3WalDefaultHook, SQLITE_INT_TO_PTR(nFrame)); }else{ @@ -1718,6 +1818,12 @@ void *sqlite3_wal_hook( ){ #ifndef SQLITE_OMIT_WAL void *pRet; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pWalArg; db->xWalCallback = xCallback; @@ -1745,6 +1851,10 @@ int sqlite3_wal_checkpoint_v2( int rc; /* Return code */ int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + /* Initialize the output variables to -1 in case an error occurs. */ if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; @@ -2141,6 +2251,12 @@ static const int aHardLimit[] = { int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ int oldLimit; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif /* EVIDENCE-OF: R-30189-54097 For each limit category SQLITE_LIMIT_NAME ** there is a hard upper bound set at compile-time by a C preprocessor @@ -2426,6 +2542,9 @@ static int openDatabase( char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); @@ -2715,13 +2834,15 @@ int sqlite3_open16( sqlite3_value *pVal; int rc; - assert( zFilename ); - assert( ppDb ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppDb==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppDb = 0; #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); if( rc ) return rc; #endif + if( zFilename==0 ) zFilename = "\000\000"; pVal = sqlite3ValueNew(0); sqlite3ValueSetStr(pVal, -1, zFilename, SQLITE_UTF16NATIVE, SQLITE_STATIC); zFilename8 = sqlite3ValueText(pVal, SQLITE_UTF8); @@ -2766,6 +2887,10 @@ int sqlite3_create_collation_v2( void(*xDel)(void*) ){ int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); rc = createCollation(db, zName, (u8)enc, pCtx, xCompare, xDel); @@ -2787,6 +2912,10 @@ int sqlite3_create_collation16( ){ int rc = SQLITE_OK; char *zName8; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zName8 = sqlite3Utf16to8(db, zName, -1, SQLITE_UTF16NATIVE); @@ -2809,6 +2938,9 @@ int sqlite3_collation_needed( void *pCollNeededArg, void(*xCollNeeded)(void*,sqlite3*,int eTextRep,const char*) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xCollNeeded = xCollNeeded; db->xCollNeeded16 = 0; @@ -2827,6 +2959,9 @@ int sqlite3_collation_needed16( void *pCollNeededArg, void(*xCollNeeded16)(void*,sqlite3*,int eTextRep,const void*) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->xCollNeeded = 0; db->xCollNeeded16 = xCollNeeded16; @@ -2853,6 +2988,12 @@ int sqlite3_global_recover(void){ ** by the next COMMIT or ROLLBACK. */ int sqlite3_get_autocommit(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif return db->autoCommit; } @@ -3035,6 +3176,9 @@ int sqlite3_sleep(int ms){ ** Enable or disable the extended result codes. */ int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); db->errMask = onoff ? 0xffffffff : 0xff; sqlite3_mutex_leave(db->mutex); @@ -3048,6 +3192,9 @@ int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ int rc = SQLITE_ERROR; Btree *pBtree; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); pBtree = sqlite3DbNameToBtree(db, zDbName); if( pBtree ){ @@ -3390,7 +3537,7 @@ int sqlite3_test_control(int op, ...){ ** returns a NULL pointer. */ const char *sqlite3_uri_parameter(const char *zFilename, const char *zParam){ - if( zFilename==0 ) return 0; + if( zFilename==0 || zParam==0 ) return 0; zFilename += sqlite3Strlen30(zFilename) + 1; while( zFilename[0] ){ int x = strcmp(zFilename, zParam); @@ -3446,6 +3593,12 @@ Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ ** connection. */ const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif Btree *pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeGetFilename(pBt) : 0; } @@ -3455,6 +3608,12 @@ const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ ** no such database exists. */ int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif Btree *pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; } diff --git a/src/malloc.c b/src/malloc.c index 6fb9d53d1b..c3c644fd6a 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -475,6 +475,9 @@ sqlite3_uint64 sqlite3_msize(void *p){ ** Free memory previously obtained from sqlite3Malloc(). */ void sqlite3_free(void *p){ +#if defined(SQLITE_ENABLE_API_ARMOR) && !defined(SQLITE_OMIT_AUTOINIT) + if( sqlite3_initialize() ) return; +#endif if( p==0 ) return; /* IMP: R-49053-54554 */ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); diff --git a/src/mutex_unix.c b/src/mutex_unix.c index c8663144e8..c936914d8a 100644 --- a/src/mutex_unix.c +++ b/src/mutex_unix.c @@ -175,8 +175,12 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ break; } default: { - assert( iType-2 >= 0 ); - assert( iType-2 < ArraySize(staticMutexes) ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif p = &staticMutexes[iType-2]; #if SQLITE_MUTEX_NREF p->id = iType; diff --git a/src/os.c b/src/os.c index b6c28a1dc4..2a2cf13c5e 100644 --- a/src/os.c +++ b/src/os.c @@ -361,6 +361,10 @@ int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ int rc = sqlite3_initialize(); if( rc ) return rc; #endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( pVfs==0 ) return SQLITE_MISUSE_BKPT; +#endif + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) sqlite3_mutex_enter(mutex); vfsUnlink(pVfs); diff --git a/src/prepare.c b/src/prepare.c index a05e619f3e..ca9c64b441 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -709,9 +709,12 @@ static int sqlite3LockAndPrepare( const char **pzTail /* OUT: End of parsed string */ ){ int rc; - assert( ppStmt!=0 ); + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); @@ -818,9 +821,11 @@ static int sqlite3Prepare16( const char *zTail8 = 0; int rc = SQLITE_OK; - assert( ppStmt ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif *ppStmt = 0; - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } if( nBytes>=0 ){ diff --git a/src/printf.c b/src/printf.c index 1df287fbb6..f000da7fcc 100644 --- a/src/printf.c +++ b/src/printf.c @@ -223,6 +223,13 @@ void sqlite3VXPrintf( PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ char buf[etBUFSIZE]; /* Conversion buffer */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( ap==0 ){ + (void)SQLITE_MISUSE_BKPT; + sqlite3StrAccumReset(pAccum); + return; + } +#endif bufpt = 0; if( bFlags ){ if( (bArgList = (bFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ @@ -943,6 +950,13 @@ char *sqlite3_vmprintf(const char *zFormat, va_list ap){ char *z; char zBase[SQLITE_PRINT_BUF_SIZE]; StrAccum acc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zFormat==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -985,6 +999,13 @@ char *sqlite3_mprintf(const char *zFormat, ...){ char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ StrAccum acc; if( n<=0 ) return zBuf; +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf && n>0 ) zBuf[0] = 0; + return zBuf; + } +#endif sqlite3StrAccumInit(&acc, zBuf, n, 0); acc.useMalloc = 0; sqlite3VXPrintf(&acc, 0, zFormat, ap); diff --git a/src/random.c b/src/random.c index b82566524c..7bca0d71af 100644 --- a/src/random.c +++ b/src/random.c @@ -34,6 +34,11 @@ void sqlite3_randomness(int N, void *pBuf){ unsigned char t; unsigned char *zBuf = pBuf; +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return; +#endif + if( pBuf==0 || N<=0 ) return; + /* The "wsdPrng" macro will resolve to the pseudo-random number generator ** state vector. If writable static data is unsupported on the target, ** we have to locate the state vector at run-time. In the more common @@ -52,12 +57,6 @@ void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); #endif - if( N<=0 ){ - wsdPrng.isInit = 0; - sqlite3_mutex_leave(mutex); - return; - } - /* Initialize the state of the random number generator once, ** the first time this routine is called. The seed value does ** not need to contain a lot of randomness since we are not diff --git a/src/status.c b/src/status.c index 79a8001b8a..4c2eabb661 100644 --- a/src/status.c +++ b/src/status.c @@ -86,6 +86,9 @@ int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ if( op<0 || op>=ArraySize(wsdStat.nowValue) ){ return SQLITE_MISUSE_BKPT; } +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT; +#endif *pCurrent = wsdStat.nowValue[op]; *pHighwater = wsdStat.mxValue[op]; if( resetFlag ){ @@ -105,6 +108,11 @@ int sqlite3_db_status( int resetFlag /* Reset high-water mark if true */ ){ int rc = SQLITE_OK; /* Return code */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || pCurrent==0|| pHighwater==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { diff --git a/src/table.c b/src/table.c index c435b2bc02..6e1df30643 100644 --- a/src/table.c +++ b/src/table.c @@ -126,6 +126,9 @@ int sqlite3_get_table( int rc; TabResult res; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pazResult==0 ) return SQLITE_MISUSE_BKPT; +#endif *pazResult = 0; if( pnColumn ) *pnColumn = 0; if( pnRow ) *pnRow = 0; diff --git a/src/util.c b/src/util.c index 9bb8d89157..ab409fa256 100644 --- a/src/util.c +++ b/src/util.c @@ -251,6 +251,11 @@ int sqlite3Dequote(char *z){ */ int sqlite3_stricmp(const char *zLeft, const char *zRight){ register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } a = (unsigned char *)zLeft; b = (unsigned char *)zRight; while( *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } @@ -258,6 +263,11 @@ int sqlite3_stricmp(const char *zLeft, const char *zRight){ } int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ register unsigned char *a, *b; + if( zLeft==0 ){ + return zRight ? -1 : 0; + }else if( zRight==0 ){ + return 1; + } a = (unsigned char *)zLeft; b = (unsigned char *)zRight; while( N-- > 0 && *a!=0 && UpperToLower[*a]==UpperToLower[*b]){ a++; b++; } diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 0ab76e0784..daf7b101d2 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -966,11 +966,19 @@ static const void *columnName( const void *(*xFunc)(Mem*), int useType ){ - const void *ret = 0; - Vdbe *p = (Vdbe *)pStmt; + const void *ret; + Vdbe *p; int n; - sqlite3 *db = p->db; - + sqlite3 *db; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pStmt==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + ret = 0; + p = (Vdbe *)pStmt; + db = p->db; assert( db!=0 ); n = sqlite3_column_count(pStmt); if( N=0 ){ @@ -1435,6 +1443,12 @@ int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ */ sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ sqlite3_stmt *pNext; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(pDb) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif sqlite3_mutex_enter(pDb->mutex); if( pStmt==0 ){ pNext = (sqlite3_stmt*)pDb->pVdbe; @@ -1450,7 +1464,14 @@ sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ */ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ Vdbe *pVdbe = (Vdbe*)pStmt; - u32 v = pVdbe->aCounter[op]; + u32 v; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !pStmt ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + v = pVdbe->aCounter[op]; if( resetFlag ) pVdbe->aCounter[op] = 0; return (int)v; } diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 71bd8816d5..0cf2b0652b 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -153,6 +153,11 @@ int sqlite3_blob_open( Parse *pParse = 0; Incrblob *pBlob = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || ppBlob==0 || zTable==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif flags = !!flags; /* flags = (flags ? 1 : 0); */ *ppBlob = 0; diff --git a/src/vtab.c b/src/vtab.c index faee4ae478..334de9aacb 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -81,6 +81,9 @@ int sqlite3_create_module( const sqlite3_module *pModule, /* The definition of the module */ void *pAux /* Context pointer for xCreate/xConnect */ ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif return createModule(db, zName, pModule, pAux, 0); } @@ -94,6 +97,9 @@ int sqlite3_create_module_v2( void *pAux, /* Context pointer for xCreate/xConnect */ void (*xDestroy)(void *) /* Module destructor function */ ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) || zName==0 ) return SQLITE_MISUSE_BKPT; +#endif return createModule(db, zName, pModule, pAux, xDestroy); } @@ -698,6 +704,9 @@ int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ Table *pTab; char *zErr = 0; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); if( !db->pVtabCtx || !(pTab = db->pVtabCtx->pTab) ){ sqlite3Error(db, SQLITE_MISUSE); @@ -1054,6 +1063,9 @@ int sqlite3_vtab_on_conflict(sqlite3 *db){ static const unsigned char aMap[] = { SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE }; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif assert( OE_Rollback==1 && OE_Abort==2 && OE_Fail==3 ); assert( OE_Ignore==4 && OE_Replace==5 ); assert( db->vtabOnConflict>=1 && db->vtabOnConflict<=5 ); @@ -1069,8 +1081,10 @@ int sqlite3_vtab_config(sqlite3 *db, int op, ...){ va_list ap; int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); - va_start(ap, op); switch( op ){ case SQLITE_VTAB_CONSTRAINT_SUPPORT: { From 5a5d120bcc08af809b8687ff9fe39fce27e7796c Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 12:37:00 +0000 Subject: [PATCH 476/710] Fix two problems. Tests now passing. FossilOrigin-Name: 1c220b806d56e163842e17038c3331f71861bd9c --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/malloc.c | 3 --- src/random.c | 7 ++++++- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 0b02968a4f..da62517a33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_ENABLE_API_ARMOR\scompile-time\soption.\s\sThis\sis\sa\swork\sin\nprogress\sand\sis\snot\syet\scompletely\sfunctional. -D 2014-10-24T00:35:58.052 +C Fix\stwo\sproblems.\s\sTests\snow\spassing. +D 2014-10-24T12:37:00.827 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 F src/main.c 2b882f64580ea72e2d972a5296f9eaa75a353161 -F src/malloc.c 5e02eab3e70cfcc265817f4de008f55e768dca51 +F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -224,7 +224,7 @@ F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 -F src/random.c 689c95a50b90e7a8b3e6088e33d1d2d4212e6e88 +F src/random.c b8a058131851de1a37801b5587845ee73411c064 F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b @@ -1205,10 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e -R 98ebc598759882f163c17892cc0d9e3e -T *branch * api-armor -T *sym-api-armor * -T -sym-trunk * +P c297a84bc678f81ffc0aa9139ab73f0ca87c1971 +R 1cc86c78be65c50602286adf14ad3d43 U drh -Z 48418cab28475295c0204ad2ebe00aca +Z 4a2cbf72bfaa58a272d60444a54fd92d diff --git a/manifest.uuid b/manifest.uuid index 64b7efe90b..0375d856fe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c297a84bc678f81ffc0aa9139ab73f0ca87c1971 \ No newline at end of file +1c220b806d56e163842e17038c3331f71861bd9c \ No newline at end of file diff --git a/src/malloc.c b/src/malloc.c index c3c644fd6a..6fb9d53d1b 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -475,9 +475,6 @@ sqlite3_uint64 sqlite3_msize(void *p){ ** Free memory previously obtained from sqlite3Malloc(). */ void sqlite3_free(void *p){ -#if defined(SQLITE_ENABLE_API_ARMOR) && !defined(SQLITE_OMIT_AUTOINIT) - if( sqlite3_initialize() ) return; -#endif if( p==0 ) return; /* IMP: R-49053-54554 */ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); assert( sqlite3MemdebugNoType(p, ~MEMTYPE_HEAP) ); diff --git a/src/random.c b/src/random.c index 7bca0d71af..c4241362de 100644 --- a/src/random.c +++ b/src/random.c @@ -37,7 +37,6 @@ void sqlite3_randomness(int N, void *pBuf){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return; #endif - if( pBuf==0 || N<=0 ) return; /* The "wsdPrng" macro will resolve to the pseudo-random number generator ** state vector. If writable static data is unsupported on the target, @@ -57,6 +56,12 @@ void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); #endif + if( N<=0 || pBuf==0 ){ + wsdPrng.isInit = 0; + sqlite3_mutex_leave(mutex); + return; + } + /* Initialize the state of the random number generator once, ** the first time this routine is called. The seed value does ** not need to contain a lot of randomness since we are not From 9769efcc4c785dc7552a2e22782f12e3c56e0169 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 14:32:21 +0000 Subject: [PATCH 477/710] Get the likelihood() functions working on operators like BETWEEN that create virtual terms in the WHERE-clause analysis. FossilOrigin-Name: 03d0498d0f24bec2383d5d79edf25069effecd59 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 29 ++++++++++++++++------------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 5b87a53662..67132856cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Implement\ssqlite3_create_collation()\sby\sinvoking\s\nsqlite3_create_collation_v2()\swith\sa\sNULL\sdestructor\sargument.\s\sThis\ssaves\na\slittle\sspace. -D 2014-10-23T01:01:26.636 +C Get\sthe\slikelihood()\sfunctions\sworking\son\soperators\slike\sBETWEEN\sthat\ncreate\svirtual\sterms\sin\sthe\sWHERE-clause\sanalysis. +D 2014-10-24T14:32:21.050 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2cd9e0af718d736459ae9d3b0f4532b4a80640d0 +F src/where.c 746d4f22d75c1f000d6dae251dc65bf755c395a8 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e580470db77d6da970c755102790e603fb26b3c6 -R f159389d59e4350424d5f438676d787b +P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e +R f460b87a81ee1fb775b7700e4252ece6 U drh -Z 177632e852e703fce9e831bfbe27f418 +Z 1428b6fe81396b49c1c6b7022fde286d diff --git a/manifest.uuid b/manifest.uuid index b0a7f1f074..59bf5ae4e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9762ad0639cca2fc1ef0573113fb613ce9e7e83e \ No newline at end of file +03d0498d0f24bec2383d5d79edf25069effecd59 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 198a6c3fdd..d4969fad1e 100644 --- a/src/where.c +++ b/src/where.c @@ -756,6 +756,15 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ } } +/* +** Mark term iChild as being a child of term iParent +*/ +static void markTermAsChild(WhereClause *pWC, int iChild, int iParent){ + pWC->a[iChild].iParent = iParent; + pWC->a[iChild].truthProb = pWC->a[iParent].truthProb; + pWC->a[iParent].nChild++; +} + #if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) /* ** Analyze a term that consists of two or more OR-connected @@ -1053,8 +1062,7 @@ static void exprAnalyzeOrTerm( testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; - pTerm->nChild = 1; + markTermAsChild(pWC, idxNew, idxTerm); }else{ sqlite3ExprListDelete(db, pList); } @@ -1156,9 +1164,8 @@ static void exprAnalyze( idxNew = whereClauseInsert(pWC, pDup, TERM_VIRTUAL|TERM_DYNAMIC); if( idxNew==0 ) return; pNew = &pWC->a[idxNew]; - pNew->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; if( pExpr->op==TK_EQ && !ExprHasProperty(pExpr, EP_FromJoin) @@ -1215,9 +1222,8 @@ static void exprAnalyze( testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); pTerm = &pWC->a[idxTerm]; - pWC->a[idxNew].iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); } - pTerm->nChild = 2; } #endif /* SQLITE_OMIT_BETWEEN_OPTIMIZATION */ @@ -1292,9 +1298,8 @@ static void exprAnalyze( exprAnalyze(pSrc, pWC, idxNew2); pTerm = &pWC->a[idxTerm]; if( isComplete ){ - pWC->a[idxNew1].iParent = idxTerm; - pWC->a[idxNew2].iParent = idxTerm; - pTerm->nChild = 2; + markTermAsChild(pWC, idxNew1, idxTerm); + markTermAsChild(pWC, idxNew2, idxTerm); } } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ @@ -1327,9 +1332,8 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_MATCH; - pNewTerm->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; pNewTerm->prereqAll = pTerm->prereqAll; } @@ -1369,9 +1373,8 @@ static void exprAnalyze( pNewTerm->leftCursor = pLeft->iTable; pNewTerm->u.leftColumn = pLeft->iColumn; pNewTerm->eOperator = WO_GT; - pNewTerm->iParent = idxTerm; + markTermAsChild(pWC, idxNew, idxTerm); pTerm = &pWC->a[idxTerm]; - pTerm->nChild = 1; pTerm->wtFlags |= TERM_COPIED; pNewTerm->prereqAll = pTerm->prereqAll; } From 4dd96a8315b3f3b2a516513b06754e1887e426eb Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 15:26:29 +0000 Subject: [PATCH 478/710] Honor a high likelihood() on range constraints. FossilOrigin-Name: 401235edf40fcd665eaf426cf5155ac6855e8537 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 67132856cf..e98be71834 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sthe\slikelihood()\sfunctions\sworking\son\soperators\slike\sBETWEEN\sthat\ncreate\svirtual\sterms\sin\sthe\sWHERE-clause\sanalysis. -D 2014-10-24T14:32:21.050 +C Honor\sa\shigh\slikelihood()\son\srange\sconstraints. +D 2014-10-24T15:26:29.800 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 746d4f22d75c1f000d6dae251dc65bf755c395a8 +F src/where.c f5c13d9c1929bcc9d571f1e7bf7bfeb8b872ef99 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9762ad0639cca2fc1ef0573113fb613ce9e7e83e -R f460b87a81ee1fb775b7700e4252ece6 +P 03d0498d0f24bec2383d5d79edf25069effecd59 +R eb013b99b9ff12a41d5ffc947e4a6227 U drh -Z 1428b6fe81396b49c1c6b7022fde286d +Z 3215dd92b9608b7b9136fb91eb8178e8 diff --git a/manifest.uuid b/manifest.uuid index 59bf5ae4e6..4ee03d34ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03d0498d0f24bec2383d5d79edf25069effecd59 \ No newline at end of file +401235edf40fcd665eaf426cf5155ac6855e8537 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d4969fad1e..e13b223366 100644 --- a/src/where.c +++ b/src/where.c @@ -2275,12 +2275,15 @@ static int whereRangeScanEst( nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); - /* TUNING: If there is both an upper and lower limit, assume the range is + /* TUNING: If there is both an upper and lower limit and neither limit + ** has an application-defined likelihood(), assume the range is ** reduced by an additional 75%. This means that, by default, an open-ended ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to ** match 1/64 of the index. */ - if( pLower && pUpper ) nNew -= 20; + if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ + nNew -= 20; + } nOut -= (pLower!=0) + (pUpper!=0); if( nNew<10 ) nNew = 10; From e6593d8e8c51fd05c762f34c444b19a74137fedc Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Oct 2014 16:40:49 +0000 Subject: [PATCH 479/710] Fix some minor formatting and code organization issues. FossilOrigin-Name: eab8706dc47aa0a44caf73619de858397c3e0b4e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 16 +++++++--------- src/pager.c | 15 +++++++++++---- src/pager.h | 2 +- 5 files changed, 28 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 396fe366d0..b52c190c13 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\swith\sthis\sbranch. -D 2014-10-22T18:42:31.680 +C Fix\ssome\sminor\sformatting\sand\scode\sorganization\sissues. +D 2014-10-24T16:40:49.448 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 37229f3134416f24828af823a3a6f8535923d45b +F src/btree.c 56381ce7614853ec0e32bb187e85db4da774c7c5 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -215,8 +215,8 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 18be49e363106bdb4c982fa27dbee7a8cd297113 -F src/pager.h 8b6707cb32c788cf36bfc3d63f6d4b4fa689e7c2 +F src/pager.c d02833adf331a5913226595306d64731a3da33f6 +F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 58d7793bd5d608ba9fc3a2cd44b9d9512e0332ba 3c933bf95f291f7957580d823dce92c981375a5c -R 738ce949461e4c95ef4d450eb06d976a +P 854a54c6c21e800b0cd999023014813f7c50b23f +R b3aec92126a25950ff876c6cdf2fb4df U dan -Z 1bc59f5d05ced43bcf6fa4690adf0fcc +Z 9e7a15bc5953228881d999c99d9f0682 diff --git a/manifest.uuid b/manifest.uuid index b48e7f9fc1..5987e57b67 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -854a54c6c21e800b0cd999023014813f7c50b23f \ No newline at end of file +eab8706dc47aa0a44caf73619de858397c3e0b4e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index d1457c5f39..b5e8cb2e0c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1233,7 +1233,7 @@ static int defragmentPage(MemPage *pPage){ ** ** This function may detect corruption within pPg. If it does and argument ** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. -** Or, if corruption is detected by pRc is NULL, NULL is returned and the +** Or, if corruption is detected and pRc is NULL, NULL is returned and the ** corruption goes unreported. */ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ @@ -6345,7 +6345,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ } #endif /* SQLITE_OMIT_QUICKBALANCE */ -#if 1 +#if 0 /* ** This function does not contribute anything to the operation of SQLite. ** it is sometimes activated temporarily while debugging code responsible @@ -6522,10 +6522,9 @@ static int balance_nonroot( u16 *szCell; /* Local size of all cells in apCell[] */ u8 *aSpace1; /* Space for copies of dividers cells */ Pgno pgno; /* Temp var to store a page number in */ - - u8 abDone[NB+2]; - Pgno aPgno[NB+2]; - u16 aPgFlags[NB+2]; + u8 abDone[NB+2]; /* True after i'th new page is populated */ + Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ + u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ memset(abDone, 0, sizeof(abDone)); pBt = pParent->pBt; @@ -6868,8 +6867,7 @@ static int balance_nonroot( } } if( apNew[i]->pgno!=iMin ){ - apNew[i]->pDbPage->flags = flags; - sqlite3PagerRekey(apNew[i]->pDbPage, iMin); + sqlite3PagerRekey(apNew[i]->pDbPage, iMin, flags); apNew[i]->pgno = iMin; } } @@ -7090,7 +7088,7 @@ static int balance_nonroot( freePage(apOld[i], &rc); } -#if 1 +#if 0 if( ISAUTOVACUUM && rc==SQLITE_OK && apNew[0]->isInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that ** all pointer map pages are set correctly. This is helpful while diff --git a/src/pager.c b/src/pager.c index 74667afab0..dc79e6754a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6845,16 +6845,23 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ return SQLITE_OK; } +#endif -void sqlite3PagerRekey(DbPage *pPage, Pgno iNew){ +/* +** The page handle passed as the first argument refers to a dirty page +** with a page number other than iNew. This function changes the page's +** page number to iNew and sets the value of the PgHdr.flags field to +** the value passed as the third parameter. +*/ +void sqlite3PagerRekey(DbPage *pPage, Pgno iNew, u16 flags){ PgHdr *pPg = (PgHdr*)pPage; - assert( pPg->flags & PGHDR_DIRTY ); + assert( (flags & PGHDR_DIRTY) && (pPg->flags & PGHDR_DIRTY) ); assert( !subjRequiresPage(pPg) ); + assert( pPg->pgno!=iNew ); + pPg->flags = flags; sqlite3PcacheMove(pPg, iNew); } -#endif - /* ** Return a pointer to the data for the specified page. */ diff --git a/src/pager.h b/src/pager.h index f3a04bbca6..764c7bd197 100644 --- a/src/pager.h +++ b/src/pager.h @@ -188,7 +188,7 @@ int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ void sqlite3PagerTruncateImage(Pager*,Pgno); -void sqlite3PagerRekey(DbPage*, Pgno); +void sqlite3PagerRekey(DbPage*, Pgno, u16); #if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) void *sqlite3PagerCodec(DbPage *); From 23eba45d23291617e5f69ffb15c0cfbb1bab5121 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Oct 2014 18:43:57 +0000 Subject: [PATCH 480/710] Fix some issues in the new code on this branch related to the handling of corrupt databases. FossilOrigin-Name: 19736dd9fbbb7e252c4f8715e2277d48ac41f5bc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 14 ++++++++++---- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index b52c190c13..10be86a2cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sminor\sformatting\sand\scode\sorganization\sissues. -D 2014-10-24T16:40:49.448 +C Fix\ssome\sissues\sin\sthe\snew\scode\son\sthis\sbranch\srelated\sto\sthe\shandling\sof\scorrupt\sdatabases. +D 2014-10-24T18:43:57.313 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 56381ce7614853ec0e32bb187e85db4da774c7c5 +F src/btree.c df15daf116e9ef1a7ff88257f6fb678ba385b5ea F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 854a54c6c21e800b0cd999023014813f7c50b23f -R b3aec92126a25950ff876c6cdf2fb4df +P eab8706dc47aa0a44caf73619de858397c3e0b4e +R 8a34c36d0ed8bbbc0bb7360d0dc30d47 U dan -Z 9e7a15bc5953228881d999c99d9f0682 +Z 610a2814ec6cdf214c41e43dd2355678 diff --git a/manifest.uuid b/manifest.uuid index 5987e57b67..11462e357c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eab8706dc47aa0a44caf73619de858397c3e0b4e \ No newline at end of file +19736dd9fbbb7e252c4f8715e2277d48ac41f5bc \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index b5e8cb2e0c..f326f713fb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1386,7 +1386,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( iStart>=pPage->hdrOffset+6+pPage->childPtrSize ); - assert( iEnd <= pPage->pBt->usableSize ); + assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ assert( iStart<=iLast ); @@ -6065,7 +6065,7 @@ static int pageInsertArray( u8 *aData = pPg->aData; u8 *pData = *ppData; const int bFreelist = aData[1] || aData[2]; - assert( pPg->hdrOffset==0 ); /* Never called on page 1 */ + assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ for(i=0; iaCellIdx[i*2]); if( pCell>=aData && pCell<&aData[pPg->pBt->usableSize] ){ @@ -6866,7 +6866,13 @@ static int balance_nonroot( flags = aPgFlags[j]; } } - if( apNew[i]->pgno!=iMin ){ + if( iMin==0 ){ + /* This case can only occur if aPgno[] contains duplicate page + ** numbers. Which can only happen if the database is corrupt. */ + assert( CORRUPT_DB ); + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + }else if( apNew[i]->pgno!=iMin ){ sqlite3PagerRekey(apNew[i]->pDbPage, iMin, flags); apNew[i]->pgno = iMin; } From 059b2d50e1c6a57ca301f3c9639f92f7e16ff96e Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Oct 2014 19:28:09 +0000 Subject: [PATCH 481/710] Enhance the automatic index logic so that it creates a partial index when doing so gives the same answer for less work. FossilOrigin-Name: d95d0313c447f5baeabdb17284d8606331ab7d49 --- manifest | 19 +++++----- manifest.uuid | 2 +- src/expr.c | 85 +++++++++++++++++++++++++++----------------- src/resolve.c | 4 +-- src/sqliteInt.h | 5 ++- src/where.c | 23 ++++++++++-- test/autoindex4.test | 52 +++++++++++++++++++++++++++ 7 files changed, 143 insertions(+), 47 deletions(-) create mode 100644 test/autoindex4.test diff --git a/manifest b/manifest index e98be71834..e831ecb526 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Honor\sa\shigh\slikelihood()\son\srange\sconstraints. -D 2014-10-24T15:26:29.800 +C Enhance\sthe\sautomatic\sindex\slogic\sso\sthat\sit\screates\sa\spartial\sindex\swhen\ndoing\sso\sgives\sthe\ssame\sanswer\sfor\sless\swork. +D 2014-10-24T19:28:09.216 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f -F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d +F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee @@ -225,14 +225,14 @@ F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 +F src/resolve.c 57d5ad93913beb43ad9b8ade435a54e5fb8ccd40 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 6e9e125698c1e5c78a51050ea61f179a281c766d +F src/sqliteInt.h 123b28f3552d4ffdd3e53707fe8120a069df69e4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c f5c13d9c1929bcc9d571f1e7bf7bfeb8b872ef99 +F src/where.c cc0733c59bd8bf6027d28b7d24b1887f4a870215 F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -346,6 +346,7 @@ F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 F test/autoindex1.test 6ff78b94f43a59616c06c11c55b12935173506d7 F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de F test/autoindex3.test 8254f689c3241081fad52b7bea18ba53e07e14a2 +F test/autoindex4.test fc807f9efd158bec60f5dfdf34ebe46fb274612d F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 @@ -1205,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 03d0498d0f24bec2383d5d79edf25069effecd59 -R eb013b99b9ff12a41d5ffc947e4a6227 +P 401235edf40fcd665eaf426cf5155ac6855e8537 +R def8f7d87d0f5784d9eaab2c56b2a690 U drh -Z 3215dd92b9608b7b9136fb91eb8178e8 +Z cb611c4b71e8fc233ea4e1005f56a9e0 diff --git a/manifest.uuid b/manifest.uuid index 4ee03d34ab..7417ef4fc1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -401235edf40fcd665eaf426cf5155ac6855e8537 \ No newline at end of file +d95d0313c447f5baeabdb17284d8606331ab7d49 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 1ad9a879a3..13a9cb46fd 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1210,20 +1210,24 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ } /* -** These routines are Walker callbacks. Walker.u.pi is a pointer -** to an integer. These routines are checking an expression to see -** if it is a constant. Set *Walker.u.i to 0 if the expression is -** not constant. +** These routines are Walker callbacks used to check expressions to +** see if they are "constant" for some definition of constant. The +** Walker.eCode value determines the type of "constant" we are looking +** for. ** ** These callback routines are used to implement the following: ** -** sqlite3ExprIsConstant() pWalker->u.i==1 -** sqlite3ExprIsConstantNotJoin() pWalker->u.i==2 -** sqlite3ExprIsConstantOrFunction() pWalker->u.i==3 or 4 +** sqlite3ExprIsConstant() pWalker->eCode==1 +** sqlite3ExprIsConstantNotJoin() pWalker->eCode==2 +** sqlite3ExprRefOneTableOnly() pWalker->eCode==3 +** sqlite3ExprIsConstantOrFunction() pWalker->eCode==4 or 5 +** +** In all cases, the callbacks set Walker.eCode=0 and abort if the expression +** is found to not be a constant. ** ** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions -** in a CREATE TABLE statement. The Walker.u.i value is 4 when parsing -** an existing schema and 3 when processing a new statement. A bound +** in a CREATE TABLE statement. The Walker.eCode value is 5 when parsing +** an existing schema and 4 when processing a new statement. A bound ** parameter raises an error for new statements, but is silently converted ** to NULL for existing schemas. This allows sqlite_master tables that ** contain a bound parameter because they were generated by older versions @@ -1232,23 +1236,25 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ - /* If pWalker->u.i is 2 then any term of the expression that comes from - ** the ON or USING clauses of a join disqualifies the expression + /* If pWalker->eCode is 2 then any term of the expression that comes from + ** the ON or USING clauses of a left join disqualifies the expression ** from being considered constant. */ - if( pWalker->u.i==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ - pWalker->u.i = 0; + if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ + pWalker->eCode = 0; return WRC_Abort; } switch( pExpr->op ){ /* Consider functions to be constant if all their arguments are constant - ** and either pWalker->u.i==3 or 4 or the function as the SQLITE_FUNC_CONST - ** flag. */ + ** and either pWalker->eCode==4 or 5 or the function has the + ** SQLITE_FUNC_CONST flag. */ case TK_FUNCTION: - if( pWalker->u.i>=3 || ExprHasProperty(pExpr,EP_Constant) ){ + if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_Constant) ){ return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; } - /* Fall through */ case TK_ID: case TK_COLUMN: case TK_AGG_FUNCTION: @@ -1257,18 +1263,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); testcase( pExpr->op==TK_AGG_COLUMN ); - pWalker->u.i = 0; - return WRC_Abort; + if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){ + return WRC_Continue; + }else{ + pWalker->eCode = 0; + return WRC_Abort; + } case TK_VARIABLE: - if( pWalker->u.i==4 ){ + if( pWalker->eCode==5 ){ /* Silently convert bound parameters that appear inside of CREATE ** statements into a NULL when parsing the CREATE statement text out ** of the sqlite_master table */ pExpr->op = TK_NULL; - }else if( pWalker->u.i==3 ){ + }else if( pWalker->eCode==4 ){ /* A bound parameter in a CREATE statement that originates from ** sqlite3_prepare() causes an error */ - pWalker->u.i = 0; + pWalker->eCode = 0; return WRC_Abort; } /* Fall through */ @@ -1280,21 +1290,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ } static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ UNUSED_PARAMETER(NotUsed); - pWalker->u.i = 0; + pWalker->eCode = 0; return WRC_Abort; } -static int exprIsConst(Expr *p, int initFlag){ +static int exprIsConst(Expr *p, int initFlag, int iCur){ Walker w; memset(&w, 0, sizeof(w)); - w.u.i = initFlag; + w.eCode = initFlag; w.xExprCallback = exprNodeIsConstant; w.xSelectCallback = selectNodeIsConstant; + w.u.iCur = iCur; sqlite3WalkExpr(&w, p); - return w.u.i; + return w.eCode; } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** and 0 if it involves variables or function calls. ** ** For the purposes of this function, a double-quoted string (ex: "abc") @@ -1302,21 +1313,31 @@ static int exprIsConst(Expr *p, int initFlag){ ** a constant. */ int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1); + return exprIsConst(p, 1, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression is constant ** that does no originate from the ON or USING clauses of a join. ** Return 0 if it involves variables or function calls or terms from ** an ON or USING clause. */ int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 2); + return exprIsConst(p, 2, 0); } /* -** Walk an expression tree. Return 1 if the expression is constant +** Walk an expression tree. Return non-zero if the expression constant +** for any single row of the table with cursor iCur. In other words, the +** expression must not refer to any non-deterministic function nor any +** table other than iCur. +*/ +int sqlite3ExprIsTableConstant(Expr *p, int iCur){ + return exprIsConst(p, 3, iCur); +} + +/* +** Walk an expression tree. Return non-zero if the expression is constant ** or a function call with constant arguments. Return and 0 if there ** are any variables. ** @@ -1326,7 +1347,7 @@ int sqlite3ExprIsConstantNotJoin(Expr *p){ */ int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(p, 3+isInit); + return exprIsConst(p, 4+isInit, 0); } /* diff --git a/src/resolve.c b/src/resolve.c index d6a865caef..e507ccb810 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -28,7 +28,7 @@ ** is a helper function - a callback for the tree walker. */ static int incrAggDepth(Walker *pWalker, Expr *pExpr){ - if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.i; + if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.n; return WRC_Continue; } static void incrAggFunctionDepth(Expr *pExpr, int N){ @@ -36,7 +36,7 @@ static void incrAggFunctionDepth(Expr *pExpr, int N){ Walker w; memset(&w, 0, sizeof(w)); w.xExprCallback = incrAggDepth; - w.u.i = N; + w.u.n = N; sqlite3WalkExpr(&w, pExpr); } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 1b3138be44..b7f992c34d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2892,9 +2892,11 @@ struct Walker { void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ + u8 eCode; /* A small processing code */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ - int i; /* Integer value */ + int n; /* A counter */ + int iCur; /* A cursor number */ SrcList *pSrcList; /* FROM clause */ struct SrcCount *pSrcCount; /* Counting column references */ } u; @@ -3295,6 +3297,7 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3*); int sqlite3ExprIsConstant(Expr*); int sqlite3ExprIsConstantNotJoin(Expr*); int sqlite3ExprIsConstantOrFunction(Expr*, u8); +int sqlite3ExprIsTableConstant(Expr*,int); int sqlite3ExprIsInteger(Expr*, int*); int sqlite3ExprCanBeNull(const Expr*); int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); diff --git a/src/where.c b/src/where.c index e13b223366..8d2a891870 100644 --- a/src/where.c +++ b/src/where.c @@ -1594,6 +1594,8 @@ static void constructAutomaticIndex( Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ u8 sentWarning = 0; /* True if a warnning has been issued */ + Expr *pPartial = 0; /* Partial Index Expression */ + int iContinue = 0; /* Jump here to skip excluded rows */ /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ @@ -1609,6 +1611,11 @@ static void constructAutomaticIndex( pLoop = pLevel->pWLoop; idxCols = 0; for(pTerm=pWC->a; pTermprereq==0 + && sqlite3ExprIsTableConstant(pTerm->pExpr, pSrc->iCursor) ){ + pPartial = sqlite3ExprAnd(pParse->db, pPartial, + sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); + } if( termCanDriveIndex(pTerm, pSrc, notReady) ){ int iCol = pTerm->u.leftColumn; Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); @@ -1621,7 +1628,9 @@ static void constructAutomaticIndex( sentWarning = 1; } if( (idxCols & cMask)==0 ){ - if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ) return; + if( whereLoopResize(pParse->db, pLoop, nKeyCol+1) ){ + goto end_auto_index_create; + } pLoop->aLTerm[nKeyCol++] = pTerm; idxCols |= cMask; } @@ -1654,7 +1663,7 @@ static void constructAutomaticIndex( /* Construct the Index object to describe this index */ pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); - if( pIdx==0 ) return; + if( pIdx==0 ) goto end_auto_index_create; pLoop->u.btree.pIndex = pIdx; pIdx->zName = "auto-index"; pIdx->pTable = pTable; @@ -1706,18 +1715,28 @@ static void constructAutomaticIndex( VdbeComment((v, "for %s", pTable->zName)); /* Fill the automatic index with content */ + sqlite3ExprCachePush(pParse); addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); + if( pPartial ){ + iContinue = sqlite3VdbeMakeLabel(v); + sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + } regRecord = sqlite3GetTempReg(pParse); sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ExprCachePop(pParse); /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); + +end_auto_index_create: + sqlite3ExprDelete(pParse->db, pPartial); } #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ diff --git a/test/autoindex4.test b/test/autoindex4.test new file mode 100644 index 0000000000..6d0865bf72 --- /dev/null +++ b/test/autoindex4.test @@ -0,0 +1,52 @@ +# 2014-10-24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# +# This file implements regression tests for SQLite library. The +# focus of this script is testing automatic index creation logic, +# and specifically creation of automatic partial indexes. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test autoindex4-1.0 { + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(123,'abc'),(234,'def'),(234,'ghi'),(345,'jkl'); + CREATE TABLE t2(x,y); + INSERT INTO t2 VALUES(987,'zyx'),(654,'wvu'),(987,'rqp'); + + SELECT *, '|' FROM t1, t2 WHERE a=234 AND x=987 ORDER BY +b; +} {234 def 987 rqp | 234 def 987 zyx | 234 ghi 987 rqp | 234 ghi 987 zyx |} +do_execsql_test autoindex4-1.1 { + SELECT *, '|' FROM t1, t2 WHERE a=234 AND x=555; +} {} + +do_execsql_test autoindex4-1.2 { + SELECT *, '|' FROM t1 LEFT JOIN t2 ON a=234 AND x=555; +} {123 abc {} {} | 234 def {} {} | 234 ghi {} {} | 345 jkl {} {} |} +do_execsql_test autoindex4-1.3 { + SELECT *, '|' FROM t1 LEFT JOIN t2 ON x=555 WHERE a=234; +} {234 def {} {} | 234 ghi {} {} |} +do_execsql_test autoindex4-1.4 { + SELECT *, '|' FROM t1 LEFT JOIN t2 WHERE a=234 AND x=555; +} {} + + +do_execsql_test autoindex4-2.0 { + CREATE TABLE t3(e,f); + INSERT INTO t3 VALUES(123,654),(555,444),(234,987); + + SELECT (SELECT count(*) FROM t1, t2 WHERE a=e AND x=f), e, f, '|' + FROM t3 + ORDER BY rowid; +} {1 123 654 | 0 555 444 | 4 234 987 |} + +finish_test From 31f4e99d449b5938f46723f6d2ec96595ff88513 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 24 Oct 2014 20:57:03 +0000 Subject: [PATCH 482/710] Ensure that the "Any prior cache entry associated with newKey is guaranteed not to be pinned" guarantee made to xRekey implementations is not violated. FossilOrigin-Name: ecc3544e712041736af7c7b4f34864a1f2e30ff7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 30 ++++++++++++------------------ src/pager.c | 5 +---- 4 files changed, 21 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 10be86a2cc..6c0464217b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\sissues\sin\sthe\snew\scode\son\sthis\sbranch\srelated\sto\sthe\shandling\sof\scorrupt\sdatabases. -D 2014-10-24T18:43:57.313 +C Ensure\sthat\sthe\s"Any\sprior\scache\sentry\sassociated\swith\snewKey\sis\sguaranteed\snot\sto\sbe\spinned"\sguarantee\smade\sto\sxRekey\simplementations\sis\snot\sviolated. +D 2014-10-24T20:57:03.500 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c df15daf116e9ef1a7ff88257f6fb678ba385b5ea +F src/btree.c 4decfb3b97d16afdd0e5a7e5f876af1f528e8a69 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -215,7 +215,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c d02833adf331a5913226595306d64731a3da33f6 +F src/pager.c 8d97b3633f098fef817656dcbf167ca904511d78 F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eab8706dc47aa0a44caf73619de858397c3e0b4e -R 8a34c36d0ed8bbbc0bb7360d0dc30d47 +P 19736dd9fbbb7e252c4f8715e2277d48ac41f5bc +R c9d1872515b77329bc9fbfe4c8e91668 U dan -Z 610a2814ec6cdf214c41e43dd2355678 +Z 931e363227b1144b3ac19bee87a82036 diff --git a/manifest.uuid b/manifest.uuid index 11462e357c..470b2345ea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19736dd9fbbb7e252c4f8715e2277d48ac41f5bc \ No newline at end of file +ecc3544e712041736af7c7b4f34864a1f2e30ff7 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f326f713fb..83ed66414c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6856,25 +6856,19 @@ static int balance_nonroot( aPgFlags[i] = apNew[i]->pDbPage->flags; } for(i=0; ipgno); - Pgno iMin = 0; - u16 flags = 0; - for(j=0; jiGt && (iMin==0 || iPgnopgno!=iMin ){ - sqlite3PagerRekey(apNew[i]->pDbPage, iMin, flags); - apNew[i]->pgno = iMin; + pgno = aPgno[iBest]; + aPgno[iBest] = 0xffffffff; + if( iBest!=i ){ + if( iBest>i ){ + sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); + } + sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); + apNew[i]->pgno = pgno; } } diff --git a/src/pager.c b/src/pager.c index dc79e6754a..997f842d0a 100644 --- a/src/pager.c +++ b/src/pager.c @@ -6853,10 +6853,7 @@ int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, int isCommit){ ** page number to iNew and sets the value of the PgHdr.flags field to ** the value passed as the third parameter. */ -void sqlite3PagerRekey(DbPage *pPage, Pgno iNew, u16 flags){ - PgHdr *pPg = (PgHdr*)pPage; - assert( (flags & PGHDR_DIRTY) && (pPg->flags & PGHDR_DIRTY) ); - assert( !subjRequiresPage(pPg) ); +void sqlite3PagerRekey(DbPage *pPg, Pgno iNew, u16 flags){ assert( pPg->pgno!=iNew ); pPg->flags = flags; sqlite3PcacheMove(pPg, iNew); From 051575cbf4c47fc472f53146e30d6ddddc9054b3 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 25 Oct 2014 12:28:25 +0000 Subject: [PATCH 483/710] Do not use virtual (and hence redundant) WHERE-clause terms to restrict the content of a automatic partial index. Show when an automatic partial index is used in the EXPLAIN QUERY PLAN output. FossilOrigin-Name: b9ad601eab1d7298d369267eb697c7fa1bc16985 --- ext/rtree/rtree6.test | 2 +- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/where.c | 4 ++++ src/whereInt.h | 1 + 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/ext/rtree/rtree6.test b/ext/rtree/rtree6.test index cec3a8da41..c9c87e8ad9 100644 --- a/ext/rtree/rtree6.test +++ b/ext/rtree/rtree6.test @@ -102,7 +102,7 @@ do_eqp_test rtree6.2.4.2 { SELECT * FROM t1,t2 WHERE v=10 and x1<10 and x2>10 } { 0 0 0 {SCAN TABLE t1 VIRTUAL TABLE INDEX 2:C0E1} - 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC COVERING INDEX (v=?)} + 0 1 1 {SEARCH TABLE t2 USING AUTOMATIC PARTIAL COVERING INDEX (v=?)} } do_eqp_test rtree6.2.5 { diff --git a/manifest b/manifest index e831ecb526..28fa8e1387 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\sautomatic\sindex\slogic\sso\sthat\sit\screates\sa\spartial\sindex\swhen\ndoing\sso\sgives\sthe\ssame\sanswer\sfor\sless\swork. -D 2014-10-24T19:28:09.216 +C Do\snot\suse\svirtual\s(and\shence\sredundant)\sWHERE-clause\sterms\sto\srestrict\sthe\ncontent\sof\sa\sautomatic\spartial\sindex.\s\sShow\swhen\san\sautomatic\spartial\sindex\nis\sused\sin\sthe\sEXPLAIN\sQUERY\sPLAN\soutput. +D 2014-10-25T12:28:25.871 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -130,7 +130,7 @@ F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba F ext/rtree/rtree3.test a494da55c30ee0bc9b01a91c80c81b387b22d2dc F ext/rtree/rtree4.test c8fe384f60ebd49540a5fecc990041bf452eb6e0 F ext/rtree/rtree5.test 6a510494f12454bf57ef28f45bc7764ea279431e -F ext/rtree/rtree6.test 0cfbdf27ee086bf16a3da2c6f2d5b3d6473cb27e +F ext/rtree/rtree6.test 773a90db2dce6a8353dd0d5b64bca69b29761196 F ext/rtree/rtree7.test 1fa710b9e6bf997a0c1a537b81be7bb6fded1971 F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 @@ -302,8 +302,8 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c cc0733c59bd8bf6027d28b7d24b1887f4a870215 -F src/whereInt.h 4b459cdbfc9b01f5f27673a35f9967e4dea917e8 +F src/where.c 3312adfda33a9ca85c8a380a642a1a5905398b06 +F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 401235edf40fcd665eaf426cf5155ac6855e8537 -R def8f7d87d0f5784d9eaab2c56b2a690 +P d95d0313c447f5baeabdb17284d8606331ab7d49 +R 9499438a1fccf595ef80fbd4c31b20b8 U drh -Z cb611c4b71e8fc233ea4e1005f56a9e0 +Z cdff8327924b867b12e561826d5fe961 diff --git a/manifest.uuid b/manifest.uuid index 7417ef4fc1..8b94233ae7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d95d0313c447f5baeabdb17284d8606331ab7d49 \ No newline at end of file +b9ad601eab1d7298d369267eb697c7fa1bc16985 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8d2a891870..2e42b8fc32 100644 --- a/src/where.c +++ b/src/where.c @@ -1612,6 +1612,7 @@ static void constructAutomaticIndex( idxCols = 0; for(pTerm=pWC->a; pTermprereq==0 + && (pTerm->wtFlags & TERM_VIRTUAL)==0 && sqlite3ExprIsTableConstant(pTerm->pExpr, pSrc->iCursor) ){ pPartial = sqlite3ExprAnd(pParse->db, pPartial, sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); @@ -1720,6 +1721,7 @@ static void constructAutomaticIndex( if( pPartial ){ iContinue = sqlite3VdbeMakeLabel(v); sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); + pLoop->wsFlags |= WHERE_PARTIALIDX; } regRecord = sqlite3GetTempReg(pParse); sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); @@ -2865,6 +2867,8 @@ static void explainOneScan( if( isSearch ){ zFmt = "PRIMARY KEY"; } + }else if( flags & WHERE_PARTIALIDX ){ + zFmt = "AUTOMATIC PARTIAL COVERING INDEX"; }else if( flags & WHERE_AUTO_INDEX ){ zFmt = "AUTOMATIC COVERING INDEX"; }else if( flags & WHERE_IDX_ONLY ){ diff --git a/src/whereInt.h b/src/whereInt.h index e9eb8b7dd2..fd4cfdf88a 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -459,3 +459,4 @@ struct WhereInfo { #define WHERE_AUTO_INDEX 0x00004000 /* Uses an ephemeral index */ #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ +#define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ From d05ab6aacf08a1fd7af581cae4b28390ecd5c44c Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 25 Oct 2014 13:42:16 +0000 Subject: [PATCH 484/710] Increase the resolution of the second parameter to the likelihood() SQL function (the probability value) so that it can handle probabilities as small as 0.00000001. Formerly, it ran out of precision at 0.001. FossilOrigin-Name: 0f08924fe0c52a85a103f67bee9809e0f8f884b0 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/resolve.c | 4 ++-- src/sqliteInt.h | 2 +- src/where.c | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 28fa8e1387..6d8395af5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\svirtual\s(and\shence\sredundant)\sWHERE-clause\sterms\sto\srestrict\sthe\ncontent\sof\sa\sautomatic\spartial\sindex.\s\sShow\swhen\san\sautomatic\spartial\sindex\nis\sused\sin\sthe\sEXPLAIN\sQUERY\sPLAN\soutput. -D 2014-10-25T12:28:25.871 +C Increase\sthe\sresolution\sof\sthe\ssecond\sparameter\sto\sthe\slikelihood()\sSQL\nfunction\s(the\sprobability\svalue)\sso\sthat\sit\scan\shandle\sprobabilities\nas\ssmall\sas\s0.00000001.\s\sFormerly,\sit\sran\sout\sof\sprecision\sat\s0.001. +D 2014-10-25T13:42:16.126 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -225,14 +225,14 @@ F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c 57d5ad93913beb43ad9b8ade435a54e5fb8ccd40 +F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 123b28f3552d4ffdd3e53707fe8120a069df69e4 +F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -302,7 +302,7 @@ F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 3312adfda33a9ca85c8a380a642a1a5905398b06 +F src/where.c 4ce8c4826b7f86d080f0ed4e7a9045bb5014be77 F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d95d0313c447f5baeabdb17284d8606331ab7d49 -R 9499438a1fccf595ef80fbd4c31b20b8 +P b9ad601eab1d7298d369267eb697c7fa1bc16985 +R 06fd31d9e4c300e561fccc4cf28a702b U drh -Z cdff8327924b867b12e561826d5fe961 +Z 628a0c5751868817bd40f1740c611aef diff --git a/manifest.uuid b/manifest.uuid index 8b94233ae7..3804da23b9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9ad601eab1d7298d369267eb697c7fa1bc16985 \ No newline at end of file +0f08924fe0c52a85a103f67bee9809e0f8f884b0 \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index e507ccb810..8fb580b3a1 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -584,7 +584,7 @@ static int exprProbability(Expr *p){ sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); assert( r>=0.0 ); if( r>1.0 ) return -1; - return (int)(r*1000.0); + return (int)(r*134217728.0); } /* @@ -716,7 +716,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent to ** likelihood(X,0.9375). */ /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ - pExpr->iTable = pDef->zName[0]=='u' ? 62 : 938; + pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120; } } #ifndef SQLITE_OMIT_AUTHORIZATION diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b7f992c34d..5114ccccf4 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2000,7 +2000,7 @@ struct Expr { int iTable; /* TK_COLUMN: cursor number of table holding column ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old - ** EP_Unlikely: 1000 times likelihood */ + ** EP_Unlikely: 134217728 times likelihood */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. ** TK_VARIABLE: variable number (always >= 1). */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ diff --git a/src/where.c b/src/where.c index 2e42b8fc32..358d508260 100644 --- a/src/where.c +++ b/src/where.c @@ -225,7 +225,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ } pTerm = &pWC->a[idx = pWC->nTerm++]; if( p && ExprHasProperty(p, EP_Unlikely) ){ - pTerm->truthProb = sqlite3LogEst(p->iTable) - 99; + pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; }else{ pTerm->truthProb = 1; } From 89ca0b3878bf964c202e1d0c984b39a9eefaafb0 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 25 Oct 2014 20:36:28 +0000 Subject: [PATCH 485/710] Further modifications to new code to better handle corrupt databases. FossilOrigin-Name: 1a8cf0a043347772ac54d150d634c32845beee8b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 30 +++++++++++++++++++++++------- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 6c0464217b..1a5be4bf57 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\sthe\s"Any\sprior\scache\sentry\sassociated\swith\snewKey\sis\sguaranteed\snot\sto\sbe\spinned"\sguarantee\smade\sto\sxRekey\simplementations\sis\snot\sviolated. -D 2014-10-24T20:57:03.500 +C Further\smodifications\sto\snew\scode\sto\sbetter\shandle\scorrupt\sdatabases. +D 2014-10-25T20:36:28.557 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 4decfb3b97d16afdd0e5a7e5f876af1f528e8a69 +F src/btree.c 2639b89f6728f5775044704c7757c0226e071bd1 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 19736dd9fbbb7e252c4f8715e2277d48ac41f5bc -R c9d1872515b77329bc9fbfe4c8e91668 +P ecc3544e712041736af7c7b4f34864a1f2e30ff7 +R 17dc589efbb3f16f6448c9d9d1ae2e24 U dan -Z 931e363227b1144b3ac19bee87a82036 +Z c615066e82cbd286f5597ce396069fd7 diff --git a/manifest.uuid b/manifest.uuid index 470b2345ea..23643921ec 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecc3544e712041736af7c7b4f34864a1f2e30ff7 \ No newline at end of file +1a8cf0a043347772ac54d150d634c32845beee8b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 83ed66414c..46ce39abc7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6099,6 +6099,7 @@ static int pageFreeArray( ){ u8 * const aData = pPg->aData; u8 * const pEnd = &aData[pPg->pBt->usableSize]; + u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; int nRet = 0; int i; u8 *pFree = 0; @@ -6106,12 +6107,13 @@ static int pageFreeArray( for(i=0; iaData && pCell=pStart && pCellpEnd ) return 0; }else{ pFree = pCell; szFree += sz; @@ -6854,6 +6856,18 @@ static int balance_nonroot( for(i=0; ipgno; aPgFlags[i] = apNew[i]->pDbPage->flags; + for(j=0; jnFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) - ); - copyNodeContent(apNew[0], pParent, &rc); - freePage(apNew[0], &rc); + rc = defragmentPage(apNew[0]); + if( rc==SQLITE_OK ){ + assert( apNew[0]->nFree == + (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + ); + copyNodeContent(apNew[0], pParent, &rc); + freePage(apNew[0], &rc); + } }else if( ISAUTOVACUUM && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken From 61e94c9deb9d0edff1fdaa443fe7668cbfacedb4 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 27 Oct 2014 08:02:16 +0000 Subject: [PATCH 486/710] If a free-slot is found within a page, but using that free-slot would fragment the page further and there are already at least 60 fragmented bytes, degragment the page. This matches the behaviour of the trunk. FossilOrigin-Name: 1f80f8c136ac970dcc7fb2337263dc5922e348c3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 18 ++++++++++++++---- 3 files changed, 21 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index bafcf0fb91..a088218356 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\swith\sthis\sbranch. -D 2014-10-27T07:01:04.796 +C If\sa\sfree-slot\sis\sfound\swithin\sa\spage,\sbut\susing\sthat\sfree-slot\swould\sfragment\sthe\spage\sfurther\sand\sthere\sare\salready\sat\sleast\s60\sfragmented\sbytes,\sdegragment\sthe\spage.\sThis\smatches\sthe\sbehaviour\sof\sthe\strunk. +D 2014-10-27T08:02:16.531 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 2639b89f6728f5775044704c7757c0226e071bd1 +F src/btree.c 9790fb4df51d36861bcbb8cd0a9b41586cbae699 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1206,7 +1206,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1a8cf0a043347772ac54d150d634c32845beee8b 0f08924fe0c52a85a103f67bee9809e0f8f884b0 -R 9858b90cbf912e211c65716d59c94e8a +P a13df3013bbac4a0d4fce5cef1376c857508c1c5 +R 05f44d14ab419727640258148977021c U dan -Z daf2b363c619dfc32da3ad287a4e25ec +Z 95c7c43e9818bc003f29ce4926da458f diff --git a/manifest.uuid b/manifest.uuid index 962b2e06bb..92680eaebb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a13df3013bbac4a0d4fce5cef1376c857508c1c5 \ No newline at end of file +1f80f8c136ac970dcc7fb2337263dc5922e348c3 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 46ce39abc7..9b14dc2b6e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1235,8 +1235,12 @@ static int defragmentPage(MemPage *pPage){ ** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. ** Or, if corruption is detected and pRc is NULL, NULL is returned and the ** corruption goes unreported. +** +** If a slot of at least nByte bytes is found but cannot be used because +** there are already at least 60 fragmented bytes on the page, return NULL. +** In this case, if pbDefrag parameter is not NULL, set *pbDefrag to true. */ -static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ +static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ const int hdr = pPg->hdrOffset; u8 * const aData = pPg->aData; int iAddr; @@ -1255,7 +1259,10 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ testcase( x==4 ); testcase( x==3 ); if( x<4 ){ - if( aData[hdr+7]>=60 ) return 0; + if( aData[hdr+7]>=60 ){ + if( pbDefrag ) *pbDefrag = 1; + return 0; + } /* Remove the slot from the free-list. Update the number of ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); @@ -1326,8 +1333,10 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap==top ); if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ int rc = SQLITE_OK; - u8 *pSpace = pageFindSlot(pPage, nByte, &rc); + int bDefrag = 0; + u8 *pSpace = pageFindSlot(pPage, nByte, &rc, &bDefrag); if( rc ) return rc; + if( bDefrag ) goto defragment_page; if( pSpace ){ *pIdx = pSpace - data; return SQLITE_OK; @@ -1339,6 +1348,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ */ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ + defragment_page: testcase( pPage->nCell==0 ); rc = defragmentPage(pPage); if( rc ) return rc; @@ -6069,7 +6079,7 @@ static int pageInsertArray( for(i=0; i Date: Mon, 27 Oct 2014 11:25:28 +0000 Subject: [PATCH 487/710] Add test file e_wal.test. FossilOrigin-Name: fc6920b5483eeeb06a474ff399a21afa51dc4859 --- manifest | 13 +-- manifest.uuid | 2 +- test/e_wal.test | 229 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 237 insertions(+), 7 deletions(-) create mode 100644 test/e_wal.test diff --git a/manifest b/manifest index 6d8395af5e..6c3a597a05 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sresolution\sof\sthe\ssecond\sparameter\sto\sthe\slikelihood()\sSQL\nfunction\s(the\sprobability\svalue)\sso\sthat\sit\scan\shandle\sprobabilities\nas\ssmall\sas\s0.00000001.\s\sFormerly,\sit\sran\sout\sof\sprecision\sat\s0.001. -D 2014-10-25T13:42:16.126 +C Add\stest\sfile\se_wal.test. +D 2014-10-27T11:25:28.528 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -466,6 +466,7 @@ F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 +F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 @@ -1206,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b9ad601eab1d7298d369267eb697c7fa1bc16985 -R 06fd31d9e4c300e561fccc4cf28a702b -U drh -Z 628a0c5751868817bd40f1740c611aef +P 0f08924fe0c52a85a103f67bee9809e0f8f884b0 +R 770a2e0b8a993a6006545c850fbc4dd8 +U dan +Z 70c394a419fcdd8591ee5c3f4c5346c4 diff --git a/manifest.uuid b/manifest.uuid index 3804da23b9..a94336a518 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f08924fe0c52a85a103f67bee9809e0f8f884b0 \ No newline at end of file +fc6920b5483eeeb06a474ff399a21afa51dc4859 \ No newline at end of file diff --git a/test/e_wal.test b/test/e_wal.test new file mode 100644 index 0000000000..a5e074f49b --- /dev/null +++ b/test/e_wal.test @@ -0,0 +1,229 @@ +# 2011 May 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_wal + +db close +testvfs oldvfs -iversion 1 + + +# EVIDENCE-OF: R-58297-14483 WAL databases can be created, read, and +# written even if shared memory is unavailable as long as the +# locking_mode is set to EXCLUSIVE before the first attempted access. +# +# EVIDENCE-OF: R-00449-33772 This feature allows WAL databases to be +# created, read, and written by legacy VFSes that lack the "version 2" +# shared-memory methods xShmMap, xShmLock, xShmBarrier, and xShmUnmap on +# the sqlite3_io_methods object. +# +# 1.1: "create" tests. +# 1.2: "read" tests. +# 1.3: "write" tests. +# +# All three done with VFS "oldvfs", which has iVersion==1 and so does +# not support shared memory. +# +sqlite3 db test.db -vfs oldvfs +do_execsql_test 1.1.1 { + PRAGMA journal_mode = WAL; +} {delete} +do_execsql_test 1.1.2 { + PRAGMA locking_mode = EXCLUSIVE; + PRAGMA journal_mode = WAL; +} {exclusive wal} +do_execsql_test 1.1.3 { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); +} {} +do_test 1.1.4 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +do_test 1.2.1 { + db close + sqlite3 db test.db -vfs oldvfs + catchsql { SELECT * FROM t1 } +} {1 {unable to open database file}} +do_test 1.2.2 { + execsql { PRAGMA locking_mode = EXCLUSIVE } + execsql { SELECT * FROM t1 } +} {1 2} +do_test 1.2.3 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +do_test 1.3.1 { + db close + sqlite3 db test.db -vfs oldvfs + catchsql { INSERT INTO t1 VALUES(3, 4) } +} {1 {unable to open database file}} +do_test 1.3.2 { + execsql { PRAGMA locking_mode = EXCLUSIVE } + execsql { INSERT INTO t1 VALUES(3, 4) } + execsql { SELECT * FROM t1 } +} {1 2 3 4} +do_test 1.3.3 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +# EVIDENCE-OF: R-31969-57825 If EXCLUSIVE locking mode is set prior to +# the first WAL-mode database access, then SQLite never attempts to call +# any of the shared-memory methods and hence no shared-memory wal-index +# is ever created. +# +db close +sqlite3 db test.db +do_execsql_test 2.1.1 { + PRAGMA locking_mode = EXCLUSIVE; + SELECT * FROM t1; +} {exclusive 1 2 3 4} +do_test 2.1.2 { + list [file exists test.db-shm] [file exists test.db-wal] +} {0 1} + +# EVIDENCE-OF: R-36328-16367 In that case, the database connection +# remains in EXCLUSIVE mode as long as the journal mode is WAL; attempts +# to change the locking mode using "PRAGMA locking_mode=NORMAL;" are +# no-ops. +# +do_execsql_test 2.2.1 { + PRAGMA locking_mode = NORMAL; + SELECT * FROM t1; +} {exclusive 1 2 3 4} +do_test 2.2.2 { + sqlite3 db2 test.db + catchsql {SELECT * FROM t1} db2 +} {1 {database is locked}} +db2 close + +# EVIDENCE-OF: R-63522-46088 The only way to change out of EXCLUSIVE +# locking mode is to first change out of WAL journal mode. +# +do_execsql_test 2.3.1 { + PRAGMA journal_mode = DELETE; + SELECT * FROM t1; +} {delete 1 2 3 4} +do_test 2.3.2 { + sqlite3 db2 test.db + catchsql {SELECT * FROM t1} db2 +} {1 {database is locked}} +do_execsql_test 2.3.3 { + PRAGMA locking_mode = NORMAL; + SELECT * FROM t1; +} {normal 1 2 3 4} +do_test 2.3.4 { + sqlite3 db2 test.db + catchsql {SELECT * FROM t1} db2 +} {0 {1 2 3 4}} +db2 close +db close + + +# EVIDENCE-OF: R-57239-11845 If NORMAL locking mode is in effect for the +# first WAL-mode database access, then the shared-memory wal-index is +# created. +# +do_test 3.0 { + sqlite3 db test.db + execsql { PRAGMA journal_mode = WAL } + db close +} {} +do_test 3.1 { + sqlite3 db test.db + execsql { SELECT * FROM t1 } + list [file exists test.db-shm] [file exists test.db-wal] +} {1 1} + +# EVIDENCE-OF: R-13779-07711 As long as exactly one connection is using +# a shared-memory wal-index, the locking mode can be changed freely +# between NORMAL and EXCLUSIVE. +# +do_execsql_test 3.2.1 { + PRAGMA locking_mode = EXCLUSIVE; + PRAGMA locking_mode = NORMAL; + PRAGMA locking_mode = EXCLUSIVE; + INSERT INTO t1 VALUES(5, 6); +} {exclusive normal exclusive} +do_test 3.2.2 { + sqlite3 db2 test.db + catchsql { SELECT * FROM t1 } db2 +} {1 {database is locked}} + +# EVIDENCE-OF: R-10993-11647 It is only when the shared-memory wal-index +# is omitted, when the locking mode is EXCLUSIVE prior to the first +# WAL-mode database access, that the locking mode is stuck in EXCLUSIVE. +# +do_execsql_test 3.2.3 { + PRAGMA locking_mode = NORMAL; + SELECT * FROM t1; +} {normal 1 2 3 4 5 6} +do_test 3.2.4 { + catchsql { SELECT * FROM t1 } db2 +} {0 {1 2 3 4 5 6}} + +do_catchsql_test 3.2.5 { + PRAGMA locking_mode = EXCLUSIVE; + INSERT INTO t1 VALUES(7, 8); +} {1 {database is locked}} + +db2 close + +# EVIDENCE-OF: R-46197-42811 This means that the underlying VFS must +# support the "version 2" shared-memory. +# +# EVIDENCE-OF: R-55316-21772 If the VFS does not support shared-memory +# methods, then the attempt to open a database that is already in WAL +# mode, or the attempt convert a database into WAL mode, will fail. +# +db close +do_test 3.4.1 { + sqlite3 db test.db -vfs oldvfs + catchsql { SELECT * FROM t1 } +} {1 {unable to open database file}} +db close +do_test 3.4.2 { + forcedelete test.db2 + sqlite3 db test.db2 -vfs oldvfs + catchsql { PRAGMA journal_mode = WAL } +} {0 delete} +db close + + +# EVIDENCE-OF: R-22428-28959 To prevent older versions of SQLite from +# trying to recover a WAL-mode database (and making matters worse) the +# database file format version numbers (bytes 18 and 19 in the database +# header) are increased from 1 to 2 in WAL mode. +# +reset_db +do_execsql_test 4.1.1 { CREATE TABLE t1(x, y) } +do_test 4.1.2 { hexio_read test.db 18 2 } {0101} +do_execsql_test 4.1.3 { PRAGMA journal_mode = wAL } {wal} +do_test 4.1.4 { hexio_read test.db 18 2 } {0202} + + +# EVIDENCE-OF: R-02535-05811 One can explicitly change out of WAL mode +# using a pragma such as this: PRAGMA journal_mode=DELETE; +# +do_execsql_test 4.2.1 { INSERT INTO t1 VALUES(1, 1); } {} +do_test 4.2.2 { file exists test.db-wal } {1} +do_execsql_test 4.2.3 { PRAGMA journal_mode = delete } {delete} +do_test 4.2.4 { file exists test.db-wal } {0} + +# EVIDENCE-OF: R-60175-02388 Deliberately changing out of WAL mode +# changes the database file format version numbers back to 1 so that +# older versions of SQLite can once again access the database file. +# +do_test 4.3 { hexio_read test.db 18 2 } {0101} + +finish_test From 8a42d1bba15c0ba571014d95b64d4d74759d39fa Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 27 Oct 2014 18:21:01 +0000 Subject: [PATCH 488/710] Fix an unused variable in btree.c:allocateSpace(). FossilOrigin-Name: 637246165a14c4808b90d0437e4d43fa5fac659e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 4 +--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 94ec16445a..3991d3d55b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimizations\saimed\sat\sreducing\sthe\snumber\sof\smemcpy()\soperations\srequired\sby\sbalance_nonroot(). -D 2014-10-27T14:26:54.508 +C Fix\san\sunused\svariable\sin\sbtree.c:allocateSpace(). +D 2014-10-27T18:21:01.498 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 9790fb4df51d36861bcbb8cd0a9b41586cbae699 +F src/btree.c 812c03daa8be68daf623dd0349ecb18e9f988967 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fc6920b5483eeeb06a474ff399a21afa51dc4859 1f80f8c136ac970dcc7fb2337263dc5922e348c3 -R ed2304a6a2e587056109284e9717f063 -U dan -Z 986c15e36469a97fa56839a4b2a55b25 +P face33bea1ba3a6d57780655fa827226b4d2baa9 +R 10f6daecfdf5443eb311db2d487f32a3 +U drh +Z a48d68bafb210b7d63c9203bfcb49102 diff --git a/manifest.uuid b/manifest.uuid index 37f4629c87..28889facf2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -face33bea1ba3a6d57780655fa827226b4d2baa9 \ No newline at end of file +637246165a14c4808b90d0437e4d43fa5fac659e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9b14dc2b6e..06dfd2cd53 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1301,7 +1301,6 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ int top; /* First byte of cell content area */ int gap; /* First byte of gap between cell pointers and cell content */ int rc; /* Integer return code */ - int usableSize; /* Usable size of the page */ assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt ); @@ -1309,8 +1308,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( nByte>=0 ); /* Minimum cell size is 4 */ assert( pPage->nFree>=nByte ); assert( pPage->nOverflow==0 ); - usableSize = pPage->pBt->usableSize; - assert( nByte < usableSize-8 ); + assert( nByte < pPage->pBt->usableSize-8 ); assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; From 98d94211ced2604da384d16978f783b4f166cfdb Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 27 Oct 2014 19:39:51 +0000 Subject: [PATCH 489/710] Fix a typo in the macro name of an #ifdef FossilOrigin-Name: 9646a136e69cf2583965dfc9fac5f056af4cdb62 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/ctime.c | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e658f3a76a..3bfe9532df 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_ENABLE_API_ARMOR\scompile-time\soption\sfor\sextra\sAPI\sparameter\nvalidation.\s\sEnhance\ssqlite3_stricmp(),\ssqlite3_strnicmp(),\sand\nsqlite3_uri_parameter()\sfor\simproved\sNULL\sparameter\shandling. -D 2014-10-27T18:34:07.031 +C Fix\sa\stypo\sin\sthe\smacro\sname\sof\san\s#ifdef +D 2014-10-27T19:39:51.561 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -178,7 +178,7 @@ F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 -F src/ctime.c dfa83bfebb4201d07b16534acb8a0149592c3a25 +F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee @@ -1207,8 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 637246165a14c4808b90d0437e4d43fa5fac659e 1c220b806d56e163842e17038c3331f71861bd9c -R 78cdca5c720e64c06c4c8ef41bcc31b7 -T +closed 1c220b806d56e163842e17038c3331f71861bd9c +P ffb9d8144bbc35bf3d929e0e13a663668fff0558 +R 517f6fa1cedd5526f59aa045554cf442 U drh -Z 1739c9dd8a9fd2ca84e2afadfbbcea12 +Z d3a8b97f7ef904d82264d3ec79cf2202 diff --git a/manifest.uuid b/manifest.uuid index 2f7fda9cba..70a2bdb89a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffb9d8144bbc35bf3d929e0e13a663668fff0558 \ No newline at end of file +9646a136e69cf2583965dfc9fac5f056af4cdb62 \ No newline at end of file diff --git a/src/ctime.c b/src/ctime.c index 36d0c266ee..59dc972d8d 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -392,7 +392,7 @@ static const char * const azCompileOpt[] = { int sqlite3_compileoption_used(const char *zOptName){ int i, n; -#ifdef SQLITE_ENABLE_API_ARMORE +#ifdef SQLITE_ENABLE_API_ARMOR if( zOptName==0 ){ (void)SQLITE_MISUSE_BKPT; return 0; From a95d8ca1fad1133a39ab65c60e7b7346f9089749 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 27 Oct 2014 19:42:02 +0000 Subject: [PATCH 490/710] Fix harmless compiler warning in an assert statement. FossilOrigin-Name: d33a1ff3aad0bfabf70a98ac338a68f82074e4fe --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3bfe9532df..c24ca49842 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\smacro\sname\sof\san\s#ifdef -D 2014-10-27T19:39:51.561 +C Fix\sharmless\scompiler\swarning\sin\san\sassert\sstatement. +D 2014-10-27T19:42:02.875 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 812c03daa8be68daf623dd0349ecb18e9f988967 +F src/btree.c 0e3262c36b69bed3c5f9260313286a2e8c060ca5 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ffb9d8144bbc35bf3d929e0e13a663668fff0558 -R 517f6fa1cedd5526f59aa045554cf442 -U drh -Z d3a8b97f7ef904d82264d3ec79cf2202 +P 9646a136e69cf2583965dfc9fac5f056af4cdb62 +R c8728e917d07299237654ca84312b584 +U mistachkin +Z 269b8116d90b241ccd2319871bd381eb diff --git a/manifest.uuid b/manifest.uuid index 70a2bdb89a..3bcc889db6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9646a136e69cf2583965dfc9fac5f056af4cdb62 \ No newline at end of file +d33a1ff3aad0bfabf70a98ac338a68f82074e4fe \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 06dfd2cd53..39d1e81a3f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1308,7 +1308,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( nByte>=0 ); /* Minimum cell size is 4 */ assert( pPage->nFree>=nByte ); assert( pPage->nOverflow==0 ); - assert( nByte < pPage->pBt->usableSize-8 ); + assert( nByte < (int)(pPage->pBt->usableSize-8) ); assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; From df9c093e2c516c011ec476cf220d477eea70983c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 27 Oct 2014 19:58:29 +0000 Subject: [PATCH 491/710] Fix compilation issue with MSVC due to a misplaced variable declaration. FossilOrigin-Name: 9588b345d09daaa49d24d7fb6cab732e64e5474e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/random.c | 16 +++++++++++----- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index c24ca49842..4a6bd095ba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sin\san\sassert\sstatement. -D 2014-10-27T19:42:02.875 +C Fix\scompilation\sissue\swith\sMSVC\sdue\sto\sa\smisplaced\svariable\sdeclaration. +D 2014-10-27T19:58:29.156 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -224,7 +224,7 @@ F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 -F src/random.c b8a058131851de1a37801b5587845ee73411c064 +F src/random.c f88232b90e308f58f1f8f10894d7a8a750d6f64d F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9646a136e69cf2583965dfc9fac5f056af4cdb62 -R c8728e917d07299237654ca84312b584 +P d33a1ff3aad0bfabf70a98ac338a68f82074e4fe +R ee98bc34f841b6720302924cb59e4bc2 U mistachkin -Z 269b8116d90b241ccd2319871bd381eb +Z 8119f44960013a2ca52459062580ce07 diff --git a/manifest.uuid b/manifest.uuid index 3bcc889db6..f68a004aea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d33a1ff3aad0bfabf70a98ac338a68f82074e4fe \ No newline at end of file +9588b345d09daaa49d24d7fb6cab732e64e5474e \ No newline at end of file diff --git a/src/random.c b/src/random.c index c4241362de..bd109e71b3 100644 --- a/src/random.c +++ b/src/random.c @@ -34,10 +34,6 @@ void sqlite3_randomness(int N, void *pBuf){ unsigned char t; unsigned char *zBuf = pBuf; -#ifndef SQLITE_OMIT_AUTOINIT - if( sqlite3_initialize() ) return; -#endif - /* The "wsdPrng" macro will resolve to the pseudo-random number generator ** state vector. If writable static data is unsupported on the target, ** we have to locate the state vector at run-time. In the more common @@ -52,13 +48,23 @@ void sqlite3_randomness(int N, void *pBuf){ #endif #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); + sqlite3_mutex *mutex; +#endif + +#ifndef SQLITE_OMIT_AUTOINIT + if( sqlite3_initialize() ) return; +#endif + +#if SQLITE_THREADSAFE + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); sqlite3_mutex_enter(mutex); #endif if( N<=0 || pBuf==0 ){ wsdPrng.isInit = 0; +#if SQLITE_THREADSAFE sqlite3_mutex_leave(mutex); +#endif return; } From d61a18a9f1493ed6ee2b6d12d7b4ca4621cc894b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 27 Oct 2014 20:14:02 +0000 Subject: [PATCH 492/710] Remove a small amount of unnecessary #ifdeffery from random.c. FossilOrigin-Name: 2b9340c8684bc382391e02813e960b3166f24daa --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/random.c | 4 +--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4a6bd095ba..31157b57d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompilation\sissue\swith\sMSVC\sdue\sto\sa\smisplaced\svariable\sdeclaration. -D 2014-10-27T19:58:29.156 +C Remove\sa\ssmall\samount\sof\sunnecessary\s#ifdeffery\sfrom\srandom.c. +D 2014-10-27T20:14:02.021 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -224,7 +224,7 @@ F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 -F src/random.c f88232b90e308f58f1f8f10894d7a8a750d6f64d +F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d33a1ff3aad0bfabf70a98ac338a68f82074e4fe -R ee98bc34f841b6720302924cb59e4bc2 -U mistachkin -Z 8119f44960013a2ca52459062580ce07 +P 9588b345d09daaa49d24d7fb6cab732e64e5474e +R e26f0edef3def3003fae07f43e971e87 +U drh +Z 3e6e6f10ca8455ff968ead5f51a9fff2 diff --git a/manifest.uuid b/manifest.uuid index f68a004aea..11afc870ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9588b345d09daaa49d24d7fb6cab732e64e5474e \ No newline at end of file +2b9340c8684bc382391e02813e960b3166f24daa \ No newline at end of file diff --git a/src/random.c b/src/random.c index bd109e71b3..179d01bef2 100644 --- a/src/random.c +++ b/src/random.c @@ -57,14 +57,12 @@ void sqlite3_randomness(int N, void *pBuf){ #if SQLITE_THREADSAFE mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_PRNG); - sqlite3_mutex_enter(mutex); #endif + sqlite3_mutex_enter(mutex); if( N<=0 || pBuf==0 ){ wsdPrng.isInit = 0; -#if SQLITE_THREADSAFE sqlite3_mutex_leave(mutex); -#endif return; } From 2d8ad51c5b53644aa40ab276a7c7342da920d465 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 27 Oct 2014 22:06:21 +0000 Subject: [PATCH 493/710] Add special handling for static mutexes in sqlite3_mutex_alloc() when automatic calls to sqlite3_initialize() are enabled. FossilOrigin-Name: 7857d27caa845e5629d94c2e66587dc89016daca --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/mutex.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 31157b57d0..ed6b523e7c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\ssmall\samount\sof\sunnecessary\s#ifdeffery\sfrom\srandom.c. -D 2014-10-27T20:14:02.021 +C Add\sspecial\shandling\sfor\sstatic\smutexes\sin\ssqlite3_mutex_alloc()\swhen\sautomatic\scalls\sto\ssqlite3_initialize()\sare\senabled. +D 2014-10-27T22:06:21.571 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -202,7 +202,7 @@ F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c 61eeb90134f9a5be6c2e68d8daae7628b25953fb F src/memjournal.c 3eb2c0b51adbd869cb6a44780323f05fa904dc85 -F src/mutex.c 84a073c9a23a8d7bdd2ea832522d1730df18812c +F src/mutex.c 19bf9acba69ca2f367c3761080f8a9f0cf4670a8 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 F src/mutex_unix.c 551e2f25f0fa0ee8fd7a43f50fc3d8be00e95dde @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9588b345d09daaa49d24d7fb6cab732e64e5474e -R e26f0edef3def3003fae07f43e971e87 -U drh -Z 3e6e6f10ca8455ff968ead5f51a9fff2 +P 2b9340c8684bc382391e02813e960b3166f24daa +R f7a6b10545dea42b1d622484e6364982 +U mistachkin +Z 6b079baac5628ec298fadd129a245092 diff --git a/manifest.uuid b/manifest.uuid index 11afc870ef..7088378432 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b9340c8684bc382391e02813e960b3166f24daa \ No newline at end of file +7857d27caa845e5629d94c2e66587dc89016daca \ No newline at end of file diff --git a/src/mutex.c b/src/mutex.c index bad5a7c113..2b45036289 100644 --- a/src/mutex.c +++ b/src/mutex.c @@ -82,6 +82,7 @@ int sqlite3MutexEnd(void){ sqlite3_mutex *sqlite3_mutex_alloc(int id){ #ifndef SQLITE_OMIT_AUTOINIT if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; + if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; #endif return sqlite3GlobalConfig.mutex.xMutexAlloc(id); } From eefaf448db0ed620f87af06a49c1bc5fbfdb9282 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 00:56:18 +0000 Subject: [PATCH 494/710] Correct the documentation on the maximum size of a scratch allocation. FossilOrigin-Name: 30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 1 + src/sqlite.h.in | 12 ++++++++---- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index ed6b523e7c..2a931c6cb3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sspecial\shandling\sfor\sstatic\smutexes\sin\ssqlite3_mutex_alloc()\swhen\sautomatic\scalls\sto\ssqlite3_initialize()\sare\senabled. -D 2014-10-27T22:06:21.571 +C Correct\sthe\sdocumentation\son\sthe\smaximum\ssize\sof\sa\sscratch\sallocation. +D 2014-10-28T00:56:18.775 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 0e3262c36b69bed3c5f9260313286a2e8c060ca5 +F src/btree.c 5189881ca403938c5ceddde496b984fef9f40c5a F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 +F src/sqlite.h.in a9f2e5a0e2472c8c7819f3a16074c14b9376608f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2b9340c8684bc382391e02813e960b3166f24daa -R f7a6b10545dea42b1d622484e6364982 -U mistachkin -Z 6b079baac5628ec298fadd129a245092 +P 7857d27caa845e5629d94c2e66587dc89016daca +R cc3ad0fbd146620828bc67546f9cb55d +U drh +Z 46e16e5efd8acf1525fc0288e922960d diff --git a/manifest.uuid b/manifest.uuid index 7088378432..bc344729d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7857d27caa845e5629d94c2e66587dc89016daca \ No newline at end of file +30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 39d1e81a3f..22b168d9e4 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6649,6 +6649,7 @@ static int balance_nonroot( nMaxCells*sizeof(u8*) /* apCell */ + nMaxCells*sizeof(u16) /* szCell */ + pBt->pageSize; /* aSpace1 */ + assert( szScratch<=16896 || szScratch<=6*pBt->pageSize ); apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f1d4e406e8..8cfe61ee1a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1539,10 +1539,14 @@ struct sqlite3_mem_methods { ** argument must be a multiple of 16. ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will use no more than two scratch buffers per thread. So -** N should be set to twice the expected maximum number of threads. -** ^SQLite will never require a scratch buffer that is more than 6 -** times the database page size. ^If SQLite needs needs additional +** ^SQLite will not use more than two scratch buffers per thread and not +** more than one scratch buffer per thread when not performing +** a [checkpoint] in [WAL mode]. +** ^SQLite will never request a scratch buffer that is more than 6 +** times the database page size, except when performing a [checkpoint] +** in [WAL mode] when the scratch buffer request size is a small fraction +** of the size of the WAL file. +** ^If SQLite needs needs additional ** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed.
    ** From 916cd23b2ca6aa4716b7b407abea6d870ea4585c Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 12:35:28 +0000 Subject: [PATCH 495/710] Bump the version number to 3.8.8 FossilOrigin-Name: 1a7e711ed10860c7985e84b97cdfff748d940b9f --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 4351a7e3a3..d7d8e42315 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.7 +3.8.8 diff --git a/configure b/configure index 9b8266d812..c0f751db62 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.7. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.8. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.7' -PACKAGE_STRING='sqlite 3.8.7' +PACKAGE_VERSION='3.8.8' +PACKAGE_STRING='sqlite 3.8.8' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.7 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.8 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.7:";; + short | recursive ) echo "Configuration of sqlite 3.8.8:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.7 +sqlite configure 3.8.8 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.7, which was +It was created by sqlite $as_me 3.8.8, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.7, which was +This file was extended by sqlite $as_me 3.8.8, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.7 +sqlite config.status 3.8.8 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index 2a931c6cb3..27680c927a 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Correct\sthe\sdocumentation\son\sthe\smaximum\ssize\sof\sa\sscratch\sallocation. -D 2014-10-28T00:56:18.775 +C Bump\sthe\sversion\snumber\sto\s3.8.8 +D 2014-10-28T12:35:28.069 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc e31dee24038965fb6269d6d61073fd6b7e331dec F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION 53a0b870e7f16d3b06623c31d233a304c163a6af +F VERSION d846487aff892625eb8e75960234e7285f0462fe F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure ad59a5f48b3c59a92b5506040a22fbe3f733a9d8 x +F configure 4343c810cc772571210af75d1a8f7c2eb711d75a x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7857d27caa845e5629d94c2e66587dc89016daca -R cc3ad0fbd146620828bc67546f9cb55d +P 30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 +R 241b510f33643f46040372d1284a7d7b U drh -Z 46e16e5efd8acf1525fc0288e922960d +Z b060cb081e36e4268c2624adfeebfbc8 diff --git a/manifest.uuid b/manifest.uuid index bc344729d5..4cdb1c73e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 \ No newline at end of file +1a7e711ed10860c7985e84b97cdfff748d940b9f \ No newline at end of file From c3ef4fa88a370dc6835d6dc4c3875d9577ae34f5 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 15:58:50 +0000 Subject: [PATCH 496/710] Trivial simplification to the automatic index logic. FossilOrigin-Name: 23073a053931de324323f631a6613086786af411 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 27680c927a..4d085dbcbc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Bump\sthe\sversion\snumber\sto\s3.8.8 -D 2014-10-28T12:35:28.069 +C Trivial\ssimplification\sto\sthe\sautomatic\sindex\slogic. +D 2014-10-28T15:58:50.607 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 4ce8c4826b7f86d080f0ed4e7a9045bb5014be77 +F src/where.c 5665df88cbd2b38eb72b4b94c8892c8afb360181 F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 30f86eb3f9ac88f83ed9e23ea6cd1fccf68e0812 -R 241b510f33643f46040372d1284a7d7b +P 1a7e711ed10860c7985e84b97cdfff748d940b9f +R b0011529707271df74d0fa481aad6091 U drh -Z b060cb081e36e4268c2624adfeebfbc8 +Z ab603a6740e2096395006a3b0e96e405 diff --git a/manifest.uuid b/manifest.uuid index 4cdb1c73e1..39c8a652a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1a7e711ed10860c7985e84b97cdfff748d940b9f \ No newline at end of file +23073a053931de324323f631a6613086786af411 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 358d508260..feccf2d11d 100644 --- a/src/where.c +++ b/src/where.c @@ -1651,7 +1651,7 @@ static void constructAutomaticIndex( ** if they go out of sync. */ extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); - mxBitCol = (pTable->nCol >= BMS-1) ? BMS-1 : pTable->nCol; + mxBitCol = MIN(BMS-1,pTable->nCol); testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); for(i=0; icolUsed & MASKBIT(BMS-1) ){ nKeyCol += pTable->nCol - BMS + 1; } - pLoop->wsFlags |= WHERE_COLUMN_EQ | WHERE_IDX_ONLY; /* Construct the Index object to describe this index */ pIdx = sqlite3AllocateIndexObject(pParse->db, nKeyCol+1, 0, &zNotUsed); From 635e57fc4893060c6c31efa3bcd936a9720212c7 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 16:19:18 +0000 Subject: [PATCH 497/710] Fix a faulty assert() in the DELETE code generator. FossilOrigin-Name: 95f8ebdbf87326f23cd38e561ac5632b5367a449 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/delete.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4d085dbcbc..a6054c1a60 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Trivial\ssimplification\sto\sthe\sautomatic\sindex\slogic. -D 2014-10-28T15:58:50.607 +C Fix\sa\sfaulty\sassert()\sin\sthe\sDELETE\scode\sgenerator. +D 2014-10-28T16:19:18.944 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 -F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f +F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1a7e711ed10860c7985e84b97cdfff748d940b9f -R b0011529707271df74d0fa481aad6091 +P 23073a053931de324323f631a6613086786af411 +R 5f4e1ea821e16c5044076b23f1aebe8a U drh -Z ab603a6740e2096395006a3b0e96e405 +Z 478016a63ef8d014d3b6d95929bf74df diff --git a/manifest.uuid b/manifest.uuid index 39c8a652a5..49ec9c5970 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -23073a053931de324323f631a6613086786af411 \ No newline at end of file +95f8ebdbf87326f23cd38e561ac5632b5367a449 \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index b97407400b..d81dd3f6b4 100644 --- a/src/delete.c +++ b/src/delete.c @@ -481,7 +481,7 @@ void sqlite3DeleteFrom( assert( nKey==nPk ); /* OP_Found will use an unpacked key */ assert( !IsVirtual(pTab) ); if( aToOpen[iDataCur-iTabCur] ){ - assert( pPk!=0 ); + assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); } From dd8c460081ef024abbb925029a18adb5deea3522 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Oct 2014 16:50:10 +0000 Subject: [PATCH 498/710] =?UTF-8?q?Fix=20a=20crash=20that=20could=20occur?= =?UTF-8?q?=20if=20the=20WHERE=20clause=20of=20an=20UPDATE=20statement=20o?= =?UTF-8?q?n=20a=20view=20that=20does=20not=20feature=20a=20column=20named?= =?UTF-8?q?=20"rowid"=20contains=20a=20term=20such=20as=20"rowid=3D=3F".?= FossilOrigin-Name: 8523670d50004f3112b7871f11c8b8b02aab96ab --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/update.c | 4 ++-- test/trigger9.test | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index a6054c1a60..8f4042bf5b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfaulty\sassert()\sin\sthe\sDELETE\scode\sgenerator. -D 2014-10-28T16:19:18.944 +C Fix\sa\scrash\sthat\scould\soccur\sif\sthe\sWHERE\sclause\sof\san\sUPDATE\sstatement\son\sa\sview\sthat\sdoes\snot\sfeature\sa\scolumn\snamed\s"rowid"\scontains\sa\sterm\ssuch\sas\s"rowid=?". +D 2014-10-28T16:50:10.527 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6de09362b657f19ba83e5fa521ee715787ce9fee F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f -F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 +F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a @@ -1051,7 +1051,7 @@ F test/trigger5.test 619391a3e9fc194081d22cefd830d811e7badf83 F test/trigger6.test 0e411654f122552da6590f0b4e6f781048a4a9b9 F test/trigger7.test b39e6dee1debe0ff9c2ef66326668f149f07c9c4 F test/trigger8.test 30cb0530bd7c4728055420e3f739aa00412eafa4 -F test/trigger9.test 5b0789f1c5c4600961f8e68511b825b87be53e31 +F test/trigger9.test 2226ec795a33b0460ab5cf8891e9054cc7edef41 F test/triggerA.test fe5597f47ee21bacb4936dc827994ed94161e332 F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test a68980c5955d62ee24be6f97129d824f199f9a4c @@ -1207,7 +1207,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 23073a053931de324323f631a6613086786af411 -R 5f4e1ea821e16c5044076b23f1aebe8a -U drh -Z 478016a63ef8d014d3b6d95929bf74df +P 95f8ebdbf87326f23cd38e561ac5632b5367a449 +R c4fade85ca4f95e680190bc9e417c1ff +U dan +Z 12d8cc683b13ea652b467efe210b5438 diff --git a/manifest.uuid b/manifest.uuid index 49ec9c5970..110e17e040 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95f8ebdbf87326f23cd38e561ac5632b5367a449 \ No newline at end of file +8523670d50004f3112b7871f11c8b8b02aab96ab \ No newline at end of file diff --git a/src/update.c b/src/update.c index f781a60ccd..3af4017f1b 100644 --- a/src/update.c +++ b/src/update.c @@ -431,8 +431,8 @@ void sqlite3Update( /* Top of the update loop */ if( okOnePass ){ - if( aToOpen[iDataCur-iBaseCur] ){ - assert( pPk!=0 ); + if( aToOpen[iDataCur-iBaseCur] && !isView ){ + assert( pPk ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); VdbeCoverageNeverTaken(v); } diff --git a/test/trigger9.test b/test/trigger9.test index f56c8acbc5..326fa63d4c 100644 --- a/test/trigger9.test +++ b/test/trigger9.test @@ -32,6 +32,7 @@ ifcapable {!trigger} { finish_test return } +set ::testprefix trigger9 proc has_rowdata {sql} { expr {[lsearch [execsql "explain $sql"] RowData]>=0} @@ -220,4 +221,36 @@ ifcapable compound { } {2} } +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(a, b); + CREATE TABLE log(x); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + CREATE VIEW v1 AS SELECT a, b FROM t1; + + CREATE TRIGGER tr1 INSTEAD OF DELETE ON v1 BEGIN + INSERT INTO log VALUES('delete'); + END; + + CREATE TRIGGER tr2 INSTEAD OF UPDATE ON v1 BEGIN + INSERT INTO log VALUES('update'); + END; + + CREATE TRIGGER tr3 INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO log VALUES('insert'); + END; +} + +do_execsql_test 4.2 { + DELETE FROM v1 WHERE rowid=1; +} {} + +do_execsql_test 4.3 { + UPDATE v1 SET a=b WHERE rowid=2; +} {} + + + + finish_test From c3da667b25cd513f3552652e783990734a200992 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Oct 2014 18:24:16 +0000 Subject: [PATCH 499/710] Modify the documentation for sqlite3_changes() to make it more testable. Add tests and minor fixes for the same. FossilOrigin-Name: 41cdd0c422d61533a94870cb5ad094682956d472 --- manifest | 19 +- manifest.uuid | 2 +- src/sqlite.h.in | 78 ++++---- src/vdbe.c | 1 + src/vdbeInt.h | 3 +- src/vdbeaux.c | 6 + test/e_changes.test | 441 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 499 insertions(+), 51 deletions(-) create mode 100644 test/e_changes.test diff --git a/manifest b/manifest index 8f4042bf5b..380d7ccd48 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scrash\sthat\scould\soccur\sif\sthe\sWHERE\sclause\sof\san\sUPDATE\sstatement\son\sa\sview\sthat\sdoes\snot\sfeature\sa\scolumn\snamed\s"rowid"\scontains\sa\sterm\ssuch\sas\s"rowid=?". -D 2014-10-28T16:50:10.527 +C Modify\sthe\sdocumentation\sfor\ssqlite3_changes()\sto\smake\sit\smore\stestable.\sAdd\stests\sand\sminor\sfixes\sfor\sthe\ssame. +D 2014-10-28T18:24:16.387 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in a9f2e5a0e2472c8c7819f3a16074c14b9376608f +F src/sqlite.h.in 1c5624f8b21cc8261a8b048033815581347b375f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 @@ -289,11 +289,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 5ee15a66ce07e0482b92aa29e4dd0c5827a22d79 +F src/vdbe.c 1b7e8ccaca2a23ae2804568f34b7c645adfd332d F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 +F src/vdbeInt.h acc36ac461f973f46ac7942f86abdd93d2f8cfbc F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 -F src/vdbeaux.c edbb7a9c8b2a8f7a68ac75c2475edd4040266b76 +F src/vdbeaux.c 3d6b2b412ef2193aa4729922dfc5df1efadbf6df F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -451,6 +451,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 +F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 @@ -1207,7 +1208,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 95f8ebdbf87326f23cd38e561ac5632b5367a449 -R c4fade85ca4f95e680190bc9e417c1ff +P 8523670d50004f3112b7871f11c8b8b02aab96ab +R 8192ca2d13d614d67e4c391f6685b170 U dan -Z 12d8cc683b13ea652b467efe210b5438 +Z 02a949e02b7399a00fe453126f032ba0 diff --git a/manifest.uuid b/manifest.uuid index 110e17e040..1803fe3792 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8523670d50004f3112b7871f11c8b8b02aab96ab \ No newline at end of file +41cdd0c422d61533a94870cb5ad094682956d472 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 8cfe61ee1a..bb5d90254d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1874,47 +1874,45 @@ sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); /* ** CAPI3REF: Count The Number Of Rows Modified ** -** ^This function returns the number of database rows that were changed -** or inserted or deleted by the most recently completed SQL statement -** on the [database connection] specified by the first parameter. -** ^(Only changes that are directly specified by the [INSERT], [UPDATE], -** or [DELETE] statement are counted. Auxiliary changes caused by -** triggers or [foreign key actions] are not counted.)^ Use the -** [sqlite3_total_changes()] function to find the total number of changes -** including changes caused by triggers and foreign key actions. +** ^This function returns the number of rows modified, inserted or +** deleted by the most recently completed INSERT, UPDATE or DELETE +** statement on the database connection specified by the only parameter. +** ^Executing any other type of SQL statement does not modify the value +** returned by this function. ** -** ^Changes to a view that are simulated by an [INSTEAD OF trigger] -** are not counted. Only real table changes are counted. +** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are +** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], +** [foreign key actions] or [REPLACE] constraint resolution are not counted. +** +** Changes to a view that are intercepted by +** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value +** returned by sqlite3_changes() immediately after an INSERT, UPDATE or +** DELETE statement run on a view is always zero. Only changes made to real +** tables are counted. ** -** ^(A "row change" is a change to a single row of a single table -** caused by an INSERT, DELETE, or UPDATE statement. Rows that -** are changed as side effects of [REPLACE] constraint resolution, -** rollback, ABORT processing, [DROP TABLE], or by any other -** mechanisms do not count as direct row changes.)^ -** -** A "trigger context" is a scope of execution that begins and -** ends with the script of a [CREATE TRIGGER | trigger]. -** Most SQL statements are -** evaluated outside of any trigger. This is the "top level" -** trigger context. If a trigger fires from the top level, a -** new trigger context is entered for the duration of that one -** trigger. Subtriggers create subcontexts for their duration. -** -** ^Calling [sqlite3_exec()] or [sqlite3_step()] recursively does -** not create a new trigger context. -** -** ^This function returns the number of direct row changes in the -** most recent INSERT, UPDATE, or DELETE statement within the same -** trigger context. -** -** ^Thus, when called from the top level, this function returns the -** number of changes in the most recent INSERT, UPDATE, or DELETE -** that also occurred at the top level. ^(Within the body of a trigger, -** the sqlite3_changes() interface can be called to find the number of -** changes in the most recently completed INSERT, UPDATE, or DELETE -** statement within the body of the same trigger. -** However, the number returned does not include changes -** caused by subtriggers since those have their own context.)^ +** Things are more complicated if the sqlite3_changes() function is +** executed while a trigger program is running. This may happen if the +** program uses the [changes() SQL function], or if some other callback +** function invokes sqlite3_changes() directly. Essentially: +** +**
      +**
    • ^(Before entering a trigger program the value returned by +** sqlite3_changes() function is saved. After the trigger program +** has finished, the original value is restored.)^ +** +**
    • ^(Within a trigger program each INSERT, UPDATE and DELETE +** statement sets the value returned by sqlite3_changes() +** upon completion as normal. Of course, this value will not include +** any changes performed by sub-triggers, as the sqlite3_changes() +** value will be saved and restored after each sub-trigger has run.)^ +**
    +** +** ^This means that if the changes() SQL function (or similar) is used +** by the first INSERT, UPDATE or DELETE statement within a trigger, it +** returns the value as set when the calling statement began executing. +** ^If it is used by the second or subsequent such statement within a trigger +** program, the value returned reflects the number of rows modified by the +** previous INSERT, UPDATE or DELETE statement within the same trigger. ** ** See also the [sqlite3_total_changes()] interface, the ** [count_changes pragma], and the [changes() SQL function]. @@ -1934,7 +1932,7 @@ int sqlite3_changes(sqlite3*); ** from all [CREATE TRIGGER | trigger] contexts and changes made by ** [foreign key actions]. However, ** the count does not include changes used to implement [REPLACE] constraints, -** do rollbacks or ABORT processing, or [DROP TABLE] processing. The +** rollbacks or [DROP TABLE] commands. The ** count does not include rows of views that fire an [INSTEAD OF trigger], ** though if the INSTEAD OF trigger makes changes of its own, those changes ** are counted.)^ diff --git a/src/vdbe.c b/src/vdbe.c index 0f9f45c456..d256c6b770 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5423,6 +5423,7 @@ case OP_Program: { /* jump */ pFrame->pParent = p->pFrame; pFrame->lastRowid = lastRowid; pFrame->nChange = p->nChange; + pFrame->nDbChange = p->db->nChange; p->nChange = 0; p->pFrame = pFrame; p->aMem = aMem = &VdbeFrameMem(pFrame)[-1]; diff --git a/src/vdbeInt.h b/src/vdbeInt.h index bb504d64a1..623c5fdde8 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -144,7 +144,8 @@ struct VdbeFrame { int nOnceFlag; /* Number of entries in aOnceFlag */ int nChildMem; /* Number of memory cells for child frame */ int nChildCsr; /* Number of cursors for child frame */ - int nChange; /* Statement changes (Vdbe.nChanges) */ + int nChange; /* Statement changes (Vdbe.nChange) */ + int nDbChange; /* Value of db->nChange */ }; #define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) diff --git a/src/vdbeaux.c b/src/vdbeaux.c index c0018bb71c..7dfb64130d 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1772,6 +1772,7 @@ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ v->nCursor = pFrame->nCursor; v->db->lastRowid = pFrame->lastRowid; v->nChange = pFrame->nChange; + v->db->nChange = pFrame->nDbChange; return pFrame->pc; } @@ -2339,6 +2340,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } } @@ -2379,6 +2381,7 @@ int sqlite3VdbeHalt(Vdbe *p){ }else if( rc!=SQLITE_OK ){ p->rc = rc; sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; }else{ db->nDeferredCons = 0; db->nDeferredImmCons = 0; @@ -2387,6 +2390,7 @@ int sqlite3VdbeHalt(Vdbe *p){ } }else{ sqlite3RollbackAll(db, SQLITE_OK); + p->nChange = 0; } db->nStatement = 0; }else if( eStatementOp==0 ){ @@ -2398,6 +2402,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } @@ -2418,6 +2423,7 @@ int sqlite3VdbeHalt(Vdbe *p){ sqlite3RollbackAll(db, SQLITE_ABORT_ROLLBACK); sqlite3CloseSavepoints(db); db->autoCommit = 1; + p->nChange = 0; } } diff --git a/test/e_changes.test b/test/e_changes.test new file mode 100644 index 0000000000..a77e22a2ee --- /dev/null +++ b/test/e_changes.test @@ -0,0 +1,441 @@ +# 2011 October 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_changes + +# Like [do_execsql_test], except it appends the value returned by +# [db changes] to the result of executing the SQL script. +# +proc do_changes_test {tn sql res} { + uplevel [list \ + do_test $tn "concat \[execsql {$sql}\] \[db changes\]" $res + ] +} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-15996-49369 This function returns the number of rows +# modified, inserted or deleted by the most recently completed INSERT, +# UPDATE or DELETE statement on the database connection specified by the +# only parameter. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(x, y, PRIMARY KEY(x, y)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(a); + CREATE INDEX i2 ON t2(y); +} +foreach {tn schema} { + 1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(b); + } + 2 { + CREATE TABLE t1(a, b, PRIMARY KEY(a, b)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } +} { + reset_db + execsql $schema + + # Insert 1 row. + do_changes_test 1.$tn.1 { INSERT INTO t1 VALUES(0, 0) } 1 + + # Insert 10 rows. + do_changes_test 1.$tn.2 { + WITH rows(i, j) AS ( + SELECT 1, 1 UNION ALL SELECT i+1, j+i FROM rows WHERE i<10 + ) + INSERT INTO t1 SELECT * FROM rows + } 10 + + # Modify 5 rows. + do_changes_test 1.$tn.3 { + UPDATE t1 SET b=b+1 WHERE a<5; + } 5 + + # Delete 4 rows + do_changes_test 1.$tn.4 { + DELETE FROM t1 WHERE a>6 + } 4 + + # Check the "on the database connecton specified" part of hte + # requirement - changes made by other connections do not show up in + # the return value of sqlite3_changes(). + do_test 1.$tn.5 { + sqlite3 db2 test.db + execsql { INSERT INTO t1 VALUES(-1, -1) } db2 + db2 changes + } 1 + do_test 1.$tn.6 { + db changes + } 4 + db2 close + + # Test that statements that modify no rows because they hit UNIQUE + # constraints set the sqlite3_changes() value to 0. Regardless of + # whether or not they are executed inside an explicit transaction. + # + # 1.$tn.8-9: outside of a transaction + # 1.$tn.10-12: inside a transaction + # + do_changes_test 1.$tn.7 { + CREATE UNIQUE INDEX i2 ON t1(a); + } 4 + do_catchsql_test 1.$tn.8 { + INSERT INTO t1 VALUES('a', 0), ('b', 0), ('c', 0), (0, 11); + } {1 {UNIQUE constraint failed: t1.a}} + do_test 1.$tn.9 { db changes } 0 + do_catchsql_test 1.$tn.10 { + BEGIN; + INSERT INTO t1 VALUES('a', 0), ('b', 0), ('c', 0), (0, 11); + } {1 {UNIQUE constraint failed: t1.a}} + do_test 1.$tn.11 { db changes } 0 + do_changes_test 1.$tn.12 COMMIT 0 + +} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-44877-05564 Executing any other type of SQL statement +# does not modify the value returned by this function. +# +reset_db +do_changes_test 2.1 { CREATE TABLE t1(x) } 0 +do_changes_test 2.2 { + WITH d(y) AS (SELECT 1 UNION ALL SELECT y+1 FROM d WHERE y<47) + INSERT INTO t1 SELECT y FROM d; +} 47 + +# The statement above set changes() to 47. Check that none of the following +# modify this. +do_changes_test 2.3 { SELECT count(x) FROM t1 } {47 47} +do_changes_test 2.4 { DROP TABLE t1 } 47 +do_changes_test 2.5 { CREATE TABLE t1(x) } 47 +do_changes_test 2.6 { ALTER TABLE t1 ADD COLUMN b } 47 + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-53938-27527 Only changes made directly by the INSERT, +# UPDATE or DELETE statement are considered - auxiliary changes caused +# by triggers, foreign key actions or REPLACE constraint resolution are +# not counted. +# +# 3.1.*: triggers +# 3.2.*: foreign key actions +# 3.3.*: replace constraints +# +reset_db +do_execsql_test 3.1.0 { + CREATE TABLE log(x); + CREATE TABLE p1(one PRIMARY KEY, two); + + CREATE TRIGGER tr_ai AFTER INSERT ON p1 BEGIN + INSERT INTO log VALUES('insert'); + END; + CREATE TRIGGER tr_bd BEFORE DELETE ON p1 BEGIN + INSERT INTO log VALUES('delete'); + END; + CREATE TRIGGER tr_au AFTER UPDATE ON p1 BEGIN + INSERT INTO log VALUES('update'); + END; + +} + +do_changes_test 3.1.1 { + INSERT INTO p1 VALUES('a', 'A'), ('b', 'B'), ('c', 'C'); +} 3 +do_changes_test 3.1.2 { + UPDATE p1 SET two = two||two; +} 3 +do_changes_test 3.1.3 { + DELETE FROM p1 WHERE one IN ('a', 'c'); +} 2 +do_execsql_test 3.1.4 { + -- None of the inserts on table log were counted. + SELECT count(*) FROM log +} 8 + +do_execsql_test 3.2.0 { + DELETE FROM p1; + INSERT INTO p1 VALUES('a', 'A'), ('b', 'B'), ('c', 'C'); + + CREATE TABLE c1(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET NULL); + CREATE TABLE c2(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET DEFAULT); + CREATE TABLE c3(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE CASCADE); + INSERT INTO c1 VALUES('a', 'aaa'); + INSERT INTO c2 VALUES('b', 'bbb'); + INSERT INTO c3 VALUES('c', 'ccc'); + + INSERT INTO p1 VALUES('d', 'D'), ('e', 'E'), ('f', 'F'); + CREATE TABLE c4(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET NULL); + CREATE TABLE c5(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET DEFAULT); + CREATE TABLE c6(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE CASCADE); + INSERT INTO c4 VALUES('d', 'aaa'); + INSERT INTO c5 VALUES('e', 'bbb'); + INSERT INTO c6 VALUES('f', 'ccc'); + + PRAGMA foreign_keys = ON; +} + +do_changes_test 3.2.1 { DELETE FROM p1 WHERE one = 'a' } 1 +do_changes_test 3.2.2 { DELETE FROM p1 WHERE one = 'b' } 1 +do_changes_test 3.2.3 { DELETE FROM p1 WHERE one = 'c' } 1 +do_execsql_test 3.2.4 { + SELECT * FROM c1; + SELECT * FROM c2; + SELECT * FROM c3; +} {{} aaa {} bbb} + +do_changes_test 3.2.5 { UPDATE p1 SET one = 'g' WHERE one = 'd' } 1 +do_changes_test 3.2.6 { UPDATE p1 SET one = 'h' WHERE one = 'e' } 1 +do_changes_test 3.2.7 { UPDATE p1 SET one = 'i' WHERE one = 'f' } 1 +do_execsql_test 3.2.8 { + SELECT * FROM c4; + SELECT * FROM c5; + SELECT * FROM c6; +} {{} aaa {} bbb i ccc} + +do_execsql_test 3.3.0 { + CREATE TABLE r1(a UNIQUE, b UNIQUE); + INSERT INTO r1 VALUES('i', 'i'); + INSERT INTO r1 VALUES('ii', 'ii'); + INSERT INTO r1 VALUES('iii', 'iii'); + INSERT INTO r1 VALUES('iv', 'iv'); + INSERT INTO r1 VALUES('v', 'v'); + INSERT INTO r1 VALUES('vi', 'vi'); + INSERT INTO r1 VALUES('vii', 'vii'); +} + +do_changes_test 3.3.1 { INSERT OR REPLACE INTO r1 VALUES('i', 1) } 1 +do_changes_test 3.3.2 { INSERT OR REPLACE INTO r1 VALUES('iv', 'v') } 1 +do_changes_test 3.3.3 { UPDATE OR REPLACE r1 SET b='v' WHERE a='iii' } 1 +do_changes_test 3.3.4 { UPDATE OR REPLACE r1 SET b='vi',a='vii' WHERE a='ii' } 1 +do_execsql_test 3.3.5 { + SELECT * FROM r1 ORDER BY a; +} {i 1 iii v vii vi} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-09813-48563 The value returned by sqlite3_changes() +# immediately after an INSERT, UPDATE or DELETE statement run on a view +# is always zero. +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE log(log); + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + + CREATE VIEW v1 AS SELECT * FROM t1; + CREATE TRIGGER v1_i INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO log VALUES('insert'); + END; + CREATE TRIGGER v1_u INSTEAD OF UPDATE ON v1 BEGIN + INSERT INTO log VALUES('update'), ('update'); + END; + CREATE TRIGGER v1_d INSTEAD OF DELETE ON v1 BEGIN + INSERT INTO log VALUES('delete'), ('delete'), ('delete'); + END; +} + +do_changes_test 4.2.1 { INSERT INTO t1 SELECT * FROM t1 } 3 +do_changes_test 4.2.2 { INSERT INTO v1 VALUES(1, 2) } 0 + +do_changes_test 4.3.1 { INSERT INTO t1 SELECT * FROM t1 } 6 +do_changes_test 4.3.2 { UPDATE v1 SET y='xyz' WHERE x=1 } 0 + +do_changes_test 4.4.1 { INSERT INTO t1 SELECT * FROM t1 } 12 +do_changes_test 4.4.2 { DELETE FROM v1 WHERE x=5 } 0 + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-32918-61474 Before entering a trigger program the value +# returned by sqlite3_changes() function is saved. After the trigger +# program has finished, the original value is restored. +# +reset_db +db func my_changes my_changes +set ::changes [list] +proc my_changes {x} { + set res [db changes] + lappend ::changes $x $res + return $res +} + +do_execsql_test 5.1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(x); + INSERT INTO t1 VALUES(1, NULL); + INSERT INTO t1 VALUES(2, NULL); + INSERT INTO t1 VALUES(3, NULL); + CREATE TRIGGER AFTER UPDATE ON t1 BEGIN + INSERT INTO t2 VALUES('a'), ('b'), ('c'); + SELECT my_changes('trigger'); + END; +} + +do_execsql_test 5.1.1 { + INSERT INTO t2 VALUES('a'), ('b'); + UPDATE t1 SET b = my_changes('update'); + SELECT * FROM t1; +} {1 2 2 2 3 2} + +# Value is being restored to "2" when the trigger program exits. +do_test 5.1.2 { + set ::changes +} {update 2 trigger 3 update 2 trigger 3 update 2 trigger 3} + + +reset_db +do_execsql_test 5.2.0 { + CREATE TABLE t1(a, b); + CREATE TABLE log(x); + INSERT INTO t1 VALUES(1, 0); + INSERT INTO t1 VALUES(2, 0); + INSERT INTO t1 VALUES(3, 0); + CREATE TRIGGER t1_a_u AFTER UPDATE ON t1 BEGIN + INSERT INTO log VALUES(old.b || ' -> ' || new.b || ' c = ' || changes() ); + END; + CREATE TABLE t2(a); + INSERT INTO t2 VALUES(1), (2), (3); + UPDATE t1 SET b = changes(); +} +do_execsql_test 5.2.1 { + SELECT * FROM t1; +} {1 3 2 3 3 3} +do_execsql_test 5.2.2 { + SELECT * FROM log; +} {{0 -> 3 c = 3} {0 -> 3 c = 3} {0 -> 3 c = 3}} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-17146-37073 Within a trigger program each INSERT, +# UPDATE and DELETE statement sets the value returned by +# sqlite3_changes() upon completion as normal. Of course, this value +# will not include any changes performed by sub-triggers, as the +# sqlite3_changes() value will be saved and restored after each +# sub-trigger has run. +reset_db +do_execsql_test 6.0 { + + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + CREATE TABLE t3(a, b); + CREATE TABLE log(x); + + CREATE TRIGGER t1_i BEFORE INSERT ON t1 BEGIN + INSERT INTO t2 VALUES(new.a, new.b), (new.a, new.b); + INSERT INTO log VALUES('t2->' || changes()); + END; + + CREATE TRIGGER t2_i AFTER INSERT ON t2 BEGIN + INSERT INTO t3 VALUES(new.a, new.b), (new.a, new.b), (new.a, new.b); + INSERT INTO log VALUES('t3->' || changes()); + END; + + CREATE TRIGGER t1_u AFTER UPDATE ON t1 BEGIN + UPDATE t2 SET b=new.b WHERE a=old.a; + INSERT INTO log VALUES('t2->' || changes()); + END; + + CREATE TRIGGER t2_u BEFORE UPDATE ON t2 BEGIN + UPDATE t3 SET b=new.b WHERE a=old.a; + INSERT INTO log VALUES('t3->' || changes()); + END; + + CREATE TRIGGER t1_d AFTER DELETE ON t1 BEGIN + DELETE FROM t2 WHERE a=old.a AND b=old.b; + INSERT INTO log VALUES('t2->' || changes()); + END; + + CREATE TRIGGER t2_d BEFORE DELETE ON t2 BEGIN + DELETE FROM t3 WHERE a=old.a AND b=old.b; + INSERT INTO log VALUES('t3->' || changes()); + END; +} + +do_changes_test 6.1 { + INSERT INTO t1 VALUES('+', 'o'); + SELECT * FROM log; +} {t3->3 t3->3 t2->2 1} + +do_changes_test 6.2 { + DELETE FROM log; + UPDATE t1 SET b='*'; + SELECT * FROM log; +} {t3->6 t3->6 t2->2 1} + +do_changes_test 6.3 { + DELETE FROM log; + DELETE FROM t1; + SELECT * FROM log; +} {t3->6 t3->0 t2->2 1} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-43399-09409 This means that if the changes() SQL +# function (or similar) is used by the first INSERT, UPDATE or DELETE +# statement within a trigger, it returns the value as set when the +# calling statement began executing. +# +# EVIDENCE-OF: R-53215-27584 If it is used by the second or subsequent +# such statement within a trigger program, the value returned reflects +# the number of rows modified by the previous INSERT, UPDATE or DELETE +# statement within the same trigger. +# +reset_db +do_execsql_test 7.1 { + CREATE TABLE q1(t); + CREATE TABLE q2(u, v); + CREATE TABLE q3(w); + + CREATE TRIGGER q2_insert BEFORE INSERT ON q2 BEGIN + + /* changes() returns value from previous I/U/D in callers context */ + INSERT INTO q1 VALUES('1:' || changes()); + + /* changes() returns value of previous I/U/D in this context */ + INSERT INTO q3 VALUES(changes()), (2), (3); + INSERT INTO q1 VALUES('2:' || changes()); + INSERT INTO q3 VALUES(changes() + 3), (changes()+4); + SELECT 'this does not affect things!'; + INSERT INTO q1 VALUES('3:' || changes()); + UPDATE q3 SET w = w+10 WHERE w%2; + INSERT INTO q1 VALUES('4:' || changes()); + DELETE FROM q3; + INSERT INTO q1 VALUES('5:' || changes()); + END; +} + +do_execsql_test 7.2 { + INSERT INTO q2 VALUES('x', 'y'); + SELECT * FROM q1; +} { + 1:0 2:3 3:2 4:3 5:5 +} + +do_execsql_test 7.3 { + DELETE FROM q1; + INSERT INTO q2 VALUES('x', 'y'); + SELECT * FROM q1; +} { + 1:5 2:3 3:2 4:3 5:5 +} + + + +finish_test From 4f41b7dec5a99cc72148cf6d0dd66b890cf82619 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 28 Oct 2014 20:35:18 +0000 Subject: [PATCH 500/710] Update the documentation on the sqlite3_randomness() interface to conform to enhancements associated with the SQLITE_ENABLE_API_ARMOR change. FossilOrigin-Name: 96e9917c350dfe2069b87860bbb961424ff1105a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 11 ++++++----- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 380d7ccd48..4c55bebcf9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sdocumentation\sfor\ssqlite3_changes()\sto\smake\sit\smore\stestable.\sAdd\stests\sand\sminor\sfixes\sfor\sthe\ssame. -D 2014-10-28T18:24:16.387 +C Update\sthe\sdocumentation\son\sthe\ssqlite3_randomness()\sinterface\sto\sconform\nto\senhancements\sassociated\swith\sthe\sSQLITE_ENABLE_API_ARMOR\schange. +D 2014-10-28T20:35:18.499 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 1c5624f8b21cc8261a8b048033815581347b375f +F src/sqlite.h.in 9c8090268644fba8d2b5b594b9d577b3da23905c F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 @@ -1208,7 +1208,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8523670d50004f3112b7871f11c8b8b02aab96ab -R 8192ca2d13d614d67e4c391f6685b170 -U dan -Z 02a949e02b7399a00fe453126f032ba0 +P 41cdd0c422d61533a94870cb5ad094682956d472 +R d8b973f48f438632851d26e03df40631 +U drh +Z d1783aa6143d26d5a0a1c09bffad4d99 diff --git a/manifest.uuid b/manifest.uuid index 1803fe3792..6cff793b96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -41cdd0c422d61533a94870cb5ad094682956d472 \ No newline at end of file +96e9917c350dfe2069b87860bbb961424ff1105a \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index bb5d90254d..20835794e8 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2417,13 +2417,14 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag); ** applications to access the same PRNG for other purposes. ** ** ^A call to this routine stores N bytes of randomness into buffer P. -** ^If N is less than one, then P can be a NULL pointer. +** ^The P parameter can be a NULL pointer. ** ** ^If this routine has not been previously called or if the previous -** call had N less than one, then the PRNG is seeded using randomness -** obtained from the xRandomness method of the default [sqlite3_vfs] object. -** ^If the previous call to this routine had an N of 1 or more then -** the pseudo-randomness is generated +** call had N less than one or a NULL pointer for P, then the PRNG is +** seeded using randomness obtained from the xRandomness method of +** the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more and a +** non-NULL P then the pseudo-randomness is generated ** internally and without recourse to the [sqlite3_vfs] xRandomness ** method. */ From aa55563d7eed0ba2e6ab0afd5625b00e30931b1d Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 28 Oct 2014 20:49:59 +0000 Subject: [PATCH 501/710] Add new test file e_totalchanges.test, containing tests of the sqlite3_total_changes() interface. FossilOrigin-Name: f84af4adcc34d7a4c72027bf5b038a1a45a4c307 --- manifest | 15 +-- manifest.uuid | 2 +- src/sqlite.h.in | 25 ++--- test/e_totalchanges.test | 213 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 233 insertions(+), 22 deletions(-) create mode 100644 test/e_totalchanges.test diff --git a/manifest b/manifest index 4c55bebcf9..cbbdd001c6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sdocumentation\son\sthe\ssqlite3_randomness()\sinterface\sto\sconform\nto\senhancements\sassociated\swith\sthe\sSQLITE_ENABLE_API_ARMOR\schange. -D 2014-10-28T20:35:18.499 +C Add\snew\stest\sfile\se_totalchanges.test,\scontaining\stests\sof\sthe\ssqlite3_total_changes()\sinterface. +D 2014-10-28T20:49:59.213 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 9c8090268644fba8d2b5b594b9d577b3da23905c +F src/sqlite.h.in 737b7dd0f3f81fe183646d22828b39f85ef3e68c F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 @@ -464,6 +464,7 @@ F test/e_reindex.test 396b7b4f0a66863b4e95116a67d93b227193e589 F test/e_resolve.test dcce9308fb13b934ce29591105d031d3e14fbba6 F test/e_select.test 52692ff3849541e828ad4661fe3773a9b8711763 F test/e_select2.test aceb80ab927d46fba5ce7586ebabf23e2bb0604f +F test/e_totalchanges.test b12ee5809d3e63aeb83238dd501a7bca7fd72c10 F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 @@ -1208,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 41cdd0c422d61533a94870cb5ad094682956d472 -R d8b973f48f438632851d26e03df40631 -U drh -Z d1783aa6143d26d5a0a1c09bffad4d99 +P 96e9917c350dfe2069b87860bbb961424ff1105a +R ff91cb59544b367ee28dfb499ecb1ca4 +U dan +Z ba0579c9ba051395a01d0b2f40d05365 diff --git a/manifest.uuid b/manifest.uuid index 6cff793b96..6aa58547c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -96e9917c350dfe2069b87860bbb961424ff1105a \ No newline at end of file +f84af4adcc34d7a4c72027bf5b038a1a45a4c307 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 20835794e8..b4081f2a02 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1926,20 +1926,17 @@ int sqlite3_changes(sqlite3*); /* ** CAPI3REF: Total Number Of Rows Modified ** -** ^This function returns the number of row changes caused by [INSERT], -** [UPDATE] or [DELETE] statements since the [database connection] was opened. -** ^(The count returned by sqlite3_total_changes() includes all changes -** from all [CREATE TRIGGER | trigger] contexts and changes made by -** [foreign key actions]. However, -** the count does not include changes used to implement [REPLACE] constraints, -** rollbacks or [DROP TABLE] commands. The -** count does not include rows of views that fire an [INSTEAD OF trigger], -** though if the INSTEAD OF trigger makes changes of its own, those changes -** are counted.)^ -** ^The sqlite3_total_changes() function counts the changes as soon as -** the statement that makes them is completed (when the statement handle -** is passed to [sqlite3_reset()] or [sqlite3_finalize()]). -** +** ^This function returns the total number of rows inserted, modified or +** deleted by all [INSERT], [UPDATE] or [DELETE] statements completed +** since the database connection was opened, including those executed as +** part of trigger programs. ^Executing any other type of SQL statement +** does not affect the value returned by sqlite3_total_changes(). +** +** ^Changes made as part of [foreign key actions] are included in the +** count, but those made as part of REPLACE constraint resolution are +** not. ^Changes to a view that are intercepted by INSTEAD OF triggers +** are not counted. +** ** See also the [sqlite3_changes()] interface, the ** [count_changes pragma], and the [total_changes() SQL function]. ** diff --git a/test/e_totalchanges.test b/test/e_totalchanges.test new file mode 100644 index 0000000000..ee163c914f --- /dev/null +++ b/test/e_totalchanges.test @@ -0,0 +1,213 @@ +# 2011 May 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_totalchanges + +# Like [do_execsql_test], except it appends the value returned by +# [db total_changes] to the result of executing the SQL script. +# +proc do_tc_test {tn sql res} { + uplevel [list \ + do_test $tn "concat \[execsql {$sql}\] \[db total_changes\]" $res + ] +} + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE INDEX t1_b ON t1(b); + CREATE TABLE t2(x, y, PRIMARY KEY(x, y)) WITHOUT ROWID; + CREATE INDEX t2_y ON t2(y); +} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-65438-26258 This function returns the total number of +# rows inserted, modified or deleted by all INSERT, UPDATE or DELETE +# statements completed since the database connection was opened, +# including those executed as part of trigger programs. +# +# 1.1.*: different types of I/U/D statements, +# 1.2.*: trigger programs. +# +do_tc_test 1.1.1 { + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + UPDATE t1 SET a = a+1; + DELETE FROM t1; +} {6} +do_tc_test 1.1.2 { + DELETE FROM t1 +} {6} + +do_tc_test 1.1.3 { + WITH data(a,b) AS ( + SELECT 0, 0 UNION ALL SELECT a+1, b+1 FROM data WHERE a<99 + ) + INSERT INTO t1 SELECT * FROM data; +} {106} + +do_tc_test 1.1.4 { + INSERT INTO t2 SELECT * FROM t1 WHERE a<50; + UPDATE t2 SET y=y+1; +} {206} + +do_tc_test 1.1.5 { + DELETE FROM t2 WHERE y<=25 +} {231} + +do_execsql_test 1.2.1 { + DELETE FROM t1; + DELETE FROM t2; +} +sqlite3 db test.db ; # To reset total_changes +do_tc_test 1.2.2 { + CREATE TABLE log(detail); + CREATE TRIGGER t1_after_insert AFTER INSERT ON t1 BEGIN + INSERT INTO log VALUES('inserted into t1'); + END; + + CREATE TRIGGER t1_before_delete BEFORE DELETE ON t1 BEGIN + INSERT INTO log VALUES('deleting from t1'); + INSERT INTO log VALUES('here we go!'); + END; + + CREATE TRIGGER t1_after_update AFTER UPDATE ON t1 BEGIN + INSERT INTO log VALUES('update'); + DELETE FROM log; + END; + + INSERT INTO t1 VALUES('a', 'b'); -- 1 + 1 + UPDATE t1 SET b='c'; -- 1 + 1 + 2 + DELETE FROM t1; -- 1 + 1 + 1 +} {9} + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-61766-15253 Executing any other type of SQL statement +# does not affect the value returned by sqlite3_total_changes(). +do_tc_test 2.1 { + INSERT INTO t1 VALUES(1, 2), (3, 4); + INSERT INTO t2 VALUES(1, 2), (3, 4); +} {15} +do_tc_test 2.2 { + SELECT count(*) FROM t1; +} {2 15} +do_tc_test 2.3 { + CREATE TABLE t4(a, b); + ALTER TABLE t4 ADD COLUMN c; + CREATE INDEX i4 ON t4(c); + ALTER TABLE t4 RENAME TO t5; + ANALYZE; + BEGIN; + DROP TABLE t2; + ROLLBACK; + VACUUM; +} {15} + + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-36043-10590 Changes made as part of foreign key +# actions are included in the count, but those made as part of REPLACE +# constraint resolution are not. +# +# 3.1.*: foreign key actions +# 3.2.*: REPLACE constraints. +# +sqlite3 db test.db ; # To reset total_changes +do_tc_test 3.1.1 { + CREATE TABLE p1(c PRIMARY KEY, d); + CREATE TABLE c1(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET NULL); + CREATE TABLE c2(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE CASCADE); + CREATE TABLE c3(a, b, FOREIGN KEY(a) REFERENCES p1 ON DELETE SET DEFAULT); + + INSERT INTO p1 VALUES(1, 'one'); + INSERT INTO p1 VALUES(2, 'two'); + INSERT INTO p1 VALUES(3, 'three'); + INSERT INTO p1 VALUES(4, 'four'); + + INSERT INTO c1 VALUES(1, 'i'); + INSERT INTO c2 VALUES(2, 'ii'); + INSERT INTO c3 VALUES(3, 'iii'); + PRAGMA foreign_keys = ON; +} {7} + +do_tc_test 3.1.2 { DELETE FROM p1 WHERE c=1; } {9} +do_tc_test 3.1.3 { DELETE FROM p1 WHERE c=2; } {11} +do_tc_test 3.1.4 { DELETE FROM p1 WHERE c=3; } {13} +do_tc_test 3.1.5 { DELETE FROM p1 WHERE c=4; } {14} ; # only 1 this time. + +sqlite3 db test.db ; # To reset total_changes +do_tc_test 3.1.6 { + DROP TABLE c1; + DROP TABLE c2; + DROP TABLE c3; + CREATE TABLE c1(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET NULL); + CREATE TABLE c2(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE CASCADE); + CREATE TABLE c3(a, b, FOREIGN KEY(a) REFERENCES p1 ON UPDATE SET DEFAULT); + + INSERT INTO p1 VALUES(1, 'one'); + INSERT INTO p1 VALUES(2, 'two'); + INSERT INTO p1 VALUES(3, 'three'); + INSERT INTO p1 VALUES(4, 'four'); + + INSERT INTO c1 VALUES(1, 'i'); + INSERT INTO c2 VALUES(2, 'ii'); + INSERT INTO c3 VALUES(3, 'iii'); + PRAGMA foreign_keys = ON; +} {7} + +do_tc_test 3.1.7 { UPDATE p1 SET c=c+4 WHERE c=1; } {9} +do_tc_test 3.1.8 { UPDATE p1 SET c=c+4 WHERE c=2; } {11} +do_tc_test 3.1.9 { UPDATE p1 SET c=c+4 WHERE c=3; } {13} +do_tc_test 3.1.10 { UPDATE p1 SET c=c+4 WHERE c=4; } {14} ; # only 1 this time. + +sqlite3 db test.db ; # To reset total_changes +do_tc_test 3.2.1 { + CREATE TABLE t3(a UNIQUE, b UNIQUE); + INSERT INTO t3 VALUES('one', 'one'); + INSERT INTO t3 VALUES('two', 'two'); + INSERT OR REPLACE INTO t3 VALUES('one', 'two'); +} {3} + +do_tc_test 3.2.2 { + INSERT INTO t3 VALUES('three', 'one'); + UPDATE OR REPLACE t3 SET b='two' WHERE b='one'; + SELECT * FROM t3; +} {three two 5} + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-54872-08741 Changes to a view that are intercepted by +# INSTEAD OF triggers are not counted. +# +sqlite3 db test.db ; # To reset total_changes +do_tc_test 4.1 { + CREATE TABLE t6(x); + CREATE VIEW v1 AS SELECT * FROM t6; + CREATE TRIGGER v1_tr1 INSTEAD OF INSERT ON v1 BEGIN + SELECT 'no-op'; + END; + + INSERT INTO v1 VALUES('a'); + INSERT INTO v1 VALUES('b'); +} {0} +do_tc_test 4.2 { + CREATE TRIGGER v1_tr2 INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO t6 VALUES(new.x); + END; + + INSERT INTO v1 VALUES('c'); + INSERT INTO v1 VALUES('d'); +} {2} + + +finish_test From 22e8d833f06835a9ea2cdf7e7d2dd55f79102ce0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 00:58:38 +0000 Subject: [PATCH 502/710] In the OP_Column opcode, when extracting a field that is past the end of a short record (because the row was originally inserted prior to ALTER TABLE ADD COLUMN) then make sure the output register is fully NULL and does not contain leftover flags (such as MEM_Ephem) from its previous use. Fix for ticket [43107840f1c02]. FossilOrigin-Name: 24780f8ddc1683fc62180e6961dc6bfe1168f4df --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- test/update.test | 14 ++++++++++++++ 4 files changed, 24 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index cbbdd001c6..209baf8beb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\se_totalchanges.test,\scontaining\stests\sof\sthe\ssqlite3_total_changes()\sinterface. -D 2014-10-28T20:49:59.213 +C In\sthe\sOP_Column\sopcode,\swhen\sextracting\sa\sfield\sthat\sis\spast\sthe\send\sof\na\sshort\srecord\s(because\sthe\srow\swas\soriginally\sinserted\sprior\sto\sALTER\sTABLE\nADD\sCOLUMN)\sthen\smake\ssure\sthe\soutput\sregister\sis\sfully\sNULL\sand\sdoes\snot\ncontain\sleftover\sflags\s(such\sas\sMEM_Ephem)\sfrom\sits\sprevious\suse.\nFix\sfor\sticket\s[43107840f1c02]. +D 2014-10-29T00:58:38.898 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 1b7e8ccaca2a23ae2804568f34b7c645adfd332d +F src/vdbe.c 49e659bc165e99b28492004b440e22146dc898ab F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h acc36ac461f973f46ac7942f86abdd93d2f8cfbc F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 @@ -1067,7 +1067,7 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 -F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb +F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 96e9917c350dfe2069b87860bbb961424ff1105a -R ff91cb59544b367ee28dfb499ecb1ca4 -U dan -Z ba0579c9ba051395a01d0b2f40d05365 +P f84af4adcc34d7a4c72027bf5b038a1a45a4c307 +R c2870e402bb1970025fb678b8c13d710 +U drh +Z 3a55744e14b55dcc32cb71f070af972c diff --git a/manifest.uuid b/manifest.uuid index 6aa58547c5..ea9bca8740 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f84af4adcc34d7a4c72027bf5b038a1a45a4c307 \ No newline at end of file +24780f8ddc1683fc62180e6961dc6bfe1168f4df \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index d256c6b770..c17bfdfe14 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2441,7 +2441,7 @@ case OP_Column: { if( pOp->p4type==P4_MEM ){ sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); } goto op_column_out; } diff --git a/test/update.test b/test/update.test index e67b0efddc..d7baf6e702 100644 --- a/test/update.test +++ b/test/update.test @@ -604,5 +604,19 @@ do_test update-14.4 { } ;# ifcapable {trigger} +# Ticket [https://www.sqlite.org/src/tktview/43107840f1c02] on 2014-10-29 +# An assertion fault on UPDATE +# +do_execsql_test update-15.1 { + CREATE TABLE t15(a INTEGER PRIMARY KEY, b); + INSERT INTO t15(a,b) VALUES(10,'abc'),(20,'def'),(30,'ghi'); + ALTER TABLE t15 ADD COLUMN c; + CREATE INDEX t15c ON t15(c); + INSERT INTO t15(a,b) + VALUES(5,'zyx'),(15,'wvu'),(25,'tsr'),(35,'qpo'); + UPDATE t15 SET c=printf("y%d",a) WHERE c IS NULL; + SELECT a,b,c,'|' FROM t15 ORDER BY a; +} {5 zyx y5 | 10 abc y10 | 15 wvu y15 | 20 def y20 | 25 tsr y25 | 30 ghi y30 | 35 qpo y35 |} + finish_test From 62ecc28caf3c45769fe4a7f8aa9e631a2280cf1d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 01:07:21 +0000 Subject: [PATCH 503/710] Call fsync() right after ftruncate() when in journal_mode=TRUNCATE and when synchronous=FULL in order to ensure that transactions are durable across a power loss that happens moments after the commit. Proposed fix for [https://bugzilla.mozilla.org/show_bug.cgi?id=1072773]. This is a cherrypick of [3e922208b68563489]. FossilOrigin-Name: a8f9bd1e890434fcffa02fcd2baf8b0fb5d4c9dd --- manifest | 19 ++++++++++--------- manifest.uuid | 2 +- src/pager.c | 8 ++++++++ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index e210943d15..a156aa4f0e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.7 -D 2014-10-17T11:24:17.839 +C Call\sfsync()\sright\safter\sftruncate()\swhen\sin\sjournal_mode=TRUNCATE\sand\nwhen\ssynchronous=FULL\sin\sorder\sto\sensure\sthat\stransactions\sare\sdurable\nacross\sa\spower\sloss\sthat\shappens\smoments\safter\sthe\scommit.\s\sProposed\nfix\sfor\s[https://bugzilla.mozilla.org/show_bug.cgi?id=1072773].\nThis\sis\sa\scherrypick\sof\s[3e922208b68563489]. +D 2014-10-29T01:07:21.172 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -215,7 +215,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c a171cf9dd09c6cb162b262c328d4dfd198e04f80 +F src/pager.c a98547ad9b1b5dbbc5e7d1c520be041b5d2c0926 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a @@ -1204,10 +1204,11 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 06c576c152c4013080c255cbbeb45bf2e298be9f -R 76bb09f8a8f8bdebeb708f5bb39aa31b -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.7 * +P e4ab094f8afce0817f4074e823fabe59fc29ebb4 +Q +3e922208b68563489c7766abb9afb4885113e7b8 +R dac8a0df55bcef8314fa6cd2a33fc123 +T *branch * branch-3.8.7 +T *sym-branch-3.8.7 * +T -sym-trunk * U drh -Z b2f516d1147acb0e1bf1c700327ee52e +Z 1e440e6f18a01c290161fa6c4a1f516b diff --git a/manifest.uuid b/manifest.uuid index f4f611fe44..bf1f77f03a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4ab094f8afce0817f4074e823fabe59fc29ebb4 \ No newline at end of file +a8f9bd1e890434fcffa02fcd2baf8b0fb5d4c9dd \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index d3a36ef484..d840a39a15 100644 --- a/src/pager.c +++ b/src/pager.c @@ -1941,6 +1941,14 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ rc = SQLITE_OK; }else{ rc = sqlite3OsTruncate(pPager->jfd, 0); + if( rc==SQLITE_OK && pPager->fullSync ){ + /* Make sure the new file size is written into the inode right away. + ** Otherwise the journal might resurrect following a power loss and + ** cause the last transaction to roll back. See + ** https://bugzilla.mozilla.org/show_bug.cgi?id=1072773 + */ + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags); + } } pPager->journalOff = 0; }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST From 739383d25dc714bf968a67e6f5aae52cc0033e2e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 01:13:58 +0000 Subject: [PATCH 504/710] In the OP_Column opcode, when extracting a field that is past the end of a short record (because the row was originally inserted prior to ALTER TABLE ADD COLUMN) then make sure the output register is fully NULL and does not contain leftover flags (such as MEM_Ephem) from its previous use. Fix for ticket [43107840f1c02]. This is a cherrypick of check-in [24780f8ddc1683fc]. FossilOrigin-Name: 304ea6ba6f4cf40a76d32d37af73a253f493ba47 --- manifest | 19 ++++++++----------- manifest.uuid | 2 +- src/vdbe.c | 2 +- test/update.test | 14 ++++++++++++++ 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index a156aa4f0e..d38979f110 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Call\sfsync()\sright\safter\sftruncate()\swhen\sin\sjournal_mode=TRUNCATE\sand\nwhen\ssynchronous=FULL\sin\sorder\sto\sensure\sthat\stransactions\sare\sdurable\nacross\sa\spower\sloss\sthat\shappens\smoments\safter\sthe\scommit.\s\sProposed\nfix\sfor\s[https://bugzilla.mozilla.org/show_bug.cgi?id=1072773].\nThis\sis\sa\scherrypick\sof\s[3e922208b68563489]. -D 2014-10-29T01:07:21.172 +C In\sthe\sOP_Column\sopcode,\swhen\sextracting\sa\sfield\sthat\sis\spast\sthe\send\sof\na\sshort\srecord\s(because\sthe\srow\swas\soriginally\sinserted\sprior\sto\sALTER\sTABLE\nADD\sCOLUMN)\sthen\smake\ssure\sthe\soutput\sregister\sis\sfully\sNULL\sand\sdoes\snot\ncontain\sleftover\sflags\s(such\sas\sMEM_Ephem)\sfrom\sits\sprevious\suse.\nFix\sfor\sticket\s[43107840f1c02].\s\sThis\sis\sa\scherrypick\sof\ncheck-in\s[24780f8ddc1683fc]. +D 2014-10-29T01:13:58.092 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 5ee15a66ce07e0482b92aa29e4dd0c5827a22d79 +F src/vdbe.c 5a1afb571853ddb911d698ac996bc4fd8ddf1eed F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -1062,7 +1062,7 @@ F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264 F test/unique2.test 41e7f83c6827605991160a31380148a9fc5f1339 F test/unixexcl.test cd6c765f75e50e8e2c2ba763149e5d340ea19825 F test/unordered.test ca7adce0419e4ca0c50f039885e76ed2c531eda8 -F test/update.test 1b6c488a8f993d090b7ee9ad0e234faa161b3aeb +F test/update.test 6c68446b8a0a33d522a7c72b320934596a2d7d32 F test/uri.test 23662b7b61958b0f0e47082de7d06341ccf85d5b F test/userauth01.test e740a2697a7b40d7c5003a7d7edaee16acd349a9 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae @@ -1204,11 +1204,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e4ab094f8afce0817f4074e823fabe59fc29ebb4 -Q +3e922208b68563489c7766abb9afb4885113e7b8 -R dac8a0df55bcef8314fa6cd2a33fc123 -T *branch * branch-3.8.7 -T *sym-branch-3.8.7 * -T -sym-trunk * +P a8f9bd1e890434fcffa02fcd2baf8b0fb5d4c9dd +Q +24780f8ddc1683fc62180e6961dc6bfe1168f4df +R d684ab06f9952cfb8bd4bb795d206cf9 U drh -Z 1e440e6f18a01c290161fa6c4a1f516b +Z 6a68e4e50056aa43b6faddb91db24dac diff --git a/manifest.uuid b/manifest.uuid index bf1f77f03a..e62a923d4d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a8f9bd1e890434fcffa02fcd2baf8b0fb5d4c9dd \ No newline at end of file +304ea6ba6f4cf40a76d32d37af73a253f493ba47 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 0f9f45c456..88fadb023e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2441,7 +2441,7 @@ case OP_Column: { if( pOp->p4type==P4_MEM ){ sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); } goto op_column_out; } diff --git a/test/update.test b/test/update.test index e67b0efddc..d7baf6e702 100644 --- a/test/update.test +++ b/test/update.test @@ -604,5 +604,19 @@ do_test update-14.4 { } ;# ifcapable {trigger} +# Ticket [https://www.sqlite.org/src/tktview/43107840f1c02] on 2014-10-29 +# An assertion fault on UPDATE +# +do_execsql_test update-15.1 { + CREATE TABLE t15(a INTEGER PRIMARY KEY, b); + INSERT INTO t15(a,b) VALUES(10,'abc'),(20,'def'),(30,'ghi'); + ALTER TABLE t15 ADD COLUMN c; + CREATE INDEX t15c ON t15(c); + INSERT INTO t15(a,b) + VALUES(5,'zyx'),(15,'wvu'),(25,'tsr'),(35,'qpo'); + UPDATE t15 SET c=printf("y%d",a) WHERE c IS NULL; + SELECT a,b,c,'|' FROM t15 ORDER BY a; +} {5 zyx y5 | 10 abc y10 | 15 wvu y15 | 20 def y20 | 25 tsr y25 | 30 ghi y30 | 35 qpo y35 |} + finish_test From bba8206be97e8b09ee9c135e94c55db10d4d19c0 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 01:18:03 +0000 Subject: [PATCH 505/710] Disable the use of strchrnul() unless specifically enabled by compile-time options. Cherrypick of check-in [e580470db77d6da9] FossilOrigin-Name: 837368adfe859c41b347d3124d5b3fdf790eec03 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/printf.c | 6 +----- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index d38979f110..f5c3788a32 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sOP_Column\sopcode,\swhen\sextracting\sa\sfield\sthat\sis\spast\sthe\send\sof\na\sshort\srecord\s(because\sthe\srow\swas\soriginally\sinserted\sprior\sto\sALTER\sTABLE\nADD\sCOLUMN)\sthen\smake\ssure\sthe\soutput\sregister\sis\sfully\sNULL\sand\sdoes\snot\ncontain\sleftover\sflags\s(such\sas\sMEM_Ephem)\sfrom\sits\sprevious\suse.\nFix\sfor\sticket\s[43107840f1c02].\s\sThis\sis\sa\scherrypick\sof\ncheck-in\s[24780f8ddc1683fc]. -D 2014-10-29T01:13:58.092 +C Disable\sthe\suse\sof\sstrchrnul()\sunless\sspecifically\senabled\sby\scompile-time\noptions.\s\sCherrypick\sof\scheck-in\s[e580470db77d6da9] +D 2014-10-29T01:18:03.185 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 6b79bbd063dcbadca4cf617a4cde255bcc13ea64 +F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -1204,8 +1204,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a8f9bd1e890434fcffa02fcd2baf8b0fb5d4c9dd -Q +24780f8ddc1683fc62180e6961dc6bfe1168f4df -R d684ab06f9952cfb8bd4bb795d206cf9 +P 304ea6ba6f4cf40a76d32d37af73a253f493ba47 +Q +e580470db77d6da970c755102790e603fb26b3c6 +R 87a2c5d1dcd7231208aa0589a51fed61 U drh -Z 6a68e4e50056aa43b6faddb91db24dac +Z b2af8bf5eda5ca802506e90d5ed9925a diff --git a/manifest.uuid b/manifest.uuid index e62a923d4d..3e0b305a64 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -304ea6ba6f4cf40a76d32d37af73a253f493ba47 \ No newline at end of file +837368adfe859c41b347d3124d5b3fdf790eec03 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index c0b3c70f6b..1df287fbb6 100644 --- a/src/printf.c +++ b/src/printf.c @@ -21,11 +21,7 @@ ** the glibc version so the glibc version is definitely preferred. */ #if !defined(HAVE_STRCHRNUL) -# if defined(linux) -# define HAVE_STRCHRNUL 1 -# else -# define HAVE_STRCHRNUL 0 -# endif +# define HAVE_STRCHRNUL 0 #endif From a26b9a77275e5ef76e2ba5ac0c480e70f5abeda4 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 01:26:25 +0000 Subject: [PATCH 506/710] Fix problems with running UPDATE and DELETE against a VIEW and referencing the rowid in the WHERE clause. This is a cherrypick of [95f8ebdbf87326f2] and [8523670d50004f3]. FossilOrigin-Name: cc33e846c8509419f0a1fbfb286807b4c137788d --- manifest | 19 ++++++++++--------- manifest.uuid | 2 +- src/delete.c | 2 +- src/update.c | 4 ++-- test/trigger9.test | 33 +++++++++++++++++++++++++++++++++ 5 files changed, 47 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index f5c3788a32..be89f4c86a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\suse\sof\sstrchrnul()\sunless\sspecifically\senabled\sby\scompile-time\noptions.\s\sCherrypick\sof\scheck-in\s[e580470db77d6da9] -D 2014-10-29T01:18:03.185 +C Fix\sproblems\swith\srunning\sUPDATE\sand\sDELETE\sagainst\sa\sVIEW\sand\sreferencing\nthe\srowid\sin\sthe\sWHERE\sclause.\s\sThis\sis\sa\scherrypick\sof\s[95f8ebdbf87326f2]\nand\s[8523670d50004f3]. +D 2014-10-29T01:26:25.633 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -180,7 +180,7 @@ F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 -F src/delete.c fae81cc2eb14b75267d4f47d3cfc9ae02aae726f +F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 @@ -285,7 +285,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6de09362b657f19ba83e5fa521ee715787ce9fee F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f -F src/update.c 729f6f18fc27740591d085e1172cebe311144bf0 +F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a @@ -1048,7 +1048,7 @@ F test/trigger5.test 619391a3e9fc194081d22cefd830d811e7badf83 F test/trigger6.test 0e411654f122552da6590f0b4e6f781048a4a9b9 F test/trigger7.test b39e6dee1debe0ff9c2ef66326668f149f07c9c4 F test/trigger8.test 30cb0530bd7c4728055420e3f739aa00412eafa4 -F test/trigger9.test 5b0789f1c5c4600961f8e68511b825b87be53e31 +F test/trigger9.test 2226ec795a33b0460ab5cf8891e9054cc7edef41 F test/triggerA.test fe5597f47ee21bacb4936dc827994ed94161e332 F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test a68980c5955d62ee24be6f97129d824f199f9a4c @@ -1204,8 +1204,9 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 304ea6ba6f4cf40a76d32d37af73a253f493ba47 -Q +e580470db77d6da970c755102790e603fb26b3c6 -R 87a2c5d1dcd7231208aa0589a51fed61 +P 837368adfe859c41b347d3124d5b3fdf790eec03 +Q +8523670d50004f3112b7871f11c8b8b02aab96ab +Q +95f8ebdbf87326f23cd38e561ac5632b5367a449 +R b3ee8c71d158066739429e7ae8387325 U drh -Z b2af8bf5eda5ca802506e90d5ed9925a +Z 624aac6d2575ef133925a49cc2066eb1 diff --git a/manifest.uuid b/manifest.uuid index 3e0b305a64..99e8b826f5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -837368adfe859c41b347d3124d5b3fdf790eec03 \ No newline at end of file +cc33e846c8509419f0a1fbfb286807b4c137788d \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index b97407400b..d81dd3f6b4 100644 --- a/src/delete.c +++ b/src/delete.c @@ -481,7 +481,7 @@ void sqlite3DeleteFrom( assert( nKey==nPk ); /* OP_Found will use an unpacked key */ assert( !IsVirtual(pTab) ); if( aToOpen[iDataCur-iTabCur] ){ - assert( pPk!=0 ); + assert( pPk!=0 || pTab->pSelect!=0 ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); } diff --git a/src/update.c b/src/update.c index f781a60ccd..3af4017f1b 100644 --- a/src/update.c +++ b/src/update.c @@ -431,8 +431,8 @@ void sqlite3Update( /* Top of the update loop */ if( okOnePass ){ - if( aToOpen[iDataCur-iBaseCur] ){ - assert( pPk!=0 ); + if( aToOpen[iDataCur-iBaseCur] && !isView ){ + assert( pPk ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); VdbeCoverageNeverTaken(v); } diff --git a/test/trigger9.test b/test/trigger9.test index f56c8acbc5..326fa63d4c 100644 --- a/test/trigger9.test +++ b/test/trigger9.test @@ -32,6 +32,7 @@ ifcapable {!trigger} { finish_test return } +set ::testprefix trigger9 proc has_rowdata {sql} { expr {[lsearch [execsql "explain $sql"] RowData]>=0} @@ -220,4 +221,36 @@ ifcapable compound { } {2} } +reset_db +do_execsql_test 4.1 { + CREATE TABLE t1(a, b); + CREATE TABLE log(x); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + CREATE VIEW v1 AS SELECT a, b FROM t1; + + CREATE TRIGGER tr1 INSTEAD OF DELETE ON v1 BEGIN + INSERT INTO log VALUES('delete'); + END; + + CREATE TRIGGER tr2 INSTEAD OF UPDATE ON v1 BEGIN + INSERT INTO log VALUES('update'); + END; + + CREATE TRIGGER tr3 INSTEAD OF INSERT ON v1 BEGIN + INSERT INTO log VALUES('insert'); + END; +} + +do_execsql_test 4.2 { + DELETE FROM v1 WHERE rowid=1; +} {} + +do_execsql_test 4.3 { + UPDATE v1 SET a=b WHERE rowid=2; +} {} + + + + finish_test From 35404253f680a8d6939a09d24ad4286ade7b5e8f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 01:27:43 +0000 Subject: [PATCH 507/710] Increase the version number to 3.8.7.1 FossilOrigin-Name: 83afe23e553e802c0947c80d0ffdd120423e7c52 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 16 +++++++--------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 20 deletions(-) diff --git a/VERSION b/VERSION index 4351a7e3a3..f53c1ed56b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.7 +3.8.7.1 diff --git a/configure b/configure index 9b8266d812..d253f21a7d 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.7. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.7.1. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.7' -PACKAGE_STRING='sqlite 3.8.7' +PACKAGE_VERSION='3.8.7.1' +PACKAGE_STRING='sqlite 3.8.7.1' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.7 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.7.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.7:";; + short | recursive ) echo "Configuration of sqlite 3.8.7.1:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.7 +sqlite configure 3.8.7.1 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.7, which was +It was created by sqlite $as_me 3.8.7.1, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.7, which was +This file was extended by sqlite $as_me 3.8.7.1, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.7 +sqlite config.status 3.8.7.1 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index be89f4c86a..916213d465 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Fix\sproblems\swith\srunning\sUPDATE\sand\sDELETE\sagainst\sa\sVIEW\sand\sreferencing\nthe\srowid\sin\sthe\sWHERE\sclause.\s\sThis\sis\sa\scherrypick\sof\s[95f8ebdbf87326f2]\nand\s[8523670d50004f3]. -D 2014-10-29T01:26:25.633 +C Increase\sthe\sversion\snumber\sto\s3.8.7.1 +D 2014-10-29T01:27:43.048 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc e31dee24038965fb6269d6d61073fd6b7e331dec F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION 53a0b870e7f16d3b06623c31d233a304c163a6af +F VERSION 5cc0baaee7eeee3238f0f7b5871398d17f79d0cd F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure ad59a5f48b3c59a92b5506040a22fbe3f733a9d8 x +F configure 56fe985cf0e59cd594f9b929099d0be40260e667 x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1204,9 +1204,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 837368adfe859c41b347d3124d5b3fdf790eec03 -Q +8523670d50004f3112b7871f11c8b8b02aab96ab -Q +95f8ebdbf87326f23cd38e561ac5632b5367a449 -R b3ee8c71d158066739429e7ae8387325 +P cc33e846c8509419f0a1fbfb286807b4c137788d +R 5381895dceb9bc7ad4d425f399561cb7 U drh -Z 624aac6d2575ef133925a49cc2066eb1 +Z 906d83e1887c766ebb0692623bc0209a diff --git a/manifest.uuid b/manifest.uuid index 99e8b826f5..abe23b5708 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc33e846c8509419f0a1fbfb286807b4c137788d \ No newline at end of file +83afe23e553e802c0947c80d0ffdd120423e7c52 \ No newline at end of file From 7abfe9d0c62966783f878c94ca07ee4f7a557b72 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 13:59:56 +0000 Subject: [PATCH 508/710] Version 3.8.7.1 FossilOrigin-Name: 3b7b72c4685aa5cf5e675c2c47ebec10d9704221 --- manifest | 11 +++++++---- manifest.uuid | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/manifest b/manifest index 916213d465..8688f8549f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.8.7.1 -D 2014-10-29T01:27:43.048 +C Version\s3.8.7.1 +D 2014-10-29T13:59:56.070 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1204,7 +1204,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cc33e846c8509419f0a1fbfb286807b4c137788d +P 83afe23e553e802c0947c80d0ffdd120423e7c52 R 5381895dceb9bc7ad4d425f399561cb7 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.8.7.1 * U drh -Z 906d83e1887c766ebb0692623bc0209a +Z e5ce942eb65e60b92fcaefcb5b84039b diff --git a/manifest.uuid b/manifest.uuid index abe23b5708..1787d659c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -83afe23e553e802c0947c80d0ffdd120423e7c52 \ No newline at end of file +3b7b72c4685aa5cf5e675c2c47ebec10d9704221 \ No newline at end of file From af8f513f9d9414b8aae11f9470e4a8789e7cf74e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 29 Oct 2014 18:20:18 +0000 Subject: [PATCH 509/710] Fix the %c format character in sqlite3VXPrintf() so that it correctly handles precisions larger than 70. FossilOrigin-Name: 08a27440f19b7fc884464832e6105af1bf008172 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/printf.c | 33 +++++++++++++++++++-------------- src/sqliteInt.h | 2 +- test/printf2.test | 20 ++++++++++++++++++++ 5 files changed, 49 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 209baf8beb..c8c7eafdd4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sOP_Column\sopcode,\swhen\sextracting\sa\sfield\sthat\sis\spast\sthe\send\sof\na\sshort\srecord\s(because\sthe\srow\swas\soriginally\sinserted\sprior\sto\sALTER\sTABLE\nADD\sCOLUMN)\sthen\smake\ssure\sthe\soutput\sregister\sis\sfully\sNULL\sand\sdoes\snot\ncontain\sleftover\sflags\s(such\sas\sMEM_Ephem)\sfrom\sits\sprevious\suse.\nFix\sfor\sticket\s[43107840f1c02]. -D 2014-10-29T00:58:38.898 +C Fix\sthe\s%c\sformat\scharacter\sin\ssqlite3VXPrintf()\sso\sthat\sit\scorrectly\nhandles\sprecisions\slarger\sthan\s70. +D 2014-10-29T18:20:18.932 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 -F src/printf.c c31012ac23e458081df4a32634b60424e0cdfaf3 +F src/printf.c 10a2493593c8e4a538915cd3674bd7a67f70c488 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -232,7 +232,7 @@ F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 F src/sqlite.h.in 737b7dd0f3f81fe183646d22828b39f85ef3e68c F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 90519c3b3e8ee90adfce013234c4bd07275d77b5 +F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -772,7 +772,7 @@ F test/permutations.test cef25f5e8499a15846eccd06785f17f4180407ab F test/pragma.test 19d0241a007bcdd77fc2606ec60fc60357e7fc8b F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 -F test/printf2.test bed79b4c3e5da08ba88ad637c0bf62586843cfb1 +F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f84af4adcc34d7a4c72027bf5b038a1a45a4c307 -R c2870e402bb1970025fb678b8c13d710 +P 24780f8ddc1683fc62180e6961dc6bfe1168f4df +R c12283d048765bdbe52bad70443f70c1 U drh -Z 3a55744e14b55dcc32cb71f070af972c +Z 8aeb3112f50d033e0580c4400e41dd66 diff --git a/manifest.uuid b/manifest.uuid index ea9bca8740..bd48a08b43 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24780f8ddc1683fc62180e6961dc6bfe1168f4df \ No newline at end of file +08a27440f19b7fc884464832e6105af1bf008172 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index f000da7fcc..85d237f0e7 100644 --- a/src/printf.c +++ b/src/printf.c @@ -212,7 +212,7 @@ void sqlite3VXPrintf( const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ - char *zExtra; /* Malloced memory used by some conversion */ + char *zExtra = 0; /* Malloced memory used by some conversion */ #ifndef SQLITE_OMIT_FLOATING_POINT int exp, e2; /* exponent of real numbers */ int nsd; /* Number of significant digits returned */ @@ -336,7 +336,6 @@ void sqlite3VXPrintf( break; } } - zExtra = 0; /* ** At this point, variables are initialized as follows: @@ -627,13 +626,16 @@ void sqlite3VXPrintf( }else{ c = va_arg(ap,int); } - buf[0] = (char)c; - if( precision>=0 ){ - for(idx=1; idx1 ){ + width -= precision-1; + if( width>1 && !flag_leftjustify ){ + sqlite3AppendChar(pAccum, width-1, ' '); + width = 0; + } + sqlite3AppendChar(pAccum, precision-1, c); } + length = 1; + buf[0] = c; bufpt = buf; break; case etSTRING: @@ -734,11 +736,14 @@ void sqlite3VXPrintf( ** the output. */ width -= length; - if( width>0 && !flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && !flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); sqlite3StrAccumAppend(pAccum, bufpt, length); - if( width>0 && flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); - if( zExtra ) sqlite3_free(zExtra); + if( zExtra ){ + sqlite3_free(zExtra); + zExtra = 0; + } }/* End for loop over the format string */ } /* End of function */ @@ -791,11 +796,11 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ } /* -** Append N space characters to the given string buffer. +** Append N copies of character c to the given string buffer. */ -void sqlite3AppendSpace(StrAccum *p, int N){ +void sqlite3AppendChar(StrAccum *p, int N, char c){ if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; - while( (N--)>0 ) p->zText[p->nChar++] = ' '; + while( (N--)>0 ) p->zText[p->nChar++] = c; } /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5114ccccf4..f4785411d5 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3543,7 +3543,7 @@ int sqlite3OpenTempDatabase(Parse *); void sqlite3StrAccumInit(StrAccum*, char*, int, int); void sqlite3StrAccumAppend(StrAccum*,const char*,int); void sqlite3StrAccumAppendAll(StrAccum*,const char*); -void sqlite3AppendSpace(StrAccum*,int); +void sqlite3AppendChar(StrAccum*,int,char); char *sqlite3StrAccumFinish(StrAccum*); void sqlite3StrAccumReset(StrAccum*); void sqlite3SelectDestInit(SelectDest*,int,int); diff --git a/test/printf2.test b/test/printf2.test index 4cb1783bfb..21deeb779d 100644 --- a/test/printf2.test +++ b/test/printf2.test @@ -95,5 +95,25 @@ do_execsql_test printf2-2.3 { SELECT printf('%s=(%d/%g/%s)',a) FROM t1 ORDER BY a; } {-1=(0/0/) 1=(0/0/) 1.5=(0/0/) abc=(0/0/)} +# The precision of the %c conversion causes the character to repeat. +# +do_execsql_test printf2-3.1 { + SELECT printf('|%110.100c|','*'); +} {{| ****************************************************************************************************|}} +do_execsql_test printf2-3.2 { + SELECT printf('|%-110.100c|','*'); +} {{|**************************************************************************************************** |}} +do_execsql_test printf2-3.3 { + SELECT printf('|%9.8c|%-9.8c|','*','*'); +} {{| ********|******** |}} +do_execsql_test printf2-3.4 { + SELECT printf('|%8.8c|%-8.8c|','*','*'); +} {|********|********|} +do_execsql_test printf2-3.5 { + SELECT printf('|%7.8c|%-7.8c|','*','*'); +} {|********|********|} + + + finish_test From ba0f99941848816581ac12bd1d5f6d03216f5338 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 30 Oct 2014 20:48:44 +0000 Subject: [PATCH 510/710] Tweaks to comments in btree.c. Minor code changes to enhance testability. FossilOrigin-Name: c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 20 ++++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index c8c7eafdd4..c8c02f1b16 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s%c\sformat\scharacter\sin\ssqlite3VXPrintf()\sso\sthat\sit\scorrectly\nhandles\sprecisions\slarger\sthan\s70. -D 2014-10-29T18:20:18.932 +C Tweaks\sto\scomments\sin\sbtree.c.\s\sMinor\scode\schanges\sto\senhance\stestability. +D 2014-10-30T20:48:44.305 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 5189881ca403938c5ceddde496b984fef9f40c5a +F src/btree.c 8d955d8ef15dd724ea5ef1cb65c17151beaff1e0 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 24780f8ddc1683fc62180e6961dc6bfe1168f4df -R c12283d048765bdbe52bad70443f70c1 +P 08a27440f19b7fc884464832e6105af1bf008172 +R 984c5e022c7b2ce03eb5bed97ff43b89 U drh -Z 8aeb3112f50d033e0580c4400e41dd66 +Z 333c45d7dfd79a88b32c00dfbc4028a5 diff --git a/manifest.uuid b/manifest.uuid index bd48a08b43..8964ddfef6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -08a27440f19b7fc884464832e6105af1bf008172 \ No newline at end of file +c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 22b168d9e4..e56cdf81ed 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1231,10 +1231,8 @@ static int defragmentPage(MemPage *pPage){ ** ** If no suitable space can be found on the free-list, return NULL. ** -** This function may detect corruption within pPg. If it does and argument -** pRc is non-NULL, then *pRc is set to SQLITE_CORRUPT and NULL is returned. -** Or, if corruption is detected and pRc is NULL, NULL is returned and the -** corruption goes unreported. +** This function may detect corruption within pPg. If corruption is +** detected then *pRc is set to SQLITE_CORRUPT and NULL is returned. ** ** If a slot of at least nByte bytes is found but cannot be used because ** there are already at least 60 fragmented bytes on the page, return NULL. @@ -1250,7 +1248,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ int size; /* Size of the free slot */ if( pc>usableSize-4 || pc usableSize ){ - if( pRc ) *pRc = SQLITE_CORRUPT_BKPT; + *pRc = SQLITE_CORRUPT_BKPT; return 0; }else{ /* The slot remains on the free-list. Reduce its size to account @@ -6076,8 +6074,9 @@ static int pageInsertArray( assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ for(i=0; i Date: Thu, 30 Oct 2014 23:14:56 +0000 Subject: [PATCH 511/710] Improvements to the wording of some comments. Reinstate an assert() that is only true for non-corrupt database files by adding an "|| CORRUPT_DB" term. FossilOrigin-Name: 67adb44838f98805f86aecca634d9a3b07370b9e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 27 ++++++++++++--------------- 3 files changed, 19 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index c8c02f1b16..98a273f2e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tweaks\sto\scomments\sin\sbtree.c.\s\sMinor\scode\schanges\sto\senhance\stestability. -D 2014-10-30T20:48:44.305 +C Improvements\sto\sthe\swording\sof\ssome\scomments.\s\sReinstate\san\sassert()\sthat\nis\sonly\strue\sfor\snon-corrupt\sdatabase\sfiles\sby\sadding\san\s"||\sCORRUPT_DB"\sterm. +D 2014-10-30T23:14:56.756 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 8d955d8ef15dd724ea5ef1cb65c17151beaff1e0 +F src/btree.c 8f7ea96935c3b1db94439204965a6ec1392f7977 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 08a27440f19b7fc884464832e6105af1bf008172 -R 984c5e022c7b2ce03eb5bed97ff43b89 +P c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 +R 230a6673a34b4786200f7ae54e553a63 U drh -Z 333c45d7dfd79a88b32c00dfbc4028a5 +Z 52f8e66a5796212d2c4dbab1cb7cef14 diff --git a/manifest.uuid b/manifest.uuid index 8964ddfef6..fd375ba77a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 \ No newline at end of file +67adb44838f98805f86aecca634d9a3b07370b9e \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e56cdf81ed..2c3cbb4adc 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6526,7 +6526,7 @@ static int balance_nonroot( u8 *apDiv[NB-1]; /* Divider cells in pParent */ int cntNew[NB+2]; /* Index in aCell[] of cell after i-th page */ int cntOld[NB+2]; /* Old index in aCell[] after i-th page */ - int szNew[NB+2]; /* Combined size of cells place on i-th page */ + int szNew[NB+2]; /* Combined size of cells placed on i-th page */ u8 **apCell = 0; /* All cells begin balanced */ u16 *szCell; /* Local size of all cells in apCell[] */ u8 *aSpace1; /* Space for copies of dividers cells */ @@ -6765,9 +6765,10 @@ static int balance_nonroot( /* ** The packing computed by the previous block is biased toward the siblings - ** on the left side. The left siblings are always nearly full, while the - ** right-most sibling might be nearly empty. This block of code attempts - ** to adjust the packing of siblings to get a better balance. + ** on the left side (siblings with smaller keys). The left siblings are + ** always nearly full, while the right-most sibling might be nearly empty. + ** The next block of code attempts to adjust the packing of siblings to + ** get a better balance. ** ** This adjustment is more than an optimization. The packing above might ** be so out of balance as to be illegal. For example, the right-most @@ -6796,18 +6797,14 @@ static int balance_nonroot( szNew[i-1] = szLeft; } - /* Either we found one or more cells (cntnew[0])>0) or pPage is - ** a virtual root page. A virtual root page is when the real root - ** page is page 1 and we are the only child of that page. - ** - ** UPDATE: The assert() below is not necessarily true if the database - ** file is corrupt. The corruption will be detected and reported later - ** in this procedure so there is no need to act upon it now. + /* Sanity check: For a non-corrupt database file one of the follwing + ** must be true: + ** (1) We found one or more cells (cntNew[0])>0), or + ** (2) pPage is a virtual root page. A virtual root page is when + ** the real root page is page 1 and we are the only child of + ** that page. */ -#if 0 - assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) ); -#endif - + assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", apOld[0]->pgno, apOld[0]->nCell, nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, From 00fe08af82c5e66a69abfbf687d9d99d46d8ad47 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 00:05:23 +0000 Subject: [PATCH 512/710] In the balance_nonroot() routine, protect the values in aPgno[] array from change during the page sort, so that aPgno[] can be used to avoid unnecessary pointer-map updates for auto_vacuum databases. FossilOrigin-Name: 69c3924fe834a78d4a8d86833626bf5f68e33a3a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 9 +++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 98a273f2e0..b3f68e9802 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\swording\sof\ssome\scomments.\s\sReinstate\san\sassert()\sthat\nis\sonly\strue\sfor\snon-corrupt\sdatabase\sfiles\sby\sadding\san\s"||\sCORRUPT_DB"\sterm. -D 2014-10-30T23:14:56.756 +C In\sthe\sbalance_nonroot()\sroutine,\sprotect\sthe\svalues\sin\saPgno[]\sarray\sfrom\nchange\sduring\sthe\spage\ssort,\sso\sthat\saPgno[]\scan\sbe\sused\sto\savoid\sunnecessary\npointer-map\supdates\sfor\sauto_vacuum\sdatabases. +D 2014-10-31T00:05:23.983 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 8f7ea96935c3b1db94439204965a6ec1392f7977 +F src/btree.c 88c87803b334807da3150245987139baacea4e6e F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c7d9aa3a1ce63e27ec94295601bc89fecf1e4977 -R 230a6673a34b4786200f7ae54e553a63 +P 67adb44838f98805f86aecca634d9a3b07370b9e +R 882446b4b0e231e5d89ec5bd9f22a940 U drh -Z 52f8e66a5796212d2c4dbab1cb7cef14 +Z 021277b745352cb31d525498ae57f1ac diff --git a/manifest.uuid b/manifest.uuid index fd375ba77a..03860ab39c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67adb44838f98805f86aecca634d9a3b07370b9e \ No newline at end of file +69c3924fe834a78d4a8d86833626bf5f68e33a3a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 2c3cbb4adc..05dbd8cec7 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6533,6 +6533,7 @@ static int balance_nonroot( Pgno pgno; /* Temp var to store a page number in */ u8 abDone[NB+2]; /* True after i'th new page is populated */ Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ + Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ memset(abDone, 0, sizeof(abDone)); @@ -6859,7 +6860,7 @@ static int balance_nonroot( ** for large insertions and deletions. */ for(i=0; ipgno; + aPgOrder[i] = aPgno[i] = apNew[i]->pgno; aPgFlags[i] = apNew[i]->pDbPage->flags; for(j=0; ji ){ sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); From 768f29002e24301bb601e5eefcf41b1a0904b78f Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 02:51:41 +0000 Subject: [PATCH 513/710] Remove an unnecessary branch from balance_nonroot(). FossilOrigin-Name: 9fc7c88e3f5221883aa6eafbf8af3be94db0c299 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 23 ++++++++++++----------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index b3f68e9802..2496dc7f00 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sbalance_nonroot()\sroutine,\sprotect\sthe\svalues\sin\saPgno[]\sarray\sfrom\nchange\sduring\sthe\spage\ssort,\sso\sthat\saPgno[]\scan\sbe\sused\sto\savoid\sunnecessary\npointer-map\supdates\sfor\sauto_vacuum\sdatabases. -D 2014-10-31T00:05:23.983 +C Remove\san\sunnecessary\sbranch\sfrom\sbalance_nonroot(). +D 2014-10-31T02:51:41.215 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 88c87803b334807da3150245987139baacea4e6e +F src/btree.c 7031b8cec28b4ba853090da021c6c456952f1f92 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1209,7 +1209,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 67adb44838f98805f86aecca634d9a3b07370b9e -R 882446b4b0e231e5d89ec5bd9f22a940 +P 69c3924fe834a78d4a8d86833626bf5f68e33a3a +R 85b30d88694d9feacf26ae3f80208908 U drh -Z 021277b745352cb31d525498ae57f1ac +Z 7bc5c33b6cb165c738435440761728db diff --git a/manifest.uuid b/manifest.uuid index 03860ab39c..d59874b337 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -69c3924fe834a78d4a8d86833626bf5f68e33a3a \ No newline at end of file +9fc7c88e3f5221883aa6eafbf8af3be94db0c299 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 05dbd8cec7..f21fce2a51 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7078,19 +7078,20 @@ static int balance_nonroot( ** sets all pointer-map entries corresponding to database image pages ** for which the pointer is stored within the content being copied. ** - ** The second assert below verifies that the child page is defragmented - ** (it must be, as it was just reconstructed using assemblePage()). This - ** is important if the parent page happens to be page 1 of the database - ** image. */ + ** It is critical that the child page be defragmented before being + ** copied into the parent, because if the parent is page 1 then it will + ** by smaller than the child due to the database header, and so all the + ** free space needs to be up front. + */ assert( nNew==1 ); rc = defragmentPage(apNew[0]); - if( rc==SQLITE_OK ){ - assert( apNew[0]->nFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) - ); - copyNodeContent(apNew[0], pParent, &rc); - freePage(apNew[0], &rc); - } + testcase( rc!=SQLITE_OK ); + assert( apNew[0]->nFree == + (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + || rc!=SQLITE_OK + ); + copyNodeContent(apNew[0], pParent, &rc); + freePage(apNew[0], &rc); }else if( ISAUTOVACUUM && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken From 9c0153457af66ca84768c08472f7424f7e6711d0 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 10:31:59 +0000 Subject: [PATCH 514/710] Add the "varint.c" utility program in the tool directory. FossilOrigin-Name: ea5d56be5fe14934e4dbe9c17d46b058f487a231 --- manifest | 11 +++-- manifest.uuid | 2 +- tool/varint.c | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 6 deletions(-) create mode 100644 tool/varint.c diff --git a/manifest b/manifest index 2496dc7f00..2c4f7bf41f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sbranch\sfrom\sbalance_nonroot(). -D 2014-10-31T02:51:41.215 +C Add\sthe\s"varint.c"\sutility\sprogram\sin\sthe\stool\sdirectory. +D 2014-10-31T10:31:59.052 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1204,12 +1204,13 @@ F tool/stack_usage.tcl f8e71b92cdb099a147dad572375595eae55eca43 F tool/symbols-mingw.sh 4dbcea7e74768305384c9fd2ed2b41bbf9f0414d F tool/symbols.sh fec58532668296d7c7dc48be9c87f75ccdb5814f F tool/tostr.awk e75472c2f98dd76e06b8c9c1367f4ab07e122d06 +F tool/varint.c 5d94cb5003db9dbbcbcc5df08d66f16071aee003 F tool/vdbe-compress.tcl 5926c71f9c12d2ab73ef35c29376e756eb68361c F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 69c3924fe834a78d4a8d86833626bf5f68e33a3a -R 85b30d88694d9feacf26ae3f80208908 +P 9fc7c88e3f5221883aa6eafbf8af3be94db0c299 +R 932c8189456e662bc00860a65a53ebcf U drh -Z 7bc5c33b6cb165c738435440761728db +Z 653e5ed37198e17f610045be3bfdae23 diff --git a/manifest.uuid b/manifest.uuid index d59874b337..337c7347f8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9fc7c88e3f5221883aa6eafbf8af3be94db0c299 \ No newline at end of file +ea5d56be5fe14934e4dbe9c17d46b058f487a231 \ No newline at end of file diff --git a/tool/varint.c b/tool/varint.c new file mode 100644 index 0000000000..f4a51118b4 --- /dev/null +++ b/tool/varint.c @@ -0,0 +1,123 @@ +/* +** A utility program to translate SQLite varints into decimal and decimal +** integers into varints. +*/ +#include +#include +#include + +#if defined(_MSC_VER) || defined(__BORLANDC__) + typedef __int64 i64; + typedef unsigned __int64 u64; +#else + typedef long long int i64; + typedef unsigned long long int u64; +#endif + +static int hexValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +static char toHex(unsigned char c){ + return "0123456789abcdef"[c&0xf]; +} + +static int putVarint(unsigned char *p, u64 v){ + int i, j, n; + unsigned char buf[10]; + if( v & (((u64)0xff000000)<<32) ){ + p[8] = (unsigned char)v; + v >>= 8; + for(i=7; i>=0; i--){ + p[i] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + } + return 9; + } + n = 0; + do{ + buf[n++] = (unsigned char)((v & 0x7f) | 0x80); + v >>= 7; + }while( v!=0 ); + buf[0] &= 0x7f; + for(i=0, j=n-1; j>=0; j--, i++){ + p[i] = buf[j]; + } + return n; +} + + +int main(int argc, char **argv){ + int i; + u64 x; + u64 uX = 0; + i64 iX; + int n; + unsigned char zHex[20]; + + if( argc==1 ){ + fprintf(stderr, + "Usage:\n" + " %s HH HH HH ... Convert varint to decimal\n" + " %s DDDDD Convert decimal to varint\n" + " Add '+' or '-' before DDDDD to disambiguate.\n", + argv[0], argv[0]); + exit(1); + } + if( argc>2 + || (strlen(argv[1])==2 && hexValue(argv[1][0])>=0 && hexValue(argv[1][1])>=0) + ){ + /* Hex to decimal */ + for(i=1; i'9' ){ + fprintf(stderr, "Not a decimal number: %s", argv[1]); + exit(1); + } + uX = uX*10 + z[0] - '0'; + z++; + } + if( sign<0 ){ + memcpy(&iX, &uX, 8); + iX = -iX; + memcpy(&uX, &iX, 8); + } + } + n = putVarint(zHex, uX); + printf("%lld =", (i64)uX); + for(i=0; i>4), toHex(zHex[i]&0x0f)); + } + printf("\n"); + return 0; +} From bec021b9fca398564dd2eccabe10d8af8fa6fc41 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 12:22:00 +0000 Subject: [PATCH 515/710] Simplify the math slightly, and reduce by one the number of loop iterations, for the loop in balance_nonroot() that moves cells between pages. FossilOrigin-Name: 2e838db82e533598b3cb00011c04fc0d5a896895 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2c4f7bf41f..e6108baa44 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"varint.c"\sutility\sprogram\sin\sthe\stool\sdirectory. -D 2014-10-31T10:31:59.052 +C Simplify\sthe\smath\sslightly,\sand\sreduce\sby\sone\sthe\snumber\sof\sloop\siterations,\nfor\sthe\sloop\sin\sbalance_nonroot()\sthat\smoves\scells\sbetween\spages. +D 2014-10-31T12:22:00.360 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 7031b8cec28b4ba853090da021c6c456952f1f92 +F src/btree.c 0f294d5e1eca7a912037d1992c904e28959a5aa5 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9fc7c88e3f5221883aa6eafbf8af3be94db0c299 -R 932c8189456e662bc00860a65a53ebcf +P ea5d56be5fe14934e4dbe9c17d46b058f487a231 +R 5dd41c1bf29a3c6eca71e97060df99dc U drh -Z 653e5ed37198e17f610045be3bfdae23 +Z 53accc48044e0d3a03ff14711aee89d8 diff --git a/manifest.uuid b/manifest.uuid index 337c7347f8..000bb51e9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ea5d56be5fe14934e4dbe9c17d46b058f487a231 \ No newline at end of file +2e838db82e533598b3cb00011c04fc0d5a896895 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f21fce2a51..4176d2c8bd 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7036,8 +7036,10 @@ static int balance_nonroot( ** ** If neither of the above apply, the page is safe to update. */ - for(i=0; i=nNew ? i-nNew : nNew-1-i); + for(i=1-nNew; i=0 && iPg=cntNew[iPg-1] || abDone[iPg-1]) && (cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1]) From d836d42383d93c549826510747b0b6a1bb55c55b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 14:26:36 +0000 Subject: [PATCH 516/710] Simplify the logic in the cell redistribution loop of balance_nonroot(). Enhance and clarify comments and add assert() statements for additional verification of correctness. FossilOrigin-Name: a07078b60007e88adea67bec5f0caf91f707ad78 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 39 ++++++++++++++++++++++++++++----------- 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index e6108baa44..5aa7ed2f46 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\smath\sslightly,\sand\sreduce\sby\sone\sthe\snumber\sof\sloop\siterations,\nfor\sthe\sloop\sin\sbalance_nonroot()\sthat\smoves\scells\sbetween\spages. -D 2014-10-31T12:22:00.360 +C Simplify\sthe\slogic\sin\sthe\scell\sredistribution\sloop\sof\sbalance_nonroot().\nEnhance\sand\sclarify\scomments\sand\sadd\sassert()\sstatements\sfor\sadditional\nverification\sof\scorrectness. +D 2014-10-31T14:26:36.417 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 0f294d5e1eca7a912037d1992c904e28959a5aa5 +F src/btree.c 61d96c2edacc5267fae6e477c24c774cd540d7f0 F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ea5d56be5fe14934e4dbe9c17d46b058f487a231 -R 5dd41c1bf29a3c6eca71e97060df99dc +P 2e838db82e533598b3cb00011c04fc0d5a896895 +R 6da135d51be6a673f0389acc80c0dfb3 U drh -Z 53accc48044e0d3a03ff14711aee89d8 +Z 452523febd435385cb08d146713a8a8f diff --git a/manifest.uuid b/manifest.uuid index 000bb51e9a..4ff01299eb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2e838db82e533598b3cb00011c04fc0d5a896895 \ No newline at end of file +a07078b60007e88adea67bec5f0caf91f707ad78 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 4176d2c8bd..9300a6a54f 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7026,28 +7026,43 @@ static int balance_nonroot( ** is important, as this code needs to avoid disrupting any page from which ** cells may still to be read. In practice, this means: ** - ** 1) If cells are to be removed from the start of the page and shifted - ** to the left-hand sibling, it is not safe to update the page until - ** the left-hand sibling (apNew[i-1]) has already been updated. + ** (1) If cells are moving left (from apNew[iPg] to apNew[iPg-1]) + ** then it is not safe to update page apNew[iPg] until after + ** the left-hand sibling apNew[iPg-1] has been updated. ** - ** 2) If cells are to be removed from the end of the page and shifted - ** to the right-hand sibling, it is not safe to update the page until - ** the right-hand sibling (apNew[i+1]) has already been updated. + ** (2) If cells are moving right (from apNew[iPg] to apNew[iPg+1]) + ** then it is not safe to update page apNew[iPg] until after + ** the right-hand sibling apNew[iPg+1] has been updated. ** ** If neither of the above apply, the page is safe to update. + ** + ** The iPg value in the following loop starts at nNew-1 goes down + ** to 0, then back up to nNew-1 again, thus making two passes over + ** the pages. On the initial downward pass, only condition (1) above + ** needs to be tested because (2) will always be true from the previous + ** step. On the upward pass, both conditions are always true, so the + ** upwards pass simply processes pages that were missed on the downward + ** pass. */ for(i=1-nNew; i=0 && iPg=cntNew[iPg-1] || abDone[iPg-1]) - && (cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1]) + if( abDone[iPg] ) continue; /* Skip pages already processed */ + if( i>=0 /* On the upwards pass, or... */ + || cntOld[iPg-1]>=cntNew[iPg-1] /* Condition (1) is true */ ){ int iNew; int iOld; int nNewCell; + /* Verify condition (1): If cells are moving left, update iPg + ** only after iPg-1 has already been updated. */ + assert( iPg==0 || cntOld[iPg-1]>=cntNew[iPg-1] || abDone[iPg-1] ); + + /* Verify condition (2): If cells are moving right, update iPg + ** only after iPg+1 has already been updated. */ + assert( cntNew[iPg]>=cntOld[iPg] || abDone[iPg+1] ); + if( iPg==0 ){ iNew = iOld = 0; nNewCell = cntNew[0]; @@ -7058,12 +7073,14 @@ static int balance_nonroot( } editPage(apNew[iPg], iOld, iNew, nNewCell, apCell, szCell); - abDone[iPg] = 1; + abDone[iPg]++; apNew[iPg]->nFree = usableSpace-szNew[iPg]; assert( apNew[iPg]->nOverflow==0 ); assert( apNew[iPg]->nCell==nNewCell ); } } + + /* All pages have been processed exactly once */ assert( memcmp(abDone, "\01\01\01\01\01", nNew)==0 ); assert( nOld>0 ); From 0fb5daed34ba6f16d761b8ad02b944ec59cf5c03 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 31 Oct 2014 14:46:51 +0000 Subject: [PATCH 517/710] Change the command-line shell man-page to use the ".tr" troff directive instead of ".cc" for escaping the initial "." characters in the ".help" output. FossilOrigin-Name: 67f0d469da28c023200239a1f3d0c6cef9ef0e45 --- manifest | 12 ++++---- manifest.uuid | 2 +- sqlite3.1 | 83 ++++++++++++++++++++++++++++----------------------- 3 files changed, 52 insertions(+), 45 deletions(-) diff --git a/manifest b/manifest index 5aa7ed2f46..ce0614fbee 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\slogic\sin\sthe\scell\sredistribution\sloop\sof\sbalance_nonroot().\nEnhance\sand\sclarify\scomments\sand\sadd\sassert()\sstatements\sfor\sadditional\nverification\sof\scorrectness. -D 2014-10-31T14:26:36.417 +C Change\sthe\scommand-line\sshell\sman-page\sto\suse\sthe\s".tr"\stroff\sdirective\ninstead\sof\s".cc"\sfor\sescaping\sthe\sinitial\s"."\scharacters\sin\sthe\s".help"\noutput. +D 2014-10-31T14:46:51.896 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -163,7 +163,7 @@ F mptest/mptest.c 499a74af4be293b7c1c7c3d40f332b67227dd739 F mptest/multiwrite01.test 499ad0310da8dff8e8f98d2e272fc2a8aa741b2e F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b -F sqlite3.1 3d8b83c91651f53472ca17599dae3457b8b89494 +F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2e838db82e533598b3cb00011c04fc0d5a896895 -R 6da135d51be6a673f0389acc80c0dfb3 +P a07078b60007e88adea67bec5f0caf91f707ad78 +R f904ce78baced6aeb6bcb91c631714db U drh -Z 452523febd435385cb08d146713a8a8f +Z c463256d5d45e3bce4a212ca4dd9f3e4 diff --git a/manifest.uuid b/manifest.uuid index 4ff01299eb..4735237141 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a07078b60007e88adea67bec5f0caf91f707ad78 \ No newline at end of file +67f0d469da28c023200239a1f3d0c6cef9ef0e45 \ No newline at end of file diff --git a/sqlite3.1 b/sqlite3.1 index 02df7f44c7..80353b0eec 100644 --- a/sqlite3.1 +++ b/sqlite3.1 @@ -2,7 +2,7 @@ .\" First parameter, NAME, should be all caps .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection .\" other parameters are allowed: see man(7), man(1) -.TH SQLITE3 1 "Mon Jan 31 11:14:00 2014" +.TH SQLITE3 1 "Fri Oct 31 10:41:31 EDT 2014" .\" Please adjust this date whenever revising the manpage. .\" .\" Some roff macros, for reference: @@ -49,7 +49,7 @@ a table named "memos" and insert a couple of records into that table: $ .B sqlite3 mydata.db .br -SQLite version 3.8.3 +SQLite version 3.8.8 .br Enter ".help" for instructions .br @@ -107,26 +107,29 @@ the '.help' command. For example: sqlite> .B .help .nf -.cc | -.backup ?DB? FILE Backup DB (default "main") to FILE -.bail ON|OFF Stop after hitting an error. Default OFF -.databases List names and files of attached databases -.dump ?TABLE? ... Dump the database in an SQL text format +.tr %. +%backup ?DB? FILE Backup DB (default "main") to FILE +%bail on|off Stop after hitting an error. Default OFF +%clone NEWDB Clone data into NEWDB from the existing database +%databases List names and files of attached databases +%dump ?TABLE? ... Dump the database in an SQL text format If TABLE specified, only dump tables matching LIKE pattern TABLE. -.echo ON|OFF Turn command echo on or off -.exit Exit this program -.explain ?ON|OFF? Turn output mode suitable for EXPLAIN on or off. +%echo on|off Turn command echo on or off +%eqp on|off Enable or disable automatic EXPLAIN QUERY PLAN +%exit Exit this program +%explain ?on|off? Turn output mode suitable for EXPLAIN on or off. With no args, it turns EXPLAIN on. -.header(s) ON|OFF Turn display of headers on or off -.help Show this message -.import FILE TABLE Import data from FILE into TABLE -.indices ?TABLE? Show names of all indices +%fullschema Show schema and the content of sqlite_stat tables +%headers on|off Turn display of headers on or off +%help Show this message +%import FILE TABLE Import data from FILE into TABLE +%indices ?TABLE? Show names of all indices If TABLE specified, only show indices for tables matching LIKE pattern TABLE. -.load FILE ?ENTRY? Load an extension library -.log FILE|off Turn logging on or off. FILE can be stderr/stdout -.mode MODE ?TABLE? Set output mode where MODE is one of: +%load FILE ?ENTRY? Load an extension library +%log FILE|off Turn logging on or off. FILE can be stderr/stdout +%mode MODE ?TABLE? Set output mode where MODE is one of: csv Comma-separated values column Left-aligned columns. (See .width) html HTML
    • code @@ -135,31 +138,35 @@ sqlite> list Values delimited by .separator string tabs Tab-separated values tcl TCL list elements -.nullvalue STRING Use STRING in place of NULL values -.open ?FILENAME? Close existing database and reopen FILENAME -.output FILENAME Send output to FILENAME -.output stdout Send output to the screen -.print STRING... Print literal STRING -.prompt MAIN CONTINUE Replace the standard prompts -.quit Exit this program -.read FILENAME Execute SQL in FILENAME -.restore ?DB? FILE Restore content of DB (default "main") from FILE -.schema ?TABLE? Show the CREATE statements +%nullvalue STRING Use STRING in place of NULL values +%once FILENAME Output for the next SQL command only to FILENAME +%open ?FILENAME? Close existing database and reopen FILENAME +%output ?FILENAME? Send output to FILENAME or stdout +%print STRING... Print literal STRING +%prompt MAIN CONTINUE Replace the standard prompts +%quit Exit this program +%read FILENAME Execute SQL in FILENAME +%restore ?DB? FILE Restore content of DB (default "main") from FILE +%save FILE Write in-memory database into FILE +%schema ?TABLE? Show the CREATE statements If TABLE specified, only show tables matching LIKE pattern TABLE. -.separator STRING Change separator used by output mode and .import -.show Show the current values for various settings -.stats ON|OFF Turn stats on or off -.tables ?TABLE? List names of tables +%separator STRING ?NL? Change separator used by output mode and .import + NL is the end-of-line mark for CSV +%shell CMD ARGS... Run CMD ARGS... in a system shell +%show Show the current values for various settings +%stats on|off Turn stats on or off +%system CMD ARGS... Run CMD ARGS... in a system shell +%tables ?TABLE? List names of tables If TABLE specified, only list tables matching LIKE pattern TABLE. -.timeout MS Try opening locked tables for MS milliseconds -.trace FILE|off Output each SQL statement as it is run -.vfsname ?AUX? Print the name of the VFS stack -.width NUM1 NUM2 ... Set column widths for "column" mode -.timer ON|OFF Turn the CPU timer measurement on or off +%timeout MS Try opening locked tables for MS milliseconds +%timer on|off Turn SQL timer on or off +%trace FILE|off Output each SQL statement as it is run +%vfsname ?AUX? Print the name of the VFS stack +%width NUM1 NUM2 ... Set column widths for "column" mode + Negative values right-justify sqlite> -|cc . .sp .fi .SH OPTIONS @@ -269,7 +276,7 @@ o If the -init option is present, the specified file is processed. o All other command line options are processed. .SH SEE ALSO -http://www.sqlite.org/ +http://www.sqlite.org/cli.html .br The sqlite3-doc package. .SH AUTHOR From 04489b6dce1b28fffb9afa524b0eb854a400da21 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 31 Oct 2014 20:11:32 +0000 Subject: [PATCH 518/710] Add the experimental sqlite3_stmt_scanstatus() API. FossilOrigin-Name: 6a9bab34aeb6a01b612211a28c140de60a3e883c --- manifest | 34 ++++++++++++--------- manifest.uuid | 2 +- src/sqlite.h.in | 36 ++++++++++++++++++++++ src/tclsqlite.c | 42 +++++++++++++++++++++++++ src/test1.c | 73 ++++++++++++++++++++++++++++++++++++++++++++ src/vdbe.c | 44 ++++++++++++++++++++++++-- src/vdbe.h | 16 ++++++++++ src/vdbeInt.h | 4 +++ src/vdbeapi.c | 37 ++++++++++++++++++++++ src/vdbeaux.c | 13 ++++++++ src/where.c | 55 +++++++++++++++++++++++++-------- test/scanstatus.test | 69 +++++++++++++++++++++++++++++++++++++++++ 12 files changed, 394 insertions(+), 31 deletions(-) create mode 100644 test/scanstatus.test diff --git a/manifest b/manifest index ce0614fbee..39eaf23ca4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\scommand-line\sshell\sman-page\sto\suse\sthe\s".tr"\stroff\sdirective\ninstead\sof\s".cc"\sfor\sescaping\sthe\sinitial\s"."\scharacters\sin\sthe\s".help"\noutput. -D 2014-10-31T14:46:51.896 +C Add\sthe\sexperimental\ssqlite3_stmt_scanstatus()\sAPI. +D 2014-10-31T20:11:32.562 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,15 +229,15 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 737b7dd0f3f81fe183646d22828b39f85ef3e68c +F src/sqlite.h.in 6e90cdb404e4fa8c0eb149ca79c11c6608a97ece F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc -F src/tclsqlite.c c67d310c833046cccc192125d64ad422ab882684 -F src/test1.c 63d4b1707c4052cf9c05c1cbb4a62666d70a0b48 +F src/tclsqlite.c 7cdd4dd3c2a4183483feca260070d73d6e22cd47 +F src/test1.c b53f4da2f386efa5c248716cd4bdc8344a87e952 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -289,11 +289,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 49e659bc165e99b28492004b440e22146dc898ab -F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 -F src/vdbeInt.h acc36ac461f973f46ac7942f86abdd93d2f8cfbc -F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 -F src/vdbeaux.c 3d6b2b412ef2193aa4729922dfc5df1efadbf6df +F src/vdbe.c 80329be857cff3c2d6e8c1f594ac689166c44ae7 +F src/vdbe.h f8e5388173dbf8408da4ae1dcbf75911caf2253d +F src/vdbeInt.h 284b2294c188474daa9b5ce2bd0200560cb0940d +F src/vdbeapi.c 2afa2e162f290879ca9c51e8795a377035f1662a +F src/vdbeaux.c 1e2561eeb94749f6b8c0a1eb02b5fecb645ed48c F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 5665df88cbd2b38eb72b4b94c8892c8afb360181 +F src/where.c 344beac082f8f4f69ec7c4a669d74294475e5abe F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -801,6 +801,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 +F test/scanstatus.test a62dad3311fc51eab1d6977b082738bd2148fe07 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1210,7 +1211,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a07078b60007e88adea67bec5f0caf91f707ad78 -R f904ce78baced6aeb6bcb91c631714db -U drh -Z c463256d5d45e3bce4a212ca4dd9f3e4 +P 67f0d469da28c023200239a1f3d0c6cef9ef0e45 +R 4c76f3e7b97f67c1dd65bda3a6e96149 +T *branch * scanstatus +T *sym-scanstatus * +T -sym-trunk * +U dan +Z 4994ca943ee260eeca8f493b4f59766a diff --git a/manifest.uuid b/manifest.uuid index 4735237141..b7984cac38 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67f0d469da28c023200239a1f3d0c6cef9ef0e45 \ No newline at end of file +6a9bab34aeb6a01b612211a28c140de60a3e883c \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index b4081f2a02..ba20cb9890 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7407,6 +7407,42 @@ int sqlite3_vtab_on_conflict(sqlite3 *); #define SQLITE_REPLACE 5 +/* +** Return status data for a single loop within query pStmt. +** +** Parameter "idx" identifies the specific loop to retrieve statistics for. +** Loops are numbered starting from zero. If idx is out of range - less than +** zero or greater than or equal to the total number of loops used to implement +** the statement - a non-zero value is returned. In this case the final value +** of all five output parameters is undefined. Otherwise, if idx is in range, +** the output parameters are populated and zero returned. +** +** Statistics may not be available for all loops in all statements. In cases +** where there exist loops with no available statistics, this function ignores +** them completely. +** +** This API is only available if the library is built with pre-processor +** symbol SQLITE_ENABLE_STMT_SCANSTATUS defined. +*/ +SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, + int idx, /* Index of loop to report on */ + sqlite3_int64 *pnLoop, /* OUT: Number of times loop was run */ + sqlite3_int64 *pnVisit, /* OUT: Number of rows visited (all loops) */ + sqlite3_int64 *pnEst, /* OUT: Number of rows estimated (per loop) */ + const char **pzName, /* OUT: Object name (table or index) */ + const char **pzExplain /* OUT: EQP string */ +); + + +/* +** Zero all sqlite3_stmt_scanstatus() related event counters. +** +** This API is only available if the library is built with pre-processor +** symbol SQLITE_ENABLE_STMT_SCANSTATUS defined. +*/ +SQLITE_EXPERIMENTAL void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); + /* ** Undo the hack that converts floating point types to integer for diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 756d0daa5a..bff4a92421 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3641,6 +3641,45 @@ static int db_use_legacy_prepare_cmd( Tcl_ResetResult(interp); return TCL_OK; } + +/* +** Tclcmd: db_last_stmt_ptr DB +** +** If the statement cache associated with database DB is not empty, +** return the text representation of the most recently used statement +** handle. +*/ +static int db_last_stmt_ptr( + ClientData cd, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + extern int sqlite3TestMakePointerStr(Tcl_Interp*, char*, void*); + Tcl_CmdInfo cmdInfo; + SqliteDb *pDb; + sqlite3_stmt *pStmt = 0; + char zBuf[100]; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB"); + return TCL_ERROR; + } + + if( !Tcl_GetCommandInfo(interp, Tcl_GetString(objv[1]), &cmdInfo) ){ + Tcl_AppendResult(interp, "no such db: ", Tcl_GetString(objv[1]), (char*)0); + return TCL_ERROR; + } + pDb = (SqliteDb*)cmdInfo.objClientData; + + if( pDb->stmtList ) pStmt = pDb->stmtList->pStmt; + if( sqlite3TestMakePointerStr(interp, zBuf, pStmt) ){ + return TCL_ERROR; + } + Tcl_SetResult(interp, zBuf, TCL_VOLATILE); + + return TCL_OK; +} #endif /* @@ -3760,6 +3799,9 @@ static void init_all(Tcl_Interp *interp){ Tcl_CreateObjCommand( interp, "db_use_legacy_prepare", db_use_legacy_prepare_cmd, 0, 0 ); + Tcl_CreateObjCommand( + interp, "db_last_stmt_ptr", db_last_stmt_ptr, 0, 0 + ); #ifdef SQLITE_SSE Sqlitetestsse_Init(interp); diff --git a/src/test1.c b/src/test1.c index 802aa99b5b..72e70513bc 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2301,6 +2301,75 @@ static int test_stmt_status( return TCL_OK; } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Usage: sqlite3_stmt_scanstatus STMT IDX +*/ +static int test_stmt_scanstatus( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt; /* First argument */ + int idx; /* Second argument */ + + const char *zName; + const char *zExplain; + sqlite3_int64 nLoop; + sqlite3_int64 nVisit; + sqlite3_int64 nEst; + int res; + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "STMT IDX"); + return TCL_ERROR; + } + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR; + + res = sqlite3_stmt_scanstatus( + pStmt, idx, &nLoop, &nVisit, &nEst, &zName, &zExplain + ); + if( res==0 ){ + Tcl_Obj *pRet = Tcl_NewObj(); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nLoop", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nLoop)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nVisit", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nVisit)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nEst", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nEst)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("zName", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zName, -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("zExplain", -1)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zExplain, -1)); + Tcl_SetObjResult(interp, pRet); + }else{ + Tcl_ResetResult(interp); + } + return TCL_OK; +} + +/* +** Usage: sqlite3_stmt_scanstatus_reset STMT +*/ +static int test_stmt_scanstatus_reset( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_stmt *pStmt; /* First argument */ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "STMT"); + return TCL_ERROR; + } + if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; + sqlite3_stmt_scanstatus_reset(pStmt); + return TCL_OK; +} +#endif + /* ** Usage: sqlite3_next_stmt DB STMT ** @@ -6868,6 +6937,10 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_user_change", test_user_change, 0 }, { "sqlite3_user_delete", test_user_delete, 0 }, #endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + { "sqlite3_stmt_scanstatus", test_stmt_scanstatus, 0 }, + { "sqlite3_stmt_scanstatus_reset", test_stmt_scanstatus_reset, 0 }, +#endif }; static int bitmask_size = sizeof(Bitmask)*8; diff --git a/src/vdbe.c b/src/vdbe.c index c17bfdfe14..a92c0509b3 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -167,6 +167,18 @@ int sqlite3_found_count = 0; /* Return true if the cursor was opened using the OP_OpenSorter opcode. */ #define isSorter(x) ((x)->pSorter!=0) +/* +** The first argument passed to the IncrementExplainCounter() macro must +** be a non-NULL pointer to an object of type VdbeCursor. The second +** argument must be either "nLoop" or "nVisit" (without the double-quotes). +*/ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define IncrementExplainCounter(pCsr, counter) \ + if( (pCsr)->pExplain ) (pCsr)->pExplain->counter++ +#else +# define IncrementExplainCounter(pCsr, counter) +#endif + /* ** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL ** if we run out of memory. @@ -3556,6 +3568,7 @@ case OP_SeekGT: { /* jump, in3 */ #ifdef SQLITE_DEBUG pC->seekOp = pOp->opcode; #endif + IncrementExplainCounter(pC, nLoop); if( pC->isTable ){ /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do @@ -3664,6 +3677,8 @@ case OP_SeekGT: { /* jump, in3 */ VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; + }else{ + IncrementExplainCounter(pC, nVisit); } break; } @@ -4449,6 +4464,8 @@ case OP_Last: { /* jump */ res = 0; assert( pCrsr!=0 ); rc = sqlite3BtreeLast(pCrsr, &res); + IncrementExplainCounter(pC, nLoop); + if( res==0 ) IncrementExplainCounter(pC, nVisit); pC->nullRow = (u8)res; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -4488,9 +4505,9 @@ case OP_Sort: { /* jump */ ** ** The next use of the Rowid or Column or Next instruction for P1 ** will refer to the first entry in the database table or index. -** If the table or index is empty and P2>0, then jump immediately to P2. -** If P2 is 0 or if the table or index is not empty, fall through -** to the following instruction. +** If the table or index is empty, jump immediately to P2. +** If the table or index is not empty, fall through to the following +** instruction. ** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is @@ -4518,11 +4535,14 @@ case OP_Rewind: { /* jump */ pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } + IncrementExplainCounter(pC, nLoop); pC->nullRow = (u8)res; assert( pOp->p2>0 && pOp->p2nOp ); VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; + }else{ + IncrementExplainCounter(pC, nVisit); } break; } @@ -4633,6 +4653,7 @@ next_tail: pC->cacheStatus = CACHE_STALE; VdbeBranchTaken(res==0,2); if( res==0 ){ + IncrementExplainCounter(pC, nVisit); pC->nullRow = 0; pc = pOp->p2 - 1; p->aCounter[pOp->p5]++; @@ -6055,7 +6076,10 @@ case OP_VFilter: { /* jump */ VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; + }else{ + IncrementExplainCounter(pCur, nVisit); } + IncrementExplainCounter(pCur, nLoop); } pCur->nullRow = 0; @@ -6148,6 +6172,7 @@ case OP_VNext: { /* jump */ if( !res ){ /* If there is data, jump to P2 */ pc = pOp->p2 - 1; + IncrementExplainCounter(pCur, nVisit); } goto check_for_interrupt; } @@ -6347,6 +6372,19 @@ case OP_Init: { /* jump */ break; } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +case OP_Explain: { + ExplainArg *pArg; + VdbeCursor *pCur; + if( pOp->p4type==P4_EXPLAIN && (pArg = pOp->p4.pExplain)->iCsr>=0 ){ + pArg = pOp->p4.pExplain; + pCur = p->apCsr[pArg->iCsr]; + pCur->pExplain = pArg; + } + break; +} +#endif + /* Opcode: Noop * * * * * ** diff --git a/src/vdbe.h b/src/vdbe.h index f975f95543..1b470441be 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -25,6 +25,7 @@ ** of this structure. */ typedef struct Vdbe Vdbe; +typedef struct ExplainArg ExplainArg; /* ** The names of the following types declared in vdbeInt.h are required @@ -59,6 +60,7 @@ struct VdbeOp { KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ + ExplainArg *pExplain; /* Used when p4type is P4_EXPLAIN */ int (*xAdvance)(BtCursor *, int *); } p4; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS @@ -100,6 +102,19 @@ struct VdbeOpList { }; typedef struct VdbeOpList VdbeOpList; +/* +** Structure used as the P4 parameter for OP_Explain opcodes. +*/ +struct ExplainArg { + int iCsr; /* Cursor number this applies to */ + i64 nLoop; /* Number of times loop has run */ + i64 nVisit; /* Total number of rows visited */ + i64 nEst; /* Estimated number of rows per scan */ + const char *zName; /* Name of table/index being scanned */ + const char *zExplain; /* EQP text for this loop */ +}; + + /* ** Allowed values of VdbeOp.p4type */ @@ -119,6 +134,7 @@ typedef struct VdbeOpList VdbeOpList; #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ #define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ +#define P4_EXPLAIN (-20) /* P4 is a pointer to an instance of ExplainArg */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 623c5fdde8..839108f6bc 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -83,6 +83,7 @@ struct VdbeCursor { i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ + ExplainArg *pExplain; /* Object to store seek/visit counts (may be NULL) */ /* Cached information about the header for the data record that the ** cursor is currently pointing to. Only valid if cacheStatus matches @@ -291,6 +292,7 @@ struct Explain { char zBase[100]; /* Initial space */ }; + /* A bitfield type for use inside of structures. Always follow with :N where ** N is the number of bits. */ @@ -368,6 +370,8 @@ struct Vdbe { int nOnceFlag; /* Size of array aOnceFlag[] */ u8 *aOnceFlag; /* Flags for OP_Once */ AuxData *pAuxData; /* Linked list of auxdata allocations */ + ExplainArg **apExplain; /* Array of pointers to P4_EXPLAIN p4 values */ + int nExplain; /* Number of entries in array apExplain */ }; /* diff --git a/src/vdbeapi.c b/src/vdbeapi.c index daf7b101d2..b09faa1fa5 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1475,3 +1475,40 @@ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ if( resetFlag ) pVdbe->aCounter[op] = 0; return (int)v; } + +/* +** Return status data for a single loop within query pStmt. +*/ +int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, + int idx, /* Index of loop to report on */ + sqlite3_int64 *pnLoop, /* OUT: Number of times loop was run */ + sqlite3_int64 *pnVisit, /* OUT: Number of rows visited (all loops) */ + sqlite3_int64 *pnEst, /* OUT: Number of rows estimated (per loop) */ + const char **pzName, /* OUT: Object name (table or index) */ + const char **pzExplain /* OUT: EQP string */ +){ + Vdbe *p = (Vdbe*)pStmt; + ExplainArg *pExplain; + if( idx<0 || idx>=p->nExplain ) return 1; + pExplain = p->apExplain[idx]; + if( pnLoop ) *pnLoop = pExplain->nLoop; + if( pnVisit ) *pnVisit = pExplain->nVisit; + if( pnEst ) *pnEst = pExplain->nEst; + if( *pzName ) *pzName = pExplain->zName; + if( *pzExplain ) *pzExplain = pExplain->zExplain; + return 0; +} + +/* +** Zero all counters associated with the sqlite3_stmt_scanstatus() data. +*/ +void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe*)pStmt; + int i; + for(i=0; inExplain; i++){ + p->apExplain[i]->nLoop = 0; + p->apExplain[i]->nVisit = 0; + } +} + diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 7dfb64130d..a8223890bd 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -672,6 +672,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( p4 ){ assert( db ); switch( p4type ){ + case P4_EXPLAIN: case P4_REAL: case P4_INT64: case P4_DYNAMIC: @@ -820,6 +821,13 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ pOp->p4type = P4_VTAB; sqlite3VtabLock((VTable *)zP4); assert( ((VTable *)zP4)->db==p->db ); + }else if( n==P4_EXPLAIN ){ + pOp->p4.p = (void*)zP4; + pOp->p4type = P4_EXPLAIN; + p->apExplain = (ExplainArg**)sqlite3DbReallocOrFree( + p->db, p->apExplain, sizeof(ExplainArg*) * p->nExplain + 1 + ); + if( p->apExplain ) p->apExplain[p->nExplain++] = (ExplainArg*)zP4; }else if( n<0 ){ pOp->p4.p = (void*)zP4; pOp->p4type = (signed char)n; @@ -1103,6 +1111,10 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ zTemp[0] = 0; break; } + case P4_EXPLAIN: { + zP4 = (char*)pOp->p4.pExplain->zExplain; + break; + } default: { zP4 = pOp->p4.z; if( zP4==0 ){ @@ -2685,6 +2697,7 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); sqlite3DbFree(db, p->pFree); + sqlite3DbFree(db, p->apExplain); } /* diff --git a/src/where.c b/src/where.c index feccf2d11d..3565327b52 100644 --- a/src/where.c +++ b/src/where.c @@ -2820,7 +2820,7 @@ static void explainOneScan( int iFrom, /* Value for "from" column of output */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ -#ifndef SQLITE_DEBUG +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) if( pParse->explain==2 ) #endif { @@ -2831,20 +2831,38 @@ static void explainOneScan( int isSearch; /* True for a SEARCH. False for SCAN. */ WhereLoop *pLoop; /* The controlling WhereLoop object */ u32 flags; /* Flags that describe this loop */ - char *zMsg; /* Text to add to EQP output */ StrAccum str; /* EQP output string */ char zBuf[100]; /* Initial space for EQP output string */ + const char *zObj; + ExplainArg *pExplain; + i64 nEstRow; /* Estimated rows per scan of pLevel */ pLoop = pLevel->pWLoop; flags = pLoop->wsFlags; if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; + sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + str.db = db; + + /* Reserve space at the start of the buffer managed by the StrAccum + ** object for *pExplain. */ + assert( sizeof(*pExplain)<=sizeof(zBuf) ); + str.nChar = sizeof(*pExplain); + + /* Append the object (table or index) name to the buffer. */ + if( pItem->pSelect ){ + zObj = ""; + }else if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + zObj = pLoop->u.btree.pIndex->zName; + }else{ + zObj = pItem->zName; + } + sqlite3StrAccumAppend(&str, zObj, sqlite3Strlen30(zObj)+1); + + /* Append the EQP text to the buffer */ isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); - - sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); - str.db = db; sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); @@ -2901,15 +2919,28 @@ static void explainOneScan( pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif + nEstRow = pLoop->nOut>=10 ? sqlite3LogEstToInt(pLoop->nOut) : 1; #ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS - if( pLoop->nOut>=10 ){ - sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); - }else{ - sqlite3StrAccumAppend(&str, " (~1 row)", 9); - } + sqlite3XPrintf(&str, 0, " (~%llu rows)", nEstRow); #endif - zMsg = sqlite3StrAccumFinish(&str); - sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg, P4_DYNAMIC); + pExplain = (ExplainArg*)sqlite3StrAccumFinish(&str); + assert( pExplain || db->mallocFailed ); + if( pExplain ){ + memset(pExplain, 0, sizeof(*pExplain)); + if( pLoop->wsFlags & WHERE_INDEXED ){ + pExplain->iCsr = pLevel->iIdxCur; + }else if( pItem->pSelect==0 ){ + pExplain->iCsr = pLevel->iTabCur; + }else{ + pExplain->iCsr = -1; + } + pExplain->nEst = nEstRow; + pExplain->zName = (const char*)&pExplain[1]; + pExplain->zExplain = &pExplain->zName[sqlite3Strlen30(pExplain->zName)+1]; + sqlite3VdbeAddOp4(v, OP_Explain, + iId, iLevel, iFrom, (char*)pExplain,P4_EXPLAIN + ); + } } } #else diff --git a/test/scanstatus.test b/test/scanstatus.test new file mode 100644 index 0000000000..69a92dcb6a --- /dev/null +++ b/test/scanstatus.test @@ -0,0 +1,69 @@ +# 2014 November 1 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix scanstatus + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t2 VALUES('a', 'b'); + INSERT INTO t2 VALUES('c', 'd'); + INSERT INTO t2 VALUES('e', 'f'); +} + +proc do_scanstatus_test {tn res} { + set stmt [db_last_stmt_ptr db] + set idx 0 + set ret [list] + while {1} { + set r [sqlite3_stmt_scanstatus $stmt $idx] + if {[llength $r]==0} break + lappend ret {*}$r + incr idx + } + + uplevel [list do_test $tn [list set {} $ret] [list {*}$res]] +} + +do_execsql_test 1.1 { SELECT count(*) FROM t1, t2; } 6 +do_scanstatus_test 1.2 { + nLoop 1 nVisit 2 nEst 1048576 zName t1 zExplain {SCAN TABLE t1} + nLoop 2 nVisit 6 nEst 1048576 zName t2 zExplain {SCAN TABLE t2} +} + +do_execsql_test 1.3 { + ANALYZE; + SELECT count(*) FROM t1, t2; +} 6 +do_scanstatus_test 1.4 { + nLoop 1 nVisit 2 nEst 2 zName t1 zExplain {SCAN TABLE t1} + nLoop 2 nVisit 6 nEst 3 zName t2 zExplain {SCAN TABLE t2} +} + +do_execsql_test 1.5 { + ANALYZE; + SELECT count(*) FROM t1, t2 WHERE t2.rowid>1; +} 4 +do_scanstatus_test 1.6 { + nLoop 1 nVisit 2 nEst 2 zName t2 zExplain + {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} + nLoop 2 nVisit 4 nEst 2 zName t1 zExplain {SCAN TABLE t1} +} + + + + +finish_test From 89e71646df320b5ff107f618169227c5c7cac8eb Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 1 Nov 2014 18:08:04 +0000 Subject: [PATCH 519/710] Minor fixes and documentation improvements for sqlite3_stmt_scanstatus(). FossilOrigin-Name: 8d8cc9608d30bb65fffcfe488e904411cbbc7f41 --- manifest | 23 ++++---- manifest.uuid | 2 +- src/sqlite.h.in | 24 +++++++- src/vdbe.c | 3 + src/where.c | 40 +++++++++---- src/whereInt.h | 1 + test/scanstatus.test | 138 ++++++++++++++++++++++++++++++++++++++++++- 7 files changed, 200 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 39eaf23ca4..8582d6b5e8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sexperimental\ssqlite3_stmt_scanstatus()\sAPI. -D 2014-10-31T20:11:32.562 +C Minor\sfixes\sand\sdocumentation\simprovements\sfor\ssqlite3_stmt_scanstatus(). +D 2014-11-01T18:08:04.130 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 6e90cdb404e4fa8c0eb149ca79c11c6608a97ece +F src/sqlite.h.in aeba29025ba5fc721a11c1b81ed8745f93029590 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 80329be857cff3c2d6e8c1f594ac689166c44ae7 +F src/vdbe.c 5240bad20ce0b707afee211f0c21a862b54b1e0b F src/vdbe.h f8e5388173dbf8408da4ae1dcbf75911caf2253d F src/vdbeInt.h 284b2294c188474daa9b5ce2bd0200560cb0940d F src/vdbeapi.c 2afa2e162f290879ca9c51e8795a377035f1662a @@ -302,8 +302,8 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 344beac082f8f4f69ec7c4a669d74294475e5abe -F src/whereInt.h 19279cd0664ce1d90b9ad3ef0108cb494acfe455 +F src/where.c 636ca646c8f2f53fd66f9ed5c773e84cb0c4b762 +F src/whereInt.h 99d324a8f921d7a40c605a8b197350c3cb18977d F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -801,7 +801,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 -F test/scanstatus.test a62dad3311fc51eab1d6977b082738bd2148fe07 +F test/scanstatus.test b4b1780bad243e1576329d05597b99f06886e4d2 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1211,10 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 67f0d469da28c023200239a1f3d0c6cef9ef0e45 -R 4c76f3e7b97f67c1dd65bda3a6e96149 -T *branch * scanstatus -T *sym-scanstatus * -T -sym-trunk * +P 6a9bab34aeb6a01b612211a28c140de60a3e883c +R d964f269915d57cf95d9198c6ba22665 U dan -Z 4994ca943ee260eeca8f493b4f59766a +Z 082decdd6298ff1a2330407f8d406dfb diff --git a/manifest.uuid b/manifest.uuid index b7984cac38..7325586095 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6a9bab34aeb6a01b612211a28c140de60a3e883c \ No newline at end of file +8d8cc9608d30bb65fffcfe488e904411cbbc7f41 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index ba20cb9890..51383426d9 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7408,6 +7408,8 @@ int sqlite3_vtab_on_conflict(sqlite3 *); /* +** CAPI3REF: Prepared Statement Scan Statuses +** ** Return status data for a single loop within query pStmt. ** ** Parameter "idx" identifies the specific loop to retrieve statistics for. @@ -7415,7 +7417,25 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** zero or greater than or equal to the total number of loops used to implement ** the statement - a non-zero value is returned. In this case the final value ** of all five output parameters is undefined. Otherwise, if idx is in range, -** the output parameters are populated and zero returned. +** zero is returned and the output parameters set as follows: +** +**
      +**
    • (*pnLoop) is set to the total number of times the loop has been run. +**
    • (*pnVisit) is set to the total number of rows visited by the loop. +**
    • (*pnEst) is set to the estimate of the number of rows visited +** by each run of the loop used by the SQL optimizer. Ideally, this +** value should be close to (*pnVisit)/(*pnLoop). +**
    • (*pzName) is set to point to a nul-terminated string containing the +** name of the index of table used by this loop. +**
    • (*pzExplain) is set to point to a nul-terminated string containing +** same text that would be returned for this loop by an EXPLAIN +** QUERY PLAN command. +**
    +** +** Output parameters *pzName and *pzExplain are set to point to buffers +** managed by the statement object. Both of these pointers may be invalidated +** by any API call on the same statement object, including an sqlite3_step() +** sqlite3_bind_*() call. ** ** Statistics may not be available for all loops in all statements. In cases ** where there exist loops with no available statistics, this function ignores @@ -7436,6 +7456,8 @@ SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( /* +** CAPI3REF: Zero Scan-Status Counters +** ** Zero all sqlite3_stmt_scanstatus() related event counters. ** ** This API is only available if the library is built with pre-processor diff --git a/src/vdbe.c b/src/vdbe.c index a92c0509b3..1abccfeeab 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3889,6 +3889,7 @@ case OP_NotExists: { /* jump, in3 */ res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); + IncrementExplainCounter(pC, nLoop); pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; @@ -3896,6 +3897,8 @@ case OP_NotExists: { /* jump, in3 */ VdbeBranchTaken(res!=0,2); if( res!=0 ){ pc = pOp->p2 - 1; + }else{ + IncrementExplainCounter(pC, nVisit); } pC->seekResult = res; break; diff --git a/src/where.c b/src/where.c index 3565327b52..ae3c12725e 100644 --- a/src/where.c +++ b/src/where.c @@ -2739,7 +2739,7 @@ static int codeAllEqualityTerms( return regBase; } -#ifndef SQLITE_OMIT_EXPLAIN +#if !defined(SQLITE_OMIT_EXPLAIN) || defined(SQLITE_ENABLE_STMT_SCANSTATUS) /* ** This routine is a helper for explainIndexRange() below ** @@ -2815,19 +2815,23 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ static void explainOneScan( Parse *pParse, /* Parse context */ SrcList *pTabList, /* Table list this loop refers to */ - WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ + WhereInfo *pWInfo, /* WHERE clause this loop belongs to */ int iLevel, /* Value for "level" column of output */ - int iFrom, /* Value for "from" column of output */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ #if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) if( pParse->explain==2 ) #endif { + WhereLevel *pLevel = &pWInfo->a[iLevel]; struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite3 *db = pParse->db; /* Database handle */ - int iId = pParse->iSelectId; /* Select id (left-most output column) */ +#ifdef SQLITE_OMIT_EXPLAIN + int iId = 0; /* Select id (left-most output column) */ +#else + int iId = pParse->iSelectId; /* Select id (left-most output column) */ +#endif int isSearch; /* True for a SEARCH. False for SCAN. */ WhereLoop *pLoop; /* The controlling WhereLoop object */ u32 flags; /* Flags that describe this loop */ @@ -2839,7 +2843,7 @@ static void explainOneScan( pLoop = pLevel->pWLoop; flags = pLoop->wsFlags; - if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return; + if( (flags&WHERE_MULTI_OR) ) return; sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); str.db = db; @@ -2865,7 +2869,11 @@ static void explainOneScan( || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ +#ifdef SQLITE_OMIT_EXPLAIN + sqlite3XPrintf(&str, 0, " SUBQUERY 0"); +#else sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); +#endif }else{ sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); } @@ -2937,15 +2945,15 @@ static void explainOneScan( pExplain->nEst = nEstRow; pExplain->zName = (const char*)&pExplain[1]; pExplain->zExplain = &pExplain->zName[sqlite3Strlen30(pExplain->zName)+1]; - sqlite3VdbeAddOp4(v, OP_Explain, - iId, iLevel, iFrom, (char*)pExplain,P4_EXPLAIN + pWInfo->iExplain = sqlite3VdbeAddOp4(v, OP_Explain, + iId, iLevel, pLevel->iFrom, (char*)pExplain,P4_EXPLAIN ); } } } #else -# define explainOneScan(u,v,w,x,y,z) -#endif /* SQLITE_OMIT_EXPLAIN */ +# define explainOneScan(v,w,x,y,z) +#endif /* !SQLITE_OMIT_EXPLAIN || SQLITE_ENABLE_STMT_SCANSTATUS */ /* @@ -3613,9 +3621,15 @@ static Bitmask codeOneLoopStart( assert( pSubWInfo || pParse->nErr || db->mallocFailed ); if( pSubWInfo ){ WhereLoop *pSubLoop; - explainOneScan( - pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 - ); + + /* If an OP_Explain was added for this sub-loop, fix the P2 and + ** P3 parameters to it so that they are relative to the current + ** context. */ + if( pSubWInfo->iExplain!=0 ){ + sqlite3VdbeChangeP2(v, pSubWInfo->iExplain, iLevel); + sqlite3VdbeChangeP3(v, pSubWInfo->iExplain, pLevel->iFrom); + } + /* This is the sub-WHERE clause body. First skip over ** duplicate rows from prior sub-WHERE clauses, and record the ** rowid (or PRIMARY KEY) for the current row so that the same @@ -6455,7 +6469,7 @@ WhereInfo *sqlite3WhereBegin( if( db->mallocFailed ) goto whereBeginError; } #endif - explainOneScan(pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags); + explainOneScan(pParse, pTabList, pWInfo, ii, wctrlFlags); pLevel->addrBody = sqlite3VdbeCurrentAddr(v); notReady = codeOneLoopStart(pWInfo, ii, notReady); pWInfo->iContinue = pLevel->addrCont; diff --git a/src/whereInt.h b/src/whereInt.h index fd4cfdf88a..dcf0e00604 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -407,6 +407,7 @@ struct WhereInfo { int iTop; /* The very beginning of the WHERE loop */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ + int iExplain; /* Address of OP_Explain (if WHERE_ONETABLE_ONLY) */ int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ diff --git a/test/scanstatus.test b/test/scanstatus.test index 69a92dcb6a..9df80f9ba8 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -53,16 +53,148 @@ do_scanstatus_test 1.4 { nLoop 2 nVisit 6 nEst 3 zName t2 zExplain {SCAN TABLE t2} } -do_execsql_test 1.5 { - ANALYZE; +do_execsql_test 1.5 { ANALYZE } +do_execsql_test 1.6 { SELECT count(*) FROM t1, t2 WHERE t2.rowid>1; } 4 -do_scanstatus_test 1.6 { +do_scanstatus_test 1.7 { nLoop 1 nVisit 2 nEst 2 zName t2 zExplain {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} nLoop 2 nVisit 4 nEst 2 zName t1 zExplain {SCAN TABLE t1} } +do_execsql_test 1.8 { + SELECT count(*) FROM t1, t2 WHERE t2.rowid>1; +} 4 + +do_scanstatus_test 1.9 { + nLoop 2 nVisit 4 nEst 2 zName t2 zExplain + {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} + nLoop 4 nVisit 8 nEst 2 zName t1 zExplain {SCAN TABLE t1} +} + +do_test 1.9 { + sqlite3_stmt_scanstatus_reset [db_last_stmt_ptr db] +} {} + +do_scanstatus_test 1.10 { + nLoop 0 nVisit 0 nEst 2 zName t2 zExplain + {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} + nLoop 0 nVisit 0 nEst 2 zName t1 zExplain {SCAN TABLE t1} +} + +#------------------------------------------------------------------------- +# Try a few different types of scans. +# +reset_db +do_execsql_test 2.1 { + CREATE TABLE x1(i INTEGER PRIMARY KEY, j); + INSERT INTO x1 VALUES(1, 'one'); + INSERT INTO x1 VALUES(2, 'two'); + INSERT INTO x1 VALUES(3, 'three'); + INSERT INTO x1 VALUES(4, 'four'); + CREATE INDEX x1j ON x1(j); + + SELECT * FROM x1 WHERE i=2; +} {2 two} + +do_scanstatus_test 2.2 { + nLoop 1 nVisit 1 nEst 1 zName x1 + zExplain {SEARCH TABLE x1 USING INTEGER PRIMARY KEY (rowid=?)} +} + +do_execsql_test 2.3.1 { + SELECT * FROM x1 WHERE j='two' +} {2 two} +do_scanstatus_test 2.3.2 { + nLoop 1 nVisit 1 nEst 10 zName x1j + zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j=?)} +} + +do_execsql_test 2.4.1 { + SELECT * FROM x1 WHERE j<'two' +} {4 four 1 one 3 three} +do_scanstatus_test 2.4.2 { + nLoop 1 nVisit 4 nEst 262144 zName x1j + zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j='two' +} {2 two} +do_scanstatus_test 2.5.2 { + nLoop 1 nVisit 1 nEst 262144 zName x1j + zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j>?)} +} + +do_execsql_test 2.6.1 { + SELECT * FROM x1 WHERE j BETWEEN 'three' AND 'two' +} {3 three 2 two} +do_scanstatus_test 2.6.2 { + nLoop 1 nVisit 2 nEst 16384 zName x1j + zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j>? AND j? AND j Date: Sat, 1 Nov 2014 18:32:18 +0000 Subject: [PATCH 520/710] Add requirements marks and make minor tweaks to documentation. FossilOrigin-Name: 49188b2bb53a92b0b0b6aaf8247edeb0c1bcd1f5 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/main.c | 15 +++++++++++---- src/sqlite.h.in | 4 ++-- src/sqliteInt.h | 7 +++---- 5 files changed, 25 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index ce0614fbee..8ab904b8d2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\scommand-line\sshell\sman-page\sto\suse\sthe\s".tr"\stroff\sdirective\ninstead\sof\s".cc"\sfor\sescaping\sthe\sinitial\s"."\scharacters\sin\sthe\s".help"\noutput. -D 2014-10-31T14:46:51.896 +C Add\srequirements\smarks\sand\smake\sminor\stweaks\sto\sdocumentation. +D 2014-11-01T18:32:18.050 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 2b882f64580ea72e2d972a5296f9eaa75a353161 +F src/main.c 5f659bdb14cdba1c3b587b6ed09167c147e308a1 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -229,10 +229,10 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 737b7dd0f3f81fe183646d22828b39f85ef3e68c +F src/sqlite.h.in c3c6180be49d91d3b0001a8d3c604684f361adc2 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 +F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a07078b60007e88adea67bec5f0caf91f707ad78 -R f904ce78baced6aeb6bcb91c631714db +P 67f0d469da28c023200239a1f3d0c6cef9ef0e45 +R dbf6a8dc4a878d1b1a7b9b3874ca58fb U drh -Z c463256d5d45e3bce4a212ca4dd9f3e4 +Z 455c35adeade418fab0456b1b175d27f diff --git a/manifest.uuid b/manifest.uuid index 4735237141..02f906627f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67f0d469da28c023200239a1f3d0c6cef9ef0e45 \ No newline at end of file +49188b2bb53a92b0b0b6aaf8247edeb0c1bcd1f5 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 5308698797..65a662ee5a 100644 --- a/src/main.c +++ b/src/main.c @@ -329,15 +329,17 @@ int sqlite3_config(int op, ...){ switch( op ){ /* Mutex configuration options are only available in a threadsafe - ** compile. + ** compile. */ -#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-54466-46756 */ case SQLITE_CONFIG_SINGLETHREAD: { /* Disable all mutexing */ sqlite3GlobalConfig.bCoreMutex = 0; sqlite3GlobalConfig.bFullMutex = 0; break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-20520-54086 */ case SQLITE_CONFIG_MULTITHREAD: { /* Disable mutexing of database connections */ /* Enable mutexing of core data structures */ @@ -345,17 +347,23 @@ int sqlite3_config(int op, ...){ sqlite3GlobalConfig.bFullMutex = 0; break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-59593-21810 */ case SQLITE_CONFIG_SERIALIZED: { /* Enable all mutexing */ sqlite3GlobalConfig.bCoreMutex = 1; sqlite3GlobalConfig.bFullMutex = 1; break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-63666-48755 */ case SQLITE_CONFIG_MUTEX: { /* Specify an alternative mutex implementation */ sqlite3GlobalConfig.mutex = *va_arg(ap, sqlite3_mutex_methods*); break; } +#endif +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE>0 /* IMP: R-14450-37597 */ case SQLITE_CONFIG_GETMUTEX: { /* Retrieve the current mutex implementation */ *va_arg(ap, sqlite3_mutex_methods*) = sqlite3GlobalConfig.mutex; @@ -363,7 +371,6 @@ int sqlite3_config(int op, ...){ } #endif - case SQLITE_CONFIG_MALLOC: { /* Specify an alternative malloc implementation */ sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); @@ -377,7 +384,7 @@ int sqlite3_config(int op, ...){ } case SQLITE_CONFIG_MEMSTATUS: { /* Enable or disable the malloc status collection */ - sqlite3GlobalConfig.bMemstat = va_arg(ap, int); + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); /* IMP: R-27464-47829 */ break; } case SQLITE_CONFIG_SCRATCH: { diff --git a/src/sqlite.h.in b/src/sqlite.h.in index b4081f2a02..d0f213f156 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1532,10 +1532,10 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_SCRATCH]]
    SQLITE_CONFIG_SCRATCH
    **
    ^This option specifies a static memory buffer that SQLite can use for -** scratch memory. There are three arguments: A pointer an 8-byte +** scratch memory. ^(There are three arguments: A pointer an 8-byte ** aligned memory buffer from which the scratch allocations will be ** drawn, the size of each scratch allocation (sz), -** and the maximum number of scratch allocations (N). The sz +** and the maximum number of scratch allocations (N).)^ The sz ** argument must be a multiple of 16. ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f4785411d5..e9715efcf7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -193,10 +193,9 @@ #endif /* -** The SQLITE_DEFAULT_MEMSTATUS macro must be defined as either 0 or 1. -** It determines whether or not the features related to -** SQLITE_CONFIG_MEMSTATUS are available by default or not. This value can -** be overridden at runtime using the sqlite3_config() API. +** EVIDENCE-OF: R-25715-37072 Memory allocation statistics are enabled by +** default unless SQLite is compiled with SQLITE_DEFAULT_MEMSTATUS=0 in +** which case memory allocation statistics are disabled by default. */ #if !defined(SQLITE_DEFAULT_MEMSTATUS) # define SQLITE_DEFAULT_MEMSTATUS 1 From 6f9702ed4dd4a6312e8a4e4ec9ae174d931298c8 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 1 Nov 2014 20:38:06 +0000 Subject: [PATCH 521/710] If SQLITE_ENABLE_STMT_SCANSTATUS is defined, record the number of times each VDBE opcode is executed. Derive the values returned by sqlite3_stmt_scanstatus() from these records on demand. FossilOrigin-Name: 9ea37422a8cc2fce51bb10508e5e90f40fd4b511 --- manifest | 26 ++++----- manifest.uuid | 2 +- src/vdbe.c | 44 +------------- src/vdbe.h | 22 ++----- src/vdbeInt.h | 18 ++++-- src/vdbeapi.c | 30 +++++----- src/vdbeaux.c | 52 ++++++++++++----- src/where.c | 134 +++++++++++++++++++++---------------------- src/whereInt.h | 3 + test/scanstatus.test | 49 +++++++++++++--- 10 files changed, 203 insertions(+), 177 deletions(-) diff --git a/manifest b/manifest index 8582d6b5e8..c81e911395 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sfixes\sand\sdocumentation\simprovements\sfor\ssqlite3_stmt_scanstatus(). -D 2014-11-01T18:08:04.130 +C If\sSQLITE_ENABLE_STMT_SCANSTATUS\sis\sdefined,\srecord\sthe\snumber\sof\stimes\seach\sVDBE\sopcode\sis\sexecuted.\sDerive\sthe\svalues\sreturned\sby\ssqlite3_stmt_scanstatus()\sfrom\sthese\srecords\son\sdemand. +D 2014-11-01T20:38:06.833 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,11 +289,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 5240bad20ce0b707afee211f0c21a862b54b1e0b -F src/vdbe.h f8e5388173dbf8408da4ae1dcbf75911caf2253d -F src/vdbeInt.h 284b2294c188474daa9b5ce2bd0200560cb0940d -F src/vdbeapi.c 2afa2e162f290879ca9c51e8795a377035f1662a -F src/vdbeaux.c 1e2561eeb94749f6b8c0a1eb02b5fecb645ed48c +F src/vdbe.c 69d025732d242d7c97282e6570a4e5eb768ebaff +F src/vdbe.h 7d603b93d128e614ba2600f12a6c541435405522 +F src/vdbeInt.h ee8d44cba5998279039bcd91ebddb6d9a2d463b8 +F src/vdbeapi.c 19e433e69fe2b27bfc9337a207b6ebf499f41d03 +F src/vdbeaux.c bad342af7cadb8d3bf2990700a78787d602dcd69 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -302,8 +302,8 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 636ca646c8f2f53fd66f9ed5c773e84cb0c4b762 -F src/whereInt.h 99d324a8f921d7a40c605a8b197350c3cb18977d +F src/where.c fb404e3db40db61c212ffb39022e885a91252498 +F src/whereInt.h a2bc22f4e3e70eeaa57272f354c288bc3b71b80b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -801,7 +801,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 -F test/scanstatus.test b4b1780bad243e1576329d05597b99f06886e4d2 +F test/scanstatus.test 40c7712c8bc0adc3fb88e0419356880679a3a5fb F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6a9bab34aeb6a01b612211a28c140de60a3e883c -R d964f269915d57cf95d9198c6ba22665 +P 8d8cc9608d30bb65fffcfe488e904411cbbc7f41 +R e1e809461d373caeeb0e83f126874426 U dan -Z 082decdd6298ff1a2330407f8d406dfb +Z 5d5debb04affeaeb7378f6b538c8f641 diff --git a/manifest.uuid b/manifest.uuid index 7325586095..f70d7e8aef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8d8cc9608d30bb65fffcfe488e904411cbbc7f41 \ No newline at end of file +9ea37422a8cc2fce51bb10508e5e90f40fd4b511 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 1abccfeeab..bc49505dd0 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -167,18 +167,6 @@ int sqlite3_found_count = 0; /* Return true if the cursor was opened using the OP_OpenSorter opcode. */ #define isSorter(x) ((x)->pSorter!=0) -/* -** The first argument passed to the IncrementExplainCounter() macro must -** be a non-NULL pointer to an object of type VdbeCursor. The second -** argument must be either "nLoop" or "nVisit" (without the double-quotes). -*/ -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -# define IncrementExplainCounter(pCsr, counter) \ - if( (pCsr)->pExplain ) (pCsr)->pExplain->counter++ -#else -# define IncrementExplainCounter(pCsr, counter) -#endif - /* ** Allocate VdbeCursor number iCur. Return a pointer to it. Return NULL ** if we run out of memory. @@ -620,6 +608,9 @@ int sqlite3VdbeExec( #endif nVmStep++; pOp = &aOp[pc]; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + if( p->pFrame==0 ) p->anExec[pc]++; +#endif /* Only allow tracing if SQLITE_DEBUG is defined. */ @@ -3568,7 +3559,6 @@ case OP_SeekGT: { /* jump, in3 */ #ifdef SQLITE_DEBUG pC->seekOp = pOp->opcode; #endif - IncrementExplainCounter(pC, nLoop); if( pC->isTable ){ /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do @@ -3677,8 +3667,6 @@ case OP_SeekGT: { /* jump, in3 */ VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; - }else{ - IncrementExplainCounter(pC, nVisit); } break; } @@ -3889,7 +3877,6 @@ case OP_NotExists: { /* jump, in3 */ res = 0; iKey = pIn3->u.i; rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); - IncrementExplainCounter(pC, nLoop); pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; pC->cacheStatus = CACHE_STALE; @@ -3897,8 +3884,6 @@ case OP_NotExists: { /* jump, in3 */ VdbeBranchTaken(res!=0,2); if( res!=0 ){ pc = pOp->p2 - 1; - }else{ - IncrementExplainCounter(pC, nVisit); } pC->seekResult = res; break; @@ -4467,8 +4452,6 @@ case OP_Last: { /* jump */ res = 0; assert( pCrsr!=0 ); rc = sqlite3BtreeLast(pCrsr, &res); - IncrementExplainCounter(pC, nLoop); - if( res==0 ) IncrementExplainCounter(pC, nVisit); pC->nullRow = (u8)res; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -4538,14 +4521,11 @@ case OP_Rewind: { /* jump */ pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; } - IncrementExplainCounter(pC, nLoop); pC->nullRow = (u8)res; assert( pOp->p2>0 && pOp->p2nOp ); VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; - }else{ - IncrementExplainCounter(pC, nVisit); } break; } @@ -4656,7 +4636,6 @@ next_tail: pC->cacheStatus = CACHE_STALE; VdbeBranchTaken(res==0,2); if( res==0 ){ - IncrementExplainCounter(pC, nVisit); pC->nullRow = 0; pc = pOp->p2 - 1; p->aCounter[pOp->p5]++; @@ -6079,10 +6058,7 @@ case OP_VFilter: { /* jump */ VdbeBranchTaken(res!=0,2); if( res ){ pc = pOp->p2 - 1; - }else{ - IncrementExplainCounter(pCur, nVisit); } - IncrementExplainCounter(pCur, nLoop); } pCur->nullRow = 0; @@ -6175,7 +6151,6 @@ case OP_VNext: { /* jump */ if( !res ){ /* If there is data, jump to P2 */ pc = pOp->p2 - 1; - IncrementExplainCounter(pCur, nVisit); } goto check_for_interrupt; } @@ -6375,19 +6350,6 @@ case OP_Init: { /* jump */ break; } -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -case OP_Explain: { - ExplainArg *pArg; - VdbeCursor *pCur; - if( pOp->p4type==P4_EXPLAIN && (pArg = pOp->p4.pExplain)->iCsr>=0 ){ - pArg = pOp->p4.pExplain; - pCur = p->apCsr[pArg->iCsr]; - pCur->pExplain = pArg; - } - break; -} -#endif - /* Opcode: Noop * * * * * ** diff --git a/src/vdbe.h b/src/vdbe.h index 1b470441be..966fdb622e 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -25,7 +25,6 @@ ** of this structure. */ typedef struct Vdbe Vdbe; -typedef struct ExplainArg ExplainArg; /* ** The names of the following types declared in vdbeInt.h are required @@ -60,7 +59,6 @@ struct VdbeOp { KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ int *ai; /* Used when p4type is P4_INTARRAY */ SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ - ExplainArg *pExplain; /* Used when p4type is P4_EXPLAIN */ int (*xAdvance)(BtCursor *, int *); } p4; #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS @@ -102,19 +100,6 @@ struct VdbeOpList { }; typedef struct VdbeOpList VdbeOpList; -/* -** Structure used as the P4 parameter for OP_Explain opcodes. -*/ -struct ExplainArg { - int iCsr; /* Cursor number this applies to */ - i64 nLoop; /* Number of times loop has run */ - i64 nVisit; /* Total number of rows visited */ - i64 nEst; /* Estimated number of rows per scan */ - const char *zName; /* Name of table/index being scanned */ - const char *zExplain; /* EQP text for this loop */ -}; - - /* ** Allowed values of VdbeOp.p4type */ @@ -134,7 +119,6 @@ struct ExplainArg { #define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ #define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ #define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ -#define P4_EXPLAIN (-20) /* P4 is a pointer to an instance of ExplainArg */ /* Error message codes for OP_Halt */ #define P5_ConstraintNotNull 1 @@ -298,4 +282,10 @@ void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); # define VDBE_OFFSET_LINENO(x) 0 #endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +void sqlite3VdbeScanCounter(Vdbe*, int, int, int, i64, const char*); +#else +# define sqlite3VdbeScanCounter(a,b,c,d,e) +#endif + #endif diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 839108f6bc..9f21ac7ebf 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -83,7 +83,6 @@ struct VdbeCursor { i64 seqCount; /* Sequence counter */ i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ VdbeSorter *pSorter; /* Sorter object for OP_SorterOpen cursors */ - ExplainArg *pExplain; /* Object to store seek/visit counts (may be NULL) */ /* Cached information about the header for the data record that the ** cursor is currently pointing to. Only valid if cacheStatus matches @@ -292,12 +291,20 @@ struct Explain { char zBase[100]; /* Initial space */ }; - /* A bitfield type for use inside of structures. Always follow with :N where ** N is the number of bits. */ typedef unsigned bft; /* Bit Field Type */ +typedef struct ScanCounter ScanCounter; +struct ScanCounter { + int addrExplain; /* OP_Explain for loop */ + int addrLoop; /* Address of "loops" counter */ + int addrVisit; /* Address of "rows visited" counter */ + i64 nEst; /* Estimated rows per loop */ + char *zName; /* Name of table or index */ +}; + /* ** An instance of the virtual machine. This structure contains the complete ** state of the virtual machine. @@ -370,8 +377,11 @@ struct Vdbe { int nOnceFlag; /* Size of array aOnceFlag[] */ u8 *aOnceFlag; /* Flags for OP_Once */ AuxData *pAuxData; /* Linked list of auxdata allocations */ - ExplainArg **apExplain; /* Array of pointers to P4_EXPLAIN p4 values */ - int nExplain; /* Number of entries in array apExplain */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + i64 *anExec; /* Number of times each op has been executed */ + int nScan; /* Entries in aScan[] */ + ScanCounter *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ +#endif }; /* diff --git a/src/vdbeapi.c b/src/vdbeapi.c index b09faa1fa5..f2e124dc51 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1476,6 +1476,7 @@ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ return (int)v; } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* ** Return status data for a single loop within query pStmt. */ @@ -1489,14 +1490,20 @@ int sqlite3_stmt_scanstatus( const char **pzExplain /* OUT: EQP string */ ){ Vdbe *p = (Vdbe*)pStmt; - ExplainArg *pExplain; - if( idx<0 || idx>=p->nExplain ) return 1; - pExplain = p->apExplain[idx]; - if( pnLoop ) *pnLoop = pExplain->nLoop; - if( pnVisit ) *pnVisit = pExplain->nVisit; - if( pnEst ) *pnEst = pExplain->nEst; - if( *pzName ) *pzName = pExplain->zName; - if( *pzExplain ) *pzExplain = pExplain->zExplain; + ScanCounter *pScan; + if( idx<0 || idx>=p->nScan ) return 1; + pScan = &p->aScan[idx]; + if( pnLoop ) *pnLoop = p->anExec[pScan->addrLoop]; + if( pnVisit ) *pnVisit = p->anExec[pScan->addrVisit]; + if( pnEst ) *pnEst = pScan->nEst; + if( *pzName ) *pzName = pScan->zName; + if( *pzExplain ){ + if( pScan->addrExplain ){ + *pzExplain = p->aOp[ pScan->addrExplain ].p4.z; + }else{ + *pzExplain = 0; + } + } return 0; } @@ -1505,10 +1512,7 @@ int sqlite3_stmt_scanstatus( */ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; - int i; - for(i=0; inExplain; i++){ - p->apExplain[i]->nLoop = 0; - p->apExplain[i]->nVisit = 0; - } + memset(p->anExec, 0, p->nOp * sizeof(i64)); } +#endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index a8223890bd..3f75cfee7a 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -597,6 +597,34 @@ int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ return addr; } +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) +/* +** Add an entry to the array of counters managed by sqlite3_stmt_scanstatus(). +*/ +void sqlite3VdbeScanCounter( + Vdbe *p, /* VM to add scanstatus() to */ + int addrExplain, /* Address of OP_Explain (or 0) */ + int addrLoop, /* Address of loop counter */ + int addrVisit, /* Address of rows visited counter */ + i64 nEst, /* Estimated number of rows */ + const char *zName /* Name of table or index being scanned */ +){ + int nByte = (p->nScan+1) * sizeof(ScanCounter); + ScanCounter *aNew; + aNew = (ScanCounter*)sqlite3DbRealloc(p->db, p->aScan, nByte); + if( aNew ){ + ScanCounter *pNew = &aNew[p->nScan++]; + pNew->addrExplain = addrExplain; + pNew->addrLoop = addrLoop; + pNew->addrVisit = addrVisit; + pNew->nEst = nEst; + pNew->zName = sqlite3DbStrDup(p->db, zName); + p->aScan = aNew; + } +} +#endif + + /* ** Change the value of the P1 operand for a specific instruction. ** This routine is useful when a large program is loaded from a @@ -672,7 +700,6 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( p4 ){ assert( db ); switch( p4type ){ - case P4_EXPLAIN: case P4_REAL: case P4_INT64: case P4_DYNAMIC: @@ -821,13 +848,6 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ pOp->p4type = P4_VTAB; sqlite3VtabLock((VTable *)zP4); assert( ((VTable *)zP4)->db==p->db ); - }else if( n==P4_EXPLAIN ){ - pOp->p4.p = (void*)zP4; - pOp->p4type = P4_EXPLAIN; - p->apExplain = (ExplainArg**)sqlite3DbReallocOrFree( - p->db, p->apExplain, sizeof(ExplainArg*) * p->nExplain + 1 - ); - if( p->apExplain ) p->apExplain[p->nExplain++] = (ExplainArg*)zP4; }else if( n<0 ){ pOp->p4.p = (void*)zP4; pOp->p4type = (signed char)n; @@ -1111,10 +1131,6 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ zTemp[0] = 0; break; } - case P4_EXPLAIN: { - zP4 = (char*)pOp->p4.pExplain->zExplain; - break; - } default: { zP4 = pOp->p4.z; if( zP4==0 ){ @@ -1714,6 +1730,10 @@ void sqlite3VdbeMakeReady( zEnd = &zCsr[nByte]; }while( nByte && !db->mallocFailed ); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + p->anExec = (i64*)sqlite3DbMallocZero(db, p->nOp*sizeof(i64)); +#endif + p->nCursor = nCursor; p->nOnceFlag = nOnce; if( p->aVar ){ @@ -2697,7 +2717,13 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->aColName); sqlite3DbFree(db, p->zSql); sqlite3DbFree(db, p->pFree); - sqlite3DbFree(db, p->apExplain); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + sqlite3DbFree(db, p->anExec); + for(i=0; inScan; i++){ + sqlite3DbFree(db, p->aScan[i].zName); + } + sqlite3DbFree(db, p->aScan); +#endif } /* diff --git a/src/where.c b/src/where.c index ae3c12725e..8151a7057b 100644 --- a/src/where.c +++ b/src/where.c @@ -2739,7 +2739,7 @@ static int codeAllEqualityTerms( return regBase; } -#if !defined(SQLITE_OMIT_EXPLAIN) || defined(SQLITE_ENABLE_STMT_SCANSTATUS) +#ifndef SQLITE_OMIT_EXPLAIN /* ** This routine is a helper for explainIndexRange() below ** @@ -2812,68 +2812,43 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ ** record is added to the output to describe the table scan strategy in ** pLevel. */ -static void explainOneScan( +static int explainOneScan( Parse *pParse, /* Parse context */ SrcList *pTabList, /* Table list this loop refers to */ - WhereInfo *pWInfo, /* WHERE clause this loop belongs to */ + WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ int iLevel, /* Value for "level" column of output */ + int iFrom, /* Value for "from" column of output */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) + int ret = 0; +#ifndef SQLITE_DEBUG if( pParse->explain==2 ) #endif { - WhereLevel *pLevel = &pWInfo->a[iLevel]; struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite3 *db = pParse->db; /* Database handle */ -#ifdef SQLITE_OMIT_EXPLAIN - int iId = 0; /* Select id (left-most output column) */ -#else - int iId = pParse->iSelectId; /* Select id (left-most output column) */ -#endif + int iId = pParse->iSelectId; /* Select id (left-most output column) */ int isSearch; /* True for a SEARCH. False for SCAN. */ WhereLoop *pLoop; /* The controlling WhereLoop object */ u32 flags; /* Flags that describe this loop */ + char *zMsg; /* Text to add to EQP output */ StrAccum str; /* EQP output string */ char zBuf[100]; /* Initial space for EQP output string */ - const char *zObj; - ExplainArg *pExplain; - i64 nEstRow; /* Estimated rows per scan of pLevel */ pLoop = pLevel->pWLoop; flags = pLoop->wsFlags; - if( (flags&WHERE_MULTI_OR) ) return; + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return 0; - sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); - str.db = db; - - /* Reserve space at the start of the buffer managed by the StrAccum - ** object for *pExplain. */ - assert( sizeof(*pExplain)<=sizeof(zBuf) ); - str.nChar = sizeof(*pExplain); - - /* Append the object (table or index) name to the buffer. */ - if( pItem->pSelect ){ - zObj = ""; - }else if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ - zObj = pLoop->u.btree.pIndex->zName; - }else{ - zObj = pItem->zName; - } - sqlite3StrAccumAppend(&str, zObj, sqlite3Strlen30(zObj)+1); - - /* Append the EQP text to the buffer */ isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); + + sqlite3StrAccumInit(&str, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + str.db = db; sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); if( pItem->pSelect ){ -#ifdef SQLITE_OMIT_EXPLAIN - sqlite3XPrintf(&str, 0, " SUBQUERY 0"); -#else sqlite3XPrintf(&str, 0, " SUBQUERY %d", pItem->iSelectId); -#endif }else{ sqlite3XPrintf(&str, 0, " TABLE %s", pItem->zName); } @@ -2927,33 +2902,48 @@ static void explainOneScan( pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif - nEstRow = pLoop->nOut>=10 ? sqlite3LogEstToInt(pLoop->nOut) : 1; #ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS - sqlite3XPrintf(&str, 0, " (~%llu rows)", nEstRow); -#endif - pExplain = (ExplainArg*)sqlite3StrAccumFinish(&str); - assert( pExplain || db->mallocFailed ); - if( pExplain ){ - memset(pExplain, 0, sizeof(*pExplain)); - if( pLoop->wsFlags & WHERE_INDEXED ){ - pExplain->iCsr = pLevel->iIdxCur; - }else if( pItem->pSelect==0 ){ - pExplain->iCsr = pLevel->iTabCur; - }else{ - pExplain->iCsr = -1; - } - pExplain->nEst = nEstRow; - pExplain->zName = (const char*)&pExplain[1]; - pExplain->zExplain = &pExplain->zName[sqlite3Strlen30(pExplain->zName)+1]; - pWInfo->iExplain = sqlite3VdbeAddOp4(v, OP_Explain, - iId, iLevel, pLevel->iFrom, (char*)pExplain,P4_EXPLAIN - ); + if( pLoop->nOut>=10 ){ + sqlite3XPrintf(&str, 0, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); + }else{ + sqlite3StrAccumAppend(&str, " (~1 row)", 9); } +#endif + zMsg = sqlite3StrAccumFinish(&str); + ret = sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg,P4_DYNAMIC); } + return ret; } #else -# define explainOneScan(v,w,x,y,z) -#endif /* !SQLITE_OMIT_EXPLAIN || SQLITE_ENABLE_STMT_SCANSTATUS */ +# define explainOneScan(u,v,w,x,y,z) 0 +#endif /* SQLITE_OMIT_EXPLAIN */ + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +static void addScanStatus( + Vdbe *v, + SrcList *pSrclist, + WhereLevel *pLvl, + int addrExplain +){ + const char *zObj = 0; + i64 nEst = 1; + WhereLoop *pLoop = pLvl->pWLoop; + if( (pLoop->wsFlags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + zObj = pLoop->u.btree.pIndex->zName; + }else{ + zObj = pSrclist->a[pLvl->iFrom].zName; + } + if( pLoop->nOut>=10 ){ + nEst = sqlite3LogEstToInt(pLoop->nOut); + } + sqlite3VdbeScanCounter( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, nEst, zObj + ); +} +#else +# define addScanStatus(a, b, c, d) +#endif + /* @@ -3621,14 +3611,10 @@ static Bitmask codeOneLoopStart( assert( pSubWInfo || pParse->nErr || db->mallocFailed ); if( pSubWInfo ){ WhereLoop *pSubLoop; - - /* If an OP_Explain was added for this sub-loop, fix the P2 and - ** P3 parameters to it so that they are relative to the current - ** context. */ - if( pSubWInfo->iExplain!=0 ){ - sqlite3VdbeChangeP2(v, pSubWInfo->iExplain, iLevel); - sqlite3VdbeChangeP3(v, pSubWInfo->iExplain, pLevel->iFrom); - } + int addrExplain = explainOneScan( + pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 + ); + addScanStatus(v, pOrTab, &pSubWInfo->a[0], addrExplain); /* This is the sub-WHERE clause body. First skip over ** duplicate rows from prior sub-WHERE clauses, and record the @@ -3760,6 +3746,10 @@ static Bitmask codeOneLoopStart( } } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pLevel->addrVisit = sqlite3VdbeCurrentAddr(v); +#endif + /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. */ @@ -6461,7 +6451,10 @@ WhereInfo *sqlite3WhereBegin( */ notReady = ~(Bitmask)0; for(ii=0; iia[ii]; + wsFlags = pLevel->pWLoop->wsFlags; #ifndef SQLITE_OMIT_AUTOMATIC_INDEX if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ constructAutomaticIndex(pParse, &pWInfo->sWC, @@ -6469,10 +6462,15 @@ WhereInfo *sqlite3WhereBegin( if( db->mallocFailed ) goto whereBeginError; } #endif - explainOneScan(pParse, pTabList, pWInfo, ii, wctrlFlags); + addrExplain = explainOneScan( + pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags + ); pLevel->addrBody = sqlite3VdbeCurrentAddr(v); notReady = codeOneLoopStart(pWInfo, ii, notReady); pWInfo->iContinue = pLevel->addrCont; + if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_ONETABLE_ONLY)==0 ){ + addScanStatus(v, pTabList, pLevel, addrExplain); + } } /* Done. */ diff --git a/src/whereInt.h b/src/whereInt.h index dcf0e00604..168e11f4c7 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -85,6 +85,9 @@ struct WhereLevel { } u; struct WhereLoop *pWLoop; /* The selected WhereLoop object */ Bitmask notReady; /* FROM entries not usable at this level */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrVisit; /* Address at which row is visited */ +#endif }; /* diff --git a/test/scanstatus.test b/test/scanstatus.test index 9df80f9ba8..e4e0bbd4a5 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -115,7 +115,7 @@ do_execsql_test 2.4.1 { SELECT * FROM x1 WHERE j<'two' } {4 four 1 one 3 three} do_scanstatus_test 2.4.2 { - nLoop 1 nVisit 4 nEst 262144 zName x1j + nLoop 1 nVisit 3 nEst 262144 zName x1j zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j? AND a? AND b? AND b? AND a Date: Sat, 1 Nov 2014 21:00:04 +0000 Subject: [PATCH 522/710] Minor performance enhancements to SQLITE_ENABLE_STMT_SCANSTATUS code. FossilOrigin-Name: f13d6ba8a72d75838c4aaf85326c1129da027f8b --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/vdbe.c | 4 +++- src/vdbeInt.h | 1 + src/vdbeaux.c | 7 ++----- src/where.c | 2 +- 6 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index c81e911395..974494b3b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sSQLITE_ENABLE_STMT_SCANSTATUS\sis\sdefined,\srecord\sthe\snumber\sof\stimes\seach\sVDBE\sopcode\sis\sexecuted.\sDerive\sthe\svalues\sreturned\sby\ssqlite3_stmt_scanstatus()\sfrom\sthese\srecords\son\sdemand. -D 2014-11-01T20:38:06.833 +C Minor\sperformance\senhancements\sto\sSQLITE_ENABLE_STMT_SCANSTATUS\scode. +D 2014-11-01T21:00:04.841 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,11 +289,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 69d025732d242d7c97282e6570a4e5eb768ebaff +F src/vdbe.c ccc626d70659160596d28b4a910b6086da788695 F src/vdbe.h 7d603b93d128e614ba2600f12a6c541435405522 -F src/vdbeInt.h ee8d44cba5998279039bcd91ebddb6d9a2d463b8 +F src/vdbeInt.h 21570e5ec8b3a385d003e6e20f3a91712b7050e5 F src/vdbeapi.c 19e433e69fe2b27bfc9337a207b6ebf499f41d03 -F src/vdbeaux.c bad342af7cadb8d3bf2990700a78787d602dcd69 +F src/vdbeaux.c 2887d02721c540b25969d3260d0d3d5668d11583 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c fb404e3db40db61c212ffb39022e885a91252498 +F src/where.c ad4a3bca9070e013ff41e8e63835788bcc43dd1c F src/whereInt.h a2bc22f4e3e70eeaa57272f354c288bc3b71b80b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8d8cc9608d30bb65fffcfe488e904411cbbc7f41 -R e1e809461d373caeeb0e83f126874426 +P 9ea37422a8cc2fce51bb10508e5e90f40fd4b511 +R 4191876729196c1564c4c28cce5855c1 U dan -Z 5d5debb04affeaeb7378f6b538c8f641 +Z 150c277a759b1b656399ef38dfe29c92 diff --git a/manifest.uuid b/manifest.uuid index f70d7e8aef..72c2809a53 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ea37422a8cc2fce51bb10508e5e90f40fd4b511 \ No newline at end of file +f13d6ba8a72d75838c4aaf85326c1129da027f8b \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index bc49505dd0..890162dddc 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -609,7 +609,7 @@ int sqlite3VdbeExec( nVmStep++; pOp = &aOp[pc]; #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - if( p->pFrame==0 ) p->anExec[pc]++; + if( p->anExec ) p->anExec[pc]++; #endif /* Only allow tracing if SQLITE_DEBUG is defined. @@ -5409,6 +5409,7 @@ case OP_Program: { /* jump */ pFrame->token = pProgram->token; pFrame->aOnceFlag = p->aOnceFlag; pFrame->nOnceFlag = p->nOnceFlag; + pFrame->anExec = p->anExec; pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ @@ -5437,6 +5438,7 @@ case OP_Program: { /* jump */ p->nOp = pProgram->nOp; p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; p->nOnceFlag = pProgram->nOnce; + p->anExec = 0; pc = -1; memset(p->aOnceFlag, 0, p->nOnceFlag); diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 9f21ac7ebf..eb2438a87c 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -132,6 +132,7 @@ struct VdbeFrame { Vdbe *v; /* VM this frame belongs to */ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ Op *aOp; /* Program instructions for parent frame */ + i64 *anExec; /* Event counters from parent frame */ Mem *aMem; /* Array of memory cells for parent frame */ u8 *aOnceFlag; /* Array of OP_Once flags for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 3f75cfee7a..548ef0ac5c 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1723,6 +1723,7 @@ void sqlite3VdbeMakeReady( p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), &zCsr, zEnd, &nByte); p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, &zCsr, zEnd, &nByte); + p->anExec = allocSpace(p->anExec, p->nOp*sizeof(i64), &zCsr, zEnd, &nByte); if( nByte ){ p->pFree = sqlite3DbMallocZero(db, nByte); } @@ -1730,10 +1731,6 @@ void sqlite3VdbeMakeReady( zEnd = &zCsr[nByte]; }while( nByte && !db->mallocFailed ); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = (i64*)sqlite3DbMallocZero(db, p->nOp*sizeof(i64)); -#endif - p->nCursor = nCursor; p->nOnceFlag = nOnce; if( p->aVar ){ @@ -1794,6 +1791,7 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ */ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; + v->anExec = pFrame->anExec; v->aOnceFlag = pFrame->aOnceFlag; v->nOnceFlag = pFrame->nOnceFlag; v->aOp = pFrame->aOp; @@ -2718,7 +2716,6 @@ void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ sqlite3DbFree(db, p->zSql); sqlite3DbFree(db, p->pFree); #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - sqlite3DbFree(db, p->anExec); for(i=0; inScan; i++){ sqlite3DbFree(db, p->aScan[i].zName); } diff --git a/src/where.c b/src/where.c index 8151a7057b..61c0ab7d3d 100644 --- a/src/where.c +++ b/src/where.c @@ -2821,7 +2821,7 @@ static int explainOneScan( u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ int ret = 0; -#ifndef SQLITE_DEBUG +#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) if( pParse->explain==2 ) #endif { From 037b5324bd634e6288d7502b560bf53f273f0c94 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Nov 2014 11:25:32 +0000 Subject: [PATCH 523/710] Remove unused variable from struct WhereInfo. Add some explanatory comments to new code. FossilOrigin-Name: f5313e0c680d9baebefb1cf50ddadedd4418a334 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/vdbe.h | 4 ++-- src/vdbeInt.h | 6 +++--- src/vdbeapi.c | 2 +- src/vdbeaux.c | 10 +++++----- src/where.c | 28 ++++++++++++++++++++-------- src/whereInt.h | 1 - test/scanstatus.test | 35 +++++++++++++++++++++++++++++++++++ 9 files changed, 79 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 974494b3b9..9b7cf5c34a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sperformance\senhancements\sto\sSQLITE_ENABLE_STMT_SCANSTATUS\scode. -D 2014-11-01T21:00:04.841 +C Remove\sunused\svariable\sfrom\sstruct\sWhereInfo.\sAdd\ssome\sexplanatory\scomments\sto\snew\scode. +D 2014-11-03T11:25:32.305 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,10 +290,10 @@ F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c ccc626d70659160596d28b4a910b6086da788695 -F src/vdbe.h 7d603b93d128e614ba2600f12a6c541435405522 -F src/vdbeInt.h 21570e5ec8b3a385d003e6e20f3a91712b7050e5 -F src/vdbeapi.c 19e433e69fe2b27bfc9337a207b6ebf499f41d03 -F src/vdbeaux.c 2887d02721c540b25969d3260d0d3d5668d11583 +F src/vdbe.h d412bd01e89f0d69991b8f46601f96bc169d28f4 +F src/vdbeInt.h 539ba284790e871f98be74a78cbdfcedfae22639 +F src/vdbeapi.c addf446ecade237bebd7e9fe769bdfb9db8d9fb1 +F src/vdbeaux.c 0aeb90cb62d7c07572798b41882838b3d4e55b44 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -302,8 +302,8 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c ad4a3bca9070e013ff41e8e63835788bcc43dd1c -F src/whereInt.h a2bc22f4e3e70eeaa57272f354c288bc3b71b80b +F src/where.c 53dae5ed6133438a9342c17bf3e95e00edbb0556 +F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test b35b4cd69fc913f90d39a575e171e1116c3a4bb7 @@ -801,7 +801,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 -F test/scanstatus.test 40c7712c8bc0adc3fb88e0419356880679a3a5fb +F test/scanstatus.test 0c0baa647e98940d753d40691bf6475345c05cc5 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9ea37422a8cc2fce51bb10508e5e90f40fd4b511 -R 4191876729196c1564c4c28cce5855c1 +P f13d6ba8a72d75838c4aaf85326c1129da027f8b +R 79a96158570090f7f8b0799c9e59db67 U dan -Z 150c277a759b1b656399ef38dfe29c92 +Z b85dee7fa48b3d6596e077d675db2dc9 diff --git a/manifest.uuid b/manifest.uuid index 72c2809a53..5972c23854 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f13d6ba8a72d75838c4aaf85326c1129da027f8b \ No newline at end of file +f5313e0c680d9baebefb1cf50ddadedd4418a334 \ No newline at end of file diff --git a/src/vdbe.h b/src/vdbe.h index 966fdb622e..1b9ad8b6bf 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -283,9 +283,9 @@ void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); #endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS -void sqlite3VdbeScanCounter(Vdbe*, int, int, int, i64, const char*); +void sqlite3VdbeScanStatus(Vdbe*, int, int, int, i64, const char*); #else -# define sqlite3VdbeScanCounter(a,b,c,d,e) +# define sqlite3VdbeScanStatus(a,b,c,d,e) #endif #endif diff --git a/src/vdbeInt.h b/src/vdbeInt.h index eb2438a87c..29117dd064 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -297,8 +297,8 @@ struct Explain { */ typedef unsigned bft; /* Bit Field Type */ -typedef struct ScanCounter ScanCounter; -struct ScanCounter { +typedef struct ScanStatus ScanStatus; +struct ScanStatus { int addrExplain; /* OP_Explain for loop */ int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ @@ -381,7 +381,7 @@ struct Vdbe { #ifdef SQLITE_ENABLE_STMT_SCANSTATUS i64 *anExec; /* Number of times each op has been executed */ int nScan; /* Entries in aScan[] */ - ScanCounter *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ + ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ #endif }; diff --git a/src/vdbeapi.c b/src/vdbeapi.c index f2e124dc51..e6eb034ae9 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1490,7 +1490,7 @@ int sqlite3_stmt_scanstatus( const char **pzExplain /* OUT: EQP string */ ){ Vdbe *p = (Vdbe*)pStmt; - ScanCounter *pScan; + ScanStatus *pScan; if( idx<0 || idx>=p->nScan ) return 1; pScan = &p->aScan[idx]; if( pnLoop ) *pnLoop = p->anExec[pScan->addrLoop]; diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 548ef0ac5c..da24291b4e 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -601,7 +601,7 @@ int sqlite3VdbeAddOpList(Vdbe *p, int nOp, VdbeOpList const *aOp, int iLineno){ /* ** Add an entry to the array of counters managed by sqlite3_stmt_scanstatus(). */ -void sqlite3VdbeScanCounter( +void sqlite3VdbeScanStatus( Vdbe *p, /* VM to add scanstatus() to */ int addrExplain, /* Address of OP_Explain (or 0) */ int addrLoop, /* Address of loop counter */ @@ -609,11 +609,11 @@ void sqlite3VdbeScanCounter( i64 nEst, /* Estimated number of rows */ const char *zName /* Name of table or index being scanned */ ){ - int nByte = (p->nScan+1) * sizeof(ScanCounter); - ScanCounter *aNew; - aNew = (ScanCounter*)sqlite3DbRealloc(p->db, p->aScan, nByte); + int nByte = (p->nScan+1) * sizeof(ScanStatus); + ScanStatus *aNew; + aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); if( aNew ){ - ScanCounter *pNew = &aNew[p->nScan++]; + ScanStatus *pNew = &aNew[p->nScan++]; pNew->addrExplain = addrExplain; pNew->addrLoop = addrLoop; pNew->addrVisit = addrVisit; diff --git a/src/where.c b/src/where.c index 61c0ab7d3d..81298b7821 100644 --- a/src/where.c +++ b/src/where.c @@ -2808,9 +2808,12 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop, Table *pTab){ /* ** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command. If the query being compiled is an EXPLAIN QUERY PLAN, a single -** record is added to the output to describe the table scan strategy in -** pLevel. +** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was +** defined at compile-time. If it is not a no-op, a single OP_Explain opcode +** is added to the output to describe the table scan strategy in pLevel. +** +** If an OP_Explain opcode is added to the VM, its address is returned. +** Otherwise, if no OP_Explain is coded, zero is returned. */ static int explainOneScan( Parse *pParse, /* Parse context */ @@ -2919,11 +2922,20 @@ static int explainOneScan( #endif /* SQLITE_OMIT_EXPLAIN */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Configure the VM passed as the first argument with an +** sqlite3_stmt_scanstatus() entry corresponding to the scan used to +** implement level pLvl. Argument pSrclist is a pointer to the FROM +** clause that the scan reads data from. +** +** If argument addrExplain is not 0, it must be the address of an +** OP_Explain instruction that describes the same loop. +*/ static void addScanStatus( - Vdbe *v, - SrcList *pSrclist, - WhereLevel *pLvl, - int addrExplain + Vdbe *v, /* Vdbe to add scanstatus entry to */ + SrcList *pSrclist, /* FROM clause pLvl reads data from */ + WhereLevel *pLvl, /* Level to add scanstatus() entry for */ + int addrExplain /* Address of OP_Explain (or 0) */ ){ const char *zObj = 0; i64 nEst = 1; @@ -2936,7 +2948,7 @@ static void addScanStatus( if( pLoop->nOut>=10 ){ nEst = sqlite3LogEstToInt(pLoop->nOut); } - sqlite3VdbeScanCounter( + sqlite3VdbeScanStatus( v, addrExplain, pLvl->addrBody, pLvl->addrVisit, nEst, zObj ); } diff --git a/src/whereInt.h b/src/whereInt.h index 168e11f4c7..2ccc6ec064 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -410,7 +410,6 @@ struct WhereInfo { int iTop; /* The very beginning of the WHERE loop */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ - int iExplain; /* Address of OP_Explain (if WHERE_ONETABLE_ONLY) */ int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ diff --git a/test/scanstatus.test b/test/scanstatus.test index e4e0bbd4a5..d4c70fbef4 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -231,4 +231,39 @@ do_scanstatus_test 3.4.2 { zExplain {SEARCH TABLE a1 USING INTEGER PRIMARY KEY (rowid=?)} } +#------------------------------------------------------------------------- +# Test that scanstatus() data is not available for searches performed +# by triggers. +# +# It is available for searches performed as part of FK processing, but +# not FK action processing. +# +do_execsql_test 4.0 { + CREATE TABLE t1(a, b, c); + CREATE TABLE t2(x PRIMARY KEY, y, z); + CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN + SELECT * FROM t2 WHERE x BETWEEN 20 AND 40; + END; + WITH d(x) AS (SELECT 1 UNION ALL SELECT x+1 AS n FROM d WHERE n<=100) + INSERT INTO t2 SELECT x, x*2, x*3 FROM d; +} + +do_execsql_test 4.1.1 { INSERT INTO t1 VALUES(1, 2, 3); } +do_scanstatus_test 4.1.2 { } + +do_execsql_test 4.2 { + CREATE TABLE p1(x PRIMARY KEY); + INSERT INTO p1 VALUES(1), (2), (3), (4); + CREATE TABLE c1(y REFERENCES p1); + INSERT INTO c1 VALUES(1), (2), (3); + PRAGMA foreign_keys=on; +} +do_execsql_test 4.2.1 { DELETE FROM p1 WHERE x=4 } +do_scanstatus_test 4.2.2 { + nLoop 1 nVisit 1 nEst 1 zName sqlite_autoindex_p1_1 + zExplain {SEARCH TABLE p1 USING INDEX sqlite_autoindex_p1_1 (x=?)} + + nLoop 1 nVisit 3 nEst 524288 zName c1 zExplain {SCAN TABLE c1} +} + finish_test From 7f5a7ecd216747e738588ad32064a182d2ce2928 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Nov 2014 13:24:12 +0000 Subject: [PATCH 524/710] When enlarging the size of a StrAccum object, use sqlite3DbMallocSize() to record the entire size of the allocation, not just the requested size. FossilOrigin-Name: 3dda3c937469ce661d1cd0e8d8963a03e698ee39 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 8ab904b8d2..3034b1c180 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\srequirements\smarks\sand\smake\sminor\stweaks\sto\sdocumentation. -D 2014-11-01T18:32:18.050 +C When\senlarging\sthe\ssize\sof\sa\sStrAccum\sobject,\suse\ssqlite3DbMallocSize()\sto\nrecord\sthe\sentire\ssize\sof\sthe\sallocation,\snot\sjust\sthe\srequested\ssize. +D 2014-11-03T13:24:12.668 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 -F src/printf.c 10a2493593c8e4a538915cd3674bd7a67f70c488 +F src/printf.c ee13daccd49ddf767d057bc98395c9e83d4e2067 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 67f0d469da28c023200239a1f3d0c6cef9ef0e45 -R dbf6a8dc4a878d1b1a7b9b3874ca58fb +P 49188b2bb53a92b0b0b6aaf8247edeb0c1bcd1f5 +R 041ffec3947ff9e90700478ea54978f5 U drh -Z 455c35adeade418fab0456b1b175d27f +Z 42c888343c3ca87538d56b9b17ae76ab diff --git a/manifest.uuid b/manifest.uuid index 02f906627f..8be6ab9856 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -49188b2bb53a92b0b0b6aaf8247edeb0c1bcd1f5 \ No newline at end of file +3dda3c937469ce661d1cd0e8d8963a03e698ee39 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 85d237f0e7..e20619f9a7 100644 --- a/src/printf.c +++ b/src/printf.c @@ -786,6 +786,7 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ assert( p->zText!=0 || p->nChar==0 ); if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); p->zText = zNew; + p->nAlloc = sqlite3DbMallocSize(p->db, zNew); }else{ sqlite3StrAccumReset(p); setStrAccumError(p, STRACCUM_NOMEM); From 7b4d780b541ac9af0f5b183c72afa12916fd5d26 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Nov 2014 14:46:29 +0000 Subject: [PATCH 525/710] Use exponential buffer size growth in StrAccum, as long as the size does not grow to large, to avoid excess memory allocation resize operations. Also, document the fact that setting scratch memory causes SQLite to try to avoid large memory allocations. FossilOrigin-Name: a518bc3318232d652349eb29303ff250aee40459 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/printf.c | 5 +++++ src/sqlite.h.in | 11 ++++++++--- src/vdbesort.c | 8 +++----- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 3034b1c180..de67a3e45a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\senlarging\sthe\ssize\sof\sa\sStrAccum\sobject,\suse\ssqlite3DbMallocSize()\sto\nrecord\sthe\sentire\ssize\sof\sthe\sallocation,\snot\sjust\sthe\srequested\ssize. -D 2014-11-03T13:24:12.668 +C Use\sexponential\sbuffer\ssize\sgrowth\sin\sStrAccum,\sas\slong\sas\sthe\ssize\sdoes\snot\ngrow\sto\slarge,\sto\savoid\sexcess\smemory\sallocation\sresize\soperations.\s\sAlso,\ndocument\sthe\sfact\sthat\ssetting\sscratch\smemory\scauses\sSQLite\sto\stry\sto\savoid\nlarge\smemory\sallocations. +D 2014-11-03T14:46:29.027 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,13 +223,13 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 -F src/printf.c ee13daccd49ddf767d057bc98395c9e83d4e2067 +F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in c3c6180be49d91d3b0001a8d3c604684f361adc2 +F src/sqlite.h.in dd0cb63fb7cb558892afe8253c3279c508c50329 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -296,7 +296,7 @@ F src/vdbeapi.c 02d8afcff710eb35e3d9e49cb677308296b00009 F src/vdbeaux.c 3d6b2b412ef2193aa4729922dfc5df1efadbf6df F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f -F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 +F src/vdbesort.c daf87ea542df088ac4607660d09976d36b6bd95d F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 @@ -1210,7 +1210,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 49188b2bb53a92b0b0b6aaf8247edeb0c1bcd1f5 -R 041ffec3947ff9e90700478ea54978f5 +P 3dda3c937469ce661d1cd0e8d8963a03e698ee39 +R e2c699e9cb6d832117ea76de0a03f26b U drh -Z 42c888343c3ca87538d56b9b17ae76ab +Z 91c9a2887aef0ce4f97740bae4c1a9ba diff --git a/manifest.uuid b/manifest.uuid index 8be6ab9856..11815fb0c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3dda3c937469ce661d1cd0e8d8963a03e698ee39 \ No newline at end of file +a518bc3318232d652349eb29303ff250aee40459 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index e20619f9a7..ac7b051631 100644 --- a/src/printf.c +++ b/src/printf.c @@ -770,6 +770,11 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ char *zOld = (p->zText==p->zBase ? 0 : p->zText); i64 szNew = p->nChar; szNew += N + 1; + if( szNew+p->nChar<=p->mxAlloc ){ + /* Force exponential buffer size growth as long as it does not overflow, + ** to avoid having to call this routine too often */ + szNew += p->nChar; + } if( szNew > p->mxAlloc ){ sqlite3StrAccumReset(p); setStrAccumError(p, STRACCUM_TOOBIG); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index d0f213f156..ffd0ade039 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1535,8 +1535,7 @@ struct sqlite3_mem_methods { ** scratch memory. ^(There are three arguments: A pointer an 8-byte ** aligned memory buffer from which the scratch allocations will be ** drawn, the size of each scratch allocation (sz), -** and the maximum number of scratch allocations (N).)^ The sz -** argument must be a multiple of 16. +** and the maximum number of scratch allocations (N).)^ ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. ** ^SQLite will not use more than two scratch buffers per thread and not @@ -1548,7 +1547,13 @@ struct sqlite3_mem_methods { ** of the size of the WAL file. ** ^If SQLite needs needs additional ** scratch memory beyond what is provided by this configuration option, then -** [sqlite3_malloc()] will be used to obtain the memory needed.
    +** [sqlite3_malloc()] will be used to obtain the memory needed.

    +** ^When the application provides any amount of scratch memory using +** SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary large +** [sqlite3_malloc|heap allocations]. +** This can help [Robson proof|prevent memory allocation failures] due to heap +** fragmentation in low-memory embedded systems. +** ** ** [[SQLITE_CONFIG_PAGECACHE]]

    SQLITE_CONFIG_PAGECACHE
    **
    ^This option specifies a static memory buffer that SQLite can use for diff --git a/src/vdbesort.c b/src/vdbesort.c index 46c9f3789d..918d840716 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -847,11 +847,9 @@ int sqlite3VdbeSorterInit( if( mxCachemxPmaSize = mxCache * pgsz; - /* If the application has not configure scratch memory using - ** SQLITE_CONFIG_SCRATCH then we assume it is OK to do large memory - ** allocations. If scratch memory has been configured, then assume - ** large memory allocations should be avoided to prevent heap - ** fragmentation. + /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of + ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary + ** large heap allocations. */ if( sqlite3GlobalConfig.pScratch==0 ){ assert( pSorter->iMemory==0 ); From e2f771b04706d0c055d5c3b7731e4cda14467600 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Nov 2014 15:33:17 +0000 Subject: [PATCH 526/710] Add further tests. Fixes so that compilation without ENABLE_STMT_SCANSTATUS works. FossilOrigin-Name: a2303c719222f1effb51acc6b37930561148c00c --- manifest | 20 +++---- manifest.uuid | 2 +- src/test_config.c | 6 ++ src/vdbe.c | 4 ++ src/vdbeaux.c | 4 ++ src/where.c | 2 +- test/scanstatus.test | 129 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 155 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 9b7cf5c34a..5c51df5fcc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunused\svariable\sfrom\sstruct\sWhereInfo.\sAdd\ssome\sexplanatory\scomments\sto\snew\scode. -D 2014-11-03T11:25:32.305 +C Add\sfurther\stests.\sFixes\sso\sthat\scompilation\swithout\sENABLE_STMT_SCANSTATUS\sworks. +D 2014-11-03T15:33:17.869 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -250,7 +250,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c a4cdebe093474c02eecc5e4008b1a22198edf975 +F src/test_config.c c8b8b50bf2fe5102de10e4c7100b746d7f6bf62f F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -289,11 +289,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c ccc626d70659160596d28b4a910b6086da788695 +F src/vdbe.c 175a360c56e75ce4eb2b60704fd7c011b93926f5 F src/vdbe.h d412bd01e89f0d69991b8f46601f96bc169d28f4 F src/vdbeInt.h 539ba284790e871f98be74a78cbdfcedfae22639 F src/vdbeapi.c addf446ecade237bebd7e9fe769bdfb9db8d9fb1 -F src/vdbeaux.c 0aeb90cb62d7c07572798b41882838b3d4e55b44 +F src/vdbeaux.c cf6b8152dd22155201d57c216e6266866b61da59 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 53dae5ed6133438a9342c17bf3e95e00edbb0556 +F src/where.c d5fa1081bf7cb70478ed06489e063695c92ee1e1 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -801,7 +801,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 -F test/scanstatus.test 0c0baa647e98940d753d40691bf6475345c05cc5 +F test/scanstatus.test 01afb2220f18ce85f9e338c20684f428d56e5c01 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f13d6ba8a72d75838c4aaf85326c1129da027f8b -R 79a96158570090f7f8b0799c9e59db67 +P f5313e0c680d9baebefb1cf50ddadedd4418a334 +R b9cde6c71acdc4fef0c5c1ac03d62018 U dan -Z b85dee7fa48b3d6596e077d675db2dc9 +Z f0600457ea98c0c998694790015a9674 diff --git a/manifest.uuid b/manifest.uuid index 5972c23854..fe7315da12 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f5313e0c680d9baebefb1cf50ddadedd4418a334 \ No newline at end of file +a2303c719222f1effb51acc6b37930561148c00c \ No newline at end of file diff --git a/src/test_config.c b/src/test_config.c index 2c11f713fb..03f4f76f6d 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -484,6 +484,12 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "stat3", "0", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + Tcl_SetVar2(interp, "sqlite_options", "scanstatus", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "scanstatus", "0", TCL_GLOBAL_ONLY); +#endif + #if !defined(SQLITE_ENABLE_LOCKING_STYLE) # if defined(__APPLE__) # define SQLITE_ENABLE_LOCKING_STYLE 1 diff --git a/src/vdbe.c b/src/vdbe.c index 890162dddc..62470a743b 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5409,7 +5409,9 @@ case OP_Program: { /* jump */ pFrame->token = pProgram->token; pFrame->aOnceFlag = p->aOnceFlag; pFrame->nOnceFlag = p->nOnceFlag; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS pFrame->anExec = p->anExec; +#endif pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; for(pMem=VdbeFrameMem(pFrame); pMem!=pEnd; pMem++){ @@ -5438,7 +5440,9 @@ case OP_Program: { /* jump */ p->nOp = pProgram->nOp; p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; p->nOnceFlag = pProgram->nOnce; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS p->anExec = 0; +#endif pc = -1; memset(p->aOnceFlag, 0, p->nOnceFlag); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index da24291b4e..7cf996ce5c 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1723,7 +1723,9 @@ void sqlite3VdbeMakeReady( p->apCsr = allocSpace(p->apCsr, nCursor*sizeof(VdbeCursor*), &zCsr, zEnd, &nByte); p->aOnceFlag = allocSpace(p->aOnceFlag, nOnce, &zCsr, zEnd, &nByte); +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS p->anExec = allocSpace(p->anExec, p->nOp*sizeof(i64), &zCsr, zEnd, &nByte); +#endif if( nByte ){ p->pFree = sqlite3DbMallocZero(db, nByte); } @@ -1791,7 +1793,9 @@ void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ */ int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS v->anExec = pFrame->anExec; +#endif v->aOnceFlag = pFrame->aOnceFlag; v->nOnceFlag = pFrame->nOnceFlag; v->aOp = pFrame->aOp; diff --git a/src/where.c b/src/where.c index 81298b7821..d7966fdfbc 100644 --- a/src/where.c +++ b/src/where.c @@ -2953,7 +2953,7 @@ static void addScanStatus( ); } #else -# define addScanStatus(a, b, c, d) +# define addScanStatus(a, b, c, d) ((void)d) #endif diff --git a/test/scanstatus.test b/test/scanstatus.test index d4c70fbef4..9b34506e91 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -14,6 +14,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix scanstatus +ifcapable !scanstatus { + finish_test + return +} + do_execsql_test 1.0 { CREATE TABLE t1(a, b); CREATE TABLE t2(x, y); @@ -266,4 +271,128 @@ do_scanstatus_test 4.2.2 { nLoop 1 nVisit 3 nEst 524288 zName c1 zExplain {SCAN TABLE c1} } +#------------------------------------------------------------------------- +# Further tests of different scan types. +# +reset_db +proc tochar {i} { + set alphabet {a b c d e f g h i j k l m n o p q r s t u v w x y z} + return [lindex $alphabet [expr $i % [llength $alphabet]]] +} +db func tochar tochar +do_execsql_test 5.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(0, 1, 'a'); + INSERT INTO t1 VALUES(1, 0, 'b'); + INSERT INTO t1 VALUES(2, 1, 'c'); + INSERT INTO t1 VALUES(3, 0, 'd'); + INSERT INTO t1 VALUES(4, 1, 'e'); + INSERT INTO t1 VALUES(5, 0, 'a'); + INSERT INTO t1 VALUES(6, 1, 'b'); + INSERT INTO t1 VALUES(7, 0, 'c'); + INSERT INTO t1 VALUES(8, 1, 'd'); + INSERT INTO t1 VALUES(9, 0, 'e'); + CREATE INDEX t1bc ON t1(b, c); + + CREATE TABLE t2(x, y); + CREATE INDEX t2xy ON t2(x, y); + WITH data(i, x, y) AS ( + SELECT 0, 0, tochar(0) + UNION ALL + SELECT i+1, (i+1)%2, tochar(i+1) FROM data WHERE i<500 + ) INSERT INTO t2 SELECT x, y FROM data; + + CREATE TABLE t3(x, y); + INSERT INTO t3 SELECT * FROM t2; + + ANALYZE; +} + +do_execsql_test 5.1.1 { + SELECT count(*) FROM t1 WHERE a IN (SELECT b FROM t1 AS ii) +} {2} +do_scanstatus_test 5.1.2 { + nLoop 1 nVisit 10 nEst 10 zName t1bc + zExplain {SCAN TABLE t1 AS ii USING COVERING INDEX t1bc} + nLoop 1 nVisit 2 nEst 8 zName sqlite_autoindex_t1_1 + zExplain {SEARCH TABLE t1 USING COVERING INDEX sqlite_autoindex_t1_1 (a=?)} +} + +do_execsql_test 5.2.1 { + SELECT count(*) FROM t1 WHERE a IN (0, 1) +} {2} +do_scanstatus_test 5.2.2 { + nLoop 1 nVisit 2 nEst 2 zName sqlite_autoindex_t1_1 + zExplain {SEARCH TABLE t1 USING COVERING INDEX sqlite_autoindex_t1_1 (a=?)} +} + +do_eqp_test 5.3.1 { + SELECT count(*) FROM t2 WHERE y = 'j'; +} {0 0 0 {SEARCH TABLE t2 USING COVERING INDEX t2xy (ANY(x) AND y=?)}} +do_execsql_test 5.3.2 { + SELECT count(*) FROM t2 WHERE y = 'j'; +} {19} +do_scanstatus_test 5.3.3 { + nLoop 1 nVisit 19 nEst 56 zName t2xy zExplain + {SEARCH TABLE t2 USING COVERING INDEX t2xy (ANY(x) AND y=?)} +} + +do_eqp_test 5.4.1 { + SELECT count(*) FROM t1, t2 WHERE y = c; +} { + 0 0 0 {SCAN TABLE t1 USING COVERING INDEX t1bc} + 0 1 1 {SEARCH TABLE t2 USING COVERING INDEX t2xy (ANY(x) AND y=?)} +} +do_execsql_test 5.4.2 { + SELECT count(*) FROM t1, t2 WHERE y = c; +} {200} +do_scanstatus_test 5.4.3 { + nLoop 1 nVisit 10 nEst 10 zName t1bc + zExplain {SCAN TABLE t1 USING COVERING INDEX t1bc} + nLoop 10 nVisit 200 nEst 56 zName t2xy + zExplain {SEARCH TABLE t2 USING COVERING INDEX t2xy (ANY(x) AND y=?)} +} + +do_eqp_test 5.5.1 { + SELECT count(*) FROM t1, t3 WHERE y = c; +} { + 0 0 1 {SCAN TABLE t3} + 0 1 0 {SEARCH TABLE t1 USING AUTOMATIC COVERING INDEX (c=?)} +} +do_execsql_test 5.5.2 { + SELECT count(*) FROM t1, t3 WHERE y = c; +} {200} +do_scanstatus_test 5.5.3 { + nLoop 1 nVisit 501 nEst 480 zName t3 zExplain {SCAN TABLE t3} + nLoop 501 nVisit 200 nEst 20 zName auto-index zExplain + {SEARCH TABLE t1 USING AUTOMATIC COVERING INDEX (c=?)} +} + +#------------------------------------------------------------------------- +# Virtual table scans +# +ifcapable fts3 { + do_execsql_test 6.0 { + CREATE VIRTUAL TABLE ft1 USING fts4; + INSERT INTO ft1 VALUES('a d c f g h e i f c'); + INSERT INTO ft1 VALUES('g c h b g b f f f g'); + INSERT INTO ft1 VALUES('h h c c h f a e d d'); + INSERT INTO ft1 VALUES('e j i j i e b c f g'); + INSERT INTO ft1 VALUES('g f b g j c h a d f'); + INSERT INTO ft1 VALUES('j i a e g f a i a c'); + INSERT INTO ft1 VALUES('f d g g j j c a h g'); + INSERT INTO ft1 VALUES('b d h a d j j j b i'); + INSERT INTO ft1 VALUES('j e a b j e c b c i'); + INSERT INTO ft1 VALUES('a d e f b j j c g d'); + } + do_execsql_test 6.1.1 { + SELECT count(*) FROM ft1 WHERE ft1 MATCH 'd' + } {6} + do_scanstatus_test 6.1.2 { + nLoop 1 nVisit 6 nEst 24 zName ft1 zExplain + {SCAN TABLE ft1 VIRTUAL TABLE INDEX 3:} + } +} + + finish_test From d1a1c23423560f4cac2cab305d92bb2d67a04d57 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Nov 2014 16:35:55 +0000 Subject: [PATCH 527/710] Refactor the interface to make it more easily extensible. FossilOrigin-Name: 7955342da4a35b57e4ae26690b8d40f7bba20e8f --- manifest | 18 +++++----- manifest.uuid | 2 +- src/sqlite.h.in | 93 ++++++++++++++++++++++++++++++------------------- src/test1.c | 8 +++-- src/vdbeapi.c | 46 +++++++++++++++--------- 5 files changed, 102 insertions(+), 65 deletions(-) diff --git a/manifest b/manifest index 5c51df5fcc..134366e0aa 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests.\sFixes\sso\sthat\scompilation\swithout\sENABLE_STMT_SCANSTATUS\sworks. -D 2014-11-03T15:33:17.869 +C Refactor\sthe\sinterface\sto\smake\sit\smore\seasily\sextensible. +D 2014-11-03T16:35:55.977 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in aeba29025ba5fc721a11c1b81ed8745f93029590 +F src/sqlite.h.in a110c6320b3af8d7c0660fa1adbc530b6caa2d95 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 @@ -237,7 +237,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 7cdd4dd3c2a4183483feca260070d73d6e22cd47 -F src/test1.c b53f4da2f386efa5c248716cd4bdc8344a87e952 +F src/test1.c 5890094c09691fe9564cf0f0d5b22d35b3218c47 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -292,7 +292,7 @@ F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 175a360c56e75ce4eb2b60704fd7c011b93926f5 F src/vdbe.h d412bd01e89f0d69991b8f46601f96bc169d28f4 F src/vdbeInt.h 539ba284790e871f98be74a78cbdfcedfae22639 -F src/vdbeapi.c addf446ecade237bebd7e9fe769bdfb9db8d9fb1 +F src/vdbeapi.c 76d62888455e3d3ffeaf70911cefa94e76a4678a F src/vdbeaux.c cf6b8152dd22155201d57c216e6266866b61da59 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f5313e0c680d9baebefb1cf50ddadedd4418a334 -R b9cde6c71acdc4fef0c5c1ac03d62018 -U dan -Z f0600457ea98c0c998694790015a9674 +P a2303c719222f1effb51acc6b37930561148c00c +R cbd06b886fdee03f70c2bf18e1342ca1 +U drh +Z 0328a60e2b5415c07361c47ade3f21c5 diff --git a/manifest.uuid b/manifest.uuid index fe7315da12..5fd1f91e84 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a2303c719222f1effb51acc6b37930561148c00c \ No newline at end of file +7955342da4a35b57e4ae26690b8d40f7bba20e8f \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 51383426d9..30ef4afb60 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7406,54 +7406,75 @@ int sqlite3_vtab_on_conflict(sqlite3 *); /* #define SQLITE_ABORT 4 // Also an error code */ #define SQLITE_REPLACE 5 +/* CAPI3REF: Prepared Statement Scan Status Opcodes +** KEYWORDS: {scanstatus option} +** +** The following constants can be used for the T parameter to the +** [sqlite3_stmt_scanstatus(S,X,T,V)] interface. Each constant designates a +** different metric for sqlite3_stmt_scanstatus() to return. +** +**
    +** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP +**
    The [sqlite3_int64] variable pointed to by the T parameter will be set to the +** total number of times that the X-th loop has run.
    +** +** [[SQLITE_SCANSTAT_NVISIT]]     		SQLITE_SCANSTAT_NVISIT +**
    The [sqlite3_int64] variable pointed to by the T parameter will be set to the +** total number of rows visited by the X-th loop.
    +** +** [[SQLITE_SCANSTAT_EST]]     		SQLITE_SCANSTAT_EST +**
    The [sqlite3_int64] variable pointed to by the T parameter will be set to the +** query planner's estimate for the number of rows visited for each +** iteration of the X-th loop. If the query planner's estimate was accurate, +** then this value should be approximately NVISIT/NLOOP. +** +** [[SQLITE_SCANSTAT_NAME]]
    		SQLITE_SCANSTAT_NAME +**
    The "const char *" variable pointed to by the T parameter will be set to +** a zero-terminated UTF-8 string containing the name of the index or table used +** for the X-th loop. +** +** [[SQLITE_SCANSTAT_EXPLAIN]]
    		SQLITE_SCANSTAT_EXPLAIN +**
    The "const char *" variable pointed to by the T parameter will be set to +** a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] description +** for the X-th loop. +** +*/ +#define SQLITE_SCANSTAT_NLOOP 0 +#define SQLITE_SCANSTAT_NVISIT 1 +#define SQLITE_SCANSTAT_NEST 2 +#define SQLITE_SCANSTAT_NAME 3 +#define SQLITE_SCANSTAT_EXPLAIN 4 /* -** CAPI3REF: Prepared Statement Scan Statuses +** CAPI3REF: Prepared Statement Scan Status ** ** Return status data for a single loop within query pStmt. ** +** The "iScanStatusOp" parameter determines which status information to return. +** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior of +** this interface is undefined. +** The requested measurement is written into a variable pointed to by +** the "pOut" parameter. ** Parameter "idx" identifies the specific loop to retrieve statistics for. ** Loops are numbered starting from zero. If idx is out of range - less than ** zero or greater than or equal to the total number of loops used to implement -** the statement - a non-zero value is returned. In this case the final value -** of all five output parameters is undefined. Otherwise, if idx is in range, -** zero is returned and the output parameters set as follows: +** the statement - a non-zero value is returned and the variable that pOut +** points to is unchanged. ** -**
      -**
    • (*pnLoop) is set to the total number of times the loop has been run. -**
    • (*pnVisit) is set to the total number of rows visited by the loop. -**
    • (*pnEst) is set to the estimate of the number of rows visited -** by each run of the loop used by the SQL optimizer. Ideally, this -** value should be close to (*pnVisit)/(*pnLoop). -**
    • (*pzName) is set to point to a nul-terminated string containing the -** name of the index of table used by this loop. -**
    • (*pzExplain) is set to point to a nul-terminated string containing -** same text that would be returned for this loop by an EXPLAIN -** QUERY PLAN command. -**
    -** -** Output parameters *pzName and *pzExplain are set to point to buffers -** managed by the statement object. Both of these pointers may be invalidated -** by any API call on the same statement object, including an sqlite3_step() -** sqlite3_bind_*() call. -** -** Statistics may not be available for all loops in all statements. In cases -** where there exist loops with no available statistics, this function ignores -** them completely. +** Statistics might not be available for all loops in all statements. In cases +** where there exist loops with no available statistics, this function behaves +** as if the loop did not exist - it returns non-zero and leave the variable +** that pOut points to unchanged. ** ** This API is only available if the library is built with pre-processor -** symbol SQLITE_ENABLE_STMT_SCANSTATUS defined. +** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. */ SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( - sqlite3_stmt *pStmt, - int idx, /* Index of loop to report on */ - sqlite3_int64 *pnLoop, /* OUT: Number of times loop was run */ - sqlite3_int64 *pnVisit, /* OUT: Number of rows visited (all loops) */ - sqlite3_int64 *pnEst, /* OUT: Number of rows estimated (per loop) */ - const char **pzName, /* OUT: Object name (table or index) */ - const char **pzExplain /* OUT: EQP string */ -); - + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + void *pOut /* Result written here */ +); /* ** CAPI3REF: Zero Scan-Status Counters @@ -7461,7 +7482,7 @@ SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( ** Zero all sqlite3_stmt_scanstatus() related event counters. ** ** This API is only available if the library is built with pre-processor -** symbol SQLITE_ENABLE_STMT_SCANSTATUS defined. +** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. */ SQLITE_EXPERIMENTAL void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); diff --git a/src/test1.c b/src/test1.c index 72e70513bc..5e526b3013 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2328,19 +2328,21 @@ static int test_stmt_scanstatus( if( getStmtPointer(interp, Tcl_GetString(objv[1]), &pStmt) ) return TCL_ERROR; if( Tcl_GetIntFromObj(interp, objv[2], &idx) ) return TCL_ERROR; - res = sqlite3_stmt_scanstatus( - pStmt, idx, &nLoop, &nVisit, &nEst, &zName, &zExplain - ); + res = sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop); if( res==0 ){ Tcl_Obj *pRet = Tcl_NewObj(); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nLoop", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nLoop)); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nVisit", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nVisit)); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_EST, (void*)&nEst); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nEst", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nEst)); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NAME, (void*)&zName); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("zName", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zName, -1)); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("zExplain", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zExplain, -1)); Tcl_SetObjResult(interp, pRet); diff --git a/src/vdbeapi.c b/src/vdbeapi.c index e6eb034ae9..d3ba4e95d2 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1481,27 +1481,42 @@ int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ ** Return status data for a single loop within query pStmt. */ int sqlite3_stmt_scanstatus( - sqlite3_stmt *pStmt, + sqlite3_stmt *pStmt, /* Prepared statement being queried */ int idx, /* Index of loop to report on */ - sqlite3_int64 *pnLoop, /* OUT: Number of times loop was run */ - sqlite3_int64 *pnVisit, /* OUT: Number of rows visited (all loops) */ - sqlite3_int64 *pnEst, /* OUT: Number of rows estimated (per loop) */ - const char **pzName, /* OUT: Object name (table or index) */ - const char **pzExplain /* OUT: EQP string */ + int iScanStatusOp, /* Which metric to return */ + void *pOut /* OUT: Write the answer here */ ){ Vdbe *p = (Vdbe*)pStmt; ScanStatus *pScan; if( idx<0 || idx>=p->nScan ) return 1; pScan = &p->aScan[idx]; - if( pnLoop ) *pnLoop = p->anExec[pScan->addrLoop]; - if( pnVisit ) *pnVisit = p->anExec[pScan->addrVisit]; - if( pnEst ) *pnEst = pScan->nEst; - if( *pzName ) *pzName = pScan->zName; - if( *pzExplain ){ - if( pScan->addrExplain ){ - *pzExplain = p->aOp[ pScan->addrExplain ].p4.z; - }else{ - *pzExplain = 0; + switch( iScanStatusOp ){ + case SQLITE_SCANSTAT_NLOOP: { + *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + break; + } + case SQLITE_SCANSTAT_NVISIT: { + *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + break; + } + case SQLITE_SCANSTAT_EST: { + *(sqlite3_int64*)pOut = pScan->nEst; + break; + } + case SQLITE_SCANSTAT_NAME: { + *(const char**)pOut = pScan->zName + break; + } + case SQLITE_SCANSTAT_EXPLAIN: { + if( pScan->addrExplain ){ + *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z; + }else{ + *(const char**)pOut = 0; + } + break; + } + default: { + return 1; } } return 0; @@ -1515,4 +1530,3 @@ void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ memset(p->anExec, 0, p->nOp * sizeof(i64)); } #endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ - From d72219da438dcc06d78fc10b9c7958cf4f12ffab Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 3 Nov 2014 16:39:37 +0000 Subject: [PATCH 528/710] Fix a typo preventing this from building with SQLITE_ENABLE_STMT_SCANSTATUS defined. FossilOrigin-Name: 4c5714ab3dba19513374c7b1478221a0b90b450c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqlite.h.in | 2 +- src/vdbeapi.c | 2 +- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 134366e0aa..75cfa9ca43 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Refactor\sthe\sinterface\sto\smake\sit\smore\seasily\sextensible. -D 2014-11-03T16:35:55.977 +C Fix\sa\stypo\spreventing\sthis\sfrom\sbuilding\swith\sSQLITE_ENABLE_STMT_SCANSTATUS\sdefined. +D 2014-11-03T16:39:37.742 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in a110c6320b3af8d7c0660fa1adbc530b6caa2d95 +F src/sqlite.h.in 26b6aa6a1825b97c0fa16ce14fc657be00c299f6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 4f86ac648ea398c1bb3db036062934cde257ea23 @@ -292,7 +292,7 @@ F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 175a360c56e75ce4eb2b60704fd7c011b93926f5 F src/vdbe.h d412bd01e89f0d69991b8f46601f96bc169d28f4 F src/vdbeInt.h 539ba284790e871f98be74a78cbdfcedfae22639 -F src/vdbeapi.c 76d62888455e3d3ffeaf70911cefa94e76a4678a +F src/vdbeapi.c 900259bdd85cd66a9f210d8ec08147a9034593bd F src/vdbeaux.c cf6b8152dd22155201d57c216e6266866b61da59 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a2303c719222f1effb51acc6b37930561148c00c -R cbd06b886fdee03f70c2bf18e1342ca1 -U drh -Z 0328a60e2b5415c07361c47ade3f21c5 +P 7955342da4a35b57e4ae26690b8d40f7bba20e8f +R c9becfda3a2880c1d6a40a42c50145fa +U dan +Z a994e9671fe739ddd2243271f124e718 diff --git a/manifest.uuid b/manifest.uuid index 5fd1f91e84..2f63e2e7f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7955342da4a35b57e4ae26690b8d40f7bba20e8f \ No newline at end of file +4c5714ab3dba19513374c7b1478221a0b90b450c \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 30ef4afb60..8f51f0e180 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7441,7 +7441,7 @@ int sqlite3_vtab_on_conflict(sqlite3 *); */ #define SQLITE_SCANSTAT_NLOOP 0 #define SQLITE_SCANSTAT_NVISIT 1 -#define SQLITE_SCANSTAT_NEST 2 +#define SQLITE_SCANSTAT_EST 2 #define SQLITE_SCANSTAT_NAME 3 #define SQLITE_SCANSTAT_EXPLAIN 4 diff --git a/src/vdbeapi.c b/src/vdbeapi.c index d3ba4e95d2..d472004590 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1504,7 +1504,7 @@ int sqlite3_stmt_scanstatus( break; } case SQLITE_SCANSTAT_NAME: { - *(const char**)pOut = pScan->zName + *(const char**)pOut = pScan->zName; break; } case SQLITE_SCANSTAT_EXPLAIN: { From d84bf205c38c9f46cee73c1e24f1dee67d32a441 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 3 Nov 2014 18:03:00 +0000 Subject: [PATCH 529/710] Updates to the sqlite3_stmt_scanstatus() documentation. No changes to code. FossilOrigin-Name: d97c324eb1d870c994911c53fbf84205f4e3e7a1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 35 +++++++++++++++++++---------------- 3 files changed, 27 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 07d97518f1..8d217ecfe0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sexperimental\ssqlite3_stmt_scanstatus()\sAPI.\sFor\scomparing\sthe\snumber\sof\srows\sactually\svisited\sby\sa\sloop\swith\sthe\sestimate\sused\sby\sthe\squery\splanner. -D 2014-11-03T16:56:43.943 +C Updates\sto\sthe\ssqlite3_stmt_scanstatus()\sdocumentation.\s\sNo\schanges\sto\scode. +D 2014-11-03T18:03:00.467 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 3b69f7d9b74e2e5d0448dc2f0d2d160517afa253 +F src/sqlite.h.in 11f33a3e968a9637d6fa4ae879b83edd171ac88f F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a518bc3318232d652349eb29303ff250aee40459 4c5714ab3dba19513374c7b1478221a0b90b450c -R ce3f2c2aee62a076b6ed59326c9cde3e -U dan -Z cd275722bcaf367ed1a07165e9a16c03 +P ab3b0fc5760c6d428dbe1f974726a7d3526640bc +R e3e372d3594a5227dc7ce2380d71aae3 +U drh +Z 7bcc3180ea33e48aa6d0e7855da7bfd4 diff --git a/manifest.uuid b/manifest.uuid index c71e9c3f37..0611c8a12e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ab3b0fc5760c6d428dbe1f974726a7d3526640bc \ No newline at end of file +d97c324eb1d870c994911c53fbf84205f4e3e7a1 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f51dca0d84..41cc439846 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7411,35 +7411,36 @@ int sqlite3_vtab_on_conflict(sqlite3 *); /* #define SQLITE_ABORT 4 // Also an error code */ #define SQLITE_REPLACE 5 -/* CAPI3REF: Prepared Statement Scan Status Opcodes -** KEYWORDS: {scanstatus option} +/* +** CAPI3REF: Prepared Statement Scan Status Opcodes +** KEYWORDS: {scanstatus options} ** ** The following constants can be used for the T parameter to the ** [sqlite3_stmt_scanstatus(S,X,T,V)] interface. Each constant designates a ** different metric for sqlite3_stmt_scanstatus() to return. ** **
    -** [[SQLITE_SCANSTAT_NLOOP]]
    		SQLITE_SCANSTAT_NLOOP -**
    The [sqlite3_int64] variable pointed to by the T parameter will be set to the +** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set to the ** total number of times that the X-th loop has run.
    ** -** [[SQLITE_SCANSTAT_NVISIT]]     		SQLITE_SCANSTAT_NVISIT -**
    The [sqlite3_int64] variable pointed to by the T parameter will be set to the +** [[SQLITE_SCANSTAT_NVISIT]]
    SQLITE_SCANSTAT_NVISIT
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set to the ** total number of rows visited by the X-th loop.
    ** -** [[SQLITE_SCANSTAT_EST]]     		SQLITE_SCANSTAT_EST -**
    The [sqlite3_int64] variable pointed to by the T parameter will be set to the +** [[SQLITE_SCANSTAT_EST]]
    SQLITE_SCANSTAT_EST
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set to the ** query planner's estimate for the number of rows visited for each ** iteration of the X-th loop. If the query planner's estimate was accurate, ** then this value should be approximately NVISIT/NLOOP. ** -** [[SQLITE_SCANSTAT_NAME]]
    		SQLITE_SCANSTAT_NAME -**
    The "const char *" variable pointed to by the T parameter will be set to +** [[SQLITE_SCANSTAT_NAME]]
    SQLITE_SCANSTAT_NAME
    +**
    ^The "const char *" variable pointed to by the T parameter will be set to ** a zero-terminated UTF-8 string containing the name of the index or table used ** for the X-th loop. ** -** [[SQLITE_SCANSTAT_EXPLAIN]]
    		SQLITE_SCANSTAT_EXPLAIN -**
    The "const char *" variable pointed to by the T parameter will be set to +** [[SQLITE_SCANSTAT_EXPLAIN]]
    SQLITE_SCANSTAT_EXPLAIN
    +**
    ^The "const char *" variable pointed to by the T parameter will be set to ** a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] description ** for the X-th loop. ** @@ -7458,21 +7459,23 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** The "iScanStatusOp" parameter determines which status information to return. ** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior of ** this interface is undefined. -** The requested measurement is written into a variable pointed to by +** ^The requested measurement is written into a variable pointed to by ** the "pOut" parameter. ** Parameter "idx" identifies the specific loop to retrieve statistics for. -** Loops are numbered starting from zero. If idx is out of range - less than +** Loops are numbered starting from zero. ^If idx is out of range - less than ** zero or greater than or equal to the total number of loops used to implement ** the statement - a non-zero value is returned and the variable that pOut ** points to is unchanged. ** -** Statistics might not be available for all loops in all statements. In cases +** ^Statistics might not be available for all loops in all statements. ^In cases ** where there exist loops with no available statistics, this function behaves ** as if the loop did not exist - it returns non-zero and leave the variable ** that pOut points to unchanged. ** ** This API is only available if the library is built with pre-processor ** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. +** +** See also: [sqlite3_stmt_scanstatus_reset()] */ SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ @@ -7484,7 +7487,7 @@ SQLITE_EXPERIMENTAL int sqlite3_stmt_scanstatus( /* ** CAPI3REF: Zero Scan-Status Counters ** -** Zero all sqlite3_stmt_scanstatus() related event counters. +** ^Zero all [sqlite3_stmt_scanstatus()] related event counters. ** ** This API is only available if the library is built with pre-processor ** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. From def6889d214a04780390df0aa77efcd5011620cf Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Nov 2014 12:11:23 +0000 Subject: [PATCH 530/710] Add the SQLITE_CONFIG_PCACHE_HDRSZ option for sqlite3_config(). FossilOrigin-Name: 6eb03e62a34e8e0964175283587247b0212db604 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/btree.c | 5 +++++ src/btree.h | 1 + src/main.c | 9 +++++++++ src/pcache.c | 7 +++++++ src/pcache.h | 4 ++++ src/pcache1.c | 5 +++++ src/sqlite.h.in | 36 ++++++++++++++++++++++++------------ 9 files changed, 68 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 8d217ecfe0..2d1da1580d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\ssqlite3_stmt_scanstatus()\sdocumentation.\s\sNo\schanges\sto\scode. -D 2014-11-03T18:03:00.467 +C Add\sthe\sSQLITE_CONFIG_PCACHE_HDRSZ\soption\sfor\ssqlite3_config(). +D 2014-11-04T12:11:23.751 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,8 +172,8 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 61d96c2edacc5267fae6e477c24c774cd540d7f0 -F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 +F src/btree.c 3cc918768d100f0efea62404b8f4e779b8c94731 +F src/btree.h 49b408be9c1cd41249076898e0673711071205d8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 5f659bdb14cdba1c3b587b6ed09167c147e308a1 +F src/main.c 8903165064534858a79ae06615ea7f4b482ad482 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -218,9 +218,9 @@ F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 8d97b3633f098fef817656dcbf167ca904511d78 F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 -F src/pcache.c 4121a0571c18581ee9f82f086d5e2030051ebd6a -F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a -F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 +F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 +F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 +F src/pcache1.c facbdd3ecc09c8f750089d941305694301328e98 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 11f33a3e968a9637d6fa4ae879b83edd171ac88f +F src/sqlite.h.in 915b5a955dd2f1d3c7e848f721af48ccc47b972b F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ab3b0fc5760c6d428dbe1f974726a7d3526640bc -R e3e372d3594a5227dc7ce2380d71aae3 +P d97c324eb1d870c994911c53fbf84205f4e3e7a1 +R 9efeed3832394f98added1ade80fb8e9 U drh -Z 7bcc3180ea33e48aa6d0e7855da7bfd4 +Z 388d5bd03930b5f393bf21b730c0228b diff --git a/manifest.uuid b/manifest.uuid index 0611c8a12e..d3cdc0284b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d97c324eb1d870c994911c53fbf84205f4e3e7a1 \ No newline at end of file +6eb03e62a34e8e0964175283587247b0212db604 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9300a6a54f..f815e85a6d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8965,3 +8965,8 @@ void sqlite3BtreeCursorHints(BtCursor *pCsr, unsigned int mask){ int sqlite3BtreeIsReadonly(Btree *p){ return (p->pBt->btsFlags & BTS_READ_ONLY)!=0; } + +/* +** Return the size of the header added to each page by this module. +*/ +int sqlite3HeaderSizeBtree(void){ return sizeof(MemPage); } diff --git a/src/btree.h b/src/btree.h index 38abdca1a2..63337b1946 100644 --- a/src/btree.h +++ b/src/btree.h @@ -196,6 +196,7 @@ void sqlite3BtreeClearCursor(BtCursor *); int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); void sqlite3BtreeCursorHints(BtCursor *, unsigned int mask); int sqlite3BtreeIsReadonly(Btree *pBt); +int sqlite3HeaderSizeBtree(void); #ifndef NDEBUG int sqlite3BtreeCursorIsValid(BtCursor*); diff --git a/src/main.c b/src/main.c index 65a662ee5a..6a896b2a3d 100644 --- a/src/main.c +++ b/src/main.c @@ -401,6 +401,15 @@ int sqlite3_config(int op, ...){ sqlite3GlobalConfig.nPage = va_arg(ap, int); break; } + case SQLITE_CONFIG_PCACHE_HDRSZ: { + /* Return the total size of all headers added to each page + ** of the page cache */ + *va_arg(ap, int*) = + sqlite3HeaderSizeBtree() + + sqlite3HeaderSizePcache() + + sqlite3HeaderSizePcache1(); + break; + } case SQLITE_CONFIG_PCACHE: { /* no-op */ diff --git a/src/pcache.c b/src/pcache.c index 191a9d00f4..13551872d1 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -651,6 +651,13 @@ void sqlite3PcacheShrink(PCache *pCache){ sqlite3GlobalConfig.pcache2.xShrink(pCache->pCache); } +/* +** Return the size of the header added by this middleware layer +** in the page-cache hierarchy. +*/ +int sqlite3HeaderSizePcache(void){ return sizeof(PgHdr); } + + #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* ** For all dirty pages currently in the cache, invoke the specified diff --git a/src/pcache.h b/src/pcache.h index dd9bfc7451..9ed62a88ff 100644 --- a/src/pcache.h +++ b/src/pcache.h @@ -160,4 +160,8 @@ void sqlite3PcacheStats(int*,int*,int*,int*); void sqlite3PCacheSetDefault(void); +/* Return the header size */ +int sqlite3HeaderSizePcache(void); +int sqlite3HeaderSizePcache1(void); + #endif /* _PCACHE_H_ */ diff --git a/src/pcache1.c b/src/pcache1.c index a8c3217382..cad41b1b73 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -981,6 +981,11 @@ void sqlite3PCacheSetDefault(void){ sqlite3_config(SQLITE_CONFIG_PCACHE2, &defaultMethods); } +/* +** Return the size of the header on each page of this PCACHE implementation. +*/ +int sqlite3HeaderSizePcache1(void){ return sizeof(PgHdr1); } + #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* ** This function is called to free superfluous dynamically allocated memory diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 41cc439846..13373afe4a 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1559,28 +1559,30 @@ struct sqlite3_mem_methods { **
    ^This option specifies a static memory buffer that SQLite can use for ** the database page cache with the default page cache implementation. ** This configuration should not be used if an application-define page -** cache implementation is loaded using the SQLITE_CONFIG_PCACHE2 option. -** There are three arguments to this option: A pointer to 8-byte aligned +** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2] +** configuration option. +** ^There are three arguments to this option: A pointer to 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page -** (a power of two between 512 and 32768) plus a little extra for each -** page header. ^The page header size is 20 to 40 bytes depending on -** the host architecture. ^It is harmless, apart from the wasted memory, -** to make sz a little too large. The first -** argument should point to an allocation of at least sz*N bytes of memory. +** (a power of two between 512 and 32768) plus some extra bytes for each +** page header. ^The number of extra bytes needed by the page header +** can be determined using the [SQLITE_CONFIG_PCACHE_HDRSZ] option +** to [sqlite3_config()]. +** ^It is harmless, apart from the wasted memory, +** for the sz parameter to be larger than necessary. The first +** argument should pointer to an 8-byte aligned block of memory that +** is at least sz*N bytes of memory, otherwise subsequent behavior is +** undefined. ** ^SQLite will use the memory provided by the first argument to satisfy its ** memory needs for the first N pages that it adds to cache. ^If additional ** page cache memory is needed beyond what is provided by this option, then -** SQLite goes to [sqlite3_malloc()] for the additional storage space. -** The pointer in the first argument must -** be aligned to an 8-byte boundary or subsequent behavior of SQLite -** will be undefined.
    +** SQLite goes to [sqlite3_malloc()] for the additional storage space. ** ** [[SQLITE_CONFIG_HEAP]]
    SQLITE_CONFIG_HEAP
    **
    ^This option specifies a static memory buffer that SQLite will use ** for all of its dynamic memory allocation needs beyond those provided ** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. -** There are three arguments: An 8-byte aligned pointer to the memory, +** ^There are three arguments: An 8-byte aligned pointer to the memory, ** the number of bytes in the memory buffer, and the minimum allocation size. ** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts ** to using its default memory allocator (the system malloc() implementation), @@ -1728,6 +1730,15 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** +** +** [[SQLITE_CONFIG_PCACHE_HDRSZ]] +**
    SQLITE_CONFIG_PCACHE_HDRSZ +**
    ^This option takes a single parameter which is a pointer to an integer +** and writes into that integer the number of extra bytes per page required +** for each page in [SQLITE_CONFIG_PAGECACHE]. The amount of +** extra space required can change depending on the compiler, +** target platform, and SQLite version. +** */ #define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ #define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ @@ -1752,6 +1763,7 @@ struct sqlite3_mem_methods { #define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ #define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ +#define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ /* ** CAPI3REF: Database Connection Configuration Options From 5279d3433c2ef6089559eb65d3ed1a3e298cc3b3 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Nov 2014 13:41:32 +0000 Subject: [PATCH 531/710] Improved documentation and addition of source-code evidence marks for the sqlite3_config() interface. FossilOrigin-Name: 681031a436fdd4cce426d6cd43cbae6b83167d26 --- manifest | 16 ++++---- manifest.uuid | 2 +- src/btree.c | 8 +++- src/main.c | 45 ++++++++++++++++++----- src/sqlite.h.in | 98 ++++++++++++++++++++++++++----------------------- 5 files changed, 103 insertions(+), 66 deletions(-) diff --git a/manifest b/manifest index 2d1da1580d..c02ddb43b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_CONFIG_PCACHE_HDRSZ\soption\sfor\ssqlite3_config(). -D 2014-11-04T12:11:23.751 +C Improved\sdocumentation\sand\saddition\sof\ssource-code\sevidence\smarks\sfor\sthe\nsqlite3_config()\sinterface. +D 2014-11-04T13:41:32.444 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 3cc918768d100f0efea62404b8f4e779b8c94731 +F src/btree.c 5b6e02a2cb69bfba5d44d0a093b8cc7468fdcf61 F src/btree.h 49b408be9c1cd41249076898e0673711071205d8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 8903165064534858a79ae06615ea7f4b482ad482 +F src/main.c c673d0972da4b6826a3e907029faf5be0840cd57 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 915b5a955dd2f1d3c7e848f721af48ccc47b972b +F src/sqlite.h.in 81c5105a19bd1c16a06d59e729ddc3bb0c26d003 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d97c324eb1d870c994911c53fbf84205f4e3e7a1 -R 9efeed3832394f98added1ade80fb8e9 +P 6eb03e62a34e8e0964175283587247b0212db604 +R a92bedb66491253840f18dd40e991665 U drh -Z 388d5bd03930b5f393bf21b730c0228b +Z 6591907c91e192439bb8b0655576863e diff --git a/manifest.uuid b/manifest.uuid index d3cdc0284b..e57823b45e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6eb03e62a34e8e0964175283587247b0212db604 \ No newline at end of file +681031a436fdd4cce426d6cd43cbae6b83167d26 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f815e85a6d..6fe8e10f3d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6649,7 +6649,13 @@ static int balance_nonroot( nMaxCells*sizeof(u8*) /* apCell */ + nMaxCells*sizeof(u16) /* szCell */ + pBt->pageSize; /* aSpace1 */ - assert( szScratch<=16896 || szScratch<=6*pBt->pageSize ); + + /* EVIDENCE-OF: R-37926-08392 SQLite will never request a scratch buffer + ** that is more than 6 times the database page size, except when + ** performing a checkpoint in WAL mode when the scratch buffer request + ** size is a small fraction of the size of the WAL file. */ + assert( szScratch<=6*pBt->pageSize ); + apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; diff --git a/src/main.c b/src/main.c index 6a896b2a3d..80800d7cf6 100644 --- a/src/main.c +++ b/src/main.c @@ -372,38 +372,54 @@ int sqlite3_config(int op, ...){ #endif case SQLITE_CONFIG_MALLOC: { - /* Specify an alternative malloc implementation */ + /* EVIDENCE-OF: R-55594-21030 The SQLITE_CONFIG_MALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The argument specifies alternative + ** low-level memory allocation routines to be used in place of the memory + ** allocation routines built into SQLite. */ sqlite3GlobalConfig.m = *va_arg(ap, sqlite3_mem_methods*); break; } case SQLITE_CONFIG_GETMALLOC: { - /* Retrieve the current malloc() implementation */ + /* EVIDENCE-OF: R-51213-46414 The SQLITE_CONFIG_GETMALLOC option takes a + ** single argument which is a pointer to an instance of the + ** sqlite3_mem_methods structure. The sqlite3_mem_methods structure is + ** filled with the currently defined memory allocation routines. */ if( sqlite3GlobalConfig.m.xMalloc==0 ) sqlite3MemSetDefault(); *va_arg(ap, sqlite3_mem_methods*) = sqlite3GlobalConfig.m; break; } case SQLITE_CONFIG_MEMSTATUS: { - /* Enable or disable the malloc status collection */ - sqlite3GlobalConfig.bMemstat = va_arg(ap, int); /* IMP: R-27464-47829 */ + /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes + ** single argument of type int, interpreted as a boolean, which enables + ** or disables the collection of memory allocation statistics. */ + sqlite3GlobalConfig.bMemstat = va_arg(ap, int); break; } case SQLITE_CONFIG_SCRATCH: { - /* Designate a buffer for scratch memory space */ + /* EVIDENCE-OF: R-08404-60887 There are three arguments to + ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from + ** which the scratch allocations will be drawn, the size of each scratch + ** allocation (sz), and the maximum number of scratch allocations (N). */ sqlite3GlobalConfig.pScratch = va_arg(ap, void*); sqlite3GlobalConfig.szScratch = va_arg(ap, int); sqlite3GlobalConfig.nScratch = va_arg(ap, int); break; } case SQLITE_CONFIG_PAGECACHE: { - /* Designate a buffer for page cache memory space */ + /* EVIDENCE-OF: R-31408-40510 There are three arguments to + ** SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned memory, the size + ** of each page buffer (sz), and the number of pages (N). */ sqlite3GlobalConfig.pPage = va_arg(ap, void*); sqlite3GlobalConfig.szPage = va_arg(ap, int); sqlite3GlobalConfig.nPage = va_arg(ap, int); break; } case SQLITE_CONFIG_PCACHE_HDRSZ: { - /* Return the total size of all headers added to each page - ** of the page cache */ + /* EVIDENCE-OF: R-39100-27317 The SQLITE_CONFIG_PCACHE_HDRSZ option takes + ** a single parameter which is a pointer to an integer and writes into + ** that integer the number of extra bytes per page required for each page + ** in SQLITE_CONFIG_PAGECACHE. */ *va_arg(ap, int*) = sqlite3HeaderSizeBtree() + sqlite3HeaderSizePcache() + @@ -422,11 +438,18 @@ int sqlite3_config(int op, ...){ } case SQLITE_CONFIG_PCACHE2: { - /* Specify an alternative page cache implementation */ + /* EVIDENCE-OF: R-63325-48378 The SQLITE_CONFIG_PCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. This object specifies the interface to a custom page cache + ** implementation. */ sqlite3GlobalConfig.pcache2 = *va_arg(ap, sqlite3_pcache_methods2*); break; } case SQLITE_CONFIG_GETPCACHE2: { + /* EVIDENCE-OF: R-22035-46182 The SQLITE_CONFIG_GETPCACHE2 option takes a + ** single argument which is a pointer to an sqlite3_pcache_methods2 + ** object. SQLite copies of the current page cache implementation into + ** that object. */ if( sqlite3GlobalConfig.pcache2.xInit==0 ){ sqlite3PCacheSetDefault(); } @@ -436,7 +459,9 @@ int sqlite3_config(int op, ...){ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) case SQLITE_CONFIG_HEAP: { - /* Designate a buffer for heap memory space */ + /* EVIDENCE-OF: R-19854-42126 There are three arguments to + ** SQLITE_CONFIG_HEAP: An 8-byte aligned pointer to the memory, the + ** number of bytes in the memory buffer, and the minimum allocation size. */ sqlite3GlobalConfig.pHeap = va_arg(ap, void*); sqlite3GlobalConfig.nHeap = va_arg(ap, int); sqlite3GlobalConfig.mnReq = va_arg(ap, int); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 13373afe4a..271b7a0ad1 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1499,25 +1499,27 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_SERIALIZED configuration option.
    ** ** [[SQLITE_CONFIG_MALLOC]]
    SQLITE_CONFIG_MALLOC
    -**
    ^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mem_methods] structure. The argument specifies +**
    ^(The SQLITE_CONFIG_MALLOC option takes a single argument which is +** a pointer to an instance of the [sqlite3_mem_methods] structure. +** The argument specifies ** alternative low-level memory allocation routines to be used in place of ** the memory allocation routines built into SQLite.)^ ^SQLite makes ** its own private copy of the content of the [sqlite3_mem_methods] structure ** before the [sqlite3_config()] call returns.
    ** ** [[SQLITE_CONFIG_GETMALLOC]]
    SQLITE_CONFIG_GETMALLOC
    -**
    ^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mem_methods] structure. The [sqlite3_mem_methods] +**
    ^(The SQLITE_CONFIG_GETMALLOC option takes a single argument which +** is a pointer to an instance of the [sqlite3_mem_methods] structure. +** The [sqlite3_mem_methods] ** structure is filled with the currently defined memory allocation routines.)^ ** This option can be used to overload the default memory allocation ** routines with a wrapper that simulations memory allocation failure or ** tracks memory usage, for example.
    ** ** [[SQLITE_CONFIG_MEMSTATUS]]
    SQLITE_CONFIG_MEMSTATUS
    -**
    ^This option takes single argument of type int, interpreted as a -** boolean, which enables or disables the collection of memory allocation -** statistics. ^(When memory allocation statistics are disabled, the +**
    ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +** interpreted as a boolean, which enables or disables the collection of +** memory allocation statistics. ^(When memory allocation statistics are disabled, the ** following SQLite interfaces become non-operational: **
      **
    • [sqlite3_memory_used()] @@ -1531,8 +1533,9 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_CONFIG_SCRATCH]]
    SQLITE_CONFIG_SCRATCH
    -**
    ^This option specifies a static memory buffer that SQLite can use for -** scratch memory. ^(There are three arguments: A pointer an 8-byte +**
    ^The SQLITE_CONFIG_SCRATCH option specifies a static memory buffer +** that SQLite can use for scratch memory. ^(There are three arguments +** to SQLITE_CONFIG_SCRATCH: A pointer an 8-byte ** aligned memory buffer from which the scratch allocations will be ** drawn, the size of each scratch allocation (sz), ** and the maximum number of scratch allocations (N).)^ @@ -1556,12 +1559,13 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_CONFIG_PAGECACHE]]
    SQLITE_CONFIG_PAGECACHE
    -**
    ^This option specifies a static memory buffer that SQLite can use for -** the database page cache with the default page cache implementation. +**
    ^The SQLITE_CONFIG_PAGECACHE option specifies a static memory buffer +** that SQLite can use for the database page cache with the default page +** cache implementation. ** This configuration should not be used if an application-define page ** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2] ** configuration option. -** ^There are three arguments to this option: A pointer to 8-byte aligned +** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page ** (a power of two between 512 and 32768) plus some extra bytes for each @@ -1579,10 +1583,11 @@ struct sqlite3_mem_methods { ** SQLite goes to [sqlite3_malloc()] for the additional storage space.
    ** ** [[SQLITE_CONFIG_HEAP]]
    SQLITE_CONFIG_HEAP
    -**
    ^This option specifies a static memory buffer that SQLite will use -** for all of its dynamic memory allocation needs beyond those provided -** for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. -** ^There are three arguments: An 8-byte aligned pointer to the memory, +**
    ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer +** that SQLite will use for all of its dynamic memory allocation needs +** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. +** ^There are three arguments to SQLITE_CONFIG_HEAP: +** An 8-byte aligned pointer to the memory, ** the number of bytes in the memory buffer, and the minimum allocation size. ** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts ** to using its default memory allocator (the system malloc() implementation), @@ -1596,9 +1601,9 @@ struct sqlite3_mem_methods { ** for the minimum allocation size are 2**5 through 2**8.
    ** ** [[SQLITE_CONFIG_MUTEX]]
    SQLITE_CONFIG_MUTEX
    -**
    ^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mutex_methods] structure. The argument specifies -** alternative low-level mutex routines to be used in place +**
    ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a +** pointer to an instance of the [sqlite3_mutex_methods] structure. +** The argument specifies alternative low-level mutex routines to be used in place ** the mutex routines built into SQLite.)^ ^SQLite makes a copy of the ** content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with @@ -1608,8 +1613,8 @@ struct sqlite3_mem_methods { ** return [SQLITE_ERROR].
    ** ** [[SQLITE_CONFIG_GETMUTEX]]
    SQLITE_CONFIG_GETMUTEX
    -**
    ^(This option takes a single argument which is a pointer to an -** instance of the [sqlite3_mutex_methods] structure. The +**
    ^(The SQLITE_CONFIG_GETMUTEX option takes a single argument which +** is a pointer to an instance of the [sqlite3_mutex_methods] structure. The ** [sqlite3_mutex_methods] ** structure is filled with the currently defined mutex routines.)^ ** This option can be used to overload the default mutex allocation @@ -1621,24 +1626,24 @@ struct sqlite3_mem_methods { ** return [SQLITE_ERROR].
    ** ** [[SQLITE_CONFIG_LOOKASIDE]]
    SQLITE_CONFIG_LOOKASIDE
    -**
    ^(This option takes two arguments that determine the default -** memory allocation for the lookaside memory allocator on each -** [database connection]. The first argument is the +**
    ^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine +** the default size of lookaside memory on each [database connection]. +** The first argument is the ** size of each lookaside buffer slot and the second is the number of -** slots allocated to each database connection.)^ ^(This option sets the -** default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] -** verb to [sqlite3_db_config()] can be used to change the lookaside +** slots allocated to each database connection.)^ ^(SQLITE_CONFIG_LOOKASIDE +** sets the default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] +** option to [sqlite3_db_config()] can be used to change the lookaside ** configuration on individual connections.)^
    ** ** [[SQLITE_CONFIG_PCACHE2]]
    SQLITE_CONFIG_PCACHE2
    -**
    ^(This option takes a single argument which is a pointer to -** an [sqlite3_pcache_methods2] object. This object specifies the interface -** to a custom page cache implementation.)^ ^SQLite makes a copy of the -** object and uses it for page cache memory allocations.
    +**
    ^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is +** a pointer to an [sqlite3_pcache_methods2] object. This object specifies +** the interface to a custom page cache implementation.)^ +** ^SQLite makes a copy of the [sqlite3_pcache_methods2] object.
    ** ** [[SQLITE_CONFIG_GETPCACHE2]]
    SQLITE_CONFIG_GETPCACHE2
    -**
    ^(This option takes a single argument which is a pointer to an -** [sqlite3_pcache_methods2] object. SQLite copies of the current +**
    ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of the current ** page cache implementation into that object.)^
    ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    @@ -1662,10 +1667,10 @@ struct sqlite3_mem_methods { ** function must be threadsafe. ** ** [[SQLITE_CONFIG_URI]]
    SQLITE_CONFIG_URI -**
    ^(This option takes a single argument of type int. If non-zero, then -** URI handling is globally enabled. If the parameter is zero, then URI handling -** is globally disabled.)^ ^If URI handling is globally enabled, all filenames -** passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +**
    ^(The SQLITE_CONFIG_URI option takes a single argument of type int. +** If non-zero, then URI handling is globally enabled. If the parameter is zero, +** then URI handling is globally disabled.)^ ^If URI handling is globally enabled, +** all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or ** specified as part of [ATTACH] commands are interpreted as URIs, regardless ** of whether or not the [SQLITE_OPEN_URI] flag is set when the database ** connection is opened. ^If it is globally disabled, filenames are @@ -1675,9 +1680,10 @@ struct sqlite3_mem_methods { ** [SQLITE_USE_URI] symbol defined.)^ ** ** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
    SQLITE_CONFIG_COVERING_INDEX_SCAN -**
    ^This option takes a single integer argument which is interpreted as -** a boolean in order to enable or disable the use of covering indices for -** full table scans in the query optimizer. ^The default setting is determined +**
    ^The SQLITE_CONFIG_COVERING_INDEX_SCAN option takes a single integer +** argument which is interpreted as a boolean in order to enable or disable +** the use of covering indices for full table scans in the query optimizer. +** ^The default setting is determined ** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" ** if that compile-time option is omitted. ** The ability to disable the use of covering indices for full table scans @@ -1725,17 +1731,17 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] **
    SQLITE_CONFIG_WIN32_HEAPSIZE -**
    ^This option is only available if SQLite is compiled for Windows -** with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. -** SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +**
    ^The SQLITE_CONFIG_WIN32_HEAPSIZE option is only available if SQLite is +** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. +** ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** ** ** [[SQLITE_CONFIG_PCACHE_HDRSZ]] **
    SQLITE_CONFIG_PCACHE_HDRSZ -**
    ^This option takes a single parameter which is a pointer to an integer -** and writes into that integer the number of extra bytes per page required -** for each page in [SQLITE_CONFIG_PAGECACHE]. The amount of +**
    ^The SQLITE_CONFIG_PCACHE_HDRSZ option takes a single parameter which +** is a pointer to an integer and writes into that integer the number of extra +** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. The amount of ** extra space required can change depending on the compiler, ** target platform, and SQLite version. ** From cbd55b0362c8dd2a0c7c3bd0bc56bd16ccd2d962 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Nov 2014 14:22:27 +0000 Subject: [PATCH 532/710] Change the definition of SQLITE_CONFIG_SCRATCH so that at most one scratch buffer is used per thread. Use the generic heap memory allocator for the WalIterator object when running a checkpoint. FossilOrigin-Name: 391c9b85abcb5ba300fb2e116384639310c69ed2 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/btree.c | 7 ++----- src/malloc.c | 11 ++++++----- src/sqlite.h.in | 8 ++------ src/wal.c | 8 ++++---- 6 files changed, 24 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index c02ddb43b9..9decec2f00 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\sdocumentation\sand\saddition\sof\ssource-code\sevidence\smarks\sfor\sthe\nsqlite3_config()\sinterface. -D 2014-11-04T13:41:32.444 +C Change\sthe\sdefinition\sof\sSQLITE_CONFIG_SCRATCH\sso\sthat\sat\smost\sone\sscratch\nbuffer\sis\sused\sper\sthread.\s\sUse\sthe\sgeneric\sheap\smemory\sallocator\sfor\sthe\nWalIterator\sobject\swhen\srunning\sa\scheckpoint. +D 2014-11-04T14:22:27.705 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 5b6e02a2cb69bfba5d44d0a093b8cc7468fdcf61 +F src/btree.c 11d1262110c2d459b68121833fa3ec6625b1d022 F src/btree.h 49b408be9c1cd41249076898e0673711071205d8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -195,7 +195,7 @@ F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 F src/main.c c673d0972da4b6826a3e907029faf5be0840cd57 -F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec +F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f F src/mem2.c f1940d9e91948dd6a908fbb9ce3835c36b5d83c3 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 -F src/sqlite.h.in 81c5105a19bd1c16a06d59e729ddc3bb0c26d003 +F src/sqlite.h.in 6e9af739d79f0bea2584b70fb1c54d3bb1a2eab6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -299,7 +299,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c daf87ea542df088ac4607660d09976d36b6bd95d F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 -F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 +F src/wal.c 825c948066c7604a07d56e67958cdab210749016 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c d5fa1081bf7cb70478ed06489e063695c92ee1e1 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6eb03e62a34e8e0964175283587247b0212db604 -R a92bedb66491253840f18dd40e991665 +P 681031a436fdd4cce426d6cd43cbae6b83167d26 +R cb98a95faa41015902ec00cd7a294542 U drh -Z 6591907c91e192439bb8b0655576863e +Z 113a55cc7642fbc8d6bb5b88b140b929 diff --git a/manifest.uuid b/manifest.uuid index e57823b45e..cd70e30b51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -681031a436fdd4cce426d6cd43cbae6b83167d26 \ No newline at end of file +391c9b85abcb5ba300fb2e116384639310c69ed2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 6fe8e10f3d..fe15c922ca 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6650,12 +6650,9 @@ static int balance_nonroot( + nMaxCells*sizeof(u16) /* szCell */ + pBt->pageSize; /* aSpace1 */ - /* EVIDENCE-OF: R-37926-08392 SQLite will never request a scratch buffer - ** that is more than 6 times the database page size, except when - ** performing a checkpoint in WAL mode when the scratch buffer request - ** size is a small fraction of the size of the WAL file. */ + /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer + ** that is more than 6 times the database page size. */ assert( szScratch<=6*pBt->pageSize ); - apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; diff --git a/src/malloc.c b/src/malloc.c index 6fb9d53d1b..4960f91e02 100644 --- a/src/malloc.c +++ b/src/malloc.c @@ -377,11 +377,12 @@ void *sqlite3ScratchMalloc(int n){ #if SQLITE_THREADSAFE==0 && !defined(NDEBUG) - /* Verify that no more than two scratch allocations per thread - ** are outstanding at one time. (This is only checked in the - ** single-threaded case since checking in the multi-threaded case - ** would be much more complicated.) */ - assert( scratchAllocOut<=1 ); + /* EVIDENCE-OF: R-12970-05880 SQLite will not use more than one scratch + ** buffers per thread. + ** + ** This can only be checked in single-threaded mode. + */ + assert( scratchAllocOut==0 ); if( p ) scratchAllocOut++; #endif diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 271b7a0ad1..44f7800ca2 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1541,13 +1541,9 @@ struct sqlite3_mem_methods { ** and the maximum number of scratch allocations (N).)^ ** The first argument must be a pointer to an 8-byte aligned buffer ** of at least sz*N bytes of memory. -** ^SQLite will not use more than two scratch buffers per thread and not -** more than one scratch buffer per thread when not performing -** a [checkpoint] in [WAL mode]. +** ^SQLite will not use more than one scratch buffers per thread. ** ^SQLite will never request a scratch buffer that is more than 6 -** times the database page size, except when performing a [checkpoint] -** in [WAL mode] when the scratch buffer request size is a small fraction -** of the size of the WAL file. +** times the database page size. ** ^If SQLite needs needs additional ** scratch memory beyond what is provided by this configuration option, then ** [sqlite3_malloc()] will be used to obtain the memory needed.

    diff --git a/src/wal.c b/src/wal.c index c0861d5be7..d2ed293a4d 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1504,7 +1504,7 @@ static void walMergesort( ** Free an iterator allocated by walIteratorInit(). */ static void walIteratorFree(WalIterator *p){ - sqlite3ScratchFree(p); + sqlite3_free(p); } /* @@ -1539,7 +1539,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ nByte = sizeof(WalIterator) + (nSegment-1)*sizeof(struct WalSegment) + iLast*sizeof(ht_slot); - p = (WalIterator *)sqlite3ScratchMalloc(nByte); + p = (WalIterator *)sqlite3_malloc(nByte); if( !p ){ return SQLITE_NOMEM; } @@ -1549,7 +1549,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ /* Allocate temporary space used by the merge-sort routine. This block ** of memory will be freed before this function returns. */ - aTmp = (ht_slot *)sqlite3ScratchMalloc( + aTmp = (ht_slot *)sqlite3_malloc( sizeof(ht_slot) * (iLast>HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) ); if( !aTmp ){ @@ -1586,7 +1586,7 @@ static int walIteratorInit(Wal *pWal, WalIterator **pp){ p->aSegment[i].aPgno = (u32 *)aPgno; } } - sqlite3ScratchFree(aTmp); + sqlite3_free(aTmp); if( rc!=SQLITE_OK ){ walIteratorFree(p); From 4d9f188f005b204e5e7aa4f60762ad67948d2f9c Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Nov 2014 17:23:24 +0000 Subject: [PATCH 533/710] Add various requirements evidence marks for sqlite3_config() options. FossilOrigin-Name: d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/global.c | 8 ++++++++ src/main.c | 25 ++++++++++++++++++++----- src/vdbesort.c | 2 +- 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 9decec2f00..e9f3481cba 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sdefinition\sof\sSQLITE_CONFIG_SCRATCH\sso\sthat\sat\smost\sone\sscratch\nbuffer\sis\sused\sper\sthread.\s\sUse\sthe\sgeneric\sheap\smemory\sallocator\sfor\sthe\nWalIterator\sobject\swhen\srunning\sa\scheckpoint. -D 2014-11-04T14:22:27.705 +C Add\svarious\srequirements\sevidence\smarks\sfor\ssqlite3_config()\soptions. +D 2014-11-04T17:23:24.610 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -185,7 +185,7 @@ F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee -F src/global.c 01c1f36ecfcf10770db648422a8852c222308bb9 +F src/global.c a50ad0b9ee328107a65aa8f5f3cd34905e74745c F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c c673d0972da4b6826a3e907029faf5be0840cd57 +F src/main.c 42f857be3cef3e1f9752d8e46d61345f80932396 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -296,7 +296,7 @@ F src/vdbeapi.c 900259bdd85cd66a9f210d8ec08147a9034593bd F src/vdbeaux.c cf6b8152dd22155201d57c216e6266866b61da59 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f -F src/vdbesort.c daf87ea542df088ac4607660d09976d36b6bd95d +F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 825c948066c7604a07d56e67958cdab210749016 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 681031a436fdd4cce426d6cd43cbae6b83167d26 -R cb98a95faa41015902ec00cd7a294542 +P 391c9b85abcb5ba300fb2e116384639310c69ed2 +R 59d979fd0dabfa4a2d3ae24d9471db8f U drh -Z 113a55cc7642fbc8d6bb5b88b140b929 +Z d72c7362e49ef35f8e5912118c20e6ff diff --git a/manifest.uuid b/manifest.uuid index cd70e30b51..969ed55bc1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -391c9b85abcb5ba300fb2e116384639310c69ed2 \ No newline at end of file +d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 \ No newline at end of file diff --git a/src/global.c b/src/global.c index e769eb425f..b5323e8df6 100644 --- a/src/global.c +++ b/src/global.c @@ -135,11 +135,19 @@ const unsigned char sqlite3CtypeMap[256] = { ** ** EVIDENCE-OF: R-38799-08373 URI filenames can be enabled or disabled ** using the SQLITE_USE_URI=1 or SQLITE_USE_URI=0 compile-time options. +** +** EVIDENCE-OF: R-43642-56306 By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** SQLITE_USE_URI symbol defined. */ #ifndef SQLITE_USE_URI # define SQLITE_USE_URI 0 #endif +/* EVIDENCE-OF: R-38720-18127 The default setting is determined by the +** SQLITE_ALLOW_COVERING_INDEX_SCAN compile-time option, or is "on" if +** that compile-time option is omitted. +*/ #ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN # define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 #endif diff --git a/src/main.c b/src/main.c index 80800d7cf6..4c33bbde9d 100644 --- a/src/main.c +++ b/src/main.c @@ -523,11 +523,19 @@ int sqlite3_config(int op, ...){ ** sqlite3_config(SQLITE_CONFIG_URI,0) configuration calls. */ case SQLITE_CONFIG_URI: { + /* EVIDENCE-OF: R-25451-61125 The SQLITE_CONFIG_URI option takes a single + ** argument of type int. If non-zero, then URI handling is globally + ** enabled. If the parameter is zero, then URI handling is globally + ** disabled. */ sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); break; } case SQLITE_CONFIG_COVERING_INDEX_SCAN: { + /* EVIDENCE-OF: R-36592-02772 The SQLITE_CONFIG_COVERING_INDEX_SCAN + ** option takes a single integer argument which is interpreted as a + ** boolean in order to enable or disable the use of covering indices for + ** full table scans in the query optimizer. */ sqlite3GlobalConfig.bUseCis = va_arg(ap, int); break; } @@ -542,20 +550,27 @@ int sqlite3_config(int op, ...){ #endif case SQLITE_CONFIG_MMAP_SIZE: { + /* EVIDENCE-OF: R-58063-38258 SQLITE_CONFIG_MMAP_SIZE takes two 64-bit + ** integer (sqlite3_int64) values that are the default mmap size limit + ** (the default setting for PRAGMA mmap_size) and the maximum allowed + ** mmap size limit. */ sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); - if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ){ - mxMmap = SQLITE_MAX_MMAP_SIZE; - } - sqlite3GlobalConfig.mxMmap = mxMmap; + /* EVIDENCE-OF: R-53367-43190 If either argument to this option is + ** negative, then that argument is changed to its compile-time default. */ + if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ) mxMmap = SQLITE_MAX_MMAP_SIZE; if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; if( szMmap>mxMmap) szMmap = mxMmap; + sqlite3GlobalConfig.mxMmap = mxMmap; sqlite3GlobalConfig.szMmap = szMmap; break; } -#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) +#if SQLITE_OS_WIN && defined(SQLITE_WIN32_MALLOC) /* IMP: R-04780-55815 */ case SQLITE_CONFIG_WIN32_HEAPSIZE: { + /* EVIDENCE-OF: R-34926-03360 SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit + ** unsigned integer value that specifies the maximum size of the created + ** heap. */ sqlite3GlobalConfig.nHeap = va_arg(ap, int); break; } diff --git a/src/vdbesort.c b/src/vdbesort.c index 918d840716..df8357a57e 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -100,7 +100,7 @@ ** The sorter is running in multi-threaded mode if (a) the library was built ** with pre-processor symbol SQLITE_MAX_WORKER_THREADS set to a value greater ** than zero, and (b) worker threads have been enabled at runtime by calling -** sqlite3_config(SQLITE_CONFIG_WORKER_THREADS, ...). +** "PRAGMA threads=N" with some value of N greater than 0. ** ** When Rewind() is called, any data remaining in memory is flushed to a ** final PMA. So at this point the data is stored in some number of sorted From 73767829b9a224197d8d046f653535770bd97174 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 4 Nov 2014 19:37:22 +0000 Subject: [PATCH 534/710] For the Win32 VFS, allow memory mapped files to work when compiled without WAL support. FossilOrigin-Name: 1fc7e2f3d34e25e7b59aa8b51d10c1e27ab4a527 --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/os_win.c | 27 +++++++++++++++++---------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index e9f3481cba..74e32143e0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\svarious\srequirements\sevidence\smarks\sfor\ssqlite3_config()\soptions. -D 2014-11-04T17:23:24.610 +C For\sthe\sWin32\sVFS,\sallow\smemory\smapped\sfiles\sto\swork\swhen\scompiled\swithout\sWAL\ssupport. +D 2014-11-04T19:37:22.576 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 -F src/os_win.c a019caaae2bcbbc0cc4c39af6e7d7e43d8426053 +F src/os_win.c a9e500dd963fb1f67d7860e58b5772abe6123862 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 8d97b3633f098fef817656dcbf167ca904511d78 F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b @@ -1211,7 +1211,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 391c9b85abcb5ba300fb2e116384639310c69ed2 -R 59d979fd0dabfa4a2d3ae24d9471db8f -U drh -Z d72c7362e49ef35f8e5912118c20e6ff +P d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 +R c0de55f8c674bc2f58a7f862c94a6470 +T *branch * winMmapNoWal +T *sym-winMmapNoWal * +T -sym-trunk * +U mistachkin +Z 8ecf95ce02a024463c10f127838cfcbc diff --git a/manifest.uuid b/manifest.uuid index 969ed55bc1..e9b06734b6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 \ No newline at end of file +1fc7e2f3d34e25e7b59aa8b51d10c1e27ab4a527 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 8ca2107d90..2a7681c73f 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -34,6 +34,11 @@ with SQLITE_OMIT_WAL." #endif +#if !SQLITE_OS_WINNT && SQLITE_MAX_MMAP_SIZE>0 +# error "Memory mapped files require support from the Windows NT kernel,\ + compile with SQLITE_MAX_MMAP_SIZE=0." +#endif + /* ** Are most of the Win32 ANSI APIs available (i.e. with certain exceptions ** based on the sub-platform)? @@ -163,10 +168,11 @@ /* ** Do we need to manually define the Win32 file mapping APIs for use with WAL -** mode (e.g. these APIs are available in the Windows CE SDK; however, they -** are not present in the header file)? +** mode or memory mapped files (e.g. these APIs are available in the Windows +** CE SDK; however, they are not present in the header file)? */ -#if SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) +#if SQLITE_WIN32_FILEMAPPING_API && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) /* ** Two of the file mapping APIs are different under WinRT. Figure out which ** set we need. @@ -194,7 +200,7 @@ WINBASEAPI LPVOID WINAPI MapViewOfFile(HANDLE, DWORD, DWORD, DWORD, SIZE_T); ** This file mapping API is common to both Win32 and WinRT. */ WINBASEAPI BOOL WINAPI UnmapViewOfFile(LPCVOID); -#endif /* SQLITE_WIN32_FILEMAPPING_API && !defined(SQLITE_OMIT_WAL) */ +#endif /* SQLITE_WIN32_FILEMAPPING_API */ /* ** Some Microsoft compilers lack this definition. @@ -487,7 +493,7 @@ static struct win_syscall { LPSECURITY_ATTRIBUTES,DWORD,DWORD,HANDLE))aSyscall[5].pCurrent) #if (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_ANSI) && \ - !defined(SQLITE_OMIT_WAL)) + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "CreateFileMappingA", (SYSCALL)CreateFileMappingA, 0 }, #else { "CreateFileMappingA", (SYSCALL)0, 0 }, @@ -497,7 +503,7 @@ static struct win_syscall { DWORD,DWORD,DWORD,LPCSTR))aSyscall[6].pCurrent) #if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && defined(SQLITE_WIN32_HAS_WIDE) && \ - !defined(SQLITE_OMIT_WAL)) + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "CreateFileMappingW", (SYSCALL)CreateFileMappingW, 0 }, #else { "CreateFileMappingW", (SYSCALL)0, 0 }, @@ -837,7 +843,8 @@ static struct win_syscall { LPOVERLAPPED))aSyscall[48].pCurrent) #endif -#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL)) +#if SQLITE_OS_WINCE || (!SQLITE_OS_WINRT && \ + (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0)) { "MapViewOfFile", (SYSCALL)MapViewOfFile, 0 }, #else { "MapViewOfFile", (SYSCALL)0, 0 }, @@ -907,7 +914,7 @@ static struct win_syscall { #define osUnlockFileEx ((BOOL(WINAPI*)(HANDLE,DWORD,DWORD,DWORD, \ LPOVERLAPPED))aSyscall[58].pCurrent) -#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINCE || !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "UnmapViewOfFile", (SYSCALL)UnmapViewOfFile, 0 }, #else { "UnmapViewOfFile", (SYSCALL)0, 0 }, @@ -970,7 +977,7 @@ static struct win_syscall { #define osGetFileInformationByHandleEx ((BOOL(WINAPI*)(HANDLE, \ FILE_INFO_BY_HANDLE_CLASS,LPVOID,DWORD))aSyscall[66].pCurrent) -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "MapViewOfFileFromApp", (SYSCALL)MapViewOfFileFromApp, 0 }, #else { "MapViewOfFileFromApp", (SYSCALL)0, 0 }, @@ -1034,7 +1041,7 @@ static struct win_syscall { #define osGetProcessHeap ((HANDLE(WINAPI*)(VOID))aSyscall[74].pCurrent) -#if SQLITE_OS_WINRT && !defined(SQLITE_OMIT_WAL) +#if SQLITE_OS_WINRT && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "CreateFileMappingFromApp", (SYSCALL)CreateFileMappingFromApp, 0 }, #else { "CreateFileMappingFromApp", (SYSCALL)0, 0 }, From fdece7bad171dbb43985dfc7f53cf4a6e368c350 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 4 Nov 2014 19:52:15 +0000 Subject: [PATCH 535/710] Skip tests that require WAL mode when it is not enabled. FossilOrigin-Name: 6fc4ead26d19b9348bbda34c3053ae1e066abc32 --- manifest | 15 ++++----- manifest.uuid | 2 +- test/mmap1.test | 86 +++++++++++++++++++++++++------------------------ 3 files changed, 51 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index 74e32143e0..cf7c330a5e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sthe\sWin32\sVFS,\sallow\smemory\smapped\sfiles\sto\swork\swhen\scompiled\swithout\sWAL\ssupport. -D 2014-11-04T19:37:22.576 +C Skip\stests\sthat\srequire\sWAL\smode\swhen\sit\sis\snot\senabled. +D 2014-11-04T19:52:15.610 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -728,7 +728,7 @@ F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 -F test/mmap1.test 93d167b328255cbe6679fe1e1a23be1b1197d07b +F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 F test/mmap3.test c92273e16eb8d23c1d55c9815b446bb72ef0512e F test/mmapfault.test d4c9eff9cd8c2dc14bc43e71e042f175b0a26fe3 @@ -1211,10 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 -R c0de55f8c674bc2f58a7f862c94a6470 -T *branch * winMmapNoWal -T *sym-winMmapNoWal * -T -sym-trunk * +P 1fc7e2f3d34e25e7b59aa8b51d10c1e27ab4a527 +R ab0636a3afda48f38dd438546ea79dde U mistachkin -Z 8ecf95ce02a024463c10f127838cfcbc +Z 673ee028789258ee8d4aa78195cdb56d diff --git a/manifest.uuid b/manifest.uuid index e9b06734b6..b10e2d0d51 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1fc7e2f3d34e25e7b59aa8b51d10c1e27ab4a527 \ No newline at end of file +6fc4ead26d19b9348bbda34c3053ae1e066abc32 \ No newline at end of file diff --git a/test/mmap1.test b/test/mmap1.test index ece3e0201e..18aec9f8fa 100644 --- a/test/mmap1.test +++ b/test/mmap1.test @@ -33,7 +33,7 @@ proc register_rblob_code {dbname seed} { set ::rcnt $seed proc rblob {n} { set ::rcnt [expr (([set ::rcnt] << 3) + [set ::rcnt] + 456) & 0xFFFFFFFF] - set str [format %.8x [expr [set ::rcnt] ^ 0xbdf20da3]] + set str [format %.8x [expr [set ::rcnt] ^ 0xbdf20da3]] string range [string repeat [set str] [expr [set n]/4]] 1 [set n] } $dbname func rblob rblob @@ -42,7 +42,7 @@ proc register_rblob_code {dbname seed} { # For cases 1.1 and 1.4, the number of pages read using xRead() is 4 on # unix and 9 on windows. The difference is that windows only ever maps -# an integer number of OS pages (i.e. creates mappings that are a multiple +# an integer number of OS pages (i.e. creates mappings that are a multiple # of 4KB in size). Whereas on unix any sized mapping may be created. # foreach {t mmap_size nRead c2init} { @@ -106,50 +106,52 @@ foreach {t mmap_size nRead c2init} { set ::rcnt 0 proc rblob {n} { set ::rcnt [expr (($::rcnt << 3) + $::rcnt + 456) & 0xFFFFFFFF] - set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] + set str [format %.8x [expr $::rcnt ^ 0xbdf20da3]] string range [string repeat $str [expr $n/4]] 1 $n } reset_db db func rblob rblob -do_execsql_test 2.1 { - PRAGMA auto_vacuum = 1; - PRAGMA mmap_size = 67108864; - PRAGMA journal_mode = wal; - CREATE TABLE t1(a, b, UNIQUE(a, b)); - INSERT INTO t1 VALUES(rblob(500), rblob(500)); - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 - PRAGMA wal_checkpoint; -} {67108864 wal 0 103 103} - -do_execsql_test 2.2 { - PRAGMA auto_vacuum; - SELECT count(*) FROM t1; -} {1 32} - -if {[permutation] != "inmemory_journal"} { - do_test 2.3 { - sqlite3 db2 test.db - db2 func rblob rblob - db2 eval { - DELETE FROM t1 WHERE (rowid%4); - PRAGMA wal_checkpoint; - } - db2 eval { - INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 - SELECT count(*) FROM t1; - } - } {16} - - do_execsql_test 2.4 { +ifcapable wal { + do_execsql_test 2.1 { + PRAGMA auto_vacuum = 1; + PRAGMA mmap_size = 67108864; + PRAGMA journal_mode = wal; + CREATE TABLE t1(a, b, UNIQUE(a, b)); + INSERT INTO t1 VALUES(rblob(500), rblob(500)); + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 2 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 4 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 8 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 32 PRAGMA wal_checkpoint; - } {0 24 24} - db2 close + } {67108864 wal 0 103 103} + + do_execsql_test 2.2 { + PRAGMA auto_vacuum; + SELECT count(*) FROM t1; + } {1 32} + + if {[permutation] != "inmemory_journal"} { + do_test 2.3 { + sqlite3 db2 test.db + db2 func rblob rblob + db2 eval { + DELETE FROM t1 WHERE (rowid%4); + PRAGMA wal_checkpoint; + } + db2 eval { + INSERT INTO t1 SELECT rblob(500), rblob(500) FROM t1; -- 16 + SELECT count(*) FROM t1; + } + } {16} + + do_execsql_test 2.4 { + PRAGMA wal_checkpoint; + } {0 24 24} + db2 close + } } reset_db @@ -227,7 +229,7 @@ do_test 4.4 { do_execsql_test 4.5 { COMMIT } #------------------------------------------------------------------------- -# Ensure that existing cursors holding xFetch() references are not +# Ensure that existing cursors holding xFetch() references are not # confused if those pages are moved to make way for the root page of a # new table or index. # @@ -296,7 +298,7 @@ foreach {tn1 mmap1 mmap2} { sql1 "PRAGMA mmap_size = $mmap1" sql2 "PRAGMA mmap_size = $mmap2" - do_test $tn1.$tn { + do_test $tn1.$tn { for {set i 1} {$i <= 100} {incr i} { if {$i % 2} { set c1 sql1 @@ -311,7 +313,7 @@ foreach {tn1 mmap1 mmap2} { UPDATE t2 SET x = (SELECT md5sum(a) FROM t1); } - set res [$c2 { + set res [$c2 { SELECT count(*) FROM t1; SELECT x == (SELECT md5sum(a) FROM t1) FROM t2; PRAGMA integrity_check; From 547fb61807b3ac19ee97d5acc99e0810e4baba5d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 4 Nov 2014 21:38:45 +0000 Subject: [PATCH 536/710] Improved output formatting for the showstat4 tool. FossilOrigin-Name: 7df82c46da437bc743576358c25e758280067df8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/showstat4.c | 9 +++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index e9f3481cba..9ad9755cc5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\svarious\srequirements\sevidence\smarks\sfor\ssqlite3_config()\soptions. -D 2014-11-04T17:23:24.610 +C Improved\soutput\sformatting\sfor\sthe\sshowstat4\stool. +D 2014-11-04T21:38:45.383 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1190,7 +1190,7 @@ F tool/restore_jrnl.tcl 6957a34f8f1f0f8285e07536225ec3b292a9024a F tool/rollback-test.c 9fc98427d1e23e84429d7e6d07d9094fbdec65a5 F tool/showdb.c bd073a78bce714a0e42d92ea474b3eb8cb53be5d F tool/showjournal.c 053eb1cc774710c6890b7dd6293300cc297b16a5 -F tool/showstat4.c c39279d6bd37cb999b634f0064f6f86ad7af008f +F tool/showstat4.c 9515faa8ec176599d4a8288293ba8ec61f7b728a F tool/showwal.c 85cb36d4fe3e93e2fbd63e786e0d1ce42d0c4fad F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 391c9b85abcb5ba300fb2e116384639310c69ed2 -R 59d979fd0dabfa4a2d3ae24d9471db8f +P d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 +R 3d8e834a0bd7bfcb019dc53478b58f7b U drh -Z d72c7362e49ef35f8e5912118c20e6ff +Z 0a79c8f46f24bb92a10190eb30b27d6d diff --git a/manifest.uuid b/manifest.uuid index 969ed55bc1..11e49b8e8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 \ No newline at end of file +7df82c46da437bc743576358c25e758280067df8 \ No newline at end of file diff --git a/tool/showstat4.c b/tool/showstat4.c index 668d2106af..215962919e 100644 --- a/tool/showstat4.c +++ b/tool/showstat4.c @@ -39,6 +39,7 @@ int main(int argc, char **argv){ int nSample; i64 iVal; const char *zSep; + int iRow = 0; if( argc!=2 ){ fprintf(stderr, "Usage: %s DATABASE-FILE\n", argv[0]); @@ -60,13 +61,13 @@ int main(int argc, char **argv){ } while( SQLITE_ROW==sqlite3_step(pStmt) ){ if( zIdx==0 || strcmp(zIdx, (const char*)sqlite3_column_text(pStmt,0))!=0 ){ - if( zIdx ) printf("\n"); + if( zIdx ) printf("\n**************************************" + "**************\n\n"); sqlite3_free(zIdx); zIdx = sqlite3_mprintf("%s", sqlite3_column_text(pStmt,0)); - printf("%s:\n", zIdx); - }else{ - printf(" -----------------------------------------------------------\n"); + iRow = 0; } + printf("%s sample %d ------------------------------------\n", zIdx, ++iRow); printf(" nEq = %s\n", sqlite3_column_text(pStmt,1)); printf(" nLt = %s\n", sqlite3_column_text(pStmt,2)); printf(" nDLt = %s\n", sqlite3_column_text(pStmt,3)); From 8d1edb92c4c5d29bb0746b79778692b66e2c1296 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Nov 2014 09:07:28 +0000 Subject: [PATCH 537/710] Add the ".scanstats on" command to the shell tool. Executing this command causes the shell tool to print values from sqlite3_stmt_scanstatus() after each query is run. FossilOrigin-Name: 7974c0ed10ffdc960a43fed89845c2bed428958d --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/shell.c | 54 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 9ad9755cc5..29e1d31cf2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\soutput\sformatting\sfor\sthe\sshowstat4\stool. -D 2014-11-04T21:38:45.383 +C Add\sthe\s".scanstats\son"\scommand\sto\sthe\sshell\stool.\sExecuting\sthis\scommand\scauses\sthe\sshell\stool\sto\sprint\svalues\sfrom\ssqlite3_stmt_scanstatus()\safter\seach\squery\sis\srun. +D 2014-11-05T09:07:28.365 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 282f8f5278e0c78eb442217531172ec9e1538796 +F src/shell.c 5ad1eb4dfcd7a57e15825207a9bd559415bf34b1 F src/sqlite.h.in 6e9af739d79f0bea2584b70fb1c54d3bb1a2eab6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d423349d2cd8bc7e04f3d90ca7bab11e1ad86e25 -R 3d8e834a0bd7bfcb019dc53478b58f7b -U drh -Z 0a79c8f46f24bb92a10190eb30b27d6d +P 7df82c46da437bc743576358c25e758280067df8 +R f4931dcb1177b970d566df3dbaa382ec +U dan +Z 44baa22d0450fc9fcacf5946d93d71c5 diff --git a/manifest.uuid b/manifest.uuid index 11e49b8e8c..ccd696efe4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7df82c46da437bc743576358c25e758280067df8 \ No newline at end of file +7974c0ed10ffdc960a43fed89845c2bed428958d \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 59cd2011e7..3423bb0143 100644 --- a/src/shell.c +++ b/src/shell.c @@ -457,6 +457,7 @@ struct ShellState { int echoOn; /* True to echo input commands */ int autoEQP; /* Run EXPLAIN QUERY PLAN prior to seach SQL stmt */ int statsOn; /* True to display memory stats before each finalize */ + int scanstatsOn; /* True to display scan stats before each finalize */ int outCount; /* Revert to stdout when reaching zero */ int cnt; /* Number of records displayed so far */ FILE *out; /* Write results here */ @@ -1185,6 +1186,42 @@ static int display_stats( return 0; } +/* +** Display scan stats. +*/ +static void display_scanstats( + sqlite3 *db, /* Database to query */ + ShellState *pArg /* Pointer to ShellState */ +){ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int i; + fprintf(pArg->out, "-------- scanstats --------\n"); + for(i=0; 1; i++){ + sqlite3_stmt *p = pArg->pStmt; + sqlite3_int64 nEst, nLoop, nVisit; + const char *zExplain; + if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ + break; + } + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&nEst); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); + + fprintf(pArg->out, "Loop %d: \"%s\"\n", i, zExplain); + fprintf(pArg->out, " nLoop=%-8lld nVisit=%-8lld nEst=%-8lld\n", + nLoop, nVisit, nEst + ); + } +#else + fprintf(pArg->out, "-------- scanstats --------\n"); + fprintf(pArg->out, + "sqlite3_stmt_scanstatus() unavailable - " + "rebuild with SQLITE_ENABLE_STMT_SCANSTATUS\n" + ); +#endif + fprintf(pArg->out, "---------------------------\n"); +} + /* ** Parameter azArray points to a zero-terminated array of strings. zStr ** points to a single nul-terminated string. Return non-zero if zStr @@ -1423,6 +1460,11 @@ static int shell_exec( display_stats(db, pArg, 0); } + /* print loop-counters if required */ + if( pArg && pArg->scanstatsOn ){ + display_scanstats(db, pArg); + } + /* Finalize the statement just executed. If this fails, save a ** copy of the error message. Otherwise, set zSql to point to the ** next statement to execute. */ @@ -3014,6 +3056,16 @@ static int do_meta_command(char *zLine, ShellState *p){ sqlite3_close(pSrc); }else + + if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ + if( nArg==2 ){ + p->scanstatsOn = booleanValue(azArg[1]); + }else{ + fprintf(stderr, "Usage: .scanstats on|off\n"); + rc = 1; + } + }else + if( c=='s' && strncmp(azArg[0], "schema", n)==0 ){ ShellState data; char *zErrMsg = 0; @@ -4140,6 +4192,8 @@ int main(int argc, char **argv){ data.autoEQP = 1; }else if( strcmp(z,"-stats")==0 ){ data.statsOn = 1; + }else if( strcmp(z,"-scanstats")==0 ){ + data.scanstatsOn = 1; }else if( strcmp(z,"-bail")==0 ){ bail_on_error = 1; }else if( strcmp(z,"-version")==0 ){ From e0de876e279cc6cfd9f809a145f1172cbae9837e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Nov 2014 13:13:13 +0000 Subject: [PATCH 538/710] Enhance whereLoopCheaperProperSubset(X,Y) so that it does not report true if X uses skip-scan less than Y, since in that case X might deserve to be cheaper even if it is a proper subset. FossilOrigin-Name: c106b755369c1f8546e897ecd2ac56fd09d6e885 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 12 +++++++----- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 29e1d31cf2..8e0066d70e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s".scanstats\son"\scommand\sto\sthe\sshell\stool.\sExecuting\sthis\scommand\scauses\sthe\sshell\stool\sto\sprint\svalues\sfrom\ssqlite3_stmt_scanstatus()\safter\seach\squery\sis\srun. -D 2014-11-05T09:07:28.365 +C Enhance\swhereLoopCheaperProperSubset(X,Y)\sso\sthat\sit\sdoes\snot\sreport\strue\nif\sX\suses\sskip-scan\sless\sthan\sY,\ssince\sin\sthat\scase\sX\smight\ndeserve\sto\sbe\scheaper\seven\sif\sit\sis\sa\sproper\ssubset. +D 2014-11-05T13:13:13.983 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 825c948066c7604a07d56e67958cdab210749016 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c d5fa1081bf7cb70478ed06489e063695c92ee1e1 +F src/where.c 240961041f35862ebcafd260587c79a1ab7347f8 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7df82c46da437bc743576358c25e758280067df8 -R f4931dcb1177b970d566df3dbaa382ec -U dan -Z 44baa22d0450fc9fcacf5946d93d71c5 +P 7974c0ed10ffdc960a43fed89845c2bed428958d +R 418c93fe7ac7140cffa87d5f3271cb48 +U drh +Z 68c42aaa0a73598a09ed4215a5cfd466 diff --git a/manifest.uuid b/manifest.uuid index ccd696efe4..da46c6359a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7974c0ed10ffdc960a43fed89845c2bed428958d \ No newline at end of file +c106b755369c1f8546e897ecd2ac56fd09d6e885 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d7966fdfbc..34614e6bb3 100644 --- a/src/where.c +++ b/src/where.c @@ -4012,10 +4012,11 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ } /* -** Return TRUE if both of the following are true: +** Return TRUE if all of the following are true: ** ** (1) X has the same or lower cost that Y ** (2) X is a proper subset of Y +** (3) X skips at least as many columns as Y ** ** By "proper subset" we mean that X uses fewer WHERE clause terms ** than Y and that every WHERE clause term used by X is also used @@ -4023,7 +4024,9 @@ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ ** ** If X is a proper subset of Y then Y is a better choice and ought ** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. +** relationship is inverted and needs to be adjusted. The third rule +** was added because if X uses skip-scan less than Y it still might +** deserve a lower cost even if it is a proper subset of Y. */ static int whereLoopCheaperProperSubset( const WhereLoop *pX, /* First WhereLoop to compare */ @@ -4033,6 +4036,7 @@ static int whereLoopCheaperProperSubset( if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ return 0; /* X is not a subset of Y */ } + if( pY->nSkip > pX->nSkip ) return 0; if( pX->rRun >= pY->rRun ){ if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ @@ -4068,9 +4072,7 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ /* Adjust pTemplate cost downward so that it is cheaper than its - ** subset p. Except, do not adjust the cost estimate downward for - ** a loop that skips more columns. */ - if( pTemplate->nSkip>p->nSkip ) continue; + ** subset p. */ WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); pTemplate->rRun = p->rRun; From 937994aa6596a0a77504f06d86d11f30bdc82fb3 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 5 Nov 2014 14:19:05 +0000 Subject: [PATCH 539/710] Add a test case to check that the previous commit is effective. FossilOrigin-Name: 948d6e5d07bc14b6de32ec2144c716a5532f894c --- manifest | 14 ++++++------ manifest.uuid | 2 +- test/skipscan6.test | 55 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8e0066d70e..c849d3cf1c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\swhereLoopCheaperProperSubset(X,Y)\sso\sthat\sit\sdoes\snot\sreport\strue\nif\sX\suses\sskip-scan\sless\sthan\sY,\ssince\sin\sthat\scase\sX\smight\ndeserve\sto\sbe\scheaper\seven\sif\sit\sis\sa\sproper\ssubset. -D 2014-11-05T13:13:13.983 +C Add\sa\stest\scase\sto\scheck\sthat\sthe\sprevious\scommit\sis\seffective. +D 2014-11-05T14:19:05.905 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -849,7 +849,7 @@ F test/skipscan1.test 7e15e1cc524524e7b2c4595ec85c75501d22f4ff F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 -F test/skipscan6.test 3a891b45d6df266ced861a2ad9d03fca2bc7fcc5 +F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test c4400e7533748f6bd7413851ff148645e82b9e2d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7974c0ed10ffdc960a43fed89845c2bed428958d -R 418c93fe7ac7140cffa87d5f3271cb48 -U drh -Z 68c42aaa0a73598a09ed4215a5cfd466 +P c106b755369c1f8546e897ecd2ac56fd09d6e885 +R 27944d6d7573adcf0b31a08e199961ff +U dan +Z 1a8729ffc04f45ef7b13c20540a14cbb diff --git a/manifest.uuid b/manifest.uuid index da46c6359a..911dbed051 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c106b755369c1f8546e897ecd2ac56fd09d6e885 \ No newline at end of file +948d6e5d07bc14b6de32ec2144c716a5532f894c \ No newline at end of file diff --git a/test/skipscan6.test b/test/skipscan6.test index 9eda9a66f3..026c4d7b00 100644 --- a/test/skipscan6.test +++ b/test/skipscan6.test @@ -141,5 +141,60 @@ do_execsql_test 2.2 { } {/INDEX good .bb=. AND aa=. AND dd>. AND dd<../} +# Create a table containing 100 rows. Column "a" contains a copy of the +# rowid value - sequentially increasing integers from 1 to 100. Column +# "b" contains the value of (a % 5). Columns "c" and "d" both contain +# constant values (i.e. the same for every row). +# +# Then create a second table t2. t2 is the same as t3 except for the +# order in which the indexes are created. +# +do_execsql_test 3.0 { + CREATE TABLE t3(a, b, c, d); + CREATE INDEX t3_ba ON t3(b, a, c); + CREATE INDEX t3_a ON t3(a); + + WITH d(a, b) AS ( + SELECT 1, 1 + UNION ALL + SELECT a+1, (a+1) % 5 FROM d WHERE a<100 + ) + INSERT INTO t3 SELECT a, b, 'c', 'd' FROM d; + + CREATE TABLE t2(a, b, c, d); + CREATE INDEX t2_a ON t2(a); + CREATE INDEX t2_ba ON t2(b, a, c); + INSERT INTO t2 SELECT * FROM t3; + + ANALYZE; + SELECT * FROM sqlite_stat1; +} { + t2 t2_ba {100 20 1 1} + t2 t2_a {100 1} + t3 t3_a {100 1} + t3 t3_ba {100 20 1 1} +} + +# Use index "t3_a", as (a=?) is expected to match only a single row. +# +do_eqp_test 3.1 { + SELECT * FROM t3 WHERE a = ? AND c = ? +} { + 0 0 0 {SEARCH TABLE t3 USING INDEX t3_a (a=?)} +} + +# The same query on table t2. This should use index "t2_a", for the +# same reason. At one point though, it was mistakenly using a skip-scan. +# +do_eqp_test 3.2 { + SELECT * FROM t2 WHERE a = ? AND c = ? +} { + 0 0 0 {SEARCH TABLE t2 USING INDEX t2_a (a=?)} +} + +finish_test + + + finish_test From 6b5631e02f2204da55742ec6e89b3633d36f51cb Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Nov 2014 15:57:39 +0000 Subject: [PATCH 540/710] Make sure that NULL results from OP_Column are fully and completely NULL and do not have the MEM_Ephem bit set. Fix for ticket [094d39a4c95ee4]. FossilOrigin-Name: 42705fd7d892c4fdfb95fbbb468c99569beece25 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- test/table.test | 18 +++++++++++++++++- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index c849d3cf1c..29f44c63b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sto\scheck\sthat\sthe\sprevious\scommit\sis\seffective. -D 2014-11-05T14:19:05.905 +C Make\ssure\sthat\sNULL\sresults\sfrom\sOP_Column\sare\sfully\sand\scompletely\sNULL\nand\sdo\snot\shave\sthe\sMEM_Ephem\sbit\sset.\s\sFix\sfor\sticket\s[094d39a4c95ee4]. +D 2014-11-05T15:57:39.984 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 175a360c56e75ce4eb2b60704fd7c011b93926f5 +F src/vdbe.c 3fd4ebd3e87b63175bfd2be747608bae1670b4df F src/vdbe.h d412bd01e89f0d69991b8f46601f96bc169d28f4 F src/vdbeInt.h 539ba284790e871f98be74a78cbdfcedfae22639 F src/vdbeapi.c 900259bdd85cd66a9f210d8ec08147a9034593bd @@ -879,7 +879,7 @@ F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/table.test 2a1d2fa52c531de5915f28023747d9a8c27b6f31 +F test/table.test 06271d61eb13871490d38168433c1ef3dd82bb2a F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c106b755369c1f8546e897ecd2ac56fd09d6e885 -R 27944d6d7573adcf0b31a08e199961ff -U dan -Z 1a8729ffc04f45ef7b13c20540a14cbb +P 948d6e5d07bc14b6de32ec2144c716a5532f894c +R 4628888308aa60120db393287d665a20 +U drh +Z e07070b00a71f6116b6f23e15e08fb5e diff --git a/manifest.uuid b/manifest.uuid index 911dbed051..8df21c81cd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -948d6e5d07bc14b6de32ec2144c716a5532f894c \ No newline at end of file +42705fd7d892c4fdfb95fbbb468c99569beece25 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 62470a743b..bf9b233a26 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2301,7 +2301,7 @@ case OP_Column: { pC->payloadSize = pC->szRow = avail = pReg->n; pC->aRow = (u8*)pReg->z; }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); goto op_column_out; } }else{ diff --git a/test/table.test b/test/table.test index 656884ca73..69f105aa6c 100644 --- a/test/table.test +++ b/test/table.test @@ -11,7 +11,6 @@ # This file implements regression tests for SQLite library. The # focus of this file is testing the CREATE TABLE statement. # -# $Id: table.test,v 1.53 2009/06/05 17:09:12 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -773,4 +772,21 @@ do_catchsql_test table-16.7 { INSERT INTO t16 DEFAULT VALUES; } {1 {unknown function: group_concat()}} +# Ticket [https://www.sqlite.org/src/info/094d39a4c95ee4abbc417f04214617675ba15c63] +# describes a assertion fault that occurs on a CREATE TABLE .. AS SELECT statement. +# the following test verifies that the problem has been fixed. +# +do_execsql_test table-17.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a TEXT); + INSERT INTO t1(a) VALUES(1),(2); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(x TEXT, y TEXT); + INSERT INTO t2(x,y) VALUES(3,4); + DROP TABLE IF EXISTS t3; + CREATE TABLE t3 AS + SELECT a AS p, coalesce(y,a) AS q FROM t1 LEFT JOIN t2 ON a=x; + SELECT p, q, '|' FROM t3 ORDER BY p; +} {1 1 | 2 2 |} + finish_test From 6d3f91d040ae42394d7b1534cce8ead374cc0f7c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 5 Nov 2014 19:26:12 +0000 Subject: [PATCH 541/710] Change the query planner to do a better job of estimating the number rows selected by a BETWEEN operator using STAT4 when both upper and lower bounds are contained within the same sample. FossilOrigin-Name: 2d36be5d9a1cdd4fd2d54fc4eeece32a81cbacc1 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 34 +++++++++++++++++++++------------- test/analyze8.test | 8 ++++---- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 29f44c63b9..d1dcbe649d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthat\sNULL\sresults\sfrom\sOP_Column\sare\sfully\sand\scompletely\sNULL\nand\sdo\snot\shave\sthe\sMEM_Ephem\sbit\sset.\s\sFix\sfor\sticket\s[094d39a4c95ee4]. -D 2014-11-05T15:57:39.984 +C Change\sthe\squery\splanner\sto\sdo\sa\sbetter\sjob\sof\sestimating\sthe\snumber\srows\nselected\sby\sa\sBETWEEN\soperator\susing\sSTAT4\swhen\sboth\supper\sand\slower\sbounds\nare\scontained\swithin\sthe\ssame\ssample. +D 2014-11-05T19:26:12.741 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 825c948066c7604a07d56e67958cdab210749016 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 240961041f35862ebcafd260587c79a1ab7347f8 +F src/where.c 2c2081c546c90227577c502765611555503ce3f7 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -321,7 +321,7 @@ F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test f1c552ce39cca4ec922a7e4e0e5d0203d6b3281f F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f -F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 +F test/analyze8.test c05a461d0a6b05991106467d0c47480f2e709c82 F test/analyze9.test 72795c8113604b5dcd47a1498a61d6d7fb5d041a F test/analyzeA.test 3335697f6700c7052295cfd0067fc5b2aacddf9a F test/analyzeB.test 8bf35ee0a548aea831bf56762cb8e7fdb1db083d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 948d6e5d07bc14b6de32ec2144c716a5532f894c -R 4628888308aa60120db393287d665a20 +P 42705fd7d892c4fdfb95fbbb468c99569beece25 +R b31ea7cd782cb3cc21bda8e8c039f16c U drh -Z e07070b00a71f6116b6f23e15e08fb5e +Z 7c6a788d1b7b1ae57295ef90b3d9da88 diff --git a/manifest.uuid b/manifest.uuid index 8df21c81cd..64b0a08a4b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42705fd7d892c4fdfb95fbbb468c99569beece25 \ No newline at end of file +2d36be5d9a1cdd4fd2d54fc4eeece32a81cbacc1 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 34614e6bb3..92f783b2e9 100644 --- a/src/where.c +++ b/src/where.c @@ -1897,7 +1897,6 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ } #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ - #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** Estimate the location of a particular key among all keys in an @@ -1906,9 +1905,10 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ ** aStat[0] Est. number of rows less than pVal ** aStat[1] Est. number of rows equal to pVal ** -** Return SQLITE_OK on success. +** Return the index of the sample that is the smallest sample that +** is greater than or equal to pRec. */ -static void whereKeyStats( +static int whereKeyStats( Parse *pParse, /* Database connection */ Index *pIdx, /* Index to consider domain of */ UnpackedRecord *pRec, /* Vector of values to consider */ @@ -1990,6 +1990,7 @@ static void whereKeyStats( } aStat[0] = iLower + iGap; } + return i; } #endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ @@ -2140,7 +2141,7 @@ static int whereRangeSkipScanEst( ** If either of the upper or lower bound is not present, then NULL is passed in ** place of the corresponding WhereTerm. ** -** The value in (pBuilder->pNew->u.btree.nEq) is the index of the index +** The value in (pBuilder->pNew->u.btree.nEq) is the number of the index ** column subject to the range constraint. Or, equivalently, the number of ** equality constraints optimized by the proposed index scan. For example, ** assuming index p is on t1(a, b), and the SQL query is: @@ -2156,7 +2157,7 @@ static int whereRangeSkipScanEst( ** ** When this function is called, *pnOut is set to the sqlite3LogEst() of the ** number of rows that the index scan is expected to visit without -** considering the range constraints. If nEq is 0, this is the number of +** considering the range constraints. If nEq is 0, then *pnOut is the number of ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) ** to account for the range constraints pLower and pUpper. ** @@ -2180,9 +2181,7 @@ static int whereRangeScanEst( Index *p = pLoop->u.btree.pIndex; int nEq = pLoop->u.btree.nEq; - if( p->nSample>0 - && nEqnSampleCol - ){ + if( p->nSample>0 && nEqnSampleCol ){ if( nEq==pBuilder->nRecValid ){ UnpackedRecord *pRec = pBuilder->pRec; tRowcnt a[2]; @@ -2198,15 +2197,19 @@ static int whereRangeScanEst( ** is not a simple variable or literal value), the lower bound of the ** range is $P. Due to a quirk in the way whereKeyStats() works, even ** if $L is available, whereKeyStats() is called for both ($P) and - ** ($P:$L) and the larger of the two returned values used. + ** ($P:$L) and the larger of the two returned values is used. ** ** Similarly, iUpper is to be set to the estimate of the number of rows ** less than the upper bound of the range query. Where the upper bound ** is either ($P) or ($P:$U). Again, even if $U is available, both values ** of iUpper are requested of whereKeyStats() and the smaller used. + ** + ** The number of rows between the two bounds is then just iUpper-iLower. */ - tRowcnt iLower; - tRowcnt iUpper; + tRowcnt iLower; /* Rows less than the lower bound */ + tRowcnt iUpper; /* Rows less than the upper bound */ + int iLwrIdx = -2; /* aSample[] for the lower bound */ + int iUprIdx = -1; /* aSample[] for the upper bound */ if( pRec ){ testcase( pRec->nField!=pBuilder->nRecValid ); @@ -2244,7 +2247,7 @@ static int whereRangeScanEst( rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); if( rc==SQLITE_OK && bOk ){ tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 0, a); + iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); if( iNew>iLower ) iLower = iNew; nOut--; @@ -2259,7 +2262,7 @@ static int whereRangeScanEst( rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); if( rc==SQLITE_OK && bOk ){ tRowcnt iNew; - whereKeyStats(pParse, p, pRec, 1, a); + iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); if( iNewiLower ){ nNew = sqlite3LogEst(iUpper - iLower); + /* TUNING: If both iUpper and iLower are derived from the same + ** sample, then assume they are 4x more selective. This brings + ** the estimated selectivity more in line with what it would be + ** if estimated without the use of STAT3/4 tables. */ + if( iLwrIdx==iUprIdx ) nNew -= 20; assert( 20==sqlite3LogEst(4) ); }else{ nNew = 10; assert( 10==sqlite3LogEst(2) ); } diff --git a/test/analyze8.test b/test/analyze8.test index 4384c39676..1079e68080 100644 --- a/test/analyze8.test +++ b/test/analyze8.test @@ -86,23 +86,23 @@ do_test 2.1 { # range. # # Test 3.2 is a little unstable. It depends on the planner estimating -# that (b BETWEEN 50 AND 54) will match more rows than (c BETWEEN +# that (b BETWEEN 30 AND 34) will match more rows than (c BETWEEN # 800000 AND 900000). Which is a pretty close call (50 vs. 32), so # the planner could get it wrong with an unlucky set of samples. This # case happens to work, but others ("b BETWEEN 40 AND 44" for example) # will fail. # do_execsql_test 3.0 { - SELECT count(*) FROM t1 WHERE b BETWEEN 50 AND 54; + SELECT count(*) FROM t1 WHERE b BETWEEN 30 AND 34; SELECT count(*) FROM t1 WHERE c BETWEEN 0 AND 100000; SELECT count(*) FROM t1 WHERE c BETWEEN 800000 AND 900000; } {50 376 32} do_test 3.1 { - eqp {SELECT * FROM t1 WHERE b BETWEEN 50 AND 54 AND c BETWEEN 0 AND 100000} + eqp {SELECT * FROM t1 WHERE b BETWEEN 30 AND 34 AND c BETWEEN 0 AND 100000} } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b? AND c Date: Wed, 5 Nov 2014 21:21:08 +0000 Subject: [PATCH 542/710] Fix harmless compiler warnings in the new balance_nonroot() routine. FossilOrigin-Name: 83a1e5db926b3a6d40f4a5cf9a8e6852b9bac9ac --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 19 ++++++++++++------- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index d1dcbe649d..6994d5e8ec 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\squery\splanner\sto\sdo\sa\sbetter\sjob\sof\sestimating\sthe\snumber\srows\nselected\sby\sa\sBETWEEN\soperator\susing\sSTAT4\swhen\sboth\supper\sand\slower\sbounds\nare\scontained\swithin\sthe\ssame\ssample. -D 2014-11-05T19:26:12.741 +C Fix\sharmless\scompiler\swarnings\sin\sthe\snew\sbalance_nonroot()\sroutine. +D 2014-11-05T21:21:08.046 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 11d1262110c2d459b68121833fa3ec6625b1d022 +F src/btree.c 4a126e2066076872ab6f37f9ad116eb5f651cd38 F src/btree.h 49b408be9c1cd41249076898e0673711071205d8 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 42705fd7d892c4fdfb95fbbb468c99569beece25 -R b31ea7cd782cb3cc21bda8e8c039f16c +P 2d36be5d9a1cdd4fd2d54fc4eeece32a81cbacc1 +R e9a3ba77188e96b62213debbd39f0d21 U drh -Z 7c6a788d1b7b1ae57295ef90b3d9da88 +Z dda139af7d076df31a05d3458397d7d6 diff --git a/manifest.uuid b/manifest.uuid index 64b0a08a4b..82de0553f1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d36be5d9a1cdd4fd2d54fc4eeece32a81cbacc1 \ No newline at end of file +83a1e5db926b3a6d40f4a5cf9a8e6852b9bac9ac \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index fe15c922ca..8b4a2a4f3b 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1297,8 +1297,8 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ u8 * const data = pPage->aData; /* Local cache of pPage->aData */ int top; /* First byte of cell content area */ + int rc = SQLITE_OK; /* Integer return code */ int gap; /* First byte of gap between cell pointers and cell content */ - int rc; /* Integer return code */ assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt ); @@ -1328,13 +1328,13 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+1==top ); testcase( gap==top ); if( gap+2<=top && (data[hdr+1] || data[hdr+2]) ){ - int rc = SQLITE_OK; int bDefrag = 0; u8 *pSpace = pageFindSlot(pPage, nByte, &rc, &bDefrag); if( rc ) return rc; if( bDefrag ) goto defragment_page; if( pSpace ){ - *pIdx = pSpace - data; + assert( pSpace>=data && (pSpace - data)<65536 ); + *pIdx = (int)(pSpace - data); return SQLITE_OK; } } @@ -6117,7 +6117,10 @@ static int pageFreeArray( if( pCell>=pStart && pCellaData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } pFree = pCell; szFree = sz; if( pFree+sz>pEnd ) return 0; @@ -6128,7 +6131,10 @@ static int pageFreeArray( nRet++; } } - if( pFree ) freeSpace(pPg, pFree - aData, szFree); + if( pFree ){ + assert( pFree>aData && (pFree - aData)<65536 ); + freeSpace(pPg, (u16)(pFree - aData), szFree); + } return nRet; } @@ -6192,7 +6198,7 @@ static void editPage( for(i=0; inOverflow; i++){ int iCell = (iOld + pPg->aiOvfl[i]) - iNew; if( iCell>=0 && iCellaCellIdx[iCell * 2]; + pCellptr = &pPg->aCellIdx[iCell * 2]; memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); nCell++; if( pageInsertArray( @@ -6881,7 +6887,6 @@ static int balance_nonroot( } for(i=0; i Date: Thu, 6 Nov 2014 03:55:10 +0000 Subject: [PATCH 543/710] Change the SQLITE_SCANSTAT_EST parameter so that it returns a double for the estimated number of output rows per loop, rather than a 64-bit integer. Revise the output format for the ".scanstats on" in the shell to make use of this new capability. FossilOrigin-Name: f9684000665ae7ef6f89c3773612b8286b8f545a --- manifest | 32 ++++++++++--------- manifest.uuid | 2 +- src/shell.c | 13 +++++--- src/sqlite.h.in | 12 ++++--- src/test1.c | 6 ++-- src/vdbe.h | 2 +- src/vdbeInt.h | 2 +- src/vdbeapi.c | 8 ++++- src/vdbeaux.c | 2 +- src/where.c | 6 +--- test/scanstatus.test | 76 ++++++++++++++++++++++---------------------- 11 files changed, 85 insertions(+), 76 deletions(-) diff --git a/manifest b/manifest index a8453434a2..75d37178fb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sto\sthe\sWindows\sVFS\sto\sallow\smemory\smapped\sfiles\sto\swork\swithout\sWAL\ssupport. -D 2014-11-05T21:34:56.096 +C Change\sthe\sSQLITE_SCANSTAT_EST\sparameter\sso\sthat\sit\sreturns\sa\sdouble\sfor\nthe\sestimated\snumber\sof\soutput\srows\sper\sloop,\srather\sthan\sa\s64-bit\sinteger.\nRevise\sthe\soutput\sformat\sfor\sthe\s".scanstats\son"\sin\sthe\sshell\sto\smake\suse\nof\sthis\snew\scapability. +D 2014-11-06T03:55:10.745 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,8 +228,8 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 5ad1eb4dfcd7a57e15825207a9bd559415bf34b1 -F src/sqlite.h.in 6e9af739d79f0bea2584b70fb1c54d3bb1a2eab6 +F src/shell.c 908ff96ef1551b28b940aaf4c886ba2681057209 +F src/sqlite.h.in e13a7b64efa8d6a591577e6a5281fb22783c0133 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -237,7 +237,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 7cdd4dd3c2a4183483feca260070d73d6e22cd47 -F src/test1.c 5890094c09691fe9564cf0f0d5b22d35b3218c47 +F src/test1.c ac7f3bad83ef4508d5efc85b32e86da48db8ed7e F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -290,10 +290,10 @@ F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 3fd4ebd3e87b63175bfd2be747608bae1670b4df -F src/vdbe.h d412bd01e89f0d69991b8f46601f96bc169d28f4 -F src/vdbeInt.h 539ba284790e871f98be74a78cbdfcedfae22639 -F src/vdbeapi.c 900259bdd85cd66a9f210d8ec08147a9034593bd -F src/vdbeaux.c cf6b8152dd22155201d57c216e6266866b61da59 +F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 +F src/vdbeInt.h c32c1de25e3821a5b53d73abdb23ccc644ec5b63 +F src/vdbeapi.c 6a126fd8ed297ff0542bfbf7891b92977b5ed653 +F src/vdbeaux.c 9b0a251b6dfab349dd6c6efb40062eb7386b26f5 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a @@ -302,7 +302,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 825c948066c7604a07d56e67958cdab210749016 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 2c2081c546c90227577c502765611555503ce3f7 +F src/where.c 3862a1173ae2716bde12f1ab3fb649f1d85b05c2 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -801,7 +801,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 -F test/scanstatus.test 01afb2220f18ce85f9e338c20684f428d56e5c01 +F test/scanstatus.test a6dd739bc4d9638e8f5c2493b518057f2b681655 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1211,8 +1211,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 83a1e5db926b3a6d40f4a5cf9a8e6852b9bac9ac 6fc4ead26d19b9348bbda34c3053ae1e066abc32 -R 3ad47b2ac5693ba63ff5c591cf540d89 -T +closed 6fc4ead26d19b9348bbda34c3053ae1e066abc32 +P 272fddc14cc322655eeba670bc0f9fc30e5a804c +R c0f850c2227a7a137fbc73d5c1c5136e +T *branch * scanstatus +T *sym-scanstatus * +T -sym-trunk * U drh -Z ee3b32335a4c8630acaf81919b28b64e +Z f39cd1607e4aacbb491c044bd0bb317e diff --git a/manifest.uuid b/manifest.uuid index b558f09274..d125168b7d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -272fddc14cc322655eeba670bc0f9fc30e5a804c \ No newline at end of file +f9684000665ae7ef6f89c3773612b8286b8f545a \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 3423bb0143..4dd0ffa1a1 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1195,21 +1195,24 @@ static void display_scanstats( ){ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS int i; + double rEstLoop = 1.0; fprintf(pArg->out, "-------- scanstats --------\n"); for(i=0; 1; i++){ sqlite3_stmt *p = pArg->pStmt; - sqlite3_int64 nEst, nLoop, nVisit; + sqlite3_int64 nLoop, nVisit; + double rEst; const char *zExplain; if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ break; } sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&nEst); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); - fprintf(pArg->out, "Loop %d: \"%s\"\n", i, zExplain); - fprintf(pArg->out, " nLoop=%-8lld nVisit=%-8lld nEst=%-8lld\n", - nLoop, nVisit, nEst + fprintf(pArg->out, "Loop %2d: \"%s\"\n", i, zExplain); + rEstLoop *= rEst; + fprintf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", + nLoop, nVisit, (sqlite3_int64)rEstLoop, rEst ); } #else diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 44f7800ca2..c309dc1c5b 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7440,13 +7440,15 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** ** [[SQLITE_SCANSTAT_NVISIT]]

    SQLITE_SCANSTAT_NVISIT
    **
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set to the -** total number of rows visited by the X-th loop.
    +** total number of rows examined by all iterations of the X-th loop. ** ** [[SQLITE_SCANSTAT_EST]]
    SQLITE_SCANSTAT_EST
    -**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set to the -** query planner's estimate for the number of rows visited for each -** iteration of the X-th loop. If the query planner's estimate was accurate, -** then this value should be approximately NVISIT/NLOOP. +**
    ^The "double" variable pointed to by the T parameter will be set to the +** query planner's estimate for the average number of rows output from each +** iteration of the X-th loop. If the query planner's estimates was accurate, +** then this value will approximate the quotient NVISIT/NLOOP and the +** product of this value for the first N-1 loops will approximate +** the NLOOP value for the N-th loop. ** ** [[SQLITE_SCANSTAT_NAME]]
    SQLITE_SCANSTAT_NAME
    **
    ^The "const char *" variable pointed to by the T parameter will be set to diff --git a/src/test1.c b/src/test1.c index 5e526b3013..fcc1a23de1 100644 --- a/src/test1.c +++ b/src/test1.c @@ -2318,7 +2318,7 @@ static int test_stmt_scanstatus( const char *zExplain; sqlite3_int64 nLoop; sqlite3_int64 nVisit; - sqlite3_int64 nEst; + double rEst; int res; if( objc!=3 ){ @@ -2336,9 +2336,9 @@ static int test_stmt_scanstatus( sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nVisit", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nVisit)); - sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_EST, (void*)&nEst); + sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_EST, (void*)&rEst); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("nEst", -1)); - Tcl_ListObjAppendElement(0, pRet, Tcl_NewWideIntObj(nEst)); + Tcl_ListObjAppendElement(0, pRet, Tcl_NewDoubleObj(rEst)); sqlite3_stmt_scanstatus(pStmt, idx, SQLITE_SCANSTAT_NAME, (void*)&zName); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj("zName", -1)); Tcl_ListObjAppendElement(0, pRet, Tcl_NewStringObj(zName, -1)); diff --git a/src/vdbe.h b/src/vdbe.h index 1b9ad8b6bf..b715241b41 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -283,7 +283,7 @@ void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); #endif #ifdef SQLITE_ENABLE_STMT_SCANSTATUS -void sqlite3VdbeScanStatus(Vdbe*, int, int, int, i64, const char*); +void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); #else # define sqlite3VdbeScanStatus(a,b,c,d,e) #endif diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 29117dd064..a42ca3dffc 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -302,7 +302,7 @@ struct ScanStatus { int addrExplain; /* OP_Explain for loop */ int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ - i64 nEst; /* Estimated rows per loop */ + LogEst nEst; /* Estimated output rows per loop */ char *zName; /* Name of table or index */ }; diff --git a/src/vdbeapi.c b/src/vdbeapi.c index d472004590..d907afee00 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1500,7 +1500,13 @@ int sqlite3_stmt_scanstatus( break; } case SQLITE_SCANSTAT_EST: { - *(sqlite3_int64*)pOut = pScan->nEst; + double r = 1.0; + LogEst x = pScan->nEst; + while( x<100 ){ + x += 10; + r *= 0.5; + } + *(double*)pOut = r*sqlite3LogEstToInt(x); break; } case SQLITE_SCANSTAT_NAME: { diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 7cf996ce5c..d8ee5c8e8c 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -606,7 +606,7 @@ void sqlite3VdbeScanStatus( int addrExplain, /* Address of OP_Explain (or 0) */ int addrLoop, /* Address of loop counter */ int addrVisit, /* Address of rows visited counter */ - i64 nEst, /* Estimated number of rows */ + LogEst nEst, /* Estimated number of output rows */ const char *zName /* Name of table or index being scanned */ ){ int nByte = (p->nScan+1) * sizeof(ScanStatus); diff --git a/src/where.c b/src/where.c index 92f783b2e9..c3641c7cc7 100644 --- a/src/where.c +++ b/src/where.c @@ -2946,18 +2946,14 @@ static void addScanStatus( int addrExplain /* Address of OP_Explain (or 0) */ ){ const char *zObj = 0; - i64 nEst = 1; WhereLoop *pLoop = pLvl->pWLoop; if( (pLoop->wsFlags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ zObj = pLoop->u.btree.pIndex->zName; }else{ zObj = pSrclist->a[pLvl->iFrom].zName; } - if( pLoop->nOut>=10 ){ - nEst = sqlite3LogEstToInt(pLoop->nOut); - } sqlite3VdbeScanStatus( - v, addrExplain, pLvl->addrBody, pLvl->addrVisit, nEst, zObj + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj ); } #else diff --git a/test/scanstatus.test b/test/scanstatus.test index 9b34506e91..7713bae5fc 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -45,8 +45,8 @@ proc do_scanstatus_test {tn res} { do_execsql_test 1.1 { SELECT count(*) FROM t1, t2; } 6 do_scanstatus_test 1.2 { - nLoop 1 nVisit 2 nEst 1048576 zName t1 zExplain {SCAN TABLE t1} - nLoop 2 nVisit 6 nEst 1048576 zName t2 zExplain {SCAN TABLE t2} + nLoop 1 nVisit 2 nEst 1048576.0 zName t1 zExplain {SCAN TABLE t1} + nLoop 2 nVisit 6 nEst 1048576.0 zName t2 zExplain {SCAN TABLE t2} } do_execsql_test 1.3 { @@ -54,8 +54,8 @@ do_execsql_test 1.3 { SELECT count(*) FROM t1, t2; } 6 do_scanstatus_test 1.4 { - nLoop 1 nVisit 2 nEst 2 zName t1 zExplain {SCAN TABLE t1} - nLoop 2 nVisit 6 nEst 3 zName t2 zExplain {SCAN TABLE t2} + nLoop 1 nVisit 2 nEst 2.0 zName t1 zExplain {SCAN TABLE t1} + nLoop 2 nVisit 6 nEst 3.0 zName t2 zExplain {SCAN TABLE t2} } do_execsql_test 1.5 { ANALYZE } @@ -63,9 +63,9 @@ do_execsql_test 1.6 { SELECT count(*) FROM t1, t2 WHERE t2.rowid>1; } 4 do_scanstatus_test 1.7 { - nLoop 1 nVisit 2 nEst 2 zName t2 zExplain + nLoop 1 nVisit 2 nEst 2.0 zName t2 zExplain {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} - nLoop 2 nVisit 4 nEst 2 zName t1 zExplain {SCAN TABLE t1} + nLoop 2 nVisit 4 nEst 2.0 zName t1 zExplain {SCAN TABLE t1} } do_execsql_test 1.8 { @@ -73,9 +73,9 @@ do_execsql_test 1.8 { } 4 do_scanstatus_test 1.9 { - nLoop 2 nVisit 4 nEst 2 zName t2 zExplain + nLoop 2 nVisit 4 nEst 2.0 zName t2 zExplain {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} - nLoop 4 nVisit 8 nEst 2 zName t1 zExplain {SCAN TABLE t1} + nLoop 4 nVisit 8 nEst 2.0 zName t1 zExplain {SCAN TABLE t1} } do_test 1.9 { @@ -83,9 +83,9 @@ do_test 1.9 { } {} do_scanstatus_test 1.10 { - nLoop 0 nVisit 0 nEst 2 zName t2 zExplain + nLoop 0 nVisit 0 nEst 2.0 zName t2 zExplain {SEARCH TABLE t2 USING INTEGER PRIMARY KEY (rowid>?)} - nLoop 0 nVisit 0 nEst 2 zName t1 zExplain {SCAN TABLE t1} + nLoop 0 nVisit 0 nEst 2.0 zName t1 zExplain {SCAN TABLE t1} } #------------------------------------------------------------------------- @@ -104,7 +104,7 @@ do_execsql_test 2.1 { } {2 two} do_scanstatus_test 2.2 { - nLoop 1 nVisit 1 nEst 1 zName x1 + nLoop 1 nVisit 1 nEst 1.0 zName x1 zExplain {SEARCH TABLE x1 USING INTEGER PRIMARY KEY (rowid=?)} } @@ -112,7 +112,7 @@ do_execsql_test 2.3.1 { SELECT * FROM x1 WHERE j='two' } {2 two} do_scanstatus_test 2.3.2 { - nLoop 1 nVisit 1 nEst 10 zName x1j + nLoop 1 nVisit 1 nEst 10.0 zName x1j zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j=?)} } @@ -120,7 +120,7 @@ do_execsql_test 2.4.1 { SELECT * FROM x1 WHERE j<'two' } {4 four 1 one 3 three} do_scanstatus_test 2.4.2 { - nLoop 1 nVisit 3 nEst 262144 zName x1j + nLoop 1 nVisit 3 nEst 262144.0 zName x1j zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j='two' } {2 two} do_scanstatus_test 2.5.2 { - nLoop 1 nVisit 1 nEst 262144 zName x1j + nLoop 1 nVisit 1 nEst 262144.0 zName x1j zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j>?)} } @@ -136,7 +136,7 @@ do_execsql_test 2.6.1 { SELECT * FROM x1 WHERE j BETWEEN 'three' AND 'two' } {3 three 2 two} do_scanstatus_test 2.6.2 { - nLoop 1 nVisit 2 nEst 16384 zName x1j + nLoop 1 nVisit 2 nEst 16384.0 zName x1j zExplain {SEARCH TABLE x1 USING COVERING INDEX x1j (j>? AND j? AND j? AND a? AND b? AND b? AND a Date: Thu, 6 Nov 2014 04:42:20 +0000 Subject: [PATCH 544/710] Add the SQLITE_SCANSTAT_SELECTID metric. Use it to improve the ".stmtscan on" output in the shell. FossilOrigin-Name: 64ad5761a841f71530d41565b9fbe9d19c2d6aff --- manifest | 21 +++++++++------------ manifest.uuid | 2 +- src/shell.c | 22 +++++++++++++++++----- src/sqlite.h.in | 12 ++++++++++-- src/vdbeInt.h | 1 + src/vdbeapi.c | 8 ++++++++ 6 files changed, 46 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 75d37178fb..25d04501fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sSQLITE_SCANSTAT_EST\sparameter\sso\sthat\sit\sreturns\sa\sdouble\sfor\nthe\sestimated\snumber\sof\soutput\srows\sper\sloop,\srather\sthan\sa\s64-bit\sinteger.\nRevise\sthe\soutput\sformat\sfor\sthe\s".scanstats\son"\sin\sthe\sshell\sto\smake\suse\nof\sthis\snew\scapability. -D 2014-11-06T03:55:10.745 +C Add\sthe\sSQLITE_SCANSTAT_SELECTID\smetric.\s\sUse\sit\sto\simprove\sthe\n".stmtscan\son"\soutput\sin\sthe\sshell. +D 2014-11-06T04:42:20.310 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,8 +228,8 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 908ff96ef1551b28b940aaf4c886ba2681057209 -F src/sqlite.h.in e13a7b64efa8d6a591577e6a5281fb22783c0133 +F src/shell.c 74768f90bd0f8880937d52e2eb756655dba0015a +F src/sqlite.h.in 087d30a4c7ec7ae19bcaa03a9db9d6ee7a73b0b3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -291,8 +291,8 @@ F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a F src/vdbe.c 3fd4ebd3e87b63175bfd2be747608bae1670b4df F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 -F src/vdbeInt.h c32c1de25e3821a5b53d73abdb23ccc644ec5b63 -F src/vdbeapi.c 6a126fd8ed297ff0542bfbf7891b92977b5ed653 +F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 +F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 F src/vdbeaux.c 9b0a251b6dfab349dd6c6efb40062eb7386b26f5 F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f @@ -1211,10 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 272fddc14cc322655eeba670bc0f9fc30e5a804c -R c0f850c2227a7a137fbc73d5c1c5136e -T *branch * scanstatus -T *sym-scanstatus * -T -sym-trunk * +P f9684000665ae7ef6f89c3773612b8286b8f545a +R 34f47ade33b56f41a439ae08635a112b U drh -Z f39cd1607e4aacbb491c044bd0bb317e +Z 216d325be8d3eddf6fd433a7ff464585 diff --git a/manifest.uuid b/manifest.uuid index d125168b7d..13ff805ffa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f9684000665ae7ef6f89c3773612b8286b8f545a \ No newline at end of file +64ad5761a841f71530d41565b9fbe9d19c2d6aff \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 4dd0ffa1a1..b6ae6e15a6 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1195,12 +1195,14 @@ static void display_scanstats( ){ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS int i; - double rEstLoop = 1.0; + double *arEstLoop = 0; + int nEstLoop = 0; fprintf(pArg->out, "-------- scanstats --------\n"); for(i=0; 1; i++){ sqlite3_stmt *p = pArg->pStmt; sqlite3_int64 nLoop, nVisit; - double rEst; + double rEst, rLoop; + int iSid; const char *zExplain; if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ break; @@ -1208,13 +1210,23 @@ static void display_scanstats( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); - + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); + if( iSid>=nEstLoop ){ + arEstLoop = sqlite3_realloc(arEstLoop, sizeof(arEstLoop[0])*(iSid+1) ); + while( nEstLoop<=iSid ) arEstLoop[nEstLoop++] = 1.0; + } + if( iSid>=0 ){ + arEstLoop[iSid] *= rEst; + rLoop = arEstLoop[iSid]; + }else{ + rLoop = rEst; + } fprintf(pArg->out, "Loop %2d: \"%s\"\n", i, zExplain); - rEstLoop *= rEst; fprintf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", - nLoop, nVisit, (sqlite3_int64)rEstLoop, rEst + nLoop, nVisit, (sqlite3_int64)rLoop, rEst ); } + sqlite3_free(arEstLoop); #else fprintf(pArg->out, "-------- scanstats --------\n"); fprintf(pArg->out, diff --git a/src/sqlite.h.in b/src/sqlite.h.in index c309dc1c5b..00bf5e2d98 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7447,8 +7447,8 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** query planner's estimate for the average number of rows output from each ** iteration of the X-th loop. If the query planner's estimates was accurate, ** then this value will approximate the quotient NVISIT/NLOOP and the -** product of this value for the first N-1 loops will approximate -** the NLOOP value for the N-th loop. +** product of this value for all prior loops with the same SELECTID will +** be the NLOOP value for the current loop. ** ** [[SQLITE_SCANSTAT_NAME]]
    SQLITE_SCANSTAT_NAME
    **
    ^The "const char *" variable pointed to by the T parameter will be set to @@ -7459,6 +7459,13 @@ int sqlite3_vtab_on_conflict(sqlite3 *); **
    ^The "const char *" variable pointed to by the T parameter will be set to ** a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] description ** for the X-th loop. +** +** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    +**
    ^The "int" variable pointed to by the T parameter will be set to the +** "select-id" for the X-th loop. The select-id identifies which query or +** subquery the loop is part of. The main query has a select-id of zero. +** The select-id is the same value as is output in the first column +** of an [EXPLAIN QUERY PLAN] query. ** */ #define SQLITE_SCANSTAT_NLOOP 0 @@ -7466,6 +7473,7 @@ int sqlite3_vtab_on_conflict(sqlite3 *); #define SQLITE_SCANSTAT_EST 2 #define SQLITE_SCANSTAT_NAME 3 #define SQLITE_SCANSTAT_EXPLAIN 4 +#define SQLITE_SCANSTAT_SELECTID 5 /* ** CAPI3REF: Prepared Statement Scan Status diff --git a/src/vdbeInt.h b/src/vdbeInt.h index a42ca3dffc..1a7297e946 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -302,6 +302,7 @@ struct ScanStatus { int addrExplain; /* OP_Explain for loop */ int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ + int iSelectID; /* The "Select-ID" for this loop */ LogEst nEst; /* Estimated output rows per loop */ char *zName; /* Name of table or index */ }; diff --git a/src/vdbeapi.c b/src/vdbeapi.c index d907afee00..5744c28632 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -1521,6 +1521,14 @@ int sqlite3_stmt_scanstatus( } break; } + case SQLITE_SCANSTAT_SELECTID: { + if( pScan->addrExplain ){ + *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; + }else{ + *(int*)pOut = -1; + } + break; + } default: { return 1; } From 42f30bce1158008097d271fd06c31398cf631aea Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Nov 2014 12:08:21 +0000 Subject: [PATCH 545/710] Changes the formatting of ".scanstats on" in the shell so that the stats for subqueries are grouped together and occur after the main query. FossilOrigin-Name: eacbbd8849db9b023eff15ef1cb42ec941299433 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 54 +++++++++++++++++++++++---------------------------- 3 files changed, 31 insertions(+), 37 deletions(-) diff --git a/manifest b/manifest index 25d04501fe..af1ed3e99c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_SCANSTAT_SELECTID\smetric.\s\sUse\sit\sto\simprove\sthe\n".stmtscan\son"\soutput\sin\sthe\sshell. -D 2014-11-06T04:42:20.310 +C Changes\sthe\sformatting\sof\s".scanstats\son"\sin\sthe\sshell\sso\sthat\sthe\sstats\sfor\nsubqueries\sare\sgrouped\stogether\sand\soccur\safter\sthe\smain\squery. +D 2014-11-06T12:08:21.237 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 74768f90bd0f8880937d52e2eb756655dba0015a +F src/shell.c 22c7c693f322091b26e9333a8fa50c56e4aba667 F src/sqlite.h.in 087d30a4c7ec7ae19bcaa03a9db9d6ee7a73b0b3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f9684000665ae7ef6f89c3773612b8286b8f545a -R 34f47ade33b56f41a439ae08635a112b +P 64ad5761a841f71530d41565b9fbe9d19c2d6aff +R 301cd68395834791ac0b2ebb68f39e69 U drh -Z 216d325be8d3eddf6fd433a7ff464585 +Z c6dcae9ced19f9079c51a47717359be1 diff --git a/manifest.uuid b/manifest.uuid index 13ff805ffa..6061834c99 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -64ad5761a841f71530d41565b9fbe9d19c2d6aff \ No newline at end of file +eacbbd8849db9b023eff15ef1cb42ec941299433 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index b6ae6e15a6..3505e59d4b 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1194,39 +1194,33 @@ static void display_scanstats( ShellState *pArg /* Pointer to ShellState */ ){ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - int i; - double *arEstLoop = 0; - int nEstLoop = 0; + int i, k, n = 1; fprintf(pArg->out, "-------- scanstats --------\n"); - for(i=0; 1; i++){ - sqlite3_stmt *p = pArg->pStmt; - sqlite3_int64 nLoop, nVisit; - double rEst, rLoop; - int iSid; - const char *zExplain; - if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ - break; + for(k=0; n>0; k++){ + double rEstLoop = 1.0; + for(i=n=0; 1; i++){ + sqlite3_stmt *p = pArg->pStmt; + sqlite3_int64 nLoop, nVisit; + double rEst; + int iSid; + const char *zExplain; + if( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NLOOP, (void*)&nLoop) ){ + break; + } + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); + if( iSid!=k ) continue; + if( n==0 && k>0 ) fprintf(pArg->out, "-------- subquery %d --------\n", k); + n++; + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); + sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); + fprintf(pArg->out, "Loop %2d: %s\n", n, zExplain); + rEstLoop *= rEst; + fprintf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", + nLoop, nVisit, (sqlite3_int64)rEstLoop, rEst + ); } - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); - sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); - if( iSid>=nEstLoop ){ - arEstLoop = sqlite3_realloc(arEstLoop, sizeof(arEstLoop[0])*(iSid+1) ); - while( nEstLoop<=iSid ) arEstLoop[nEstLoop++] = 1.0; - } - if( iSid>=0 ){ - arEstLoop[iSid] *= rEst; - rLoop = arEstLoop[iSid]; - }else{ - rLoop = rEst; - } - fprintf(pArg->out, "Loop %2d: \"%s\"\n", i, zExplain); - fprintf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", - nLoop, nVisit, (sqlite3_int64)rLoop, rEst - ); } - sqlite3_free(arEstLoop); #else fprintf(pArg->out, "-------- scanstats --------\n"); fprintf(pArg->out, From 179bac3a14924c42656efdfddc5bbfd8e8a2be32 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Nov 2014 12:17:24 +0000 Subject: [PATCH 546/710] On the ".scanstats on" output in the shell, initialize the estimated count for the first loop of each subquery to the actual loop count. FossilOrigin-Name: d1c51c8455d5ce972a77720c2d56228646ced27c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index af1ed3e99c..05212e91e7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sthe\sformatting\sof\s".scanstats\son"\sin\sthe\sshell\sso\sthat\sthe\sstats\sfor\nsubqueries\sare\sgrouped\stogether\sand\soccur\safter\sthe\smain\squery. -D 2014-11-06T12:08:21.237 +C On\sthe\s".scanstats\son"\soutput\sin\sthe\sshell,\sinitialize\sthe\sestimated\scount\sfor\nthe\sfirst\sloop\sof\seach\ssubquery\sto\sthe\sactual\sloop\scount. +D 2014-11-06T12:17:24.789 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 22c7c693f322091b26e9333a8fa50c56e4aba667 +F src/shell.c 64a941c079837fd1a0d920273832e6275b777402 F src/sqlite.h.in 087d30a4c7ec7ae19bcaa03a9db9d6ee7a73b0b3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 64ad5761a841f71530d41565b9fbe9d19c2d6aff -R 301cd68395834791ac0b2ebb68f39e69 +P eacbbd8849db9b023eff15ef1cb42ec941299433 +R c65acc2e5374aae8ca471af05c87c9aa U drh -Z c6dcae9ced19f9079c51a47717359be1 +Z b18ed09265a53348fca40701c374436f diff --git a/manifest.uuid b/manifest.uuid index 6061834c99..fc422bc396 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eacbbd8849db9b023eff15ef1cb42ec941299433 \ No newline at end of file +d1c51c8455d5ce972a77720c2d56228646ced27c \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 3505e59d4b..ca49e00a69 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1209,7 +1209,10 @@ static void display_scanstats( } sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); if( iSid!=k ) continue; - if( n==0 && k>0 ) fprintf(pArg->out, "-------- subquery %d --------\n", k); + if( n==0 ){ + rEstLoop = (double)nLoop; + if( k>0 ) fprintf(pArg->out, "-------- subquery %d --------\n", k); + } n++; sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EST, (void*)&rEst); From 15f23c2cf0da0ecd46f588a156af251fb58c3234 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 6 Nov 2014 12:46:16 +0000 Subject: [PATCH 547/710] Further improvements to the ".scanstats on" display in the shell. Be sure to show the results of all subqueries even if there are gaps in the SELECTID values. Add ".scanstats" to the ".help" output. FossilOrigin-Name: ee922682bb7235dbcd23a22fcfdfa188f6d3228a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 20 ++++++++++---------- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 05212e91e7..3bbd6d3bd3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C On\sthe\s".scanstats\son"\soutput\sin\sthe\sshell,\sinitialize\sthe\sestimated\scount\sfor\nthe\sfirst\sloop\sof\seach\ssubquery\sto\sthe\sactual\sloop\scount. -D 2014-11-06T12:17:24.789 +C Further\simprovements\sto\sthe\s".scanstats\son"\sdisplay\sin\sthe\sshell.\s\sBe\ssure\nto\sshow\sthe\sresults\sof\sall\ssubqueries\seven\sif\sthere\sare\sgaps\sin\sthe\s\nSELECTID\svalues.\s\sAdd\s".scanstats"\sto\sthe\s".help"\soutput. +D 2014-11-06T12:46:16.708 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 64a941c079837fd1a0d920273832e6275b777402 +F src/shell.c 48fe276aada42a15722aee2584e6321345ed4609 F src/sqlite.h.in 087d30a4c7ec7ae19bcaa03a9db9d6ee7a73b0b3 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eacbbd8849db9b023eff15ef1cb42ec941299433 -R c65acc2e5374aae8ca471af05c87c9aa +P d1c51c8455d5ce972a77720c2d56228646ced27c +R 1f66f00948a1acda590c099646f9e3d3 U drh -Z b18ed09265a53348fca40701c374436f +Z 2bc412ec02d784f50574dd160207db0a diff --git a/manifest.uuid b/manifest.uuid index fc422bc396..5ac436c534 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d1c51c8455d5ce972a77720c2d56228646ced27c \ No newline at end of file +ee922682bb7235dbcd23a22fcfdfa188f6d3228a \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index ca49e00a69..915b69263b 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1194,9 +1194,10 @@ static void display_scanstats( ShellState *pArg /* Pointer to ShellState */ ){ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - int i, k, n = 1; + int i, k, n, mx; fprintf(pArg->out, "-------- scanstats --------\n"); - for(k=0; n>0; k++){ + mx = 0; + for(k=0; k<=mx; k++){ double rEstLoop = 1.0; for(i=n=0; 1; i++){ sqlite3_stmt *p = pArg->pStmt; @@ -1208,10 +1209,11 @@ static void display_scanstats( break; } sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_SELECTID, (void*)&iSid); + if( iSid>mx ) mx = iSid; if( iSid!=k ) continue; if( n==0 ){ rEstLoop = (double)nLoop; - if( k>0 ) fprintf(pArg->out, "-------- subquery %d --------\n", k); + if( k>0 ) fprintf(pArg->out, "-------- subquery %d -------\n", k); } n++; sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_NVISIT, (void*)&nVisit); @@ -1224,14 +1226,8 @@ static void display_scanstats( ); } } -#else - fprintf(pArg->out, "-------- scanstats --------\n"); - fprintf(pArg->out, - "sqlite3_stmt_scanstatus() unavailable - " - "rebuild with SQLITE_ENABLE_STMT_SCANSTATUS\n" - ); -#endif fprintf(pArg->out, "---------------------------\n"); +#endif } /* @@ -1687,6 +1683,7 @@ static char zHelp[] = ".read FILENAME Execute SQL in FILENAME\n" ".restore ?DB? FILE Restore content of DB (default \"main\") from FILE\n" ".save FILE Write in-memory database into FILE\n" + ".scanstats on|off Turn sqlite3_stmt_scanstatus() metrics on or off\n" ".schema ?TABLE? Show the CREATE statements\n" " If TABLE specified, only show tables matching\n" " LIKE pattern TABLE.\n" @@ -3072,6 +3069,9 @@ static int do_meta_command(char *zLine, ShellState *p){ if( c=='s' && strncmp(azArg[0], "scanstats", n)==0 ){ if( nArg==2 ){ p->scanstatsOn = booleanValue(azArg[1]); +#ifndef SQLITE_ENABLE_STMT_SCANSTATUS + fprintf(stderr, "Warning: .scanstats not available in this build.\n"); +#endif }else{ fprintf(stderr, "Usage: .scanstats on|off\n"); rc = 1; From 8790b6e860fb5c9dacc3ddc1ecb27b384dd71c2a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Nov 2014 01:43:56 +0000 Subject: [PATCH 548/710] Update documentation on sqlite3_config() and add corresponding evidence marks. FossilOrigin-Name: 360c8ca11c3315c8e08c7c52ff5468e3f723e562 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/main.c | 32 ++++++++++++++++++++++---------- src/sqlite.h.in | 10 ++++++---- 4 files changed, 36 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index 277b0baf4c..2ad90a7f0d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Added\sSQLITE_SCANSTAT_SELECTID.\s\sChange\sthe\svalue\sreturned\sby\nSQLITE_SCANSTAT_EST\sfrom\ssqlite3_int64\sto\sdouble.\s\sEnhanced\sthe\sformatting\nand\sdisplay\sof\sscan\sstatistics\susing\sthe\s".scanstats\son"\scommand\sin\sthe\nshell. -D 2014-11-06T14:43:53.299 +C Update\sdocumentation\son\ssqlite3_config()\sand\sadd\scorresponding\sevidence\smarks. +D 2014-11-07T01:43:56.762 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 42f857be3cef3e1f9752d8e46d61345f80932396 +F src/main.c f88ed28716cbbada0f3d81479e6d43823b553de6 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 48fe276aada42a15722aee2584e6321345ed4609 -F src/sqlite.h.in 087d30a4c7ec7ae19bcaa03a9db9d6ee7a73b0b3 +F src/sqlite.h.in 83e34312bc974a99f03e1b040854c18145d7b662 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -1211,8 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 272fddc14cc322655eeba670bc0f9fc30e5a804c ee922682bb7235dbcd23a22fcfdfa188f6d3228a -R 1f66f00948a1acda590c099646f9e3d3 -T +closed ee922682bb7235dbcd23a22fcfdfa188f6d3228a +P 20c7614addb8494cd7f40263a50fa6f428cce1c7 +R ccc709fd5070dead531e2a6ad6b5aa29 U drh -Z 1da4b43998d0836d48029478818e14c6 +Z 3b414dc01fd5a3ebe6b3ed14cc9e0f06 diff --git a/manifest.uuid b/manifest.uuid index 1e9e58656e..01580f60c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -20c7614addb8494cd7f40263a50fa6f428cce1c7 \ No newline at end of file +360c8ca11c3315c8e08c7c52ff5468e3f723e562 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 4c33bbde9d..ae3753c139 100644 --- a/src/main.c +++ b/src/main.c @@ -457,6 +457,9 @@ int sqlite3_config(int op, ...){ break; } +/* EVIDENCE-OF: R-06626-12911 The SQLITE_CONFIG_HEAP option is only +** available if SQLite is compiled with either SQLITE_ENABLE_MEMSYS3 or +** SQLITE_ENABLE_MEMSYS5 and returns SQLITE_ERROR if invoked otherwise. */ #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) case SQLITE_CONFIG_HEAP: { /* EVIDENCE-OF: R-19854-42126 There are three arguments to @@ -474,17 +477,19 @@ int sqlite3_config(int op, ...){ } if( sqlite3GlobalConfig.pHeap==0 ){ - /* If the heap pointer is NULL, then restore the malloc implementation - ** back to NULL pointers too. This will cause the malloc to go - ** back to its default implementation when sqlite3_initialize() is - ** run. + /* EVIDENCE-OF: R-49920-60189 If the first pointer (the memory pointer) + ** is NULL, then SQLite reverts to using its default memory allocator + ** (the system malloc() implementation), undoing any prior invocation of + ** SQLITE_CONFIG_MALLOC. + ** + ** Setting sqlite3GlobalConfig.m to all zeros will cause malloc to + ** revert to its default implementation when sqlite3_initialize() is run */ memset(&sqlite3GlobalConfig.m, 0, sizeof(sqlite3GlobalConfig.m)); }else{ - /* The heap pointer is not NULL, then install one of the - ** mem5.c/mem3.c methods. The enclosing #if guarantees at - ** least one of these methods is currently enabled. - */ + /* EVIDENCE-OF: R-61006-08918 If the memory pointer is not NULL then the + ** alternative memory allocator is engaged to handle all of SQLites + ** memory allocation needs. */ #ifdef SQLITE_ENABLE_MEMSYS3 sqlite3GlobalConfig.m = *sqlite3MemGetMemsys3(); #endif @@ -557,7 +562,13 @@ int sqlite3_config(int op, ...){ sqlite3_int64 szMmap = va_arg(ap, sqlite3_int64); sqlite3_int64 mxMmap = va_arg(ap, sqlite3_int64); /* EVIDENCE-OF: R-53367-43190 If either argument to this option is - ** negative, then that argument is changed to its compile-time default. */ + ** negative, then that argument is changed to its compile-time default. + ** + ** EVIDENCE-OF: R-34993-45031 The maximum allowed mmap size will be + ** silently truncated if necessary so that it does not exceed the + ** compile-time maximum mmap size set by the SQLITE_MAX_MMAP_SIZE + ** compile-time option. + */ if( mxMmap<0 || mxMmap>SQLITE_MAX_MMAP_SIZE ) mxMmap = SQLITE_MAX_MMAP_SIZE; if( szMmap<0 ) szMmap = SQLITE_DEFAULT_MMAP_SIZE; if( szMmap>mxMmap) szMmap = mxMmap; @@ -2389,7 +2400,8 @@ int sqlite3ParseUri( assert( *pzErrMsg==0 ); - if( ((flags & SQLITE_OPEN_URI) || sqlite3GlobalConfig.bOpenUri) + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ ){ char *zOpt; diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 00bf5e2d98..192280e726 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1582,14 +1582,16 @@ struct sqlite3_mem_methods { **
    ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer ** that SQLite will use for all of its dynamic memory allocation needs ** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. +** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled +** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns +** [SQLITE_ERROR] if invoked otherwise. ** ^There are three arguments to SQLITE_CONFIG_HEAP: ** An 8-byte aligned pointer to the memory, ** the number of bytes in the memory buffer, and the minimum allocation size. ** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts ** to using its default memory allocator (the system malloc() implementation), ** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. ^If the -** memory pointer is not NULL and either [SQLITE_ENABLE_MEMSYS3] or -** [SQLITE_ENABLE_MEMSYS5] are defined, then the alternative memory +** memory pointer is not NULL then the alternative memory ** allocator is engaged to handle all of SQLites memory allocation needs. ** The first pointer (the memory pointer) must be aligned to an 8-byte ** boundary or subsequent behavior of SQLite will be undefined. @@ -1719,8 +1721,8 @@ struct sqlite3_mem_methods { ** ^The default setting can be overridden by each database connection using ** either the [PRAGMA mmap_size] command, or by using the ** [SQLITE_FCNTL_MMAP_SIZE] file control. ^(The maximum allowed mmap size -** cannot be changed at run-time. Nor may the maximum allowed mmap size -** exceed the compile-time maximum mmap size set by the +** will be silently truncated if necessary so that it does not exceed the +** compile-time maximum mmap size set by the ** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^ ** ^If either argument to this option is negative, then that argument is ** changed to its compile-time default. From 6137138ded9c5ca02d65d62d0ec4f3ff7cde05d2 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Nov 2014 11:39:16 +0000 Subject: [PATCH 549/710] Fix typo in sqlite3.h reported on the mailing list. FossilOrigin-Name: 402703212a8488f8b571ce170b3b6c7374bd7daa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2ad90a7f0d..b80e3b8250 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sdocumentation\son\ssqlite3_config()\sand\sadd\scorresponding\sevidence\smarks. -D 2014-11-07T01:43:56.762 +C Fix\stypo\sin\ssqlite3.h\sreported\son\sthe\smailing\slist. +D 2014-11-07T11:39:16.680 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 48fe276aada42a15722aee2584e6321345ed4609 -F src/sqlite.h.in 83e34312bc974a99f03e1b040854c18145d7b662 +F src/sqlite.h.in f433227d7f619887a1064913fa66cefa3da4349a F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 20c7614addb8494cd7f40263a50fa6f428cce1c7 -R ccc709fd5070dead531e2a6ad6b5aa29 +P 360c8ca11c3315c8e08c7c52ff5468e3f723e562 +R 367fba2464fa0866db1d5cae2a24beee U drh -Z 3b414dc01fd5a3ebe6b3ed14cc9e0f06 +Z 3885b799624859bdb0af87865d5681ed diff --git a/manifest.uuid b/manifest.uuid index 01580f60c0..185fd7e51d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -360c8ca11c3315c8e08c7c52ff5468e3f723e562 \ No newline at end of file +402703212a8488f8b571ce170b3b6c7374bd7daa \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 192280e726..fc741b7291 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -52,7 +52,7 @@ extern "C" { /* ** These no-op macros are used in front of interfaces to mark those ** interfaces as either deprecated or experimental. New applications -** should not use deprecated interfaces - they are support for backwards +** should not use deprecated interfaces - they are supported for backwards ** compatibility only. Application writers should be aware that ** experimental interfaces are subject to change in point releases. ** From 86a11b8a6a671c1f0d55c7adb47994f2cec37f01 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Nov 2014 13:24:29 +0000 Subject: [PATCH 550/710] Fix harmless typos in comments. FossilOrigin-Name: 94c564da4c2cf5dffe58fdf7a180e9ba4cc3de69 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/date.c | 8 ++++---- src/global.c | 4 ++-- src/vacuum.c | 2 +- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index b80e3b8250..ace0366e33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\ssqlite3.h\sreported\son\sthe\smailing\slist. -D 2014-11-07T11:39:16.680 +C Fix\sharmless\stypos\sin\scomments. +D 2014-11-07T13:24:29.246 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -179,13 +179,13 @@ F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a -F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 +F src/date.c 93594514aae68de117ca4a2a0d6cc63eddf26744 F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee -F src/global.c a50ad0b9ee328107a65aa8f5f3cd34905e74745c +F src/global.c 6ded36dda9466fc1c9a3c5492ded81d79bf3977d F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 @@ -288,7 +288,7 @@ F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 -F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a +F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c F src/vdbe.c 3fd4ebd3e87b63175bfd2be747608bae1670b4df F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 360c8ca11c3315c8e08c7c52ff5468e3f723e562 -R 367fba2464fa0866db1d5cae2a24beee +P 402703212a8488f8b571ce170b3b6c7374bd7daa +R 9f78be6a67e175be94522748bee570f6 U drh -Z 3885b799624859bdb0af87865d5681ed +Z 1fff294369c9fe8b2c9ed9357ff2fdd6 diff --git a/manifest.uuid b/manifest.uuid index 185fd7e51d..3f84a89f53 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -402703212a8488f8b571ce170b3b6c7374bd7daa \ No newline at end of file +94c564da4c2cf5dffe58fdf7a180e9ba4cc3de69 \ No newline at end of file diff --git a/src/date.c b/src/date.c index 11b04ea004..10d9006263 100644 --- a/src/date.c +++ b/src/date.c @@ -16,7 +16,7 @@ ** sqlite3RegisterDateTimeFunctions() found at the bottom of the file. ** All other code has file scope. ** -** SQLite processes all times and dates as Julian Day numbers. The +** SQLite processes all times and dates as julian day numbers. The ** dates and times are stored as the number of days since noon ** in Greenwich on November 24, 4714 B.C. according to the Gregorian ** calendar system. @@ -31,7 +31,7 @@ ** ** The Gregorian calendar system is used for all dates and times, ** even those that predate the Gregorian calendar. Historians usually -** use the Julian calendar for dates prior to 1582-10-15 and for some +** use the julian calendar for dates prior to 1582-10-15 and for some ** dates afterwards, depending on locale. Beware of this difference. ** ** The conversion algorithms are implemented based on descriptions @@ -304,7 +304,7 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ } /* -** Attempt to parse the given string into a Julian Day Number. Return +** Attempt to parse the given string into a julian day number. Return ** the number of errors. ** ** The following are acceptable forms for the input string: @@ -875,7 +875,7 @@ static void dateFunc( ** %f ** fractional seconds SS.SSS ** %H hour 00-24 ** %j day of year 000-366 -** %J ** Julian day number +** %J ** julian day number ** %m month 01-12 ** %M minute 00-59 ** %s seconds since 1970-01-01 diff --git a/src/global.c b/src/global.c index b5323e8df6..4bc8edb3bc 100644 --- a/src/global.c +++ b/src/global.c @@ -237,8 +237,8 @@ const Token sqlite3IntTokens[] = { ** ** IMPORTANT: Changing the pending byte to any value other than ** 0x40000000 results in an incompatible database file format! -** Changing the pending byte during operating results in undefined -** and dileterious behavior. +** Changing the pending byte during operation will result in undefined +** and incorrect behavior. */ #ifndef SQLITE_OMIT_WSD int sqlite3PendingByte = 0x40000000; diff --git a/src/vacuum.c b/src/vacuum.c index 4d0c0976a1..9df8e08b22 100644 --- a/src/vacuum.c +++ b/src/vacuum.c @@ -94,7 +94,7 @@ static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ ** overwriting the database with the vacuumed content. ** ** Only 1x temporary space and only 1x writes would be required if -** the copy of step (3) were replace by deleting the original database +** the copy of step (3) were replaced by deleting the original database ** and renaming the transient database as the original. But that will ** not work if other processes are attached to the original database. ** And a power loss in between deleting the original and renaming the From 9a06d30bb5266c008ad3a2493fecf5f65c36d56a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Nov 2014 13:52:44 +0000 Subject: [PATCH 551/710] In the ".scanstats on" output from the shell, round the estRows value to the nearest integer, rather than rounding toward zero. FossilOrigin-Name: 5700508535c35ac6b158b527e1d47e529e8e28ab --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ace0366e33..b3d774824c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\stypos\sin\scomments. -D 2014-11-07T13:24:29.246 +C In\sthe\s".scanstats\son"\soutput\sfrom\sthe\sshell,\sround\sthe\sestRows\svalue\sto\nthe\snearest\sinteger,\srather\sthan\srounding\stoward\szero. +D 2014-11-07T13:52:44.718 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 48fe276aada42a15722aee2584e6321345ed4609 +F src/shell.c 0500307f63e3675dc4ab21112c6086b4d146fc08 F src/sqlite.h.in f433227d7f619887a1064913fa66cefa3da4349a F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 402703212a8488f8b571ce170b3b6c7374bd7daa -R 9f78be6a67e175be94522748bee570f6 +P 94c564da4c2cf5dffe58fdf7a180e9ba4cc3de69 +R bf25c89aa0a2170169c258f353b46c64 U drh -Z 1fff294369c9fe8b2c9ed9357ff2fdd6 +Z 9c39d53d1fb35cc72c3ed2e42a4f819b diff --git a/manifest.uuid b/manifest.uuid index 3f84a89f53..7b8b4f079c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -94c564da4c2cf5dffe58fdf7a180e9ba4cc3de69 \ No newline at end of file +5700508535c35ac6b158b527e1d47e529e8e28ab \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 915b69263b..a33e65b1f8 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1222,7 +1222,7 @@ static void display_scanstats( fprintf(pArg->out, "Loop %2d: %s\n", n, zExplain); rEstLoop *= rEst; fprintf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", - nLoop, nVisit, (sqlite3_int64)rEstLoop, rEst + nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst ); } } From 97d3898335b39b07e8753d0d705dd4b396bd85a2 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 7 Nov 2014 14:37:32 +0000 Subject: [PATCH 552/710] Fix another harmless comment typo. FossilOrigin-Name: b45bc80bb16f07192d84fd14433bb724a84d4146 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqliteInt.h | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index b3d774824c..579589c94b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\s".scanstats\son"\soutput\sfrom\sthe\sshell,\sround\sthe\sestRows\svalue\sto\nthe\snearest\sinteger,\srather\sthan\srounding\stoward\szero. -D 2014-11-07T13:52:44.718 +C Fix\sanother\sharmless\scomment\stypo. +D 2014-11-07T14:37:32.349 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -232,7 +232,7 @@ F src/shell.c 0500307f63e3675dc4ab21112c6086b4d146fc08 F src/sqlite.h.in f433227d7f619887a1064913fa66cefa3da4349a F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 8f67ca79e957b8ece7453b8e320b6a996e1b4761 +F src/sqliteInt.h 71b0bf1a7fc55b5cb374f7579fd140e730a6e0f4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -1211,7 +1211,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 94c564da4c2cf5dffe58fdf7a180e9ba4cc3de69 -R bf25c89aa0a2170169c258f353b46c64 +P 5700508535c35ac6b158b527e1d47e529e8e28ab +R 95ce072bedb368f2e0f6da4bf10906cf U drh -Z 9c39d53d1fb35cc72c3ed2e42a4f819b +Z 8c052ffec320f3367f2daa07942512b6 diff --git a/manifest.uuid b/manifest.uuid index 7b8b4f079c..13afcc03f7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5700508535c35ac6b158b527e1d47e529e8e28ab \ No newline at end of file +b45bc80bb16f07192d84fd14433bb724a84d4146 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e9715efcf7..4d272b06a1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -562,7 +562,7 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ ** gives a possible range of values of approximately 1.0e986 to 1e-986. ** But the allowed values are "grainy". Not every value is representable. ** For example, quantities 16 and 17 are both represented by a LogEst -** of 40. However, since LogEst quantaties are suppose to be estimates, +** of 40. However, since LogEst quantities are suppose to be estimates, ** not exact values, this imprecision is not a problem. ** ** "LogEst" is short for "Logarithmic Estimate". From b391b944319fad869ec86b4798a0e62daf04d48c Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 7 Nov 2014 14:41:11 +0000 Subject: [PATCH 553/710] Add new test file e_blobopen.test, containing tests for sqlite3_blob_open(). FossilOrigin-Name: ecbccd0e594d22b3ae7fabc8037951dc49570bc3 --- main.mk | 1 + manifest | 30 ++- manifest.uuid | 2 +- src/sqlite.h.in | 58 +++-- src/tclsqlite.c | 2 + src/test1.c | 152 ----------- src/test_blob.c | 319 +++++++++++++++++++++++ src/test_config.c | 6 + src/vdbe.c | 5 +- test/e_blobopen.test | 549 +++++++++++++++++++++++++++++++++++++++ test/fkey7.test | 17 ++ test/without_rowid5.test | 3 +- 12 files changed, 950 insertions(+), 194 deletions(-) create mode 100644 src/test_blob.c create mode 100644 test/e_blobopen.test diff --git a/main.mk b/main.mk index 4a7ac02710..c8bf72b48b 100644 --- a/main.mk +++ b/main.mk @@ -246,6 +246,7 @@ TESTSRC = \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ $(TOP)/src/test_backup.c \ + $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ $(TOP)/src/test_demovfs.c \ diff --git a/manifest b/manifest index 579589c94b..e08707974e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\sharmless\scomment\stypo. -D 2014-11-07T14:37:32.349 +C Add\snew\stest\sfile\se_blobopen.test,\scontaining\stests\sfor\ssqlite3_blob_open(). +D 2014-11-07T14:41:11.404 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -151,7 +151,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk bbc8b6000ed143a1a8d31d3b4995c359a3188fa1 +F main.mk 3fececc835c5c23637c3e0af970e27a81a9ba476 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -229,15 +229,15 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 0500307f63e3675dc4ab21112c6086b4d146fc08 -F src/sqlite.h.in f433227d7f619887a1064913fa66cefa3da4349a +F src/sqlite.h.in 5abfd7d6c9bd2054105820de5ce12b1e491b1dc9 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 71b0bf1a7fc55b5cb374f7579fd140e730a6e0f4 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc -F src/tclsqlite.c 7cdd4dd3c2a4183483feca260070d73d6e22cd47 -F src/test1.c ac7f3bad83ef4508d5efc85b32e86da48db8ed7e +F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 +F src/test1.c cce6ad0effe8c66a62d2634d9714ffdc7372ef11 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -249,8 +249,9 @@ F src/test9.c bea1e8cf52aa93695487badedd6e1886c321ea60 F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e +F src/test_blob.c 1f2e3e25255b731c4fcf15ee7990d06347cb6c09 F src/test_btree.c 2e9978eca99a9a4bfa8cae949efb00886860a64f -F src/test_config.c c8b8b50bf2fe5102de10e4c7100b746d7f6bf62f +F src/test_config.c 035c17a173937d019b8dfc1d524f9d3fc8123504 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -289,7 +290,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c 3fd4ebd3e87b63175bfd2be747608bae1670b4df +F src/vdbe.c d5dab22208e36e5689e9fb553aea3613921054ec F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -451,6 +452,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 +F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 w test/e_blob.test F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a @@ -491,7 +493,7 @@ F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d F test/fkey5.test 8a1fde4e7721ae00b05b3178888833726ca2df8d F test/fkey6.test abb59f866c1b44926fd02d1fdd217d831fe04f48 -F test/fkey7.test e31d0e71a41c1d29349a16448d6c420e2c53a8fc +F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13 F test/fkey_malloc.test 594a7ea1fbab553c036c70813cd8bd9407d63749 F test/format4.test 1f0cac8ff3895e9359ed87e41aaabee982a812eb F test/fts-9fd058691.test 78b887e30ae6816df0e1fed6259de4b5a64ad33c @@ -1155,7 +1157,7 @@ F test/without_rowid1.test 7862e605753c8d25329f665fa09072e842183151 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 F test/without_rowid3.test 1081aabf60a1e1123b7f9a8f6ae19954351843b0 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a -F test/without_rowid5.test b4a639a367f04d382d20e8f44fc1be4f2d57d107 +F test/without_rowid5.test 61256715b686359df48ca1742db50cc7e3e7b862 F test/wordcount.c 9915e06cb33d8ca8109b8700791afe80d305afda F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac @@ -1211,7 +1213,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5700508535c35ac6b158b527e1d47e529e8e28ab -R 95ce072bedb368f2e0f6da4bf10906cf -U drh -Z 8c052ffec320f3367f2daa07942512b6 +P b45bc80bb16f07192d84fd14433bb724a84d4146 +R 0ead7b89f3ed1286937ae097bba351c9 +U dan +Z d785e78de9d268050b7beac3067db32e diff --git a/manifest.uuid b/manifest.uuid index 13afcc03f7..6227627e36 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b45bc80bb16f07192d84fd14433bb724a84d4146 \ No newline at end of file +ecbccd0e594d22b3ae7fabc8037951dc49570bc3 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index fc741b7291..bed64bfae2 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5657,26 +5657,42 @@ typedef struct sqlite3_blob sqlite3_blob; ** SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow; ** )^ ** +** ^(Parameter zDb is not the filename that contains the database, but +** rather the symbolic name of the database. For attached databases, this is +** the name that appears after the AS keyword in the [ATTACH] statement. +** For the main database file, the database name is "main". For TEMP +** tables, the database name is "temp".)^ +** ** ^If the flags parameter is non-zero, then the BLOB is opened for read -** and write access. ^If it is zero, the BLOB is opened for read access. -** ^It is not possible to open a column that is part of an index or primary -** key for writing. ^If [foreign key constraints] are enabled, it is -** not possible to open a column that is part of a [child key] for writing. +** and write access. ^If the flags parameter is zero, the BLOB is opened for +** read-only access. ** -** ^Note that the database name is not the filename that contains -** the database but rather the symbolic name of the database that -** appears after the AS keyword when the database is connected using [ATTACH]. -** ^For the main database file, the database name is "main". -** ^For TEMP tables, the database name is "temp". +** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is stored +** in *ppBlob. Otherwise an [error code] is returned and, unless the error +** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided +** the API is not misused, it is always safe to call [sqlite3_blob_close()] +** on *ppBlob after this function it returns. +** +** This function fails with SQLITE_ERROR if any of the following are true: +**
      +**
    • ^(Database zDb does not exist)^, +**
    • ^(Table zTable does not exist within database zDb)^, +**
    • ^(Table zTable is a WITHOUT ROWID table)^, +**
    • ^(Column zColumn does not exist)^, +**
    • ^(Row iRow is not present in the table)^, +**
    • ^(The specified column of row iRow contains a value that is not +** a TEXT or BLOB value)^, +**
    • ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE +** constraint and the blob is being opened for read/write access)^, +**
    • ^([foreign key constraints | Foreign key constraints] are enabled, +** column zColumn is part of a [child key] definition and the blob is +** being opened for read/write access)^. +**
    +** +** ^Unless it returns SQLITE_MISUSE, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** -** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is written -** to *ppBlob. Otherwise an [error code] is returned and *ppBlob is set -** to be a null pointer.)^ -** ^This function sets the [database connection] error code and message -** accessible via [sqlite3_errcode()] and [sqlite3_errmsg()] and related -** functions. ^Note that the *ppBlob variable is always initialized in a -** way that makes it safe to invoke [sqlite3_blob_close()] on *ppBlob -** regardless of the success or failure of this routine. ** ** ^(If the row that a BLOB handle points to is modified by an ** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects @@ -5694,13 +5710,9 @@ typedef struct sqlite3_blob sqlite3_blob; ** interface. Use the [UPDATE] SQL command to change the size of a ** blob. ** -** ^The [sqlite3_blob_open()] interface will fail for a [WITHOUT ROWID] -** table. Incremental BLOB I/O is not possible on [WITHOUT ROWID] tables. -** ** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces -** and the built-in [zeroblob] SQL function can be used, if desired, -** to create an empty, zero-filled blob in which to read or write using -** this interface. +** and the built-in [zeroblob] SQL function may be used to create a +** zero-filled blob to read or write using the incremental-blob interface. ** ** To avoid a resource leak, every open [BLOB handle] should eventually ** be released by a call to [sqlite3_blob_close()]. diff --git a/src/tclsqlite.c b/src/tclsqlite.c index bff4a92421..4fb78a5d12 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -3725,6 +3725,7 @@ static void init_all(Tcl_Interp *interp){ extern int Sqlitetest9_Init(Tcl_Interp*); extern int Sqlitetestasync_Init(Tcl_Interp*); extern int Sqlitetest_autoext_Init(Tcl_Interp*); + extern int Sqlitetest_blob_Init(Tcl_Interp*); extern int Sqlitetest_demovfs_Init(Tcl_Interp *); extern int Sqlitetest_func_Init(Tcl_Interp*); extern int Sqlitetest_hexio_Init(Tcl_Interp*); @@ -3768,6 +3769,7 @@ static void init_all(Tcl_Interp *interp){ Sqlitetest9_Init(interp); Sqlitetestasync_Init(interp); Sqlitetest_autoext_Init(interp); + Sqlitetest_blob_Init(interp); Sqlitetest_demovfs_Init(interp); Sqlitetest_func_Init(interp); Sqlitetest_hexio_Init(interp); diff --git a/src/test1.c b/src/test1.c index fcc1a23de1..e7a5929323 100644 --- a/src/test1.c +++ b/src/test1.c @@ -1651,154 +1651,6 @@ static int blobHandleFromObj( return TCL_OK; } -/* -** sqlite3_blob_bytes CHANNEL -*/ -static int test_blob_bytes( - ClientData clientData, /* Not used */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3_blob *pBlob; - int nByte; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL"); - return TCL_ERROR; - } - - if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; - nByte = sqlite3_blob_bytes(pBlob); - Tcl_SetObjResult(interp, Tcl_NewIntObj(nByte)); - - return TCL_OK; -} - -/* -** sqlite3_blob_close CHANNEL -*/ -static int test_blob_close( - ClientData clientData, /* Not used */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3_blob *pBlob; - - if( objc!=2 ){ - Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL"); - return TCL_ERROR; - } - - if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; - sqlite3_blob_close(pBlob); - - return TCL_OK; -} - -/* -** sqlite3_blob_read CHANNEL OFFSET N -** -** This command is used to test the sqlite3_blob_read() in ways that -** the Tcl channel interface does not. The first argument should -** be the name of a valid channel created by the [incrblob] method -** of a database handle. This function calls sqlite3_blob_read() -** to read N bytes from offset OFFSET from the underlying SQLite -** blob handle. -** -** On success, a byte-array object containing the read data is -** returned. On failure, the interpreter result is set to the -** text representation of the returned error code (i.e. "SQLITE_NOMEM") -** and a Tcl exception is thrown. -*/ -static int test_blob_read( - ClientData clientData, /* Not used */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3_blob *pBlob; - int nByte; - int iOffset; - unsigned char *zBuf = 0; - int rc; - - if( objc!=4 ){ - Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET N"); - return TCL_ERROR; - } - - if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; - if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) - || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &nByte) - ){ - return TCL_ERROR; - } - - if( nByte>0 ){ - zBuf = (unsigned char *)Tcl_Alloc(nByte); - } - rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset); - if( rc==SQLITE_OK ){ - Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(zBuf, nByte)); - }else{ - Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); - } - Tcl_Free((char *)zBuf); - - return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); -} - -/* -** sqlite3_blob_write CHANNEL OFFSET DATA ?NDATA? -** -** This command is used to test the sqlite3_blob_write() in ways that -** the Tcl channel interface does not. The first argument should -** be the name of a valid channel created by the [incrblob] method -** of a database handle. This function calls sqlite3_blob_write() -** to write the DATA byte-array to the underlying SQLite blob handle. -** at offset OFFSET. -** -** On success, an empty string is returned. On failure, the interpreter -** result is set to the text representation of the returned error code -** (i.e. "SQLITE_NOMEM") and a Tcl exception is thrown. -*/ -static int test_blob_write( - ClientData clientData, /* Not used */ - Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ - int objc, /* Number of arguments */ - Tcl_Obj *CONST objv[] /* Command arguments */ -){ - sqlite3_blob *pBlob; - int iOffset; - int rc; - - unsigned char *zBuf; - int nBuf; - - if( objc!=4 && objc!=5 ){ - Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET DATA ?NDATA?"); - return TCL_ERROR; - } - - if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; - if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) ){ - return TCL_ERROR; - } - - zBuf = Tcl_GetByteArrayFromObj(objv[3], &nBuf); - if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &nBuf) ){ - return TCL_ERROR; - } - rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset); - if( rc!=SQLITE_OK ){ - Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); - } - - return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); -} - static int test_blob_reopen( ClientData clientData, /* Not used */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ @@ -6910,11 +6762,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_table_column_metadata", test_table_column_metadata, 0 }, #endif #ifndef SQLITE_OMIT_INCRBLOB - { "sqlite3_blob_read", test_blob_read, 0 }, - { "sqlite3_blob_write", test_blob_write, 0 }, { "sqlite3_blob_reopen", test_blob_reopen, 0 }, - { "sqlite3_blob_bytes", test_blob_bytes, 0 }, - { "sqlite3_blob_close", test_blob_close, 0 }, #endif { "pcache_stats", test_pcache_stats, 0 }, #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY diff --git a/src/test_blob.c b/src/test_blob.c new file mode 100644 index 0000000000..d88c91366a --- /dev/null +++ b/src/test_blob.c @@ -0,0 +1,319 @@ +/* +** 2014 October 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ +#include "sqliteInt.h" +#include "tcl.h" +#include +#include +#include + +/* These functions are implemented in main.c. */ +extern const char *sqlite3ErrName(int); + +/* From test1.c: */ +extern int getDbPointer(Tcl_Interp *interp, const char *zA, sqlite3 **ppDb); +extern void *sqlite3TestTextToPtr(const char *z); + +/* +** Return a pointer to a buffer containing a text representation of the +** pointer passed as the only argument. The original pointer may be extracted +** from the text using sqlite3TestTextToPtr(). +*/ +static char *ptrToText(void *p){ + static char buf[100]; + sqlite3_snprintf(sizeof(buf)-1, buf, "%p", p); + return buf; +} + +/* +** Attempt to extract a blob handle (type sqlite3_blob*) from the Tcl +** object passed as the second argument. If successful, set *ppBlob to +** point to the blob handle and return TCL_OK. Otherwise, store an error +** message in the tcl interpreter and return TCL_ERROR. The final value +** of *ppBlob is undefined in this case. +** +** If the object contains a string that begins with "incrblob_", then it +** is assumed to be the name of a Tcl channel opened using the [db incrblob] +** command (see tclsqlite.c). Otherwise, it is assumed to be a pointer +** encoded using the ptrToText() routine or similar. +*/ +static int blobHandleFromObj( + Tcl_Interp *interp, + Tcl_Obj *pObj, + sqlite3_blob **ppBlob +){ + char *z; + int n; + + z = Tcl_GetStringFromObj(pObj, &n); + if( n==0 ){ + *ppBlob = 0; + }else if( n>9 && 0==memcmp("incrblob_", z, 9) ){ + int notUsed; + Tcl_Channel channel; + ClientData instanceData; + + channel = Tcl_GetChannel(interp, z, ¬Used); + if( !channel ) return TCL_ERROR; + + Tcl_Flush(channel); + Tcl_Seek(channel, 0, SEEK_SET); + + instanceData = Tcl_GetChannelInstanceData(channel); + *ppBlob = *((sqlite3_blob **)instanceData); + }else{ + *ppBlob = (sqlite3_blob*)sqlite3TestTextToPtr(z); + } + + return TCL_OK; +} + +/* +** Like Tcl_GetString(), except that if the string is 0 bytes in size, a +** NULL Pointer is returned. +*/ +static char *blobStringFromObj(Tcl_Obj *pObj){ + int n; + char *z; + z = Tcl_GetStringFromObj(pObj, &n); + return (n ? z : 0); +} + +/* +** sqlite3_blob_open DB DATABASE TABLE COLUMN ROWID FLAGS VARNAME +** +** Tcl test harness for the sqlite3_blob_open() function. +*/ +static int test_blob_open( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* Calling TCL interpreter */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + const char *zDb; + const char *zTable; + const char *zColumn; + sqlite_int64 iRowid; + int flags; + const char *zVarname; + int nVarname; + + sqlite3_blob *pBlob = (sqlite3_blob*)0xFFFFFFFF; + int rc; + + if( objc!=8 ){ + const char *zUsage = "DB DATABASE TABLE COLUMN ROWID FLAGS VARNAME"; + Tcl_WrongNumArgs(interp, 1, objv, zUsage); + return TCL_ERROR; + } + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; + zDb = Tcl_GetString(objv[2]); + zTable = blobStringFromObj(objv[3]); + zColumn = Tcl_GetString(objv[4]); + if( Tcl_GetWideIntFromObj(interp, objv[5], &iRowid) ) return TCL_ERROR; + if( Tcl_GetIntFromObj(interp, objv[6], &flags) ) return TCL_ERROR; + zVarname = Tcl_GetStringFromObj(objv[7], &nVarname); + + if( nVarname>0 ){ + rc = sqlite3_blob_open(db, zDb, zTable, zColumn, iRowid, flags, &pBlob); + Tcl_SetVar(interp, zVarname, ptrToText(pBlob), 0); + }else{ + rc = sqlite3_blob_open(db, zDb, zTable, zColumn, iRowid, flags, 0); + } + + if( rc==SQLITE_OK ){ + Tcl_ResetResult(interp); + }else{ + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + return TCL_ERROR; + } + return TCL_OK; +} + + +/* +** sqlite3_blob_close HANDLE +*/ +static int test_blob_close( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_blob *pBlob; + int rc; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; + rc = sqlite3_blob_close(pBlob); + + if( rc ){ + Tcl_SetResult(interp, (char*)sqlite3ErrName(rc), TCL_VOLATILE); + }else{ + Tcl_ResetResult(interp); + } + return TCL_OK; +} + +/* +** sqlite3_blob_bytes HANDLE +*/ +static int test_blob_bytes( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_blob *pBlob; + int nByte; + + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE"); + return TCL_ERROR; + } + + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; + nByte = sqlite3_blob_bytes(pBlob); + Tcl_SetObjResult(interp, Tcl_NewIntObj(nByte)); + + return TCL_OK; +} + +/* +** sqlite3_blob_read CHANNEL OFFSET N +** +** This command is used to test the sqlite3_blob_read() in ways that +** the Tcl channel interface does not. The first argument should +** be the name of a valid channel created by the [incrblob] method +** of a database handle. This function calls sqlite3_blob_read() +** to read N bytes from offset OFFSET from the underlying SQLite +** blob handle. +** +** On success, a byte-array object containing the read data is +** returned. On failure, the interpreter result is set to the +** text representation of the returned error code (i.e. "SQLITE_NOMEM") +** and a Tcl exception is thrown. +*/ +static int test_blob_read( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_blob *pBlob; + int nByte; + int iOffset; + unsigned char *zBuf = 0; + int rc; + + if( objc!=4 ){ + Tcl_WrongNumArgs(interp, 1, objv, "CHANNEL OFFSET N"); + return TCL_ERROR; + } + + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; + if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) + || TCL_OK!=Tcl_GetIntFromObj(interp, objv[3], &nByte) + ){ + return TCL_ERROR; + } + + if( nByte>0 ){ + zBuf = (unsigned char *)Tcl_Alloc(nByte); + } + rc = sqlite3_blob_read(pBlob, zBuf, nByte, iOffset); + if( rc==SQLITE_OK ){ + Tcl_SetObjResult(interp, Tcl_NewByteArrayObj(zBuf, nByte)); + }else{ + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); + } + Tcl_Free((char *)zBuf); + + return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); +} + +/* +** sqlite3_blob_write HANDLE OFFSET DATA ?NDATA? +** +** This command is used to test the sqlite3_blob_write() in ways that +** the Tcl channel interface does not. The first argument should +** be the name of a valid channel created by the [incrblob] method +** of a database handle. This function calls sqlite3_blob_write() +** to write the DATA byte-array to the underlying SQLite blob handle. +** at offset OFFSET. +** +** On success, an empty string is returned. On failure, the interpreter +** result is set to the text representation of the returned error code +** (i.e. "SQLITE_NOMEM") and a Tcl exception is thrown. +*/ +static int test_blob_write( + ClientData clientData, /* Not used */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3_blob *pBlob; + int iOffset; + int rc; + + unsigned char *zBuf; + int nBuf; + + if( objc!=4 && objc!=5 ){ + Tcl_WrongNumArgs(interp, 1, objv, "HANDLE OFFSET DATA ?NDATA?"); + return TCL_ERROR; + } + + if( blobHandleFromObj(interp, objv[1], &pBlob) ) return TCL_ERROR; + if( TCL_OK!=Tcl_GetIntFromObj(interp, objv[2], &iOffset) ){ + return TCL_ERROR; + } + + zBuf = Tcl_GetByteArrayFromObj(objv[3], &nBuf); + if( objc==5 && Tcl_GetIntFromObj(interp, objv[4], &nBuf) ){ + return TCL_ERROR; + } + rc = sqlite3_blob_write(pBlob, zBuf, nBuf, iOffset); + if( rc!=SQLITE_OK ){ + Tcl_SetResult(interp, (char *)sqlite3ErrName(rc), TCL_VOLATILE); + } + + return (rc==SQLITE_OK ? TCL_OK : TCL_ERROR); +} + + +/* +** Register commands with the TCL interpreter. +*/ +int Sqlitetest_blob_Init(Tcl_Interp *interp){ + static struct { + char *zName; + Tcl_ObjCmdProc *xProc; + } aObjCmd[] = { + { "sqlite3_blob_open", test_blob_open }, + { "sqlite3_blob_close", test_blob_close }, + { "sqlite3_blob_bytes", test_blob_bytes }, + { "sqlite3_blob_read", test_blob_read }, + { "sqlite3_blob_write", test_blob_write }, + }; + int i; + for(i=0; ipKeyInfo, aTempRec, sizeof(aTempRec), &pFree - ); + ); if( pIdxKey==0 ) goto no_mem; assert( pIn3->flags & MEM_Blob ); - assert( (pIn3->flags & MEM_Zero)==0 ); /* zeroblobs already expanded */ + /* assert( (pIn3->flags & MEM_Zero)==0 ); // zeroblobs already expanded */ + ExpandBlob(pIn3); sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); } pIdxKey->default_rc = 0; diff --git a/test/e_blobopen.test b/test/e_blobopen.test new file mode 100644 index 0000000000..01f62cdd78 --- /dev/null +++ b/test/e_blobopen.test @@ -0,0 +1,549 @@ +# 2014 October 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_blobopen + +forcedelete test.db2 + +do_execsql_test 1.0 { + ATTACH 'test.db2' AS aux; + + CREATE TABLE main.t1(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + CREATE TEMP TABLE t1(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + CREATE TABLE aux.t1(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + + CREATE TABLE main.x1(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + CREATE TEMP TABLE x2(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + CREATE TABLE aux.x3(a INTEGER PRIMARY KEY, b TEXT, c BLOB); + + INSERT INTO main.t1 VALUES(1, 'main one', X'0101'); + INSERT INTO main.t1 VALUES(2, 'main two', X'0102'); + INSERT INTO main.t1 VALUES(3, 'main three', X'0103'); + INSERT INTO main.t1 VALUES(4, 'main four', X'0104'); + INSERT INTO main.t1 VALUES(5, 'main five', X'0105'); + + INSERT INTO main.x1 VALUES(1, 'x main one', X'000101'); + INSERT INTO main.x1 VALUES(2, 'x main two', X'000102'); + INSERT INTO main.x1 VALUES(3, 'x main three', X'000103'); + INSERT INTO main.x1 VALUES(4, 'x main four', X'000104'); + INSERT INTO main.x1 VALUES(5, 'x main five', X'000105'); + + INSERT INTO temp.t1 VALUES(1, 'temp one', X'0201'); + INSERT INTO temp.t1 VALUES(2, 'temp two', X'0202'); + INSERT INTO temp.t1 VALUES(3, 'temp three', X'0203'); + INSERT INTO temp.t1 VALUES(4, 'temp four', X'0204'); + INSERT INTO temp.t1 VALUES(5, 'temp five', X'0205'); + + INSERT INTO temp.x2 VALUES(1, 'x temp one', X'000201'); + INSERT INTO temp.x2 VALUES(2, 'x temp two', X'000202'); + INSERT INTO temp.x2 VALUES(3, 'x temp three', X'000203'); + INSERT INTO temp.x2 VALUES(4, 'x temp four', X'000204'); + INSERT INTO temp.x2 VALUES(5, 'x temp five', X'000205'); + + INSERT INTO aux.t1 VALUES(1, 'aux one', X'0301'); + INSERT INTO aux.t1 VALUES(2, 'aux two', X'0302'); + INSERT INTO aux.t1 VALUES(3, 'aux three', X'0303'); + INSERT INTO aux.t1 VALUES(4, 'aux four', X'0304'); + INSERT INTO aux.t1 VALUES(5, 'aux five', X'0305'); + + INSERT INTO aux.x3 VALUES(1, 'x aux one', X'000301'); + INSERT INTO aux.x3 VALUES(2, 'x aux two', X'000302'); + INSERT INTO aux.x3 VALUES(3, 'x aux three', X'000303'); + INSERT INTO aux.x3 VALUES(4, 'x aux four', X'000304'); + INSERT INTO aux.x3 VALUES(5, 'x aux five', X'000305'); +} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-37639-55938 This interfaces opens a handle to the BLOB +# located in row iRow, column zColumn, table zTable in database zDb; in +# other words, the same BLOB that would be selected by: SELECT zColumn +# FROM zDb.zTable WHERE rowid = iRow; +# +proc read_blob {zDb zTab zCol iRow} { + sqlite3_blob_open db $zDb $zTab $zCol $iRow 0 B + set nByte [sqlite3_blob_bytes $B] + set data [sqlite3_blob_read $B 0 $nByte] + sqlite3_blob_close $B + return $data +} + +do_test 1.1.1 { read_blob main t1 b 1 } "main one" +do_test 1.1.2 { read_blob main t1 c 1 } "\01\01" +do_test 1.1.3 { read_blob temp t1 b 1 } "temp one" +do_test 1.1.4 { read_blob temp t1 c 1 } "\02\01" +do_test 1.1.6 { read_blob aux t1 b 1 } "aux one" +do_test 1.1.7 { read_blob aux t1 c 1 } "\03\01" + +do_test 1.2.1 { read_blob main t1 b 4 } "main four" +do_test 1.2.2 { read_blob main t1 c 4 } "\01\04" +do_test 1.2.3 { read_blob temp t1 b 4 } "temp four" +do_test 1.2.4 { read_blob temp t1 c 4 } "\02\04" +do_test 1.2.6 { read_blob aux t1 b 4 } "aux four" +do_test 1.2.7 { read_blob aux t1 c 4 } "\03\04" + +do_test 1.3.1 { read_blob main x1 b 2 } "x main two" +do_test 1.3.2 { read_blob main x1 c 2 } "\00\01\02" +do_test 1.3.3 { read_blob temp x2 b 2 } "x temp two" +do_test 1.3.4 { read_blob temp x2 c 2 } "\00\02\02" +do_test 1.3.6 { read_blob aux x3 b 2 } "x aux two" +do_test 1.3.7 { read_blob aux x3 c 2 } "\00\03\02" + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-27234-05761 Parameter zDb is not the filename that +# contains the database, but rather the symbolic name of the database. +# For attached databases, this is the name that appears after the AS +# keyword in the ATTACH statement. For the main database file, the +# database name is "main". For TEMP tables, the database name is "temp". +# +# The test cases immediately above demonstrate that the database name +# for the main db, for TEMP tables and for those in attached databases +# is correct. The following tests check that filenames cannot be +# used as well. +# +do_test 2.1 { + list [catch { sqlite3_blob_open db "test.db" t1 b 1 0 B } msg] $msg +} {1 SQLITE_ERROR} +do_test 2.2 { + list [catch { sqlite3_blob_open db "test.db2" t1 b 1 0 B } msg] $msg +} {1 SQLITE_ERROR} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-50854-53979 If the flags parameter is non-zero, then +# the BLOB is opened for read and write access. +# +# EVIDENCE-OF: R-03922-41160 If the flags parameter is zero, the BLOB is +# opened for read-only access. +# +foreach {tn iRow flags} { + 1 1 0 + 2 2 1 + 3 3 -1 + 4 4 2147483647 + 5 5 -2147483648 +} { + do_test 3.$tn.1 { + sqlite3_blob_open db main x1 c $iRow $flags B + set n [sqlite3_blob_bytes $B] + sqlite3_blob_read $B 0 $n + } [binary format ccc 0 1 $iRow] + + if {$flags==0} { + # Blob was opened for read-only access - writing returns an error. + do_test 3.$tn.2 { + list [catch { sqlite3_blob_write $B 0 xxx 3 } msg] $msg + } {1 SQLITE_READONLY} + + do_execsql_test 3.$tn.3 { + SELECT c FROM x1 WHERE a=$iRow; + } [binary format ccc 0 1 $iRow] + } else { + # Blob was opened for read/write access - writing succeeds + do_test 3.$tn.4 { + list [catch { sqlite3_blob_write $B 0 xxx 3 } msg] $msg + } {0 {}} + + do_execsql_test 3.$tn.5 { + SELECT c FROM x1 WHERE a=$iRow; + } {xxx} + } + + sqlite3_blob_close $B +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 4.0 { + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES('abcd', 152); + INSERT INTO t1 VALUES(NULL, X'00010203'); + INSERT INTO t1 VALUES('', 154.2); + + CREATE TABLE t2(x PRIMARY KEY, y) WITHOUT ROWID; + INSERT INTO t2 VALUES(1, 'blob'); + + CREATE TABLE t3(a PRIMARY KEY, b, c, d, e, f, UNIQUE(e, f)); + INSERT INTO t3 VALUES('aaaa', 'bbbb', 'cccc', 'dddd', 'eeee', 'ffff'); + CREATE INDEX t3b ON t3(b); + + CREATE TABLE p1(x PRIMARY KEY); + INSERT INTO p1 VALUES('abc'); + + CREATE TABLE c1(a INTEGER PRIMARY KEY, b REFERENCES p1); + INSERT INTO c1 VALUES(45, 'abc'); +} + +proc test_blob_open {tn zDb zTab zCol iRow flags errcode errmsg} { + global B + set B "0x1234" + + if {$errcode=="SQLITE_OK"} { + set expected "0 {}" + } else { + set expected "1 $errcode" + } + + set ::res [list [ + catch { sqlite3_blob_open db $zDb $zTab $zCol $iRow $flags B } msg + ] $msg] + do_test 4.$tn.1 { set ::res } $expected + + # EVIDENCE-OF: R-08940-21305 Unless it returns SQLITE_MISUSE, this + # function sets the database connection error code and message + # accessible via sqlite3_errcode() and sqlite3_errmsg() and related + # functions. + # + # This proc (test_blob_open) is used below to test various error and + # non-error conditions. But never SQLITE_MISUSE conditions. So these + # test cases are considered as partly verifying the requirement above. + # See below for a test of the SQLITE_MISUSE case. + # + do_test 4.$tn.2 { + sqlite3_errcode db + } $errcode + do_test 4.$tn.3 { + sqlite3_errmsg db + } $errmsg + + # EVIDENCE-OF: R-31086-35521 On success, SQLITE_OK is returned and the + # new BLOB handle is stored in *ppBlob. Otherwise an error code is + # returned and, unless the error code is SQLITE_MISUSE, *ppBlob is set + # to NULL. + # + do_test 4.$tn.4 { + expr {$B == "0"} + } [expr {$errcode != "SQLITE_OK"}] + + # EVIDENCE-OF: R-63421-15521 This means that, provided the API is not + # misused, it is always safe to call sqlite3_blob_close() on *ppBlob + # after this function it returns. + do_test 4.$tn.5 { + sqlite3_blob_close $B + } {} +} + +# EVIDENCE-OF: R-31204-44780 Database zDb does not exist +test_blob_open 1 nosuchdb t1 x 1 0 SQLITE_ERROR "no such table: nosuchdb.t1" + +# EVIDENCE-OF: R-28676-08005 Table zTable does not exist within database zDb +test_blob_open 2 main tt1 x 1 0 SQLITE_ERROR "no such table: main.tt1" + +# EVIDENCE-OF: R-40134-30296 Table zTable is a WITHOUT ROWID table +test_blob_open 3 main t2 y 1 0 SQLITE_ERROR \ + "cannot open table without rowid: t2" + +# EVIDENCE-OF: R-56376-21261 Column zColumn does not exist +test_blob_open 4 main t1 z 2 0 SQLITE_ERROR "no such column: \"z\"" + +# EVIDENCE-OF: R-28258-23166 Row iRow is not present in the table +test_blob_open 5 main t1 y 6 0 SQLITE_ERROR "no such rowid: 6" + +# EVIDENCE-OF: R-11683-62380 The specified column of row iRow contains a +# value that is not a TEXT or BLOB value +test_blob_open 6 main t1 x 2 0 SQLITE_ERROR "cannot open value of type null" +test_blob_open 7 main t1 y 1 0 SQLITE_ERROR "cannot open value of type integer" +test_blob_open 8 main t1 y 3 0 SQLITE_ERROR "cannot open value of type real" + +# EVIDENCE-OF: R-34146-30782 Column zColumn is part of an index, PRIMARY +# KEY or UNIQUE constraint and the blob is being opened for read/write +# access +# +# Test cases 8.1.* show that such columns can be opened for read-access. +# Tests 8.2.* show that read-write access is different. Columns "c" and "c" +# are not part of an index, PK or UNIQUE constraint, so they work in both +# cases. +# +test_blob_open 8.1.1 main t3 a 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.2 main t3 b 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.3 main t3 c 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.4 main t3 d 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.5 main t3 e 1 0 SQLITE_OK "not an error" +test_blob_open 8.1.6 main t3 f 1 0 SQLITE_OK "not an error" + +set cannot "cannot open indexed column for writing" +test_blob_open 8.2.1 main t3 a 1 8 SQLITE_ERROR $cannot +test_blob_open 8.2.2 main t3 b 1 8 SQLITE_ERROR $cannot +test_blob_open 8.2.3 main t3 c 1 8 SQLITE_OK "not an error" +test_blob_open 8.2.4 main t3 d 1 8 SQLITE_OK "not an error" +test_blob_open 8.2.5 main t3 e 1 8 SQLITE_ERROR $cannot +test_blob_open 8.2.6 main t3 f 1 8 SQLITE_ERROR $cannot + +# EVIDENCE-OF: R-50117-55204 Foreign key constraints are enabled, column +# zColumn is part of a child key definition and the blob is being opened +# for read/write access +# +# 9.1: FK disabled, read-only access. +# 9.2: FK disabled, read-only access. +# 9.3: FK enabled, read/write access. +# 9.4: FK enabled, read/write access. +# +test_blob_open 9.1 main c1 b 45 0 SQLITE_OK "not an error" +test_blob_open 9.2 main c1 b 45 1 SQLITE_OK "not an error" +execsql { PRAGMA foreign_keys = ON } +test_blob_open 9.3 main c1 b 45 0 SQLITE_OK "not an error" +test_blob_open 9.4 main c1 b 45 1 SQLITE_ERROR \ + "cannot open foreign key column for writing" + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-08940-21305 Unless it returns SQLITE_MISUSE, this +# function sets the database connection error code and message +# accessible via sqlite3_errcode() and sqlite3_errmsg() and related +# functions. +# +# This requirement is partially verified by the many uses of test +# command [test_blob_open] above. All that is left is to verify the +# SQLITE_MISUSE case. +# +# SQLITE_MISUSE is only returned if SQLITE_ENABLE_API_ARMOR is defined +# during compilation. +# +ifcapable api_armor { + sqlite3_blob_open db main t1 x 1 0 B + + do_test 10.1.1 { + list [catch {sqlite3_blob_open $B main t1 x 1 0 B2} msg] $msg + } {1 SQLITE_MISUSE} + do_test 10.1.2 { + list [sqlite3_errcode db] [sqlite3_errmsg db] + } {SQLITE_OK {not an error}} + sqlite3_blob_close $B + + do_test 10.2.1 { + list [catch {sqlite3_blob_open db main {} x 1 0 B} msg] $msg + } {1 SQLITE_MISUSE} + do_test 10.2.2 { + list [sqlite3_errcode db] [sqlite3_errmsg db] + } {SQLITE_OK {not an error}} +} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-50542-62589 If the row that a BLOB handle points to is +# modified by an UPDATE, DELETE, or by ON CONFLICT side-effects then the +# BLOB handle is marked as "expired". This is true if any column of the +# row is changed, even a column other than the one the BLOB handle is +# open on. +# +# EVIDENCE-OF: R-48367-20048 Calls to sqlite3_blob_read() and +# sqlite3_blob_write() for an expired BLOB handle fail with a return +# code of SQLITE_ABORT. +# +# 11.2: read-only handle, DELETE. +# 11.3: read-only handle, UPDATE. +# 11.4: read-only handle, REPLACE. +# 11.5: read/write handle, DELETE. +# 11.6: read/write handle, UPDATE. +# 11.7: read/write handle, REPLACE. +# +do_execsql_test 11.1 { + CREATE TABLE b1(a INTEGER PRIMARY KEY, b, c UNIQUE); + INSERT INTO b1 VALUES(1, '1234567890', 1); + INSERT INTO b1 VALUES(2, '1234567890', 2); + INSERT INTO b1 VALUES(3, '1234567890', 3); + INSERT INTO b1 VALUES(4, '1234567890', 4); + INSERT INTO b1 VALUES(5, '1234567890', 5); + INSERT INTO b1 VALUES(6, '1234567890', 6); + + CREATE TABLE b2(a INTEGER PRIMARY KEY, b, c UNIQUE); + INSERT INTO b2 VALUES(1, '1234567890', 1); + INSERT INTO b2 VALUES(2, '1234567890', 2); + INSERT INTO b2 VALUES(3, '1234567890', 3); + INSERT INTO b2 VALUES(4, '1234567890', 4); + INSERT INTO b2 VALUES(5, '1234567890', 5); + INSERT INTO b2 VALUES(6, '1234567890', 6); +} + +do_test 11.2.1 { + sqlite3_blob_open db main b1 b 2 0 B + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.2.2 { + # Deleting a different row does not invalidate the blob handle. + execsql { DELETE FROM b1 WHERE a = 1 } + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.2.3 { + execsql { DELETE FROM b1 WHERE a = 2 } + list [catch { sqlite3_blob_read $B 0 10 } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.2.4 { + sqlite3_blob_close $B +} {} + +do_test 11.3.1 { + sqlite3_blob_open db main b1 b 3 0 B + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.3.2 { + # Updating a different row + execsql { UPDATE b1 SET c = 42 WHERE a=4 } + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.3.3 { + execsql { UPDATE b1 SET c = 43 WHERE a=3 } + list [catch { sqlite3_blob_read $B 0 10 } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.3.4 { + sqlite3_blob_close $B +} {} + +do_test 11.4.1 { + sqlite3_blob_open db main b1 b 6 0 B + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.4.2 { + # Replace a different row + execsql { INSERT OR REPLACE INTO b1 VALUES(10, 'abcdefghij', 5) } + sqlite3_blob_read $B 0 10 +} {1234567890} +do_test 11.4.3 { + execsql { INSERT OR REPLACE INTO b1 VALUES(11, 'abcdefghij', 6) } + list [catch { sqlite3_blob_read $B 0 10 } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.4.4 { + sqlite3_blob_close $B +} {} + +do_test 11.4.1 { + sqlite3_blob_open db main b2 b 2 1 B + sqlite3_blob_write $B 0 "abcdefghij" +} {} +do_test 11.4.2 { + # Deleting a different row does not invalidate the blob handle. + execsql { DELETE FROM b2 WHERE a = 1 } + sqlite3_blob_write $B 0 "ABCDEFGHIJ" +} {} +do_test 11.4.3 { + execsql { DELETE FROM b2 WHERE a = 2 } + list [catch { sqlite3_blob_write $B 0 "0987654321" } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.4.4 { + sqlite3_blob_close $B +} {} + +do_test 11.5.1 { + sqlite3_blob_open db main b2 b 3 1 B + sqlite3_blob_write $B 0 "abcdefghij" +} {} +do_test 11.5.2 { + # Updating a different row + execsql { UPDATE b2 SET c = 42 WHERE a=4 } + sqlite3_blob_write $B 0 "ABCDEFGHIJ" +} {} +do_test 11.5.3 { + execsql { UPDATE b2 SET c = 43 WHERE a=3 } + list [catch { sqlite3_blob_write $B 0 "0987654321" } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.5.4 { + sqlite3_blob_close $B +} {} + +do_test 11.6.1 { + sqlite3_blob_open db main b2 b 6 1 B + sqlite3_blob_write $B 0 "abcdefghij" +} {} +do_test 11.6.2 { + # Replace a different row + execsql { INSERT OR REPLACE INTO b2 VALUES(10, 'abcdefghij', 5) } + sqlite3_blob_write $B 0 "ABCDEFGHIJ" +} {} +do_test 11.6.3 { + execsql { INSERT OR REPLACE INTO b2 VALUES(11, 'abcdefghij', 6) } + list [catch { sqlite3_blob_write $B 0 "0987654321" } msg] $msg +} {1 SQLITE_ABORT} +do_test 11.6.4 { + sqlite3_blob_close $B +} {} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-45408-40694 Changes written into a BLOB prior to the +# BLOB expiring are not rolled back by the expiration of the BLOB. Such +# changes will eventually commit if the transaction continues to +# completion. +# +do_execsql_test 12.1 { + CREATE TABLE b3(x INTEGER PRIMARY KEY, y TEXT, z INTEGER); + INSERT INTO b3 VALUES(22, '..........', NULL); +} +do_test 12.2 { + sqlite3_blob_open db main b3 y 22 1 B + sqlite3_blob_write $B 0 "xxxxx" 5 +} {} +do_execsql_test 12.3 { + UPDATE b3 SET z = 'not null'; +} +do_test 12.4 { + list [catch {sqlite3_blob_write $B 5 "xxxxx" 5} msg] $msg +} {1 SQLITE_ABORT} +do_execsql_test 12.5 { + SELECT * FROM b3; +} {22 xxxxx..... {not null}} +do_test 12.5 { + sqlite3_blob_close $B +} {} +do_execsql_test 12.6 { + SELECT * FROM b3; +} {22 xxxxx..... {not null}} + +#------------------------------------------------------------------------- +# EVIDENCE-OF: R-58813-55036 The sqlite3_bind_zeroblob() and +# sqlite3_result_zeroblob() interfaces and the built-in zeroblob SQL +# function may be used to create a zero-filled blob to read or write +# using the incremental-blob interface. +# +do_execsql_test 13.1 { + CREATE TABLE c2(i INTEGER PRIMARY KEY, j); + INSERT INTO c2 VALUES(10, zeroblob(24)); +} + +do_test 13.2 { + set stmt [sqlite3_prepare_v2 db "INSERT INTO c2 VALUES(11, ?)" -1] + sqlite3_bind_zeroblob $stmt 1 45 + sqlite3_step $stmt + sqlite3_finalize $stmt +} {SQLITE_OK} + +# The blobs can be read: +# +do_test 13.3.1 { + sqlite3_blob_open db main c2 j 10 1 B + sqlite3_blob_open db main c2 j 11 1 B2 + list [sqlite3_blob_bytes $B] [sqlite3_blob_bytes $B2] +} {24 45} +do_test 13.3.2 { + sqlite3_blob_read $B 0 24 +} [string repeat [binary format c 0] 24] +do_test 13.3.3 { + sqlite3_blob_read $B2 0 45 +} [string repeat [binary format c 0] 45] + +# And also written: +# +do_test 13.4.1 { + sqlite3_blob_write $B 0 [string repeat [binary format c 1] 24] +} {} +do_test 13.4.2 { + sqlite3_blob_write $B2 0 [string repeat [binary format c 1] 45] +} {} +do_test 13.5 { + sqlite3_blob_close $B + sqlite3_blob_close $B2 + execsql { SELECT j FROM c2 } +} [list \ + [string repeat [binary format c 1] 24] \ + [string repeat [binary format c 1] 45] \ +] + + +finish_test + diff --git a/test/fkey7.test b/test/fkey7.test index c2682edbe5..6c646a9a7f 100644 --- a/test/fkey7.test +++ b/test/fkey7.test @@ -50,5 +50,22 @@ do_tblsread_test 1.3 { UPDATE par SET a=? WHERE b=? } {c1 c2 par} do_tblsread_test 1.4 { UPDATE par SET c=? WHERE b=? } {c3 par} do_tblsread_test 1.5 { UPDATE par SET a=?,b=?,c=? WHERE b=? } {c1 c2 c3 par s1} +ifcapable incrblob { + do_execsql_test 2.0 { + CREATE TABLE pX(x PRIMARY KEY); + CREATE TABLE cX(a INTEGER PRIMARY KEY, b REFERENCES pX); + } + + do_catchsql_test 2.1 { + INSERT INTO cX VALUES(11, zeroblob(40)); + } {1 {FOREIGN KEY constraint failed}} + + do_test 2.2 { + set stmt [sqlite3_prepare_v2 db "INSERT INTO cX VALUES(11, ?)" -1] + sqlite3_bind_zeroblob $stmt 1 45 + sqlite3_step $stmt + sqlite3_finalize $stmt + } {SQLITE_CONSTRAINT} +} finish_test diff --git a/test/without_rowid5.test b/test/without_rowid5.test index 45e047befe..d163d9c1bc 100644 --- a/test/without_rowid5.test +++ b/test/without_rowid5.test @@ -185,8 +185,7 @@ do_execsql_test without_rowid5-5.9 { # EVIDENCE-OF: R-12643-30541 The incremental blob I/O mechanism does not # work for WITHOUT ROWID tables. # -# EVIDENCE-OF: R-25760-33257 The sqlite3_blob_open() interface will fail -# for a WITHOUT ROWID table. +# EVIDENCE-OF: R-40134-30296 Table zTable is a WITHOUT ROWID table # do_execsql_test without_rowid5-6.1 { CREATE TABLE b1(a INTEGER PRIMARY KEY, b BLOB) WITHOUT ROWID; From 4ace5362c250de7d712ef107275427aaa8c2c304 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Nov 2014 14:42:28 +0000 Subject: [PATCH 554/710] Shorten over-length source code lines in shell.c. FossilOrigin-Name: 7f3819f6422badd344c1264b0cd2f2c7afe077df --- manifest | 16 ++++---- manifest.uuid | 2 +- src/shell.c | 100 +++++++++++++++++++++++++++++++++----------------- 3 files changed, 76 insertions(+), 42 deletions(-) diff --git a/manifest b/manifest index e08707974e..5e62c4081e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\se_blobopen.test,\scontaining\stests\sfor\ssqlite3_blob_open(). -D 2014-11-07T14:41:11.404 +C Shorten\sover-length\ssource\scode\slines\sin\sshell.c. +D 2014-11-10T14:42:28.114 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c 0500307f63e3675dc4ab21112c6086b4d146fc08 +F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd F src/sqlite.h.in 5abfd7d6c9bd2054105820de5ce12b1e491b1dc9 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -452,7 +452,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 -F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 w test/e_blob.test +F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a @@ -1213,7 +1213,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b45bc80bb16f07192d84fd14433bb724a84d4146 -R 0ead7b89f3ed1286937ae097bba351c9 -U dan -Z d785e78de9d268050b7beac3067db32e +P ecbccd0e594d22b3ae7fabc8037951dc49570bc3 +R 1a35da25c621486ba7906429ccbf946c +U drh +Z 427c2830707548878070627d5502fc48 diff --git a/manifest.uuid b/manifest.uuid index 6227627e36..541a9c512c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecbccd0e594d22b3ae7fabc8037951dc49570bc3 \ No newline at end of file +7f3819f6422badd344c1264b0cd2f2c7afe077df \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index a33e65b1f8..bbd37c7a04 100644 --- a/src/shell.c +++ b/src/shell.c @@ -172,7 +172,8 @@ static HANDLE hProcess; static FILETIME ftKernelBegin; static FILETIME ftUserBegin; static sqlite3_int64 ftWallBegin; -typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, LPFILETIME, LPFILETIME); +typedef BOOL (WINAPI *GETPROCTIMES)(HANDLE, LPFILETIME, LPFILETIME, + LPFILETIME, LPFILETIME); static GETPROCTIMES getProcessTimesAddr = NULL; /* @@ -183,15 +184,16 @@ static int hasTimer(void){ if( getProcessTimesAddr ){ return 1; } else { - /* GetProcessTimes() isn't supported in WIN95 and some other Windows versions. - ** See if the version we are running on has it, and if it does, save off - ** a pointer to it and the current process handle. + /* GetProcessTimes() isn't supported in WIN95 and some other Windows + ** versions. See if the version we are running on has it, and if it + ** does, save off a pointer to it and the current process handle. */ hProcess = GetCurrentProcess(); if( hProcess ){ HINSTANCE hinstLib = LoadLibrary(TEXT("Kernel32.dll")); if( NULL != hinstLib ){ - getProcessTimesAddr = (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); + getProcessTimesAddr = + (GETPROCTIMES) GetProcAddress(hinstLib, "GetProcessTimes"); if( NULL != getProcessTimesAddr ){ return 1; } @@ -208,7 +210,8 @@ static int hasTimer(void){ static void beginTimer(void){ if( enableTimer && getProcessTimesAddr ){ FILETIME ftCreation, ftExit; - getProcessTimesAddr(hProcess, &ftCreation, &ftExit, &ftKernelBegin, &ftUserBegin); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit, + &ftKernelBegin,&ftUserBegin); ftWallBegin = timeOfDay(); } } @@ -227,7 +230,7 @@ static void endTimer(void){ if( enableTimer && getProcessTimesAddr){ FILETIME ftCreation, ftExit, ftKernelEnd, ftUserEnd; sqlite3_int64 ftWallEnd = timeOfDay(); - getProcessTimesAddr(hProcess, &ftCreation, &ftExit, &ftKernelEnd, &ftUserEnd); + getProcessTimesAddr(hProcess,&ftCreation,&ftExit,&ftKernelEnd,&ftUserEnd); printf("Run Time: real %.3f user %f sys %f\n", (ftWallEnd - ftWallBegin)*0.001, timeDiff(&ftUserBegin, &ftUserEnd), @@ -726,7 +729,13 @@ static void interrupt_handler(int NotUsed){ ** This is the callback routine that the shell ** invokes for each row of a query result. */ -static int shell_callback(void *pArg, int nArg, char **azArg, char **azCol, int *aiType){ +static int shell_callback( + void *pArg, + int nArg, /* Number of result columns */ + char **azArg, /* Text of each result column */ + char **azCol, /* Column names */ + int *aiType /* Column types */ +){ int i; ShellState *p = (ShellState*)pArg; @@ -1105,57 +1114,77 @@ static int display_stats( iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_MEMORY_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Memory Used: %d (max %d) bytes\n", iCur, iHiwtr); + fprintf(pArg->out, + "Memory Used: %d (max %d) bytes\n", + iCur, iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Outstanding Allocations: %d (max %d)\n", iCur, iHiwtr); + fprintf(pArg->out, "Number of Outstanding Allocations: %d (max %d)\n", + iCur, iHiwtr); if( pArg->shellFlgs & SHFLG_Pagecache ){ iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PAGECACHE_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Pcache Pages Used: %d (max %d) pages\n", iCur, iHiwtr); + fprintf(pArg->out, + "Number of Pcache Pages Used: %d (max %d) pages\n", + iCur, iHiwtr); } iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PAGECACHE_OVERFLOW, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", iCur, iHiwtr); + fprintf(pArg->out, + "Number of Pcache Overflow Bytes: %d (max %d) bytes\n", + iCur, iHiwtr); if( pArg->shellFlgs & SHFLG_Scratch ){ iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_SCRATCH_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Scratch Allocations Used: %d (max %d)\n", iCur, iHiwtr); + fprintf(pArg->out, "Number of Scratch Allocations Used: %d (max %d)\n", + iCur, iHiwtr); } iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_SCRATCH_OVERFLOW, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", iCur, iHiwtr); + fprintf(pArg->out, + "Number of Scratch Overflow Bytes: %d (max %d) bytes\n", + iCur, iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_MALLOC_SIZE, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Largest Allocation: %d bytes\n", iHiwtr); + fprintf(pArg->out, "Largest Allocation: %d bytes\n", + iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PAGECACHE_SIZE, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Largest Pcache Allocation: %d bytes\n", iHiwtr); + fprintf(pArg->out, "Largest Pcache Allocation: %d bytes\n", + iHiwtr); iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_SCRATCH_SIZE, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Largest Scratch Allocation: %d bytes\n", iHiwtr); + fprintf(pArg->out, "Largest Scratch Allocation: %d bytes\n", + iHiwtr); #ifdef YYTRACKMAXSTACKDEPTH iHiwtr = iCur = -1; sqlite3_status(SQLITE_STATUS_PARSER_STACK, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Deepest Parser Stack: %d (max %d)\n", iCur, iHiwtr); + fprintf(pArg->out, "Deepest Parser Stack: %d (max %d)\n", + iCur, iHiwtr); #endif } if( pArg && pArg->out && db ){ if( pArg->shellFlgs & SHFLG_Lookaside ){ iHiwtr = iCur = -1; - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Lookaside Slots Used: %d (max %d)\n", iCur, iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, &iCur, &iHiwtr, bReset); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_USED, + &iCur, &iHiwtr, bReset); + fprintf(pArg->out, "Lookaside Slots Used: %d (max %d)\n", + iCur, iHiwtr); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_HIT, + &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Successful lookaside attempts: %d\n", iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &iCur, &iHiwtr, bReset); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, + &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Lookaside failures due to size: %d\n", iHiwtr); - sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &iCur, &iHiwtr, bReset); + sqlite3_db_status(db, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, + &iCur, &iHiwtr, bReset); fprintf(pArg->out, "Lookaside failures due to OOM: %d\n", iHiwtr); } iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Pager Heap Usage: %d bytes\n", iCur); iHiwtr = iCur = -1; + fprintf(pArg->out, "Pager Heap Usage: %d bytes\n",iCur); + iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_CACHE_HIT, &iCur, &iHiwtr, 1); fprintf(pArg->out, "Page cache hits: %d\n", iCur); iHiwtr = iCur = -1; @@ -1166,18 +1195,19 @@ static int display_stats( fprintf(pArg->out, "Page cache writes: %d\n", iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_SCHEMA_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Schema Heap Usage: %d bytes\n", iCur); + fprintf(pArg->out, "Schema Heap Usage: %d bytes\n",iCur); iHiwtr = iCur = -1; sqlite3_db_status(db, SQLITE_DBSTATUS_STMT_USED, &iCur, &iHiwtr, bReset); - fprintf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n", iCur); + fprintf(pArg->out, "Statement Heap/Lookaside Usage: %d bytes\n",iCur); } if( pArg && pArg->out && db && pArg->pStmt ){ - iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, bReset); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_FULLSCAN_STEP, + bReset); fprintf(pArg->out, "Fullscan Steps: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_SORT, bReset); fprintf(pArg->out, "Sort Operations: %d\n", iCur); - iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX, bReset); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_AUTOINDEX,bReset); fprintf(pArg->out, "Autoindex Inserts: %d\n", iCur); iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_VM_STEP, bReset); fprintf(pArg->out, "Virtual Machine Steps: %d\n", iCur); @@ -1221,7 +1251,8 @@ static void display_scanstats( sqlite3_stmt_scanstatus(p, i, SQLITE_SCANSTAT_EXPLAIN, (void*)&zExplain); fprintf(pArg->out, "Loop %2d: %s\n", n, zExplain); rEstLoop *= rEst; - fprintf(pArg->out, " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", + fprintf(pArg->out, + " nLoop=%-8lld nRow=%-8lld estRow=%-8lld estRow/Loop=%-8g\n", nLoop, nVisit, (sqlite3_int64)(rEstLoop+0.5), rEst ); } @@ -1271,7 +1302,8 @@ static void explain_data_prepare(ShellState *p, sqlite3_stmt *pSql){ const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", "NextIfOpen", "PrevIfOpen", 0 }; - const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", "Rewind", 0 }; + const char *azYield[] = { "Yield", "SeekLT", "SeekGT", "RowSetRead", + "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; /* Try to figure out if this is really an EXPLAIN statement. If this @@ -1384,7 +1416,8 @@ static int shell_exec( /* Show the EXPLAIN QUERY PLAN if .eqp is on */ if( pArg && pArg->autoEQP ){ sqlite3_stmt *pExplain; - char *zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", sqlite3_sql(pStmt)); + char *zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", + sqlite3_sql(pStmt)); rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0); if( rc==SQLITE_OK ){ while( sqlite3_step(pExplain)==SQLITE_ROW ){ @@ -3335,7 +3368,7 @@ static int do_meta_command(char *zLine, ShellState *p){ for(i=0; iout, "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j] : ""); + fprintf(p->out, "%s%-*s", zSp, maxlen, azResult[j] ? azResult[j]:""); } fprintf(p->out, "\n"); } @@ -3805,7 +3838,8 @@ static char *find_home_dir(void){ static char *home_dir = NULL; if( home_dir ) return home_dir; -#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) && !defined(__RTP__) && !defined(_WRS_KERNEL) +#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \ + && !defined(__RTP__) && !defined(_WRS_KERNEL) { struct passwd *pwent; uid_t uid = getuid(); From 1728bcb07f9e6784bdf5c223ffe2c035011f9123 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Nov 2014 16:49:56 +0000 Subject: [PATCH 555/710] Add the eval() SQL function extension in ext/misc/eval.c. FossilOrigin-Name: 27cf665b957f2c0ced403e3032099e80c295598f --- Makefile.in | 1 + Makefile.msc | 1 + ext/misc/eval.c | 119 ++++++++++++++++++++++++++++++++++++++++++++++++ main.mk | 1 + manifest | 20 ++++---- manifest.uuid | 2 +- src/test1.c | 2 + test/misc8.test | 41 +++++++++++++++++ 8 files changed, 177 insertions(+), 10 deletions(-) create mode 100644 ext/misc/eval.c create mode 100644 test/misc8.test diff --git a/Makefile.in b/Makefile.in index a2213e89e5..646cb39756 100644 --- a/Makefile.in +++ b/Makefile.in @@ -394,6 +394,7 @@ TESTSRC = \ TESTSRC += \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/closure.c \ + $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ diff --git a/Makefile.msc b/Makefile.msc index 4173eaae27..4aaa2894a7 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -863,6 +863,7 @@ TESTSRC = \ TESTEXT = \ $(TOP)\ext\misc\amatch.c \ $(TOP)\ext\misc\closure.c \ + $(TOP)\ext\misc\eval.c \ $(TOP)\ext\misc\fileio.c \ $(TOP)\ext\misc\fuzzer.c \ $(TOP)\ext\misc\ieee754.c \ diff --git a/ext/misc/eval.c b/ext/misc/eval.c new file mode 100644 index 0000000000..a5e297ad38 --- /dev/null +++ b/ext/misc/eval.c @@ -0,0 +1,119 @@ +/* +** 2014-11-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements SQL function eval() which runs +** SQL statements recursively. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** Structure used to accumulate the output +*/ +struct EvalResult { + char *z; /* Accumulated output */ + const char *zSep; /* Separator */ + int szSep; /* Size of the separator string */ + int nAlloc; /* Number of bytes allocated for z[] */ + int nUsed; /* Number of bytes of z[] actually used */ +}; + +/* +** Callback from sqlite_exec() for the eval() function. +*/ +static int callback(void *pCtx, int argc, char **argv, char **colnames){ + struct EvalResult *p = (struct EvalResult*)pCtx; + int i; + for(i=0; inUsed+p->szSep+1 > p->nAlloc ){ + char *zNew; + p->nAlloc = p->nAlloc*2 + sz + p->szSep + 1; + zNew = sqlite3_realloc(p->z, p->nAlloc); + if( zNew==0 ){ + sqlite3_free(p->z); + memset(p, 0, sizeof(*p)); + return 1; + } + p->z = zNew; + } + if( p->nUsed>0 ){ + memcpy(&p->z[p->nUsed], p->zSep, p->szSep); + p->nUsed += p->szSep; + } + memcpy(&p->z[p->nUsed], z, sz); + p->nUsed += sz; + } + return 0; +} + +/* +** Implementation of the eval(X) and eval(X,Y) SQL functions. +** +** Evaluate the SQL text in X. Return the results, using string +** Y as the separator. If Y is omitted, use a single space character. +*/ +static void sqlEvalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zSql; + sqlite3 *db; + char *zErr = 0; + int rc; + struct EvalResult x; + + memset(&x, 0, sizeof(x)); + x.zSep = " "; + zSql = (const char*)sqlite3_value_text(argv[0]); + if( zSql==0 ) return; + if( argc>1 ){ + x.zSep = (const char*)sqlite3_value_text(argv[1]); + if( x.zSep==0 ) return; + } + x.szSep = (int)strlen(x.zSep); + db = sqlite3_context_db_handle(context); + rc = sqlite3_exec(db, zSql, callback, &x, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + }else if( x.zSep==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(x.z); + }else{ + sqlite3_result_text(context, x.z, x.nUsed, sqlite3_free); + } +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_eval_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "eval", 1, SQLITE_UTF8, 0, + sqlEvalFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "eval", 2, SQLITE_UTF8, 0, + sqlEvalFunc, 0, 0); + } + return rc; +} diff --git a/main.mk b/main.mk index c8bf72b48b..cac996864d 100644 --- a/main.mk +++ b/main.mk @@ -281,6 +281,7 @@ TESTSRC = \ TESTSRC += \ $(TOP)/ext/misc/amatch.c \ $(TOP)/ext/misc/closure.c \ + $(TOP)/ext/misc/eval.c \ $(TOP)/ext/misc/fileio.c \ $(TOP)/ext/misc/fuzzer.c \ $(TOP)/ext/misc/ieee754.c \ diff --git a/manifest b/manifest index 5e62c4081e..7f7fe158b9 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Shorten\sover-length\ssource\scode\slines\sin\sshell.c. -D 2014-11-10T14:42:28.114 +C Add\sthe\seval()\sSQL\sfunction\sextension\sin\sext/misc/eval.c. +D 2014-11-10T16:49:56.620 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a +F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc e31dee24038965fb6269d6d61073fd6b7e331dec +F Makefile.msc 788f1288633a0c3c3cbbe0f3e4827d033f7ba530 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION d846487aff892625eb8e75960234e7285f0462fe @@ -109,6 +109,7 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 678056a4bfcd83c4e82dea81d37543cd1d6dbee1 F ext/misc/closure.c 636024302cde41b2bf0c542f81c40c624cfb7012 F ext/misc/compress.c 76e45655f4046e756064ab10c62e18f2eb846b9f +F ext/misc/eval.c 04e630bde869aa1fec6b993d40591f963be2f868 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 136533c53cfce0957f0b48fa11dba27e21c5c01d F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e @@ -151,7 +152,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 3fececc835c5c23637c3e0af970e27a81a9ba476 +F main.mk 084976077a4aa3bd985154b5423e7aed88e4a2e9 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -237,7 +238,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 -F src/test1.c cce6ad0effe8c66a62d2634d9714ffdc7372ef11 +F src/test1.c 6b0469b8e06c77b1de1d3e4a3834cf26edea9cc7 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -729,6 +730,7 @@ F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 +F test/misc8.test e538d122d5faacadbaa6fc02403f1e1427df684f F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 @@ -1213,7 +1215,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ecbccd0e594d22b3ae7fabc8037951dc49570bc3 -R 1a35da25c621486ba7906429ccbf946c +P 7f3819f6422badd344c1264b0cd2f2c7afe077df +R a579eb1c0cce07efa12ad49b9c702f07 U drh -Z 427c2830707548878070627d5502fc48 +Z 4009c695bea2e084a946ee86095e7182 diff --git a/manifest.uuid b/manifest.uuid index 541a9c512c..f90ad54213 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7f3819f6422badd344c1264b0cd2f2c7afe077df \ No newline at end of file +27cf665b957f2c0ced403e3032099e80c295598f \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index e7a5929323..ca3b54a513 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6260,6 +6260,7 @@ static int tclLoadStaticExtensionCmd( ){ extern int sqlite3_amatch_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_closure_init(sqlite3*,char**,const sqlite3_api_routines*); + extern int sqlite3_eval_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_fileio_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_fuzzer_init(sqlite3*,char**,const sqlite3_api_routines*); extern int sqlite3_ieee_init(sqlite3*,char**,const sqlite3_api_routines*); @@ -6275,6 +6276,7 @@ static int tclLoadStaticExtensionCmd( } aExtension[] = { { "amatch", sqlite3_amatch_init }, { "closure", sqlite3_closure_init }, + { "eval", sqlite3_eval_init }, { "fileio", sqlite3_fileio_init }, { "fuzzer", sqlite3_fuzzer_init }, { "ieee754", sqlite3_ieee_init }, diff --git a/test/misc8.test b/test/misc8.test new file mode 100644 index 0000000000..06ee314b60 --- /dev/null +++ b/test/misc8.test @@ -0,0 +1,41 @@ +# 2014-11-10 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# The focus of this script is testing the "eval.c" loadable extension. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +load_static_extension db eval +do_execsql_test misc8-1.0 { + CREATE TABLE t1(a,b,c); + INSERT INTO t1 VALUES(1,2,3),(4,5,6); + SELECT quote(eval('SELECT * FROM t1 ORDER BY a','-abc-')); +} {'1-abc-2-abc-3-abc-4-abc-5-abc-6'} +do_execsql_test misc8-1.1 { + SELECT quote(eval('SELECT * FROM t1 ORDER BY a')); +} {{'1 2 3 4 5 6'}} +do_catchsql_test misc8-1.2 { + SELECT quote(eval('SELECT d FROM t1 ORDER BY a')); +} {1 {no such column: d}} +do_execsql_test misc8-1.3 { + INSERT INTO t1 VALUES(7,null,9); + SELECT eval('SELECT * FROM t1 ORDER BY a',','); +} {1,2,3,4,5,6,7,,9} +do_catchsql_test misc8-1.4 { + BEGIN; + INSERT INTO t1 VALUES(10,11,12); + SELECT coalesce(b, eval('ROLLBACK')) FROM t1 ORDER BY a; +} {1 {abort due to ROLLBACK}} + + +finish_test From 923c4b35be963675b00b33a1af34dda0e6688dfa Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 10 Nov 2014 17:53:03 +0000 Subject: [PATCH 556/710] Add test file e_blobwrite.test, containing tests for the sqlite3_blob_write() interface. FossilOrigin-Name: 1df77e5f1bd82de4dc92fe28359c3e56ab3f9ed4 --- manifest | 17 ++-- manifest.uuid | 2 +- src/sqlite.h.in | 25 +++--- src/vdbeblob.c | 3 +- test/e_blobwrite.test | 204 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 229 insertions(+), 22 deletions(-) create mode 100644 test/e_blobwrite.test diff --git a/manifest b/manifest index 7f7fe158b9..d14dec8c51 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\seval()\sSQL\sfunction\sextension\sin\sext/misc/eval.c. -D 2014-11-10T16:49:56.620 +C Add\stest\sfile\se_blobwrite.test,\scontaining\stests\sfor\sthe\ssqlite3_blob_write()\sinterface. +D 2014-11-10T17:53:03.345 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd -F src/sqlite.h.in 5abfd7d6c9bd2054105820de5ce12b1e491b1dc9 +F src/sqlite.h.in 5531c4c69ee0a351c2aa5ad9f3f0f2424a57a9f4 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 71b0bf1a7fc55b5cb374f7579fd140e730a6e0f4 @@ -296,7 +296,7 @@ F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 F src/vdbeaux.c 9b0a251b6dfab349dd6c6efb40062eb7386b26f5 -F src/vdbeblob.c 8b5442ff0954c44b45cbabbe2e94091a2e16fdef +F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 @@ -454,6 +454,7 @@ F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 +F test/e_blobwrite.test 615b405f29feb2cfb5a1f03dab7933258294fa26 F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 F test/e_createtable.test c7e67b49e6cf92473c8fb30ab26143e9e2128cf7 F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a @@ -1215,7 +1216,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7f3819f6422badd344c1264b0cd2f2c7afe077df -R a579eb1c0cce07efa12ad49b9c702f07 -U drh -Z 4009c695bea2e084a946ee86095e7182 +P 27cf665b957f2c0ced403e3032099e80c295598f +R 37c32da8fb4d2f1d3e3ebc03749cb035 +U dan +Z 2886cb98deb64368f4cf93641ea335c8 diff --git a/manifest.uuid b/manifest.uuid index f90ad54213..719ea3ef1b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27cf665b957f2c0ced403e3032099e80c295598f \ No newline at end of file +1df77e5f1bd82de4dc92fe28359c3e56ab3f9ed4 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index bed64bfae2..b5585defd6 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5821,21 +5821,27 @@ int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); /* ** CAPI3REF: Write Data Into A BLOB Incrementally ** -** ^This function is used to write data into an open [BLOB handle] from a -** caller-supplied buffer. ^N bytes of data are copied from the buffer Z -** into the open BLOB, starting at offset iOffset. +** ^(This function is used to write data into an open [BLOB handle] from a +** caller-supplied buffer. N bytes of data are copied from the buffer Z +** into the open BLOB, starting at offset iOffset.)^ +** +** ^(On success, sqlite3_blob_write() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** ^Unless SQLITE_MISUSE is returned, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** ** ^If the [BLOB handle] passed as the first argument was not opened for ** writing (the flags parameter to [sqlite3_blob_open()] was zero), ** this function returns [SQLITE_READONLY]. ** -** ^This function may only modify the contents of the BLOB; it is +** This function may only modify the contents of the BLOB; it is ** not possible to increase the size of a BLOB using this API. ** ^If offset iOffset is less than N bytes from the end of the BLOB, -** [SQLITE_ERROR] is returned and no data is written. ^If N is -** less than zero [SQLITE_ERROR] is returned and no data is written. -** The size of the BLOB (and hence the maximum value of N+iOffset) -** can be determined using the [sqlite3_blob_bytes()] interface. +** [SQLITE_ERROR] is returned and no data is written. The size of the +** BLOB (and hence the maximum value of N+iOffset) can be determined +** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less +** than zero [SQLITE_ERROR] is returned and no data is written. ** ** ^An attempt to write to an expired [BLOB handle] fails with an ** error code of [SQLITE_ABORT]. ^Writes to the BLOB that occurred @@ -5844,9 +5850,6 @@ int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); ** have been overwritten by the statement that expired the BLOB handle ** or by other independent statements. ** -** ^(On success, sqlite3_blob_write() returns SQLITE_OK. -** Otherwise, an [error code] or an [extended error code] is returned.)^ -** ** This routine only works on a [BLOB handle] which has been created ** by a prior successful call to [sqlite3_blob_open()] and which has not ** been closed by [sqlite3_blob_close()]. Passing any other pointer in diff --git a/src/vdbeblob.c b/src/vdbeblob.c index 0cf2b0652b..cf1eb59054 100644 --- a/src/vdbeblob.c +++ b/src/vdbeblob.c @@ -376,7 +376,6 @@ static int blobReadWrite( if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){ /* Request is out of range. Return a transient error. */ rc = SQLITE_ERROR; - sqlite3Error(db, SQLITE_ERROR); }else if( v==0 ){ /* If there is no statement handle, then the blob-handle has ** already been invalidated. Return SQLITE_ABORT in this case. @@ -394,10 +393,10 @@ static int blobReadWrite( sqlite3VdbeFinalize(v); p->pStmt = 0; }else{ - db->errCode = rc; v->rc = rc; } } + sqlite3Error(db, rc); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; diff --git a/test/e_blobwrite.test b/test/e_blobwrite.test new file mode 100644 index 0000000000..a0d33336de --- /dev/null +++ b/test/e_blobwrite.test @@ -0,0 +1,204 @@ +# 2014 October 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_blobwrite + +#-------------------------------------------------------------------------- +# EVIDENCE-OF: R-62898-22698 This function is used to write data into an +# open BLOB handle from a caller-supplied buffer. N bytes of data are +# copied from the buffer Z into the open BLOB, starting at offset +# iOffset. +# +set dots [string repeat . 40] +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, t TEXT); + INSERT INTO t1 VALUES(-1, $dots); + INSERT INTO t1 VALUES(-2, $dots); + INSERT INTO t1 VALUES(-3, $dots); + INSERT INTO t1 VALUES(-4, $dots); + INSERT INTO t1 VALUES(-5, $dots); + INSERT INTO t1 VALUES(-6, $dots); +} + +proc blob_write_test {tn id iOffset blob nData final} { + sqlite3_blob_open db main t1 t $id 1 B + + # EVIDENCE-OF: R-45864-01884 On success, sqlite3_blob_write() returns + # SQLITE_OK. Otherwise, an error code or an extended error code is + # returned. + # + # This block tests the SQLITE_OK case in the requirement above (the + # Tcl sqlite3_blob_write() wrapper uses an empty string in place of + # "SQLITE_OK"). The error cases are tested by the "blob_write_error_test" + # tests below. + # + set res [sqlite3_blob_write $B $iOffset $blob $nData] + uplevel [list do_test $tn.1 [list set {} $res] {}] + + sqlite3_blob_close $B + uplevel [list do_execsql_test $tn.3 "SELECT t FROM t1 WHERE a=$id" $final] +} + +set blob "0123456789012345678901234567890123456789" +blob_write_test 1.1 -1 0 $blob 10 { 0123456789.............................. } +blob_write_test 1.2 -2 8 $blob 10 { ........0123456789...................... } +blob_write_test 1.3 -3 8 $blob 1 { ........0............................... } +blob_write_test 1.4 -4 18 $blob 22 { ..................0123456789012345678901 } +blob_write_test 1.5 -5 18 $blob 0 { ........................................ } +blob_write_test 1.6 -6 0 $blob 40 { 0123456789012345678901234567890123456789 } + + +proc blob_write_error_test {tn B iOffset blob nData errcode errmsg} { + + # In cases where the underlying sqlite3_blob_write() function returns + # SQLITE_OK, the Tcl wrapper returns an empty string. If the underlying + # function returns an error, the Tcl wrapper throws an exception with + # the error code as the Tcl exception message. + # + if {$errcode=="SQLITE_OK"} { + set ret "" + set isError 0 + } else { + set ret $errcode + set isError 1 + } + + set cmd [list sqlite3_blob_write $B $iOffset $blob $nData] + uplevel [list do_test $tn.1 [subst -nocommands { + list [catch {$cmd} msg] [set msg] + }] [list $isError $ret]] + + # EVIDENCE-OF: R-34782-18311 Unless SQLITE_MISUSE is returned, this + # function sets the database connection error code and message + # accessible via sqlite3_errcode() and sqlite3_errmsg() and related + # functions. + # + if {$errcode == "SQLITE_MISUSE"} { error "test proc misuse!" } + uplevel [list do_test $tn.2 [list sqlite3_errcode db] $errcode] + uplevel [list do_test $tn.3 [list sqlite3_errmsg db] $errmsg] +} + +do_execsql_test 2.0 { + CREATE TABLE t2(a TEXT, b INTEGER PRIMARY KEY); + INSERT INTO t2 VALUES($dots, 43); + INSERT INTO t2 VALUES($dots, 44); + INSERT INTO t2 VALUES($dots, 45); +} + +# EVIDENCE-OF: R-63341-57517 If the BLOB handle passed as the first +# argument was not opened for writing (the flags parameter to +# sqlite3_blob_open() was zero), this function returns SQLITE_READONLY. +# +sqlite3_blob_open db main t2 a 43 0 B +blob_write_error_test 2.1 $B 0 $blob 10 \ + SQLITE_READONLY {attempt to write a readonly database} +sqlite3_blob_close $B + +# EVIDENCE-OF: R-29804-27366 If offset iOffset is less than N bytes from +# the end of the BLOB, SQLITE_ERROR is returned and no data is written. +# +sqlite3_blob_open db main t2 a 44 3 B +blob_write_error_test 2.2.1 $B 31 $blob 10 \ + SQLITE_ERROR {SQL logic error or missing database} + +# Make a successful write to the blob handle. This shows that the +# sqlite3_errcode() and sqlite3_errmsg() values are set even if the +# blob_write() call succeeds (see requirement in the [blob_write_error_test] +# proc). +blob_write_error_test 2.2.1 $B 30 $blob 10 SQLITE_OK {not an error} + +# EVIDENCE-OF: R-58570-38916 If N or iOffset are less than zero +# SQLITE_ERROR is returned and no data is written. +# +blob_write_error_test 2.2.2 $B 31 $blob -1 \ + SQLITE_ERROR {SQL logic error or missing database} +blob_write_error_test 2.2.3 $B 20 $blob 10 SQLITE_OK {not an error} +blob_write_error_test 2.2.4 $B -1 $blob 10 \ + SQLITE_ERROR {SQL logic error or missing database} +sqlite3_blob_close $B + +# EVIDENCE-OF: R-20958-54138 An attempt to write to an expired BLOB +# handle fails with an error code of SQLITE_ABORT. +# +do_test 2.3 { + sqlite3_blob_open db main t2 a 43 0 B + execsql { DELETE FROM t2 WHERE b=43 } +} {} +blob_write_error_test 2.3.1 $B 5 $blob 5 \ + SQLITE_ABORT {callback requested query abort} +do_test 2.3.2 { + execsql { SELECT 1, 2, 3 } + sqlite3_errcode db +} {SQLITE_OK} +blob_write_error_test 2.3.3 $B 5 $blob 5 \ + SQLITE_ABORT {callback requested query abort} +sqlite3_blob_close $B + +# EVIDENCE-OF: R-08382-59936 Writes to the BLOB that occurred before the +# BLOB handle expired are not rolled back by the expiration of the +# handle, though of course those changes might have been overwritten by +# the statement that expired the BLOB handle or by other independent +# statements. +# +# 3.1.*: not rolled back, +# 3.2.*: overwritten. +# +do_execsql_test 3.0 { + CREATE TABLE t3(i INTEGER PRIMARY KEY, j TEXT, k TEXT); + INSERT INTO t3 VALUES(1, $dots, $dots); + INSERT INTO t3 VALUES(2, $dots, $dots); + SELECT * FROM t3 WHERE i=1; +} { + 1 + ........................................ + ........................................ +} +sqlite3_blob_open db main t3 j 1 1 B +blob_write_error_test 3.1.1 $B 5 $blob 10 SQLITE_OK {not an error} +do_execsql_test 3.1.2 { + UPDATE t3 SET k = 'xyz' WHERE i=1; + SELECT * FROM t3 WHERE i=1; +} { + 1 .....0123456789......................... xyz +} +blob_write_error_test 3.1.3 $B 15 $blob 10 \ + SQLITE_ABORT {callback requested query abort} +sqlite3_blob_close $B +do_execsql_test 3.1.4 { + SELECT * FROM t3 WHERE i=1; +} { + 1 .....0123456789......................... xyz +} + +sqlite3_blob_open db main t3 j 2 1 B +blob_write_error_test 3.2.1 $B 5 $blob 10 SQLITE_OK {not an error} +do_execsql_test 3.2.2 { + UPDATE t3 SET j = 'xyz' WHERE i=2; + SELECT * FROM t3 WHERE i=2; +} { + 2 xyz ........................................ +} +blob_write_error_test 3.2.3 $B 15 $blob 10 \ + SQLITE_ABORT {callback requested query abort} +sqlite3_blob_close $B +do_execsql_test 3.2.4 { + SELECT * FROM t3 WHERE i=2; +} { + 2 xyz ........................................ +} + + + +finish_test + From de58f4fe7f33438a9d1b41666bb35a578d156885 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 10 Nov 2014 19:16:59 +0000 Subject: [PATCH 557/710] New test cases for deleting content out from under a SELECT statement. FossilOrigin-Name: 8289c3e9b47f7c2a606a88839f6bf615f8904ac2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/misc8.test | 11 +++++++++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d14dec8c51..762ded40f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stest\sfile\se_blobwrite.test,\scontaining\stests\sfor\sthe\ssqlite3_blob_write()\sinterface. -D 2014-11-10T17:53:03.345 +C New\stest\scases\sfor\sdeleting\scontent\sout\sfrom\sunder\sa\sSELECT\sstatement. +D 2014-11-10T19:16:59.654 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -731,7 +731,7 @@ F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 -F test/misc8.test e538d122d5faacadbaa6fc02403f1e1427df684f +F test/misc8.test 2439b3576628bacab12cd6c73abed65de0979fd7 F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 @@ -1216,7 +1216,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 27cf665b957f2c0ced403e3032099e80c295598f -R 37c32da8fb4d2f1d3e3ebc03749cb035 -U dan -Z 2886cb98deb64368f4cf93641ea335c8 +P 1df77e5f1bd82de4dc92fe28359c3e56ab3f9ed4 +R 4b8ad408c58c17649b0e1468e9b897aa +U drh +Z 1daa82bfd52bfa0c612ccd059afbb463 diff --git a/manifest.uuid b/manifest.uuid index 719ea3ef1b..7f7352c14d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1df77e5f1bd82de4dc92fe28359c3e56ab3f9ed4 \ No newline at end of file +8289c3e9b47f7c2a606a88839f6bf615f8904ac2 \ No newline at end of file diff --git a/test/misc8.test b/test/misc8.test index 06ee314b60..da8f86970e 100644 --- a/test/misc8.test +++ b/test/misc8.test @@ -36,6 +36,17 @@ do_catchsql_test misc8-1.4 { INSERT INTO t1 VALUES(10,11,12); SELECT coalesce(b, eval('ROLLBACK')) FROM t1 ORDER BY a; } {1 {abort due to ROLLBACK}} +do_catchsql_test misc8-1.5 { + INSERT INTO t1 VALUES(10,11,12); + SELECT a, coalesce(b, eval('SELECT ''bam''')), c + FROM t1 + ORDER BY rowid; +} {0 {1 2 3 4 5 6 7 bam 9 10 11 12}} +do_catchsql_test misc8-1.6 { + SELECT a, coalesce(b, eval('DELETE FROM t1; SELECT ''bam''')), c + FROM t1 + ORDER BY rowid; +} {0 {1 2 3 4 5 6 7 bam {}}} finish_test From 47b7fc784347f2817c7669b245e5c4d65e6a3747 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Nov 2014 01:33:57 +0000 Subject: [PATCH 558/710] Experimental changes that permit read operations to continue after a ROLLBACK, as long as the schema is unchanged. FossilOrigin-Name: fa6e6a9ae276cad60e9a4abc1bc23cf2809ea786 --- manifest | 41 ++++++++++++++++++---------------- manifest.uuid | 2 +- src/backup.c | 2 +- src/btree.c | 46 ++++++++++++++++++++++----------------- src/btree.h | 4 ++-- src/main.c | 9 +++++--- src/vdbe.c | 11 +++++++--- src/wal.c | 3 +-- test/capi3.test | 6 ++--- test/capi3c.test | 6 ++--- test/misc8.test | 13 +++++++++-- test/rollback.test | 4 ++-- test/savepoint.test | 2 +- test/savepoint7.test | 6 +++-- test/tkt-f777251dc7a.test | 4 +++- test/trans3.test | 2 +- 16 files changed, 95 insertions(+), 66 deletions(-) diff --git a/manifest b/manifest index 762ded40f6..5ef3b08ad7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scases\sfor\sdeleting\scontent\sout\sfrom\sunder\sa\sSELECT\sstatement. -D 2014-11-10T19:16:59.654 +C Experimental\schanges\sthat\spermit\sread\soperations\sto\scontinue\safter\sa\nROLLBACK,\sas\slong\sas\sthe\sschema\sis\sunchanged. +D 2014-11-11T01:33:57.231 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -170,11 +170,11 @@ F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 -F src/backup.c 7f841396adfd47507ff670a471162d2bfcda3136 +F src/backup.c 86b311af1ad35decd1f2f0cee457d344acb83c3b F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 4a126e2066076872ab6f37f9ad116eb5f651cd38 -F src/btree.h 49b408be9c1cd41249076898e0673711071205d8 +F src/btree.c eaef0003bbfe740c62189355dabc818fc3a98999 +F src/btree.h d24fc2f3bc53be220b79b95800bdb2ee207b1089 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c f88ed28716cbbada0f3d81479e6d43823b553de6 +F src/main.c d3310d5ed56e246bf1589e47eeaca8be582bd4b8 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -291,7 +291,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c d5dab22208e36e5689e9fb553aea3613921054ec +F src/vdbe.c c71d819bb34269c3dbccd92e6bb308f0ec025b5d F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -301,7 +301,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 -F src/wal.c 825c948066c7604a07d56e67958cdab210749016 +F src/wal.c fa090966140602f03a621f87d82ee69e66ca63b5 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 3862a1173ae2716bde12f1ab3fb649f1d85b05c2 @@ -381,9 +381,9 @@ F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 -F test/capi3.test 71bcf2fbd36a9732f617766dfd752552c8e491b5 +F test/capi3.test f0718f4f90d0efdc980119bfbdf1d7f1541ee5ef F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 -F test/capi3c.test a21869e4d50d5dbb7e566e328fc0bc7c2efa6a32 +F test/capi3c.test fdc0d67a2cb8e8fc400d5b7735e330161ea057a2 F test/capi3d.test c84af0c49267f9c3fbf4c1c46aa647646023811e F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 @@ -731,7 +731,7 @@ F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 -F test/misc8.test 2439b3576628bacab12cd6c73abed65de0979fd7 +F test/misc8.test e838ec20c9c988bc94812fdb89af26409c20931b F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 @@ -794,18 +794,18 @@ F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a -F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c +F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 -F test/savepoint.test 51d3900dc071a7c2ad4248578a5925631b476313 +F test/savepoint.test c671fdbd34cd3bfe1518a777526ada595180cf8d F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 -F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 +F test/savepoint7.test 1c8f26b1e2a4221b0214e222ce12a97a59918eb2 F test/scanstatus.test a6dd739bc4d9638e8f5c2493b518057f2b681655 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 @@ -954,7 +954,7 @@ F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test d5a0126118142d13e27f6ce9f4c47096e9321c00 F test/tkt-f67b41381a.test a23bc124c981662db712167bacd0ed8ad11abac9 -F test/tkt-f777251dc7a.test af6531446c64bfd268416f07b4df7be7f9c749d2 +F test/tkt-f777251dc7a.test 6295d235a03c82160549da4841a83dc8e758932f F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 28ef85c7f015477916795246d8286aeda39d4ead F test/tkt-fa7bf5ec.test 9102dfea58aa371d78969da735f9392c57e2e035 @@ -1049,7 +1049,7 @@ F test/trace.test 73a5508100f7fccfbc3f8018d5f6963ed478eea0 F test/trace2.test 93b47ca6996c66b47f57224cfb146f34e07df382 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 -F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 +F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94 F test/transitive1.test 03f532954f46cdf5608f7766bff0b0c52bf2a7cd F test/trigger1.test dc47573ac79ffe0ee3eecaa517d70d8dacbccd03 F test/trigger2.test 5cd7d69a7ba1143ee045e4ae2963ff32ae4c87a6 @@ -1216,7 +1216,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1df77e5f1bd82de4dc92fe28359c3e56ab3f9ed4 -R 4b8ad408c58c17649b0e1468e9b897aa +P 8289c3e9b47f7c2a606a88839f6bf615f8904ac2 +R 703cbe5a59d81119b89af85a339f9302 +T *branch * read-after-rollback +T *sym-read-after-rollback * +T -sym-trunk * U drh -Z 1daa82bfd52bfa0c612ccd059afbb463 +Z dd5cfd6556a10d97eb1c5e11a770b9f2 diff --git a/manifest.uuid b/manifest.uuid index 7f7352c14d..0b7de2e6e0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8289c3e9b47f7c2a606a88839f6bf615f8904ac2 \ No newline at end of file +fa6e6a9ae276cad60e9a4abc1bc23cf2809ea786 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index da4303e5fd..57f7f447ea 100644 --- a/src/backup.c +++ b/src/backup.c @@ -607,7 +607,7 @@ int sqlite3_backup_finish(sqlite3_backup *p){ } /* If a transaction is still open on the Btree, roll it back. */ - sqlite3BtreeRollback(p->pDest, SQLITE_OK); + sqlite3BtreeRollback(p->pDest, SQLITE_OK, 0); /* Set the error code of the destination database handle. */ rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; diff --git a/src/btree.c b/src/btree.c index 8b4a2a4f3b..70fae7c707 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2213,7 +2213,7 @@ int sqlite3BtreeClose(Btree *p){ ** The call to sqlite3BtreeRollback() drops any table-locks held by ** this handle. */ - sqlite3BtreeRollback(p, SQLITE_OK); + sqlite3BtreeRollback(p, SQLITE_OK, 0); sqlite3BtreeLeave(p); /* If there are still other outstanding references to the shared-btree @@ -3506,27 +3506,28 @@ int sqlite3BtreeCommit(Btree *p){ /* ** This routine sets the state to CURSOR_FAULT and the error -** code to errCode for every cursor on BtShared that pBtree -** references. +** code to errCode for every cursor on any BtShared that pBtree +** references. Or if the writeOnly flag is set to 1, then only +** trip write cursors and leave read cursors unchanged. ** -** Every cursor is tripped, including cursors that belong -** to other database connections that happen to be sharing -** the cache with pBtree. +** Every cursor is a candidate to be tripped, including cursors +** that belong to other database connections that happen to be +** sharing the cache with pBtree. ** -** This routine gets called when a rollback occurs. -** All cursors using the same cache must be tripped -** to prevent them from trying to use the btree after -** the rollback. The rollback may have deleted tables -** or moved root pages, so it is not sufficient to -** save the state of the cursor. The cursor must be -** invalidated. +** This routine gets called when a rollback occurs. The writeOnly +** flag is set to 1 if the transaction did not make any schema +** changes, in which case the read cursors can continue operating. +** If schema changes did occur in the transaction, then both read +** and write cursors must both be tripped. */ -void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ +void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ BtCursor *p; + assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); if( pBtree==0 ) return; sqlite3BtreeEnter(pBtree); for(p=pBtree->pBt->pCursor; p; p=p->pNext){ int i; + if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ) continue; sqlite3BtreeClearCursor(p); p->eState = CURSOR_FAULT; p->skipNext = errCode; @@ -3539,27 +3540,32 @@ void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ } /* -** Rollback the transaction in progress. All cursors will be -** invalided by this operation. Any attempt to use a cursor -** that was open at the beginning of this operation will result -** in an error. +** Rollback the transaction in progress. +** +** If tripCode is not SQLITE_OK then cursors will be invalidated (tripped). +** Only write cursors are tripped if writeOnly is true but all cursors are +** tripped if writeOnly is false. Any attempt to use +** a tripped cursor will result in an error. ** ** This will release the write lock on the database file. If there ** are no active cursors, it also releases the read lock. */ -int sqlite3BtreeRollback(Btree *p, int tripCode){ +int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ int rc; BtShared *pBt = p->pBt; MemPage *pPage1; + assert( writeOnly==1 || writeOnly==0 ); + assert( tripCode==SQLITE_ABORT_ROLLBACK || tripCode==SQLITE_OK ); sqlite3BtreeEnter(p); if( tripCode==SQLITE_OK ){ rc = tripCode = saveAllCursors(pBt, 0, 0); + if( rc ) writeOnly = 0; }else{ rc = SQLITE_OK; } if( tripCode ){ - sqlite3BtreeTripAllCursors(p, tripCode); + sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); } btreeIntegrity(p); diff --git a/src/btree.h b/src/btree.h index 63337b1946..a6093a5edd 100644 --- a/src/btree.h +++ b/src/btree.h @@ -83,7 +83,7 @@ int sqlite3BtreeBeginTrans(Btree*,int); int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); int sqlite3BtreeCommitPhaseTwo(Btree*, int); int sqlite3BtreeCommit(Btree*); -int sqlite3BtreeRollback(Btree*,int); +int sqlite3BtreeRollback(Btree*,int,int); int sqlite3BtreeBeginStmt(Btree*,int); int sqlite3BtreeCreateTable(Btree*, int*, int flags); int sqlite3BtreeIsInTrans(Btree*); @@ -116,7 +116,7 @@ int sqlite3BtreeIncrVacuum(Btree *); int sqlite3BtreeDropTable(Btree*, int, int*); int sqlite3BtreeClearTable(Btree*, int, int*); int sqlite3BtreeClearTableOfCursor(BtCursor*); -void sqlite3BtreeTripAllCursors(Btree*, int); +void sqlite3BtreeTripAllCursors(Btree*, int, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); diff --git a/src/main.c b/src/main.c index ae3753c139..f223b71f74 100644 --- a/src/main.c +++ b/src/main.c @@ -1111,13 +1111,15 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* ** Rollback all database files. If tripCode is not SQLITE_OK, then -** any open cursors are invalidated ("tripped" - as in "tripping a circuit +** any write cursors are invalidated ("tripped" - as in "tripping a circuit ** breaker") and made to return tripCode if there are any further -** attempts to use that cursor. +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. */ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int i; int inTrans = 0; + int schemaChange; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); @@ -1128,6 +1130,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** the database rollback and schema reset, which can cause false ** corruption reports in some cases. */ sqlite3BtreeEnterAll(db); + schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; for(i=0; inDb; i++){ Btree *p = db->aDb[i].pBt; @@ -1135,7 +1138,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ if( sqlite3BtreeIsInTrans(p) ){ inTrans = 1; } - sqlite3BtreeRollback(p, tripCode); + sqlite3BtreeRollback(p, tripCode, !schemaChange); } } sqlite3VtabRollback(db); diff --git a/src/vdbe.c b/src/vdbe.c index f80e7787af..715862e73c 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2825,11 +2825,16 @@ case OP_Savepoint: { db->isTransactionSavepoint = 0; rc = p->rc; }else{ + int isSchemaChange; iSavepoint = db->nSavepoint - iSavepoint - 1; if( p1==SAVEPOINT_ROLLBACK ){ + isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; for(ii=0; iinDb; ii++){ - sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT); + sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, + isSchemaChange==0); } + }else{ + isSchemaChange = 0; } for(ii=0; iinDb; ii++){ rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint); @@ -2837,7 +2842,7 @@ case OP_Savepoint: { goto abort_due_to_error; } } - if( p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){ + if( isSchemaChange ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); db->flags = (db->flags | SQLITE_InternChanges); @@ -3234,7 +3239,7 @@ case OP_OpenWrite: { || p->readOnly==0 ); if( p->expired ){ - rc = SQLITE_ABORT; + rc = SQLITE_ABORT_ROLLBACK; break; } diff --git a/src/wal.c b/src/wal.c index d2ed293a4d..6fed8f3c6c 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2506,7 +2506,7 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); for(iFrame=pWal->hdr.mxFrame+1; - ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; + rc==SQLITE_OK && iFrame<=iMax; iFrame++ ){ /* This call cannot fail. Unless the page for which the page number @@ -2525,7 +2525,6 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ } if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); } - assert( rc==SQLITE_OK ); return rc; } diff --git a/test/capi3.test b/test/capi3.test index cbaa5c5e5b..9f3d6f6916 100644 --- a/test/capi3.test +++ b/test/capi3.test @@ -912,7 +912,7 @@ do_test capi3-11.9.3 { } 1 do_test capi3-11.10 { sqlite3_step $STMT -} {SQLITE_ERROR} +} {SQLITE_ROW} ifcapable !autoreset { # If SQLITE_OMIT_AUTORESET is defined, then the statement must be # reset() before it can be passed to step() again. @@ -921,11 +921,11 @@ ifcapable !autoreset { } do_test capi3-11.11 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_DONE} do_test capi3-11.12 { sqlite3_step $STMT sqlite3_step $STMT -} {SQLITE_DONE} +} {SQLITE_ROW} do_test capi3-11.13 { sqlite3_finalize $STMT } {SQLITE_OK} diff --git a/test/capi3c.test b/test/capi3c.test index 6388717e00..6ab3bc24f6 100644 --- a/test/capi3c.test +++ b/test/capi3c.test @@ -863,7 +863,7 @@ do_test capi3c-11.9.3 { } 1 do_test capi3c-11.10 { sqlite3_step $STMT -} {SQLITE_ABORT} +} {SQLITE_ROW} ifcapable !autoreset { # If SQLITE_OMIT_AUTORESET is defined, then the statement must be # reset() before it can be passed to step() again. @@ -872,11 +872,11 @@ ifcapable !autoreset { } do_test capi3c-11.11 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_DONE} do_test capi3c-11.12 { sqlite3_step $STMT sqlite3_step $STMT -} {SQLITE_DONE} +} {SQLITE_ROW} do_test capi3c-11.13 { sqlite3_finalize $STMT } {SQLITE_OK} diff --git a/test/misc8.test b/test/misc8.test index da8f86970e..8c0c126a61 100644 --- a/test/misc8.test +++ b/test/misc8.test @@ -34,8 +34,9 @@ do_execsql_test misc8-1.3 { do_catchsql_test misc8-1.4 { BEGIN; INSERT INTO t1 VALUES(10,11,12); - SELECT coalesce(b, eval('ROLLBACK')) FROM t1 ORDER BY a; -} {1 {abort due to ROLLBACK}} + SELECT a, coalesce(b, eval('ROLLBACK; SELECT ''bam'';')), c + FROM t1 ORDER BY a; +} {0 {1 2 3 4 5 6 7 bam 9}} do_catchsql_test misc8-1.5 { INSERT INTO t1 VALUES(10,11,12); SELECT a, coalesce(b, eval('SELECT ''bam''')), c @@ -47,6 +48,14 @@ do_catchsql_test misc8-1.6 { FROM t1 ORDER BY rowid; } {0 {1 2 3 4 5 6 7 bam {}}} +do_catchsql_test misc8-1.7 { + INSERT INTO t1 VALUES(1,2,3),(4,5,6),(7,null,9); + BEGIN; + CREATE TABLE t2(x); + SELECT a, coalesce(b, eval('ROLLBACK; SELECT ''bam''')), c + FROM t1 + ORDER BY rowid; +} {1 {abort due to ROLLBACK}} finish_test diff --git a/test/rollback.test b/test/rollback.test index c339c5d7d6..7abafece61 100644 --- a/test/rollback.test +++ b/test/rollback.test @@ -60,11 +60,11 @@ ifcapable conflict { # do_test rollback-1.5 { sqlite3_step $STMT - } {SQLITE_ERROR} + } {SQLITE_ROW} # Restart the SELECT statement # - do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_ABORT} + do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_OK} } else { do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_OK} } diff --git a/test/savepoint.test b/test/savepoint.test index 9362c8fe19..8055e61d9e 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -315,7 +315,7 @@ ifcapable incrblob { do_test savepoint-5.3.2.3 { set rc [catch {seek $fd 0; read $fd} res] set rc - } {1} + } {0} do_test savepoint-5.3.3 { catchsql {RELEASE def} } {0 {}} diff --git a/test/savepoint7.test b/test/savepoint7.test index bc99187d27..908ec571f5 100644 --- a/test/savepoint7.test +++ b/test/savepoint7.test @@ -30,6 +30,7 @@ do_test savepoint7-1.1 { db eval {SELECT * FROM t1} { db eval { SAVEPOINT x2; + CREATE TABLE IF NOT EXISTS t3(xyz); INSERT INTO t2 VALUES($a,$b,$c); RELEASE x2; } @@ -46,7 +47,7 @@ do_test savepoint7-1.2 { RELEASE x2; } } - db eval {SELECT * FROM t2} + db eval {SELECT * FROM t2;} } {1 2 3 4 5 6 7 8 9} do_test savepoint7-1.3 { @@ -65,7 +66,7 @@ do_test savepoint7-1.3 { # queries in outer contexts. # do_test savepoint7-2.1 { - db eval {DELETE FROM t2; SAVEPOINT x1;} + db eval {DELETE FROM t2; SAVEPOINT x1; CREATE TABLE t4(abc);} set rc [catch { db eval {SELECT * FROM t1} { db eval { @@ -85,6 +86,7 @@ do_test savepoint7-2.2 { db eval {SELECT * FROM t1} { db eval { SAVEPOINT x2; + CREATE TABLE t5(pqr); INSERT INTO t2 VALUES($a,$b,$c); ROLLBACK TO x2; } diff --git a/test/tkt-f777251dc7a.test b/test/tkt-f777251dc7a.test index af6f71ad96..f814d246bf 100644 --- a/test/tkt-f777251dc7a.test +++ b/test/tkt-f777251dc7a.test @@ -38,8 +38,10 @@ proc force_rollback {} { db function force_rollback force_rollback do_test tkt-f7772-1.2 { +breakpoint catchsql { BEGIN IMMEDIATE; + CREATE TABLE xyzzy(abc); SELECT x, force_rollback(), EXISTS(SELECT 1 FROM t3 WHERE w=x) FROM t2; } } {1 {abort due to ROLLBACK}} @@ -67,7 +69,7 @@ do_test tkt-f7772-2.2 { catchsql { SELECT x, force_rollback(), EXISTS(SELECT 1 FROM t3 WHERE w=x) FROM t2 } -} {1 {callback requested query abort}} +} {1 {abort due to ROLLBACK}} do_test tkt-f7772-2.3 { sqlite3_get_autocommit db } {1} diff --git a/test/trans3.test b/test/trans3.test index d5b316bec8..e828442415 100644 --- a/test/trans3.test +++ b/test/trans3.test @@ -52,7 +52,7 @@ do_test trans3-1.4 { db eval {SELECT * FROM t1} } {1 2 3 4} do_test trans3-1.5 { - db eval BEGIN + db eval {BEGIN; CREATE TABLE xyzzy(abc);} db eval {INSERT INTO t1 VALUES(5);} set ::ecode {} set x [catch { From 43f4066e102d096958e3d3211a4e44b98d6d83df Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 11 Nov 2014 12:20:35 +0000 Subject: [PATCH 559/710] Add new test file e_blobclose.test, containing tests for sqlite3_blob_close(). FossilOrigin-Name: 5a1eac2419b1462e6f21595a3fff26d9cc49d203 --- manifest | 15 ++-- manifest.uuid | 2 +- src/sqlite.h.in | 30 ++++---- test/e_blobclose.test | 171 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 194 insertions(+), 24 deletions(-) create mode 100644 test/e_blobclose.test diff --git a/manifest b/manifest index 762ded40f6..089246851e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C New\stest\scases\sfor\sdeleting\scontent\sout\sfrom\sunder\sa\sSELECT\sstatement. -D 2014-11-10T19:16:59.654 +C Add\snew\stest\sfile\se_blobclose.test,\scontaining\stests\sfor\ssqlite3_blob_close(). +D 2014-11-11T12:20:35.141 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd -F src/sqlite.h.in 5531c4c69ee0a351c2aa5ad9f3f0f2424a57a9f4 +F src/sqlite.h.in 0e6612f84936cca29166f2c66068e0227a13fdf6 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 71b0bf1a7fc55b5cb374f7579fd140e730a6e0f4 @@ -453,6 +453,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 +F test/e_blobclose.test df756753f571bc30e42e3a6cba2807576e49e716 F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 F test/e_blobwrite.test 615b405f29feb2cfb5a1f03dab7933258294fa26 F test/e_changes.test fd66105385153dbf21fdb35eb8ef6c3e1eade579 @@ -1216,7 +1217,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1df77e5f1bd82de4dc92fe28359c3e56ab3f9ed4 -R 4b8ad408c58c17649b0e1468e9b897aa -U drh -Z 1daa82bfd52bfa0c612ccd059afbb463 +P 8289c3e9b47f7c2a606a88839f6bf615f8904ac2 +R 5495040e5aa9cb71b6dcdaffde6bd935 +U dan +Z 0fca1fa0cec9033f254d39e73d650ca5 diff --git a/manifest.uuid b/manifest.uuid index 7f7352c14d..dbade0f26a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8289c3e9b47f7c2a606a88839f6bf615f8904ac2 \ No newline at end of file +5a1eac2419b1462e6f21595a3fff26d9cc49d203 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index b5585defd6..2812bf511d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5754,24 +5754,22 @@ SQLITE_EXPERIMENTAL int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); /* ** CAPI3REF: Close A BLOB Handle ** -** ^Closes an open [BLOB handle]. +** ^This function closes an open [BLOB handle]. ^(The BLOB handle is closed +** unconditionally. Even if this routine returns an error code, the +** handle is still closed.)^ ** -** ^Closing a BLOB shall cause the current transaction to commit -** if there are no other BLOBs, no pending prepared statements, and the -** database connection is in [autocommit mode]. -** ^If any writes were made to the BLOB, they might be held in cache -** until the close operation if they will fit. +** ^If the blob handle being closed was opened for read-write access, and if +** the database is in auto-commit mode and there are no other open read-write +** blob handles or active write statements, the current transaction is +** committed. ^If an error occurs while committing the transaction, an error +** code is returned and the transaction rolled back. ** -** ^(Closing the BLOB often forces the changes -** out to disk and so if any I/O errors occur, they will likely occur -** at the time when the BLOB is closed. Any errors that occur during -** closing are reported as a non-zero return value.)^ -** -** ^(The BLOB is closed unconditionally. Even if this routine returns -** an error code, the BLOB is still closed.)^ -** -** ^Calling this routine with a null pointer (such as would be returned -** by a failed call to [sqlite3_blob_open()]) is a harmless no-op. +** Calling this function with an argument that is not a NULL pointer or an +** open blob handle results in undefined behaviour. ^Calling this routine +** with a null pointer (such as would be returned by a failed call to +** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function +** is passed a valid open blob handle, the values returned by the +** sqlite3_errcode() and sqlite3_errmsg() functions are set before returning. */ int sqlite3_blob_close(sqlite3_blob *); diff --git a/test/e_blobclose.test b/test/e_blobclose.test new file mode 100644 index 0000000000..a5d432d3b5 --- /dev/null +++ b/test/e_blobclose.test @@ -0,0 +1,171 @@ +# 2014 October 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_blobclose + +set dots [string repeat . 40] +do_execsql_test 1.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b DOTS); + INSERT INTO x1 VALUES(-1, $dots); + INSERT INTO x1 VALUES(-10, $dots); + INSERT INTO x1 VALUES(-100, $dots); + INSERT INTO x1 VALUES(-1000, $dots); + INSERT INTO x1 VALUES(-10000, $dots); +} + +# EVIDENCE-OF: R-03145-46390 This function closes an open BLOB handle. +# +# It's not clear how to test that a blob handle really is closed. +# Attempting to use a closed blob handle will likely crash the process. +# Assume here that if the SHARED lock on the db file is released, +# the blob handle has been closed. +# +do_execsql_test 1.1 { PRAGMA lock_status } {main unlocked temp closed} +sqlite3_blob_open db main x1 b -1 0 B +do_execsql_test 1.2 { PRAGMA lock_status } {main shared temp closed} +sqlite3_blob_close $B +do_execsql_test 1.3 { PRAGMA lock_status } {main unlocked temp closed} + + +# EVIDENCE-OF: R-34027-00617 If the blob handle being closed was opened +# for read-write access, and if the database is in auto-commit mode and +# there are no other open read-write blob handles or active write +# statements, the current transaction is committed. +# +# 2.1.*: Transaction is not committed if there are other open +# read-write blob handles. +# +# 2.2.*: Transaction is not committed if not in auto-commit mode. +# +# 2.3.*: Active write statements. +# +do_test 2.1.1 { + sqlite3_blob_open db main x1 b -100 1 B1 + sqlite3_blob_open db main x1 b -1000 1 B2 + sqlite3_blob_open db main x1 b -10000 1 B3 + sqlite3_blob_open db main x1 b -10000 0 B4 ;# B4 is read-only! + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.1.2 { + sqlite3_blob_close $B1 + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.1.3 { + sqlite3_blob_close $B2 + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.1.4 { + sqlite3_blob_close $B3 + execsql { PRAGMA lock_status } +} {main shared temp closed} +do_test 2.1.5 { + sqlite3_blob_close $B4 + execsql { PRAGMA lock_status } +} {main unlocked temp closed} + +do_test 2.2.1 { + sqlite3_blob_open db main x1 b -100 1 B1 + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.2.2 { + execsql { BEGIN } + sqlite3_blob_close $B1 + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.2.3 { + execsql { COMMIT } + execsql { PRAGMA lock_status } +} {main unlocked temp closed} + +proc val {} { + sqlite3_blob_close $::B + db eval { PRAGMA lock_status } +} +db func val val +do_test 2.3.1 { + sqlite3_blob_open db main x1 b -100 1 B + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.3.2 { + execsql { INSERT INTO x1 VALUES(15, val()) } + execsql { PRAGMA lock_status } +} {main unlocked temp closed} +do_test 2.3.3 { + execsql { SELECT * FROM x1 WHERE a = 15 } +} {15 {main reserved temp closed}} + +# A reader does not inhibit commit. +do_test 2.3.4 { + sqlite3_blob_open db main x1 b -100 1 B + execsql { PRAGMA lock_status } +} {main reserved temp closed} +do_test 2.3.5 { + execsql { SELECT a, val() FROM x1 LIMIT 1 } +} {-10000 {main shared temp closed}} + + +do_test 3.1 { + sqlite3_blob_open db main x1 b -10 1 B + execsql { + INSERT INTO x1 VALUES(1, 'abc'); + SELECT * FROM x1 WHERE a=1; + } +} {1 abc} +do_test 3.2 { + sqlite3_blob_write $B 0 "abcdefghij" 10 + execsql { SELECT * FROM x1 WHERE a=-10 } +} {-10 abcdefghij..............................} + +do_test 3.3 { + sqlite3 db2 test.db + execsql { BEGIN ; SELECT * FROM x1 } db2 + sqlite3_blob_close $B +} {SQLITE_BUSY} + +# EVIDENCE-OF: R-41959-38737 Otherwise, if this function is passed a +# valid open blob handle, the values returned by the sqlite3_errcode() +# and sqlite3_errmsg() functions are set before returning. +# +do_test 3.4 { + list [sqlite3_errcode db] [sqlite3_errmsg db] +} {SQLITE_BUSY {database is locked}} + +# EVIDENCE-OF: R-37801-37633 The BLOB handle is closed unconditionally. +# Even if this routine returns an error code, the handle is still +# closed. +# +# Test that the lock has been released. Assume this means the handle +# is closed, even though blob_close() returned SQLITE_BUSY. +# +do_execsql_test 3.4 { PRAGMA lock_status } {main unlocked temp closed} + +# EVIDENCE-OF: R-35111-05628 If an error occurs while committing the +# transaction, an error code is returned and the transaction rolled +# back. +# +# Row 1 is removed (it was inserted this transaction) and row -10 +# is restored to its original state. Transaction has been rolled back. +# +do_execsql_test 3.5 { + SELECT * FROM x1 WHERE a IN (1, -10); +} {-10 ........................................} + +# EVIDENCE-OF: R-25894-51060 Calling this routine with a null pointer +# (such as would be returned by a failed call to sqlite3_blob_open()) is +# a harmless no-op. +# +do_test 4.0 { sqlite3_blob_close 0 } {} + +finish_test + From 2d8e3caa2e54c4856db48998ba71a2c56b47e956 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 11 Nov 2014 16:11:04 +0000 Subject: [PATCH 560/710] Add tests for sqlite3_blob_bytes(). FossilOrigin-Name: a066a3832a7c6de65c3016e77e49ac00e09db749 --- manifest | 14 ++++---- manifest.uuid | 2 +- test/e_blobbytes.test | 76 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 test/e_blobbytes.test diff --git a/manifest b/manifest index 51c9f2ee01..c994394f6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Permit\sread\soperations\sto\scontinue\safter\sa\sROLLBACK\sas\slong\sas\sthe\sschema\ndoes\snot\schange. -D 2014-11-11T14:59:31.172 +C Add\stests\sfor\ssqlite3_blob_bytes(). +D 2014-11-11T16:11:04.911 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -453,6 +453,7 @@ F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 +F test/e_blobbytes.test 9bea1d3e2b20f3010b04abba58f6ba172301f49f F test/e_blobclose.test df756753f571bc30e42e3a6cba2807576e49e716 F test/e_blobopen.test 234f960d90235a9b51ec3ca1e062e8541dd558d8 F test/e_blobwrite.test 615b405f29feb2cfb5a1f03dab7933258294fa26 @@ -1217,8 +1218,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5a1eac2419b1462e6f21595a3fff26d9cc49d203 fa6e6a9ae276cad60e9a4abc1bc23cf2809ea786 -R 90c0902351a625ca12308516be116ef0 -T +closed fa6e6a9ae276cad60e9a4abc1bc23cf2809ea786 -U drh -Z 3e9637efba5697b65ccb54a130a31d2a +P b5df5ac0529f7b0d1e880a7f4a307e7d77b7fa6c +R be9d88d05dd4414f44a05008ae6eba4c +U dan +Z 2fc1cd52a3184d003db0d33d71426d71 diff --git a/manifest.uuid b/manifest.uuid index 699edcd220..0982b7b00f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5df5ac0529f7b0d1e880a7f4a307e7d77b7fa6c \ No newline at end of file +a066a3832a7c6de65c3016e77e49ac00e09db749 \ No newline at end of file diff --git a/test/e_blobbytes.test b/test/e_blobbytes.test new file mode 100644 index 0000000000..a6283ab852 --- /dev/null +++ b/test/e_blobbytes.test @@ -0,0 +1,76 @@ +# 2014 October 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix e_blobbytes + +do_execsql_test 1.0 { + CREATE TABLE q1(r INTEGER PRIMARY KEY, s TEXT); + WITH d(a, b) AS ( + SELECT 0, '' + UNION ALL + SELECT a+1, b||'.' FROM d WHERE a<10000 + ) + INSERT INTO q1 SELECT * FROM d; +} + + +# EVIDENCE-OF: R-07796-55423 Returns the size in bytes of the BLOB +# accessible via the successfully opened BLOB handle in its only +# argument. +# +proc check_blob_size {tn rowid bytes} { + uplevel [list do_test $tn [subst -nocommands { + sqlite3_blob_open db main q1 s $rowid 0 B + set res [sqlite3_blob_bytes [set B]] + sqlite3_blob_close [set B] + set res + }] $bytes] +} +check_blob_size 1.1 43 43 +check_blob_size 1.2 391 391 +check_blob_size 1.3 6349 6349 +check_blob_size 1.4 2621 2621 +check_blob_size 1.5 7771 7771 +check_blob_size 1.6 7949 7949 +check_blob_size 1.7 4374 4374 +check_blob_size 1.8 2578 2578 +check_blob_size 1.9 7004 7004 +check_blob_size 1.10 2180 2180 +check_blob_size 1.11 3796 3796 +check_blob_size 1.12 7101 7101 +check_blob_size 1.13 7449 7449 +check_blob_size 1.14 7224 7224 +check_blob_size 1.15 3038 3038 +check_blob_size 1.16 1083 1083 +check_blob_size 1.17 5157 5157 +check_blob_size 1.18 6686 6686 +check_blob_size 1.19 6592 6592 +check_blob_size 1.20 0 0 + + +# EVIDENCE-OF: R-53088-19343 The incremental blob I/O routines can only +# read or overwriting existing blob content; they cannot change the size +# of a blob. +# +# Also demonstrated in other e_blobXXX.test files. +# +do_test 2.1 { + sqlite3_blob_open db main q1 s 86 1 B + list [catch { sqlite3_blob_write $B 86 "1" 1 } msg] $msg +} {1 SQLITE_ERROR} +sqlite3_blob_close $B + +finish_test + + From bfa395d085fbadc6436bb4baf40e96c384c64463 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 11 Nov 2014 19:07:56 +0000 Subject: [PATCH 561/710] Remove some calls to the 'breakpoint' test command. FossilOrigin-Name: 1412fcc480799ecbd68d44dd18d5bad40e20ccf1 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- test/capi3d.test | 1 - test/corruptH.test | 1 - test/sort2.test | 1 - test/tkt-f777251dc7a.test | 1 - 6 files changed, 11 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index c994394f6f..97192e581d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\ssqlite3_blob_bytes(). -D 2014-11-11T16:11:04.911 +C Remove\ssome\scalls\sto\sthe\s'breakpoint'\stest\scommand. +D 2014-11-11T19:07:56.372 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -384,7 +384,7 @@ F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 F test/capi3.test f0718f4f90d0efdc980119bfbdf1d7f1541ee5ef F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 F test/capi3c.test fdc0d67a2cb8e8fc400d5b7735e330161ea057a2 -F test/capi3d.test c84af0c49267f9c3fbf4c1c46aa647646023811e +F test/capi3d.test a82b6321c50a1cfc848e386fa2c851893606f68c F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 F test/check.test 5831ddb6f2c687782eaf2e1a07b6e17f24c4f763 @@ -423,7 +423,7 @@ F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 -F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb +F test/corruptH.test 5dd4fa98c6c1ed33b178f9e8a48c4fdd3cfc9067 F test/corruptI.test 221ad8b7f0a9ac6b80fc577e73b5ad8cdea31243 F test/cost.test 19d314526616ce4473eb4e4e450fcb94499ce318 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 @@ -860,7 +860,7 @@ F test/skipscan6.test 5866039d03a56f5bd0b3d172a012074a1d90a15b F test/soak.test 0b5b6375c9f4110c828070b826b3b4b0bb65cd5f F test/softheap1.test 40562fe6cac6d9827b7b42b86d45aedf12c15e24 F test/sort.test c4400e7533748f6bd7413851ff148645e82b9e2d -F test/sort2.test 269f4f50c6e468cc32b302ae7ff0add8338ec6de +F test/sort2.test 84a92eebf697feee9c6697746918c7d67373eea1 F test/sort3.test 6178ade30810ac9166fcdf14b7065e49c0f534e2 F test/sort4.test 6c37d85f7cd28d50cce222fcab84ccd771e105cb F test/sort5.test a448240a42b49239edc00f85d6d7ac7a1b261e1f @@ -956,7 +956,7 @@ F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test d5a0126118142d13e27f6ce9f4c47096e9321c00 F test/tkt-f67b41381a.test a23bc124c981662db712167bacd0ed8ad11abac9 -F test/tkt-f777251dc7a.test 6295d235a03c82160549da4841a83dc8e758932f +F test/tkt-f777251dc7a.test d1a8fc3eefb7a9e64d19ff24d5c8c94c34a632fb F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 28ef85c7f015477916795246d8286aeda39d4ead F test/tkt-fa7bf5ec.test 9102dfea58aa371d78969da735f9392c57e2e035 @@ -1218,7 +1218,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b5df5ac0529f7b0d1e880a7f4a307e7d77b7fa6c -R be9d88d05dd4414f44a05008ae6eba4c -U dan -Z 2fc1cd52a3184d003db0d33d71426d71 +P a066a3832a7c6de65c3016e77e49ac00e09db749 +R e8fc41a7f82a02ab8618a341f7ef7735 +U mistachkin +Z 90417f4c358533080e79b8017b4e7453 diff --git a/manifest.uuid b/manifest.uuid index 0982b7b00f..f90bf2d8c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a066a3832a7c6de65c3016e77e49ac00e09db749 \ No newline at end of file +1412fcc480799ecbd68d44dd18d5bad40e20ccf1 \ No newline at end of file diff --git a/test/capi3d.test b/test/capi3d.test index fb8abe86d2..1459c5abfe 100644 --- a/test/capi3d.test +++ b/test/capi3d.test @@ -155,7 +155,6 @@ do_execsql_test capi3d-4.1 { } do_test capi3d-4.2.1 { - breakpoint set ::s1 [sqlite3_prepare_v2 db "ROLLBACK" -1 notused] sqlite3_step $::s1 } {SQLITE_DONE} diff --git a/test/corruptH.test b/test/corruptH.test index ee2bb1ee48..5c83cb3b90 100644 --- a/test/corruptH.test +++ b/test/corruptH.test @@ -64,7 +64,6 @@ do_test 1.2 { } {} do_test 1.3 { -breakpoint db eval { PRAGMA secure_delete=1 } list [catch { db eval { SELECT * FROM t1 WHERE a IN (1, 2) } { diff --git a/test/sort2.test b/test/sort2.test index 29001f0099..a4c55c9f24 100644 --- a/test/sort2.test +++ b/test/sort2.test @@ -62,7 +62,6 @@ foreach {tn script} { do_execsql_test $tn.2.4 { PRAGMA integrity_check } {ok} - breakpoint do_execsql_test $tn.3 { PRAGMA cache_size = 5; WITH r(x,y) AS ( diff --git a/test/tkt-f777251dc7a.test b/test/tkt-f777251dc7a.test index f814d246bf..b91e438da5 100644 --- a/test/tkt-f777251dc7a.test +++ b/test/tkt-f777251dc7a.test @@ -38,7 +38,6 @@ proc force_rollback {} { db function force_rollback force_rollback do_test tkt-f7772-1.2 { -breakpoint catchsql { BEGIN IMMEDIATE; CREATE TABLE xyzzy(abc); From 85fabf1444ff49f7f1e2060d0b93d9df6f8f9f6f Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 11 Nov 2014 22:55:26 +0000 Subject: [PATCH 562/710] This is a cherry-pick of version [b5df5ac052]. FossilOrigin-Name: d4b2d5d066891e06f2bf4337902b44b000fa9fd2 --- manifest | 40 ++++++++++++++++------------------ manifest.uuid | 2 +- src/backup.c | 2 +- src/btree.c | 46 ++++++++++++++++++++++----------------- src/btree.h | 4 ++-- src/main.c | 9 +++++--- src/vdbe.c | 11 +++++++--- src/wal.c | 3 +-- test/capi3.test | 6 ++--- test/capi3c.test | 6 ++--- test/rollback.test | 4 ++-- test/savepoint.test | 2 +- test/savepoint7.test | 6 +++-- test/tkt-f777251dc7a.test | 4 +++- test/trans3.test | 2 +- 15 files changed, 81 insertions(+), 66 deletions(-) diff --git a/manifest b/manifest index 8688f8549f..0e3e6a9ff4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.7.1 -D 2014-10-29T13:59:56.070 +C This\sis\sa\scherry-pick\sof\sversion\s[b5df5ac052]. +D 2014-11-11T22:55:26.891 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,11 +169,11 @@ F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c 8c322e1ecc08909526dbd5ab4421889d05f2263d F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 -F src/backup.c a31809c65623cc41849b94d368917f8bb66e6a7e +F src/backup.c 8cdfeb0c8a6d8bdad3faefae418eb3dc767051b6 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 1b1123cba0c65caa0baa51e71b8c089e3167c3ed -F src/btree.h a79aa6a71e7f1055f01052b7f821bd1c2dce95c8 +F src/btree.c 944b84f72fd9048acde31400d89439976bf766b0 +F src/btree.h 97c4d0ef860f151e096e97eef78b87f7b4a8d899 F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -194,7 +194,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c bbe872b0ac0007bed0ebe1936fc493b039ad4f51 +F src/main.c 1bdabb62205af168498a17460bdb7533b2a4a915 F src/malloc.c 3c3ac67969612493d435e14b6832793209afd2ec F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 5a1afb571853ddb911d698ac996bc4fd8ddf1eed +F src/vdbe.c 4c77cdf16be330bd5227691919332b42d557e211 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -299,7 +299,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de -F src/wal.c 10e7de7ce90865a68153f001a61f1d985cd17983 +F src/wal.c 73051f1222321712fa4280c495780ba81d302dad F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 2947912f1f3d6a7766fe087fd532a5d688d745b1 @@ -378,9 +378,9 @@ F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 -F test/capi3.test 71bcf2fbd36a9732f617766dfd752552c8e491b5 +F test/capi3.test f0718f4f90d0efdc980119bfbdf1d7f1541ee5ef F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 -F test/capi3c.test a21869e4d50d5dbb7e566e328fc0bc7c2efa6a32 +F test/capi3c.test fdc0d67a2cb8e8fc400d5b7735e330161ea057a2 F test/capi3d.test c84af0c49267f9c3fbf4c1c46aa647646023811e F test/capi3e.test 3d49c01ef2a1a55f41d73cba2b23b5059ec460fe F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 @@ -785,18 +785,18 @@ F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a -F test/rollback.test e9504a009a202c3ed711da2e6879ff60c5a4669c +F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 F test/run-wordcount.sh 891e89c4c2d16e629cd45951d4ed899ad12afc09 -F test/savepoint.test 51d3900dc071a7c2ad4248578a5925631b476313 +F test/savepoint.test c671fdbd34cd3bfe1518a777526ada595180cf8d F test/savepoint2.test 9b8543940572a2f01a18298c3135ad0c9f4f67d7 F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 -F test/savepoint7.test fbf319a7b2dda089ec5be30a424a0e95f121d423 +F test/savepoint7.test 1c8f26b1e2a4221b0214e222ce12a97a59918eb2 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -943,7 +943,7 @@ F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 F test/tkt-f3e5abed55.test d5a0126118142d13e27f6ce9f4c47096e9321c00 F test/tkt-f67b41381a.test a23bc124c981662db712167bacd0ed8ad11abac9 -F test/tkt-f777251dc7a.test af6531446c64bfd268416f07b4df7be7f9c749d2 +F test/tkt-f777251dc7a.test 6295d235a03c82160549da4841a83dc8e758932f F test/tkt-f7b4edec.test d998a08ff2b18b7f62edce8e3044317c45efe6c7 F test/tkt-f973c7ac31.test 28ef85c7f015477916795246d8286aeda39d4ead F test/tkt-fa7bf5ec.test 9102dfea58aa371d78969da735f9392c57e2e035 @@ -1038,7 +1038,7 @@ F test/trace.test 73a5508100f7fccfbc3f8018d5f6963ed478eea0 F test/trace2.test 93b47ca6996c66b47f57224cfb146f34e07df382 F test/trans.test 6e1b4c6a42dba31bd65f8fa5e61a2708e08ddde6 F test/trans2.test 62bd045bfc7a1c14c5ba83ba64d21ade31583f76 -F test/trans3.test 373ac5183cc56be69f48ae44090e7f672939f732 +F test/trans3.test 91a100e5412b488e22a655fe423a14c26403ab94 F test/transitive1.test 03f532954f46cdf5608f7766bff0b0c52bf2a7cd F test/trigger1.test dc47573ac79ffe0ee3eecaa517d70d8dacbccd03 F test/trigger2.test 5cd7d69a7ba1143ee045e4ae2963ff32ae4c87a6 @@ -1204,10 +1204,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 83afe23e553e802c0947c80d0ffdd120423e7c52 -R 5381895dceb9bc7ad4d425f399561cb7 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.7.1 * +P 3b7b72c4685aa5cf5e675c2c47ebec10d9704221 +Q +b5df5ac0529f7b0d1e880a7f4a307e7d77b7fa6c +R 27a95de33658c36b59b93f043d249813 U drh -Z e5ce942eb65e60b92fcaefcb5b84039b +Z ea313ee60fabf91291f912ad6dcdc285 diff --git a/manifest.uuid b/manifest.uuid index 1787d659c6..983840bd0e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3b7b72c4685aa5cf5e675c2c47ebec10d9704221 \ No newline at end of file +d4b2d5d066891e06f2bf4337902b44b000fa9fd2 \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 92c6334bde..9fcb669106 100644 --- a/src/backup.c +++ b/src/backup.c @@ -597,7 +597,7 @@ int sqlite3_backup_finish(sqlite3_backup *p){ } /* If a transaction is still open on the Btree, roll it back. */ - sqlite3BtreeRollback(p->pDest, SQLITE_OK); + sqlite3BtreeRollback(p->pDest, SQLITE_OK, 0); /* Set the error code of the destination database handle. */ rc = (p->rc==SQLITE_DONE) ? SQLITE_OK : p->rc; diff --git a/src/btree.c b/src/btree.c index 758dfe6335..9d7a05157d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2171,7 +2171,7 @@ int sqlite3BtreeClose(Btree *p){ ** The call to sqlite3BtreeRollback() drops any table-locks held by ** this handle. */ - sqlite3BtreeRollback(p, SQLITE_OK); + sqlite3BtreeRollback(p, SQLITE_OK, 0); sqlite3BtreeLeave(p); /* If there are still other outstanding references to the shared-btree @@ -3464,27 +3464,28 @@ int sqlite3BtreeCommit(Btree *p){ /* ** This routine sets the state to CURSOR_FAULT and the error -** code to errCode for every cursor on BtShared that pBtree -** references. +** code to errCode for every cursor on any BtShared that pBtree +** references. Or if the writeOnly flag is set to 1, then only +** trip write cursors and leave read cursors unchanged. ** -** Every cursor is tripped, including cursors that belong -** to other database connections that happen to be sharing -** the cache with pBtree. +** Every cursor is a candidate to be tripped, including cursors +** that belong to other database connections that happen to be +** sharing the cache with pBtree. ** -** This routine gets called when a rollback occurs. -** All cursors using the same cache must be tripped -** to prevent them from trying to use the btree after -** the rollback. The rollback may have deleted tables -** or moved root pages, so it is not sufficient to -** save the state of the cursor. The cursor must be -** invalidated. +** This routine gets called when a rollback occurs. The writeOnly +** flag is set to 1 if the transaction did not make any schema +** changes, in which case the read cursors can continue operating. +** If schema changes did occur in the transaction, then both read +** and write cursors must both be tripped. */ -void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ +void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ BtCursor *p; + assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); if( pBtree==0 ) return; sqlite3BtreeEnter(pBtree); for(p=pBtree->pBt->pCursor; p; p=p->pNext){ int i; + if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ) continue; sqlite3BtreeClearCursor(p); p->eState = CURSOR_FAULT; p->skipNext = errCode; @@ -3497,27 +3498,32 @@ void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode){ } /* -** Rollback the transaction in progress. All cursors will be -** invalided by this operation. Any attempt to use a cursor -** that was open at the beginning of this operation will result -** in an error. +** Rollback the transaction in progress. +** +** If tripCode is not SQLITE_OK then cursors will be invalidated (tripped). +** Only write cursors are tripped if writeOnly is true but all cursors are +** tripped if writeOnly is false. Any attempt to use +** a tripped cursor will result in an error. ** ** This will release the write lock on the database file. If there ** are no active cursors, it also releases the read lock. */ -int sqlite3BtreeRollback(Btree *p, int tripCode){ +int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ int rc; BtShared *pBt = p->pBt; MemPage *pPage1; + assert( writeOnly==1 || writeOnly==0 ); + assert( tripCode==SQLITE_ABORT_ROLLBACK || tripCode==SQLITE_OK ); sqlite3BtreeEnter(p); if( tripCode==SQLITE_OK ){ rc = tripCode = saveAllCursors(pBt, 0, 0); + if( rc ) writeOnly = 0; }else{ rc = SQLITE_OK; } if( tripCode ){ - sqlite3BtreeTripAllCursors(p, tripCode); + sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); } btreeIntegrity(p); diff --git a/src/btree.h b/src/btree.h index 38abdca1a2..f724269a3e 100644 --- a/src/btree.h +++ b/src/btree.h @@ -83,7 +83,7 @@ int sqlite3BtreeBeginTrans(Btree*,int); int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); int sqlite3BtreeCommitPhaseTwo(Btree*, int); int sqlite3BtreeCommit(Btree*); -int sqlite3BtreeRollback(Btree*,int); +int sqlite3BtreeRollback(Btree*,int,int); int sqlite3BtreeBeginStmt(Btree*,int); int sqlite3BtreeCreateTable(Btree*, int*, int flags); int sqlite3BtreeIsInTrans(Btree*); @@ -116,7 +116,7 @@ int sqlite3BtreeIncrVacuum(Btree *); int sqlite3BtreeDropTable(Btree*, int, int*); int sqlite3BtreeClearTable(Btree*, int, int*); int sqlite3BtreeClearTableOfCursor(BtCursor*); -void sqlite3BtreeTripAllCursors(Btree*, int); +void sqlite3BtreeTripAllCursors(Btree*, int, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); diff --git a/src/main.c b/src/main.c index ea03f2639f..46b3ccdf58 100644 --- a/src/main.c +++ b/src/main.c @@ -1016,13 +1016,15 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ /* ** Rollback all database files. If tripCode is not SQLITE_OK, then -** any open cursors are invalidated ("tripped" - as in "tripping a circuit +** any write cursors are invalidated ("tripped" - as in "tripping a circuit ** breaker") and made to return tripCode if there are any further -** attempts to use that cursor. +** attempts to use that cursor. Read cursors remain open and valid +** but are "saved" in case the table pages are moved around. */ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ int i; int inTrans = 0; + int schemaChange; assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); @@ -1033,6 +1035,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ ** the database rollback and schema reset, which can cause false ** corruption reports in some cases. */ sqlite3BtreeEnterAll(db); + schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; for(i=0; inDb; i++){ Btree *p = db->aDb[i].pBt; @@ -1040,7 +1043,7 @@ void sqlite3RollbackAll(sqlite3 *db, int tripCode){ if( sqlite3BtreeIsInTrans(p) ){ inTrans = 1; } - sqlite3BtreeRollback(p, tripCode); + sqlite3BtreeRollback(p, tripCode, !schemaChange); } } sqlite3VtabRollback(db); diff --git a/src/vdbe.c b/src/vdbe.c index 88fadb023e..6e152cd27e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2822,11 +2822,16 @@ case OP_Savepoint: { db->isTransactionSavepoint = 0; rc = p->rc; }else{ + int isSchemaChange; iSavepoint = db->nSavepoint - iSavepoint - 1; if( p1==SAVEPOINT_ROLLBACK ){ + isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; for(ii=0; iinDb; ii++){ - sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT); + sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, + isSchemaChange==0); } + }else{ + isSchemaChange = 0; } for(ii=0; iinDb; ii++){ rc = sqlite3BtreeSavepoint(db->aDb[ii].pBt, p1, iSavepoint); @@ -2834,7 +2839,7 @@ case OP_Savepoint: { goto abort_due_to_error; } } - if( p1==SAVEPOINT_ROLLBACK && (db->flags&SQLITE_InternChanges)!=0 ){ + if( isSchemaChange ){ sqlite3ExpirePreparedStatements(db); sqlite3ResetAllSchemasOfConnection(db); db->flags = (db->flags | SQLITE_InternChanges); @@ -3231,7 +3236,7 @@ case OP_OpenWrite: { || p->readOnly==0 ); if( p->expired ){ - rc = SQLITE_ABORT; + rc = SQLITE_ABORT_ROLLBACK; break; } diff --git a/src/wal.c b/src/wal.c index c0861d5be7..24540a2cd5 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2506,7 +2506,7 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); for(iFrame=pWal->hdr.mxFrame+1; - ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; + rc==SQLITE_OK && iFrame<=iMax; iFrame++ ){ /* This call cannot fail. Unless the page for which the page number @@ -2525,7 +2525,6 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ } if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); } - assert( rc==SQLITE_OK ); return rc; } diff --git a/test/capi3.test b/test/capi3.test index cbaa5c5e5b..9f3d6f6916 100644 --- a/test/capi3.test +++ b/test/capi3.test @@ -912,7 +912,7 @@ do_test capi3-11.9.3 { } 1 do_test capi3-11.10 { sqlite3_step $STMT -} {SQLITE_ERROR} +} {SQLITE_ROW} ifcapable !autoreset { # If SQLITE_OMIT_AUTORESET is defined, then the statement must be # reset() before it can be passed to step() again. @@ -921,11 +921,11 @@ ifcapable !autoreset { } do_test capi3-11.11 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_DONE} do_test capi3-11.12 { sqlite3_step $STMT sqlite3_step $STMT -} {SQLITE_DONE} +} {SQLITE_ROW} do_test capi3-11.13 { sqlite3_finalize $STMT } {SQLITE_OK} diff --git a/test/capi3c.test b/test/capi3c.test index 6388717e00..6ab3bc24f6 100644 --- a/test/capi3c.test +++ b/test/capi3c.test @@ -863,7 +863,7 @@ do_test capi3c-11.9.3 { } 1 do_test capi3c-11.10 { sqlite3_step $STMT -} {SQLITE_ABORT} +} {SQLITE_ROW} ifcapable !autoreset { # If SQLITE_OMIT_AUTORESET is defined, then the statement must be # reset() before it can be passed to step() again. @@ -872,11 +872,11 @@ ifcapable !autoreset { } do_test capi3c-11.11 { sqlite3_step $STMT -} {SQLITE_ROW} +} {SQLITE_DONE} do_test capi3c-11.12 { sqlite3_step $STMT sqlite3_step $STMT -} {SQLITE_DONE} +} {SQLITE_ROW} do_test capi3c-11.13 { sqlite3_finalize $STMT } {SQLITE_OK} diff --git a/test/rollback.test b/test/rollback.test index c339c5d7d6..7abafece61 100644 --- a/test/rollback.test +++ b/test/rollback.test @@ -60,11 +60,11 @@ ifcapable conflict { # do_test rollback-1.5 { sqlite3_step $STMT - } {SQLITE_ERROR} + } {SQLITE_ROW} # Restart the SELECT statement # - do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_ABORT} + do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_OK} } else { do_test rollback-1.6 { sqlite3_reset $STMT } {SQLITE_OK} } diff --git a/test/savepoint.test b/test/savepoint.test index 9362c8fe19..8055e61d9e 100644 --- a/test/savepoint.test +++ b/test/savepoint.test @@ -315,7 +315,7 @@ ifcapable incrblob { do_test savepoint-5.3.2.3 { set rc [catch {seek $fd 0; read $fd} res] set rc - } {1} + } {0} do_test savepoint-5.3.3 { catchsql {RELEASE def} } {0 {}} diff --git a/test/savepoint7.test b/test/savepoint7.test index bc99187d27..908ec571f5 100644 --- a/test/savepoint7.test +++ b/test/savepoint7.test @@ -30,6 +30,7 @@ do_test savepoint7-1.1 { db eval {SELECT * FROM t1} { db eval { SAVEPOINT x2; + CREATE TABLE IF NOT EXISTS t3(xyz); INSERT INTO t2 VALUES($a,$b,$c); RELEASE x2; } @@ -46,7 +47,7 @@ do_test savepoint7-1.2 { RELEASE x2; } } - db eval {SELECT * FROM t2} + db eval {SELECT * FROM t2;} } {1 2 3 4 5 6 7 8 9} do_test savepoint7-1.3 { @@ -65,7 +66,7 @@ do_test savepoint7-1.3 { # queries in outer contexts. # do_test savepoint7-2.1 { - db eval {DELETE FROM t2; SAVEPOINT x1;} + db eval {DELETE FROM t2; SAVEPOINT x1; CREATE TABLE t4(abc);} set rc [catch { db eval {SELECT * FROM t1} { db eval { @@ -85,6 +86,7 @@ do_test savepoint7-2.2 { db eval {SELECT * FROM t1} { db eval { SAVEPOINT x2; + CREATE TABLE t5(pqr); INSERT INTO t2 VALUES($a,$b,$c); ROLLBACK TO x2; } diff --git a/test/tkt-f777251dc7a.test b/test/tkt-f777251dc7a.test index af6f71ad96..f814d246bf 100644 --- a/test/tkt-f777251dc7a.test +++ b/test/tkt-f777251dc7a.test @@ -38,8 +38,10 @@ proc force_rollback {} { db function force_rollback force_rollback do_test tkt-f7772-1.2 { +breakpoint catchsql { BEGIN IMMEDIATE; + CREATE TABLE xyzzy(abc); SELECT x, force_rollback(), EXISTS(SELECT 1 FROM t3 WHERE w=x) FROM t2; } } {1 {abort due to ROLLBACK}} @@ -67,7 +69,7 @@ do_test tkt-f7772-2.2 { catchsql { SELECT x, force_rollback(), EXISTS(SELECT 1 FROM t3 WHERE w=x) FROM t2 } -} {1 {callback requested query abort}} +} {1 {abort due to ROLLBACK}} do_test tkt-f7772-2.3 { sqlite3_get_autocommit db } {1} diff --git a/test/trans3.test b/test/trans3.test index d5b316bec8..e828442415 100644 --- a/test/trans3.test +++ b/test/trans3.test @@ -52,7 +52,7 @@ do_test trans3-1.4 { db eval {SELECT * FROM t1} } {1 2 3 4} do_test trans3-1.5 { - db eval BEGIN + db eval {BEGIN; CREATE TABLE xyzzy(abc);} db eval {INSERT INTO t1 VALUES(5);} set ::ecode {} set x [catch { From 51a205410c5039849bc3179a6a1b1c136494f4da Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 12 Nov 2014 14:07:28 +0000 Subject: [PATCH 563/710] Make sure that NULL results from OP_Column are fully and completely NULL and do not have the MEM_Ephem bit set. Fix for ticket [094d39a4c95ee4]. FossilOrigin-Name: e1017745e183f5d7429ce787ec2feef946a24f0f --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbe.c | 2 +- test/table.test | 18 +++++++++++++++++- 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 0e3e6a9ff4..e652361975 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C This\sis\sa\scherry-pick\sof\sversion\s[b5df5ac052]. -D 2014-11-11T22:55:26.891 +C Make\ssure\sthat\sNULL\sresults\sfrom\sOP_Column\sare\sfully\sand\scompletely\sNULL\sand\sdo\snot\shave\sthe\sMEM_Ephem\sbit\sset.\sFix\sfor\sticket\s[094d39a4c95ee4]. +D 2014-11-12T14:07:28.525 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 4c77cdf16be330bd5227691919332b42d557e211 +F src/vdbe.c 0b4ffa1aeeb7a10fefc6497dd0159bf5d9828464 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -873,7 +873,7 @@ F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/table.test 2a1d2fa52c531de5915f28023747d9a8c27b6f31 +F test/table.test 06271d61eb13871490d38168433c1ef3dd82bb2a F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tableopts.test dba698ba97251017b7c80d738c198d39ab747930 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 @@ -1204,8 +1204,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3b7b72c4685aa5cf5e675c2c47ebec10d9704221 -Q +b5df5ac0529f7b0d1e880a7f4a307e7d77b7fa6c -R 27a95de33658c36b59b93f043d249813 +P d4b2d5d066891e06f2bf4337902b44b000fa9fd2 +Q +42705fd7d892c4fdfb95fbbb468c99569beece25 +R 55a6f98cd186254776492ca00fba4393 U drh -Z ea313ee60fabf91291f912ad6dcdc285 +Z efc3d14840732f527ab6a674cff641f8 diff --git a/manifest.uuid b/manifest.uuid index 983840bd0e..cfac283fb9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d4b2d5d066891e06f2bf4337902b44b000fa9fd2 \ No newline at end of file +e1017745e183f5d7429ce787ec2feef946a24f0f \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 6e152cd27e..9462bd71f8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2298,7 +2298,7 @@ case OP_Column: { pC->payloadSize = pC->szRow = avail = pReg->n; pC->aRow = (u8*)pReg->z; }else{ - MemSetTypeFlag(pDest, MEM_Null); + sqlite3VdbeMemSetNull(pDest); goto op_column_out; } }else{ diff --git a/test/table.test b/test/table.test index 656884ca73..69f105aa6c 100644 --- a/test/table.test +++ b/test/table.test @@ -11,7 +11,6 @@ # This file implements regression tests for SQLite library. The # focus of this file is testing the CREATE TABLE statement. # -# $Id: table.test,v 1.53 2009/06/05 17:09:12 drh Exp $ set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -773,4 +772,21 @@ do_catchsql_test table-16.7 { INSERT INTO t16 DEFAULT VALUES; } {1 {unknown function: group_concat()}} +# Ticket [https://www.sqlite.org/src/info/094d39a4c95ee4abbc417f04214617675ba15c63] +# describes a assertion fault that occurs on a CREATE TABLE .. AS SELECT statement. +# the following test verifies that the problem has been fixed. +# +do_execsql_test table-17.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a TEXT); + INSERT INTO t1(a) VALUES(1),(2); + DROP TABLE IF EXISTS t2; + CREATE TABLE t2(x TEXT, y TEXT); + INSERT INTO t2(x,y) VALUES(3,4); + DROP TABLE IF EXISTS t3; + CREATE TABLE t3 AS + SELECT a AS p, coalesce(y,a) AS q FROM t1 LEFT JOIN t2 ON a=x; + SELECT p, q, '|' FROM t3 ORDER BY p; +} {1 1 | 2 2 |} + finish_test From 5a2c8c885c722e52b87040330a931d36caf3d4af Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 12 Nov 2014 14:12:28 +0000 Subject: [PATCH 564/710] Fix the %c format character in sqlite3VXPrintf() so that it correctly handles precisions larger than 70. FossilOrigin-Name: 839a6df9f98b90fb593534a62145d9c913540bae --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/printf.c | 33 +++++++++++++++++++-------------- src/sqliteInt.h | 2 +- test/printf2.test | 20 ++++++++++++++++++++ 5 files changed, 50 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index e652361975..a2c5fb1bb8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthat\sNULL\sresults\sfrom\sOP_Column\sare\sfully\sand\scompletely\sNULL\sand\sdo\snot\shave\sthe\sMEM_Ephem\sbit\sset.\sFix\sfor\sticket\s[094d39a4c95ee4]. -D 2014-11-12T14:07:28.525 +C Fix\sthe\s%c\sformat\scharacter\sin\ssqlite3VXPrintf()\sso\sthat\sit\scorrectly\shandles\sprecisions\slarger\sthan\s70. +D 2014-11-12T14:12:28.051 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -223,7 +223,7 @@ F src/pcache.h 9b559127b83f84ff76d735c8262f04853be0c59a F src/pcache1.c e412cb585f777c840ddce0500eddc5c6043c2bb5 F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f F src/prepare.c 6ef0cf2f9274982988ed6b7cab1be23147e94196 -F src/printf.c 090fac0f779c93c8a95089a125339686648835e4 +F src/printf.c d83b573624f3f6bc12b800af7fd55ce90be70659 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c a3466128b52a86c466e47ac1a19e2174f7b5cf89 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e @@ -232,7 +232,7 @@ F src/shell.c 18ee8bbe9502d8848072dc2eddd1ea09254ba494 F src/sqlite.h.in 4a5e5158c189d2bcd45c7c4607c2c0eb6d25c153 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h f7812f74f2d0c6041ef6b91a99c5a45f775dd408 +F src/sqliteInt.h c97db3c4d20b34c050a801c93451ef18e4f22de1 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 961d5926e5a8fda611d385ec22c226b8635cd1cb F src/table.c 2e99ef7ef16187e17033d9398dc962ce22dab5cb @@ -768,7 +768,7 @@ F test/permutations.test cef25f5e8499a15846eccd06785f17f4180407ab F test/pragma.test 19d0241a007bcdd77fc2606ec60fc60357e7fc8b F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 -F test/printf2.test bed79b4c3e5da08ba88ad637c0bf62586843cfb1 +F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d F test/ptrchng.test ef1aa72d6cf35a2bbd0869a649b744e9d84977fc F test/queryonly.test 5f653159e0f552f0552d43259890c1089391dcca @@ -1204,8 +1204,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d4b2d5d066891e06f2bf4337902b44b000fa9fd2 -Q +42705fd7d892c4fdfb95fbbb468c99569beece25 -R 55a6f98cd186254776492ca00fba4393 +P e1017745e183f5d7429ce787ec2feef946a24f0f +Q +08a27440f19b7fc884464832e6105af1bf008172 +R 3a2fc5ef3a825f22ce8bbd1b4d69c828 U drh -Z efc3d14840732f527ab6a674cff641f8 +Z 3e707181121665e3dbca925048596208 diff --git a/manifest.uuid b/manifest.uuid index cfac283fb9..55a1630790 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1017745e183f5d7429ce787ec2feef946a24f0f \ No newline at end of file +839a6df9f98b90fb593534a62145d9c913540bae \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 1df287fbb6..387b9e90c1 100644 --- a/src/printf.c +++ b/src/printf.c @@ -212,7 +212,7 @@ void sqlite3VXPrintf( const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ - char *zExtra; /* Malloced memory used by some conversion */ + char *zExtra = 0; /* Malloced memory used by some conversion */ #ifndef SQLITE_OMIT_FLOATING_POINT int exp, e2; /* exponent of real numbers */ int nsd; /* Number of significant digits returned */ @@ -329,7 +329,6 @@ void sqlite3VXPrintf( break; } } - zExtra = 0; /* ** At this point, variables are initialized as follows: @@ -620,13 +619,16 @@ void sqlite3VXPrintf( }else{ c = va_arg(ap,int); } - buf[0] = (char)c; - if( precision>=0 ){ - for(idx=1; idx1 ){ + width -= precision-1; + if( width>1 && !flag_leftjustify ){ + sqlite3AppendChar(pAccum, width-1, ' '); + width = 0; + } + sqlite3AppendChar(pAccum, precision-1, c); } + length = 1; + buf[0] = c; bufpt = buf; break; case etSTRING: @@ -727,11 +729,14 @@ void sqlite3VXPrintf( ** the output. */ width -= length; - if( width>0 && !flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && !flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); sqlite3StrAccumAppend(pAccum, bufpt, length); - if( width>0 && flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( width>0 && flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); - if( zExtra ) sqlite3_free(zExtra); + if( zExtra ){ + sqlite3_free(zExtra); + zExtra = 0; + } }/* End for loop over the format string */ } /* End of function */ @@ -784,11 +789,11 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ } /* -** Append N space characters to the given string buffer. +** Append N copies of character c to the given string buffer. */ -void sqlite3AppendSpace(StrAccum *p, int N){ +void sqlite3AppendChar(StrAccum *p, int N, char c){ if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; - while( (N--)>0 ) p->zText[p->nChar++] = ' '; + while( (N--)>0 ) p->zText[p->nChar++] = c; } /* diff --git a/src/sqliteInt.h b/src/sqliteInt.h index cba89b03e7..40b79a79b8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3539,7 +3539,7 @@ int sqlite3OpenTempDatabase(Parse *); void sqlite3StrAccumInit(StrAccum*, char*, int, int); void sqlite3StrAccumAppend(StrAccum*,const char*,int); void sqlite3StrAccumAppendAll(StrAccum*,const char*); -void sqlite3AppendSpace(StrAccum*,int); +void sqlite3AppendChar(StrAccum*,int,char); char *sqlite3StrAccumFinish(StrAccum*); void sqlite3StrAccumReset(StrAccum*); void sqlite3SelectDestInit(SelectDest*,int,int); diff --git a/test/printf2.test b/test/printf2.test index 4cb1783bfb..21deeb779d 100644 --- a/test/printf2.test +++ b/test/printf2.test @@ -95,5 +95,25 @@ do_execsql_test printf2-2.3 { SELECT printf('%s=(%d/%g/%s)',a) FROM t1 ORDER BY a; } {-1=(0/0/) 1=(0/0/) 1.5=(0/0/) abc=(0/0/)} +# The precision of the %c conversion causes the character to repeat. +# +do_execsql_test printf2-3.1 { + SELECT printf('|%110.100c|','*'); +} {{| ****************************************************************************************************|}} +do_execsql_test printf2-3.2 { + SELECT printf('|%-110.100c|','*'); +} {{|**************************************************************************************************** |}} +do_execsql_test printf2-3.3 { + SELECT printf('|%9.8c|%-9.8c|','*','*'); +} {{| ********|******** |}} +do_execsql_test printf2-3.4 { + SELECT printf('|%8.8c|%-8.8c|','*','*'); +} {|********|********|} +do_execsql_test printf2-3.5 { + SELECT printf('|%7.8c|%-7.8c|','*','*'); +} {|********|********|} + + + finish_test From 80231042527411987a907328722e169e4e1cb47e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Nov 2014 14:56:02 +0000 Subject: [PATCH 565/710] When a transaction or savepoint rollback occurs, save the positions of all open read-cursors so that they can be restored following the rollback operation. FossilOrigin-Name: dd03a2802f3f276525f3cef9a93f825dd8606626 --- manifest | 19 ++++++------ manifest.uuid | 2 +- src/btree.c | 63 ++++++++++++++++++++++++++------------ src/btree.h | 2 +- src/vdbe.c | 3 +- test/rollback2.test | 74 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 31 deletions(-) create mode 100644 test/rollback2.test diff --git a/manifest b/manifest index 97192e581d..11b9906fa0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\scalls\sto\sthe\s'breakpoint'\stest\scommand. -D 2014-11-11T19:07:56.372 +C When\sa\stransaction\sor\ssavepoint\srollback\soccurs,\ssave\sthe\spositions\sof\sall\sopen\sread-cursors\sso\sthat\sthey\scan\sbe\srestored\sfollowing\sthe\srollback\soperation. +D 2014-11-12T14:56:02.923 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,8 +173,8 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 86b311af1ad35decd1f2f0cee457d344acb83c3b F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c eaef0003bbfe740c62189355dabc818fc3a98999 -F src/btree.h d24fc2f3bc53be220b79b95800bdb2ee207b1089 +F src/btree.c d5d991b518fa5bebc64037dfeb98a48051d864d7 +F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -291,7 +291,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c c71d819bb34269c3dbccd92e6bb308f0ec025b5d +F src/vdbe.c 5e47308836e9bb5fdb4835fdf88eeab071848d3f F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -797,6 +797,7 @@ F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea +F test/rollback2.test 552abaab8e721b6060a727d639896427059e51ec F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 @@ -1218,7 +1219,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a066a3832a7c6de65c3016e77e49ac00e09db749 -R e8fc41a7f82a02ab8618a341f7ef7735 -U mistachkin -Z 90417f4c358533080e79b8017b4e7453 +P 1412fcc480799ecbd68d44dd18d5bad40e20ccf1 +R 668a86f911283b47b6c156fecf772681 +U dan +Z cf8e25329461439611751d7913ef3ef9 diff --git a/manifest.uuid b/manifest.uuid index f90bf2d8c8..3369a7ad97 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1412fcc480799ecbd68d44dd18d5bad40e20ccf1 \ No newline at end of file +dd03a2802f3f276525f3cef9a93f825dd8606626 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 70fae7c707..f153a0cc45 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3514,29 +3514,52 @@ int sqlite3BtreeCommit(Btree *p){ ** that belong to other database connections that happen to be ** sharing the cache with pBtree. ** -** This routine gets called when a rollback occurs. The writeOnly -** flag is set to 1 if the transaction did not make any schema -** changes, in which case the read cursors can continue operating. -** If schema changes did occur in the transaction, then both read -** and write cursors must both be tripped. +** This routine gets called when a rollback occurs. If the writeOnly +** flag is true, then only write-cursors need be tripped - read-only +** cursors save their current positions so that they may continue +** following the rollback. Or, if writeOnly is false, all cursors are +** tripped. In general, writeOnly is false if the transaction being +** rolled back modified the database schema. In this case b-tree root +** pages may be moved or deleted from the database altogether, making +** it unsafe for read cursors to continue. +** +** If the writeOnly flag is true and an error is encountered while +** saving the current position of a read-only cursor, all cursors, +** including all read-cursors are tripped. +** +** SQLITE_OK is returned if successful, or if an error occurs while +** saving a cursor position, an SQLite error code. */ -void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ +int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ BtCursor *p; + int rc = SQLITE_OK; + assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); - if( pBtree==0 ) return; - sqlite3BtreeEnter(pBtree); - for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - int i; - if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ) continue; - sqlite3BtreeClearCursor(p); - p->eState = CURSOR_FAULT; - p->skipNext = errCode; - for(i=0; i<=p->iPage; i++){ - releasePage(p->apPage[i]); - p->apPage[i] = 0; + if( pBtree ){ + sqlite3BtreeEnter(pBtree); + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + int i; + if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ + if( p->eState==CURSOR_VALID ){ + int rc = saveCursorPosition(p); + if( rc!=SQLITE_OK ){ + (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); + break; + } + } + }else{ + sqlite3BtreeClearCursor(p); + p->eState = CURSOR_FAULT; + p->skipNext = errCode; + } + for(i=0; i<=p->iPage; i++){ + releasePage(p->apPage[i]); + p->apPage[i] = 0; + } } + sqlite3BtreeLeave(pBtree); } - sqlite3BtreeLeave(pBtree); + return rc; } /* @@ -3565,7 +3588,9 @@ int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ rc = SQLITE_OK; } if( tripCode ){ - sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); + int rc2 = sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); + assert( rc==SQLITE_OK || (writeOnly==0 && rc2==SQLITE_OK) ); + if( rc2!=SQLITE_OK ) rc = rc2; } btreeIntegrity(p); diff --git a/src/btree.h b/src/btree.h index a6093a5edd..281e57dafd 100644 --- a/src/btree.h +++ b/src/btree.h @@ -116,7 +116,7 @@ int sqlite3BtreeIncrVacuum(Btree *); int sqlite3BtreeDropTable(Btree*, int, int*); int sqlite3BtreeClearTable(Btree*, int, int*); int sqlite3BtreeClearTableOfCursor(BtCursor*); -void sqlite3BtreeTripAllCursors(Btree*, int, int); +int sqlite3BtreeTripAllCursors(Btree*, int, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); diff --git a/src/vdbe.c b/src/vdbe.c index 715862e73c..e4cbeb6565 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2830,8 +2830,9 @@ case OP_Savepoint: { if( p1==SAVEPOINT_ROLLBACK ){ isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; for(ii=0; iinDb; ii++){ - sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, + rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, isSchemaChange==0); + if( rc!=SQLITE_OK ) goto abort_due_to_error; } }else{ isSchemaChange = 0; diff --git a/test/rollback2.test b/test/rollback2.test new file mode 100644 index 0000000000..9637f0c0e5 --- /dev/null +++ b/test/rollback2.test @@ -0,0 +1,74 @@ +# 2014 November 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix rollback2 + +proc int2hex {i} { format %.2X $i } +db func int2hex int2hex + +do_execsql_test 1.0 { + SELECT int2hex(0), int2hex(100), int2hex(255) +} {00 64 FF} +do_execsql_test 1.1 { + CREATE TABLE t1(i, h); + CREATE INDEX i1 ON t1(h); + WITH data(a, b) AS ( + SELECT 1, int2hex(1) + UNION ALL + SELECT a+1, int2hex(a+1) FROM data WHERE a<40 + ) + INSERT INTO t1 SELECT * FROM data; +} {} + + +proc do_rollback_test {tn args} { + set A(-setup) "" + set A(-select) "" + set A(-result) "" + set A(-rollback) ROLLBACK + + array set O $args + foreach k [array names O] { + if {[info exists A($k)]==0} { error "unknown option: $k" } + set A($k) $O($k) + } + + for {set iRollback 0} 1 {incr iRollback} { + catch { db eval ROLLBACK } + set res [list] + db eval $A(-setup) + + set i 0 + db eval $A(-select) x { + if {$i==$iRollback} { db eval $A(-rollback) } + foreach k $x(*) { lappend res $x($k) } + incr i + } + + do_test $tn.$iRollback [list set {} $res] [list {*}$A(-result)] + if {$i < $iRollback} break + } +} + +do_rollback_test 2 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%2)==1; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} + +finish_test + From d7b06909ad144b993f59e3a67d214b5ad27149fe Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Nov 2014 17:45:37 +0000 Subject: [PATCH 566/710] Add further tests for rollback operations in the presence of ongoing selects. FossilOrigin-Name: eaf3aae014f59c8d37aa20aa31d54cf13f9e86fc --- manifest | 13 +++--- manifest.uuid | 2 +- test/rollback2.test | 87 ++++++++++++++++++++++++++++++++++++++++- test/rollbackfault.test | 84 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 9 deletions(-) create mode 100644 test/rollbackfault.test diff --git a/manifest b/manifest index 11b9906fa0..dddb788b2d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sa\stransaction\sor\ssavepoint\srollback\soccurs,\ssave\sthe\spositions\sof\sall\sopen\sread-cursors\sso\sthat\sthey\scan\sbe\srestored\sfollowing\sthe\srollback\soperation. -D 2014-11-12T14:56:02.923 +C Add\sfurther\stests\sfor\srollback\soperations\sin\sthe\spresence\sof\songoing\sselects. +D 2014-11-12T17:45:37.113 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -797,7 +797,8 @@ F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea -F test/rollback2.test 552abaab8e721b6060a727d639896427059e51ec +F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 +F test/rollbackfault.test 6a004f71087cc399296cffbb5429ea6da655ae65 F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 @@ -1219,7 +1220,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1412fcc480799ecbd68d44dd18d5bad40e20ccf1 -R 668a86f911283b47b6c156fecf772681 +P dd03a2802f3f276525f3cef9a93f825dd8606626 +R c0a8c133338658022754212d7071964f U dan -Z cf8e25329461439611751d7913ef3ef9 +Z c8dc7391cf2e61af731936b53ad2fd87 diff --git a/manifest.uuid b/manifest.uuid index 3369a7ad97..634cdcad8d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dd03a2802f3f276525f3cef9a93f825dd8606626 \ No newline at end of file +eaf3aae014f59c8d37aa20aa31d54cf13f9e86fc \ No newline at end of file diff --git a/test/rollback2.test b/test/rollback2.test index 9637f0c0e5..4d42dda5d6 100644 --- a/test/rollback2.test +++ b/test/rollback2.test @@ -9,6 +9,9 @@ # #*********************************************************************** # +# This file containst tests to verify that ROLLBACK or ROLLBACK TO +# operations interact correctly with ongoing SELECT statements. +# set testdir [file dirname $argv0] source $testdir/tester.tcl @@ -16,7 +19,6 @@ set ::testprefix rollback2 proc int2hex {i} { format %.2X $i } db func int2hex int2hex - do_execsql_test 1.0 { SELECT int2hex(0), int2hex(100), int2hex(255) } {00 64 FF} @@ -32,6 +34,17 @@ do_execsql_test 1.1 { } {} +# do_rollback_test ID SWITCHES +# +# where SWITCHES are: +# +# -setup SQL script to open transaction and begin writing. +# -select SELECT to execute after -setup script +# -result Expected result of -select statement +# -rollback Use this SQL command ("ROLLBACK" or "ROLLBACK TO ...") to +# rollback the transaction in the middle of the -select statment +# execution. +# proc do_rollback_test {tn args} { set A(-setup) "" set A(-select) "" @@ -61,7 +74,7 @@ proc do_rollback_test {tn args} { } } -do_rollback_test 2 -setup { +do_rollback_test 2.1 -setup { BEGIN; DELETE FROM t1 WHERE (i%2)==1; } -select { @@ -70,5 +83,75 @@ do_rollback_test 2 -setup { 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 } +do_rollback_test 2.2 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%4)==1; + SAVEPOINT one; + DELETE FROM t1 WHERE (i%2)==1; +} -rollback { + ROLLBACK TO one; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} + +#-------------------------------------------------------------------- +# Try with some index scans +# +do_eqp_test 3.1 { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h DESC; +} {0 0 0 {SCAN TABLE t1 USING INDEX i1}} +do_rollback_test 3.2 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%2)==1; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h DESC; +} -result { + 40 38 36 34 32 30 28 26 24 22 20 18 16 14 12 10 8 6 4 2 +} +do_rollback_test 3.3 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%4)==1; + SAVEPOINT one; + DELETE FROM t1 WHERE (i%2)==1; +} -rollback { + ROLLBACK TO one; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h DESC; +} -result { + 40 38 36 34 32 30 28 26 24 22 20 18 16 14 12 10 8 6 4 2 +} + +#-------------------------------------------------------------------- +# Now with some index scans that feature overflow keys. +# +set leader [string repeat "abcdefghij" 70] +do_execsql_test 4.1 { UPDATE t1 SET h = $leader || h; } + +do_eqp_test 4.2 { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h ASC; +} {0 0 0 {SCAN TABLE t1 USING INDEX i1}} +do_rollback_test 4.3 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%2)==1; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h ASC; +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} +do_rollback_test 4.4 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%4)==1; + SAVEPOINT one; + DELETE FROM t1 WHERE (i%2)==1; +} -rollback { + ROLLBACK TO one; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h ASC; +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} + finish_test diff --git a/test/rollbackfault.test b/test/rollbackfault.test new file mode 100644 index 0000000000..f248d0758d --- /dev/null +++ b/test/rollbackfault.test @@ -0,0 +1,84 @@ +# 2014-11-12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test that errors encountered during a ROLLBACK operation correctly +# affect ongoing SQL statements. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set testprefix rollbackfault + + +proc int2hex {i} { format %.2X $i } +db func int2hex int2hex +do_execsql_test 1.0 { + SELECT int2hex(0), int2hex(100), int2hex(255) +} {00 64 FF} +do_execsql_test 1.1 { + CREATE TABLE t1(i, h); + CREATE INDEX i1 ON t1(h); + WITH data(a, b) AS ( + SELECT 1, int2hex(1) + UNION ALL + SELECT a+1, int2hex(a+1) FROM data WHERE a<40 + ) + INSERT INTO t1 SELECT * FROM data; +} {} + +foreach f {oom ioerr} { + do_faultsim_test 1.2 -faults $f* -prep { + set sql1 { SELECT i FROM t1 WHERE (i%2)==0 } + set sql2 { SELECT i FROM t1 WHERE (i%2)==0 ORDER BY h } + set ::s1 [sqlite3_prepare db $sql1 -1 dummy] + set ::s2 [sqlite3_prepare db $sql2 -1 dummy] + + for {set i 0} {$i < 10} {incr i} { sqlite3_step $::s1 } + for {set i 0} {$i < 3} {incr i} { sqlite3_step $::s2 } + + execsql { + BEGIN; DELETE FROM t1 WHERE (i%2) + } + } -body { + execsql { ROLLBACK } + } -test { + + set res1 [list] + set res2 [list] + while {"SQLITE_ROW" == [sqlite3_step $::s1]} { + lappend res1 [sqlite3_column_text $::s1 0] + } + while {"SQLITE_ROW" == [sqlite3_step $::s2]} { + lappend res2 [sqlite3_column_text $::s2 0] + } + set rc1 [sqlite3_finalize $::s1] + set rc2 [sqlite3_finalize $::s2] + + catchsql { ROLLBACK } + + if {$rc1=="SQLITE_OK" && $rc2=="SQLITE_OK" + && $res1=="22 24 26 28 30 32 34 36 38 40" + && $res2=="8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40" + } { + # This is Ok. + } elseif {$rc1!="SQLITE_OK" && $rc2!="SQLITE_OK" && $res1=="" &&$res2==""} { + # Also Ok. + } else { + error "statements don't look right" + } + } +} + + +finish_test + + From 6f9c5669e2d54fb4e480b2f1f39fba6907560cc1 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 13 Nov 2014 13:42:39 +0000 Subject: [PATCH 567/710] When a transaction or savepoint rollback occurs, save the positions of all open read-cursors so that they can be restored following the rollback operation. Cherry-pick of check-in [dd03a2802f3f27] FossilOrigin-Name: 402780a9c8df9e7ea898bdca49c1191042fe387a --- manifest | 19 ++++++------ manifest.uuid | 2 +- src/btree.c | 63 ++++++++++++++++++++++++++------------ src/btree.h | 2 +- src/vdbe.c | 3 +- test/rollback2.test | 74 +++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 31 deletions(-) create mode 100644 test/rollback2.test diff --git a/manifest b/manifest index a2c5fb1bb8..1d9fe4d78f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s%c\sformat\scharacter\sin\ssqlite3VXPrintf()\sso\sthat\sit\scorrectly\shandles\sprecisions\slarger\sthan\s70. -D 2014-11-12T14:12:28.051 +C When\sa\stransaction\sor\ssavepoint\srollback\soccurs,\ssave\sthe\spositions\sof\sall\sopen\sread-cursors\sso\sthat\sthey\scan\sbe\srestored\sfollowing\sthe\srollback\soperation.\s\sCherry-pick\sof\scheck-in\s[dd03a2802f3f27] +D 2014-11-13T13:42:39.738 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,8 +172,8 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c 8cdfeb0c8a6d8bdad3faefae418eb3dc767051b6 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 944b84f72fd9048acde31400d89439976bf766b0 -F src/btree.h 97c4d0ef860f151e096e97eef78b87f7b4a8d899 +F src/btree.c 2c15850c5c9a26b10cdf92f9a29c74e299dc3674 +F src/btree.h a4afc6b06f5a1dd2076d15aa168baec44fc0121b F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 0b4ffa1aeeb7a10fefc6497dd0159bf5d9828464 +F src/vdbe.c a6b604364c7cbb079c083418e7359d1d665f2ef0 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -786,6 +786,7 @@ F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea +F test/rollback2.test 552abaab8e721b6060a727d639896427059e51ec F test/rowhash.test 0bc1d31415e4575d10cacf31e1a66b5cc0f8be81 F test/rowid.test 742b5741584a8a44fd83e856cc2896688401d645 F test/rtree.test 0c8d9dd458d6824e59683c19ab2ffa9ef946f798 @@ -1204,8 +1205,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e1017745e183f5d7429ce787ec2feef946a24f0f -Q +08a27440f19b7fc884464832e6105af1bf008172 -R 3a2fc5ef3a825f22ce8bbd1b4d69c828 +P 839a6df9f98b90fb593534a62145d9c913540bae +Q +dd03a2802f3f276525f3cef9a93f825dd8606626 +R 0801523eb21d9763755a3afd3e84ae8b U drh -Z 3e707181121665e3dbca925048596208 +Z 995ce8d43edc83cec464fe7495c36085 diff --git a/manifest.uuid b/manifest.uuid index 55a1630790..38b0ca2922 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -839a6df9f98b90fb593534a62145d9c913540bae \ No newline at end of file +402780a9c8df9e7ea898bdca49c1191042fe387a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9d7a05157d..41e097af53 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3472,29 +3472,52 @@ int sqlite3BtreeCommit(Btree *p){ ** that belong to other database connections that happen to be ** sharing the cache with pBtree. ** -** This routine gets called when a rollback occurs. The writeOnly -** flag is set to 1 if the transaction did not make any schema -** changes, in which case the read cursors can continue operating. -** If schema changes did occur in the transaction, then both read -** and write cursors must both be tripped. +** This routine gets called when a rollback occurs. If the writeOnly +** flag is true, then only write-cursors need be tripped - read-only +** cursors save their current positions so that they may continue +** following the rollback. Or, if writeOnly is false, all cursors are +** tripped. In general, writeOnly is false if the transaction being +** rolled back modified the database schema. In this case b-tree root +** pages may be moved or deleted from the database altogether, making +** it unsafe for read cursors to continue. +** +** If the writeOnly flag is true and an error is encountered while +** saving the current position of a read-only cursor, all cursors, +** including all read-cursors are tripped. +** +** SQLITE_OK is returned if successful, or if an error occurs while +** saving a cursor position, an SQLite error code. */ -void sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ +int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ BtCursor *p; + int rc = SQLITE_OK; + assert( (writeOnly==0 || writeOnly==1) && BTCF_WriteFlag==1 ); - if( pBtree==0 ) return; - sqlite3BtreeEnter(pBtree); - for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - int i; - if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ) continue; - sqlite3BtreeClearCursor(p); - p->eState = CURSOR_FAULT; - p->skipNext = errCode; - for(i=0; i<=p->iPage; i++){ - releasePage(p->apPage[i]); - p->apPage[i] = 0; + if( pBtree ){ + sqlite3BtreeEnter(pBtree); + for(p=pBtree->pBt->pCursor; p; p=p->pNext){ + int i; + if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ + if( p->eState==CURSOR_VALID ){ + int rc = saveCursorPosition(p); + if( rc!=SQLITE_OK ){ + (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); + break; + } + } + }else{ + sqlite3BtreeClearCursor(p); + p->eState = CURSOR_FAULT; + p->skipNext = errCode; + } + for(i=0; i<=p->iPage; i++){ + releasePage(p->apPage[i]); + p->apPage[i] = 0; + } } + sqlite3BtreeLeave(pBtree); } - sqlite3BtreeLeave(pBtree); + return rc; } /* @@ -3523,7 +3546,9 @@ int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ rc = SQLITE_OK; } if( tripCode ){ - sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); + int rc2 = sqlite3BtreeTripAllCursors(p, tripCode, writeOnly); + assert( rc==SQLITE_OK || (writeOnly==0 && rc2==SQLITE_OK) ); + if( rc2!=SQLITE_OK ) rc = rc2; } btreeIntegrity(p); diff --git a/src/btree.h b/src/btree.h index f724269a3e..fabedd9a53 100644 --- a/src/btree.h +++ b/src/btree.h @@ -116,7 +116,7 @@ int sqlite3BtreeIncrVacuum(Btree *); int sqlite3BtreeDropTable(Btree*, int, int*); int sqlite3BtreeClearTable(Btree*, int, int*); int sqlite3BtreeClearTableOfCursor(BtCursor*); -void sqlite3BtreeTripAllCursors(Btree*, int, int); +int sqlite3BtreeTripAllCursors(Btree*, int, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); diff --git a/src/vdbe.c b/src/vdbe.c index 9462bd71f8..1ad8aab753 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2827,8 +2827,9 @@ case OP_Savepoint: { if( p1==SAVEPOINT_ROLLBACK ){ isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; for(ii=0; iinDb; ii++){ - sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, + rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, isSchemaChange==0); + if( rc!=SQLITE_OK ) goto abort_due_to_error; } }else{ isSchemaChange = 0; diff --git a/test/rollback2.test b/test/rollback2.test new file mode 100644 index 0000000000..9637f0c0e5 --- /dev/null +++ b/test/rollback2.test @@ -0,0 +1,74 @@ +# 2014 November 12 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix rollback2 + +proc int2hex {i} { format %.2X $i } +db func int2hex int2hex + +do_execsql_test 1.0 { + SELECT int2hex(0), int2hex(100), int2hex(255) +} {00 64 FF} +do_execsql_test 1.1 { + CREATE TABLE t1(i, h); + CREATE INDEX i1 ON t1(h); + WITH data(a, b) AS ( + SELECT 1, int2hex(1) + UNION ALL + SELECT a+1, int2hex(a+1) FROM data WHERE a<40 + ) + INSERT INTO t1 SELECT * FROM data; +} {} + + +proc do_rollback_test {tn args} { + set A(-setup) "" + set A(-select) "" + set A(-result) "" + set A(-rollback) ROLLBACK + + array set O $args + foreach k [array names O] { + if {[info exists A($k)]==0} { error "unknown option: $k" } + set A($k) $O($k) + } + + for {set iRollback 0} 1 {incr iRollback} { + catch { db eval ROLLBACK } + set res [list] + db eval $A(-setup) + + set i 0 + db eval $A(-select) x { + if {$i==$iRollback} { db eval $A(-rollback) } + foreach k $x(*) { lappend res $x($k) } + incr i + } + + do_test $tn.$iRollback [list set {} $res] [list {*}$A(-result)] + if {$i < $iRollback} break + } +} + +do_rollback_test 2 -setup { + BEGIN; + DELETE FROM t1 WHERE (i%2)==1; +} -select { + SELECT i FROM t1 WHERE (i%2)==0 +} -result { + 2 4 6 8 10 12 14 16 18 20 22 24 26 28 30 32 34 36 38 40 +} + +finish_test + From fad01993b7f9a5d519dd5d344dd5e4c8a44ccc06 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 13 Nov 2014 14:18:25 +0000 Subject: [PATCH 568/710] Have calls to sqlite3_backup_init() fail if there is already a read or read-write transaction open on the destination database. FossilOrigin-Name: 169b5505498c0a7ee2b5dbb2ba13c41dfaa7c62f --- manifest | 15 ++++++----- manifest.uuid | 2 +- src/backup.c | 27 ++++++++++++++++---- test/backup.test | 1 + test/backup5.test | 65 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 97 insertions(+), 13 deletions(-) create mode 100644 test/backup5.test diff --git a/manifest b/manifest index dddb788b2d..4f0d48c405 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sfurther\stests\sfor\srollback\soperations\sin\sthe\spresence\sof\songoing\sselects. -D 2014-11-12T17:45:37.113 +C Have\scalls\sto\ssqlite3_backup_init()\sfail\sif\sthere\sis\salready\sa\sread\sor\sread-write\stransaction\sopen\son\sthe\sdestination\sdatabase. +D 2014-11-13T14:18:25.531 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -170,7 +170,7 @@ F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 -F src/backup.c 86b311af1ad35decd1f2f0cee457d344acb83c3b +F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c d5d991b518fa5bebc64037dfeb98a48051d864d7 @@ -353,9 +353,10 @@ F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 F test/avtrans.test 0252654f4295ddda3b2cce0e894812259e655a85 F test/backcompat.test 19a1f337c68419b020a7481dd272a472c4ad8ef4 -F test/backup.test c9cdd23a495864b9edf75a9fa66f5cb7e10fcf62 +F test/backup.test b79299a536a4c6d919094786595b95be56d02014 F test/backup2.test 34986ef926ea522911a51dfdb2f8e99b7b75ebcf F test/backup4.test 2a2e4a64388090b152de753fd9e123f28f6a3bd4 +F test/backup5.test ee5da6d7fe5082f5b9b0bbfa31d016f52412a2e4 F test/backup_ioerr.test 4c3c7147cee85b024ecf6e150e090c32fdbb5135 F test/backup_malloc.test 7162d604ec2b4683c4b3799a48657fb8b5e2d450 F test/badutf.test d5360fc31f643d37a973ab0d8b4fb85799c3169f @@ -1220,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dd03a2802f3f276525f3cef9a93f825dd8606626 -R c0a8c133338658022754212d7071964f +P eaf3aae014f59c8d37aa20aa31d54cf13f9e86fc +R cf416376a3614eef6301ba322bd3cc43 U dan -Z c8dc7391cf2e61af731936b53ad2fd87 +Z 499ccc085277a0250fd3c5f22e93f6be diff --git a/manifest.uuid b/manifest.uuid index 634cdcad8d..73dcaa8447 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eaf3aae014f59c8d37aa20aa31d54cf13f9e86fc \ No newline at end of file +169b5505498c0a7ee2b5dbb2ba13c41dfaa7c62f \ No newline at end of file diff --git a/src/backup.c b/src/backup.c index 57f7f447ea..e3f869035e 100644 --- a/src/backup.c +++ b/src/backup.c @@ -122,6 +122,20 @@ static int setDestPgsz(sqlite3_backup *p){ return rc; } +/* +** Check that there is no open read-transaction on the b-tree passed as the +** second argument. If there is not, return SQLITE_OK. Otherwise, if there +** is an open read-transaction, return SQLITE_ERROR and leave an error +** message in database handle db. +*/ +static int checkReadTransaction(sqlite3 *db, Btree *p){ + if( sqlite3BtreeIsInReadTrans(p) ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "destination database is in use"); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + /* ** Create an sqlite3_backup process to copy the contents of zSrcDb from ** connection handle pSrcDb to zDestDb in pDestDb. If successful, return @@ -181,12 +195,15 @@ sqlite3_backup *sqlite3_backup_init( p->iNext = 1; p->isAttached = 0; - if( 0==p->pSrc || 0==p->pDest || setDestPgsz(p)==SQLITE_NOMEM ){ + if( 0==p->pSrc || 0==p->pDest + || setDestPgsz(p)==SQLITE_NOMEM + || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK + ){ /* One (or both) of the named databases did not exist or an OOM - ** error was hit. The error has already been written into the - ** pDestDb handle. All that is left to do here is free the - ** sqlite3_backup structure. - */ + ** error was hit. Or there is a transaction open on the destination + ** database. The error has already been written into the pDestDb + ** handle. All that is left to do here is free the sqlite3_backup + ** structure. */ sqlite3_free(p); p = 0; } diff --git a/test/backup.test b/test/backup.test index 444619c68c..3b1e1db9e5 100644 --- a/test/backup.test +++ b/test/backup.test @@ -217,6 +217,7 @@ foreach nPagePerStep {1 200} { INSERT INTO ${file_dest}.t1 VALUES(1, randstr(1000,1000)) " $db_dest } + execsql COMMIT $db_dest } # Backup the source database. diff --git a/test/backup5.test b/test/backup5.test new file mode 100644 index 0000000000..c789adfa69 --- /dev/null +++ b/test/backup5.test @@ -0,0 +1,65 @@ +# 2014 November 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix backup5 + +forcedelete test2.db + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + INSERT INTO t2 VALUES(1, 1); + INSERT INTO t2 VALUES(2, 2); + INSERT INTO t2 VALUES(3, 3); +} + +do_test 1.1 { + forcecopy test.db test.db2 + db eval { + DROP TABLE t2; + INSERT INTO t1 VALUES(zeroblob(1000), zeroblob(1000)); + INSERT INTO t1 VALUES(randomblob(1000), randomblob(1000)); + } +} {} + +do_test 1.2 { + sqlite3 db2 test.db2 + set stmt [sqlite3_prepare_v2 db2 "SELECT * FROM t2" -1 dummy] + sqlite3_step $stmt +} {SQLITE_ROW} + +do_test 1.3 { + list [catch { sqlite3_backup B db2 main db main } msg] $msg +} {1 {sqlite3_backup_init() failed}} + +do_test 1.4 { + sqlite3_errmsg db2 +} {destination database is in use} + +do_test 1.5 { + sqlite3_reset $stmt + sqlite3_backup B db2 main db main + B step 200 + B finish +} {SQLITE_OK} + +do_test 1.6 { + list [sqlite3_step $stmt] [sqlite3_finalize $stmt] +} {SQLITE_ERROR SQLITE_ERROR} + +do_test 1.7 { + sqlite3_errmsg db2 +} {no such table: t2} + +finish_test From 8ac1a67eff83ed80083e94deb70a366cf1a9568e Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 13 Nov 2014 14:30:56 +0000 Subject: [PATCH 569/710] Modify the documentation for sqlite3_backup_init() to indicate that it will fail if there is already a read or read-write transaction open on the destination database. FossilOrigin-Name: ef03a203351a6002e2b1075139717e4234c816cd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 4f0d48c405..7a15358036 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\scalls\sto\ssqlite3_backup_init()\sfail\sif\sthere\sis\salready\sa\sread\sor\sread-write\stransaction\sopen\son\sthe\sdestination\sdatabase. -D 2014-11-13T14:18:25.531 +C Modify\sthe\sdocumentation\sfor\ssqlite3_backup_init()\sto\sindicate\sthat\sit\swill\sfail\sif\sthere\sis\salready\sa\sread\sor\sread-write\stransaction\sopen\son\sthe\sdestination\sdatabase. +D 2014-11-13T14:30:56.983 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd -F src/sqlite.h.in 0e6612f84936cca29166f2c66068e0227a13fdf6 +F src/sqlite.h.in 0c5c0df7e4e436dfc5592511325bf4a96f6a638d F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 71b0bf1a7fc55b5cb374f7579fd140e730a6e0f4 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eaf3aae014f59c8d37aa20aa31d54cf13f9e86fc -R cf416376a3614eef6301ba322bd3cc43 +P 169b5505498c0a7ee2b5dbb2ba13c41dfaa7c62f +R 91cc925fca98dfe2ae31ca6f9eebcf27 U dan -Z 499ccc085277a0250fd3c5f22e93f6be +Z 09c231773b0ef1121faa87d2329ad833 diff --git a/manifest.uuid b/manifest.uuid index 73dcaa8447..018b18d5f0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -169b5505498c0a7ee2b5dbb2ba13c41dfaa7c62f \ No newline at end of file +ef03a203351a6002e2b1075139717e4234c816cd \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 2812bf511d..4427f39d06 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -6848,6 +6848,10 @@ typedef struct sqlite3_backup sqlite3_backup; ** must be different or else sqlite3_backup_init(D,N,S,M) will fail with ** an error. ** +** ^A call to sqlite3_backup_init() will fail, returning SQLITE_ERROR, if +** there is already a read or read-write transaction open on the +** destination database. +** ** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is ** returned and an error code and error message are stored in the ** destination [database connection] D. From dd715f7c57f78f96fa00f8cb299d082f59ddbfb5 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Nov 2014 15:28:33 +0000 Subject: [PATCH 570/710] Do not automatically remove the DISTINCT keyword from "a IN (SELECT DISTINCT ...)" expressions. Fix for [db87229497]. FossilOrigin-Name: 55e453aadbb676dda07f0fa537d39ce184ef636c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 1 - test/in5.test | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 56 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 7a15358036..1b20e13d73 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\sdocumentation\sfor\ssqlite3_backup_init()\sto\sindicate\sthat\sit\swill\sfail\sif\sthere\sis\salready\sa\sread\sor\sread-write\stransaction\sopen\son\sthe\sdestination\sdatabase. -D 2014-11-13T14:30:56.983 +C Do\snot\sautomatically\sremove\sthe\sDISTINCT\skeyword\sfrom\s"a\sIN\s(SELECT\sDISTINCT\s...)"\sexpressions.\sFix\sfor\s[db87229497]. +D 2014-11-14T15:28:33.929 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -182,7 +182,7 @@ F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 93594514aae68de117ca4a2a0d6cc63eddf26744 F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 -F src/expr.c 0391a657df4959eaf2a2fd7d77de5ebe750686ee +F src/expr.c a3ff05db5709d628c23890db862e30f3dd9dc428 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee @@ -627,7 +627,7 @@ F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068 -F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 +F test/in5.test 1de657472fa9ac2924be25c2c959ac5ca1aae554 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 169b5505498c0a7ee2b5dbb2ba13c41dfaa7c62f -R 91cc925fca98dfe2ae31ca6f9eebcf27 +P ef03a203351a6002e2b1075139717e4234c816cd +R bb54553354aa14551393c5e5701542c8 U dan -Z 09c231773b0ef1121faa87d2329ad833 +Z f91e19e3f2d1e58cdccd86465c86ab70 diff --git a/manifest.uuid b/manifest.uuid index 018b18d5f0..c5dbef8e09 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef03a203351a6002e2b1075139717e4234c816cd \ No newline at end of file +55e453aadbb676dda07f0fa537d39ce184ef636c \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 13a9cb46fd..25f0be400f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1857,7 +1857,6 @@ int sqlite3CodeSubselect( assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); pSelect->iLimit = 0; testcase( pSelect->selFlags & SF_Distinct ); - pSelect->selFlags &= ~SF_Distinct; testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ if( sqlite3Select(pParse, pSelect, &dest) ){ sqlite3KeyInfoUnref(pKeyInfo); diff --git a/test/in5.test b/test/in5.test index 8a43b8d44a..67d212589d 100644 --- a/test/in5.test +++ b/test/in5.test @@ -12,6 +12,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix in5 do_test in5-1.1 { execsql { @@ -135,4 +136,51 @@ do_test in5-5.3 { }] } {0} +#------------------------------------------------------------------------- +# At one point SQLite was removing the DISTINCT keyword from expressions +# similar to: +# +# IN (SELECT DISTINCT FROM...) +# +# However, there are a few obscure cases where this is incorrect. For +# example, if the SELECT features a LIMIT clause, or if the collation +# sequence or affinity used by the DISTINCT does not match the one used +# by the IN(...) expression. +# +do_execsql_test 6.1.1 { + CREATE TABLE t1(a COLLATE nocase); + INSERT INTO t1 VALUES('one'); + INSERT INTO t1 VALUES('ONE'); +} +do_execsql_test 6.1.2 { + SELECT count(*) FROM t1 WHERE a COLLATE BINARY IN (SELECT DISTINCT a FROM t1) +} {1} + +do_execsql_test 6.2.1 { + CREATE TABLE t3(a, b); + INSERT INTO t3 VALUES(1, 1); + INSERT INTO t3 VALUES(1, 2); + INSERT INTO t3 VALUES(1, 3); + INSERT INTO t3 VALUES(2, 4); + INSERT INTO t3 VALUES(2, 5); + INSERT INTO t3 VALUES(2, 6); + INSERT INTO t3 VALUES(3, 7); + INSERT INTO t3 VALUES(3, 8); + INSERT INTO t3 VALUES(3, 9); +} +do_execsql_test 6.2.2 { + SELECT count(*) FROM t3 WHERE b IN (SELECT DISTINCT a FROM t3 LIMIT 5); +} {3} +do_execsql_test 6.2.3 { + SELECT count(*) FROM t3 WHERE b IN (SELECT a FROM t3 LIMIT 5); +} {2} + +do_execsql_test 6.3.1 { + CREATE TABLE x1(a); + CREATE TABLE x2(b); + INSERT INTO x1 VALUES(1), (1), (2); + INSERT INTO x2 VALUES(1), (2); + SELECT count(*) FROM x2 WHERE b IN (SELECT DISTINCT a FROM x1 LIMIT 2); +} {2} + finish_test From 0a9d9d566e24d05afcfdd15368eccb8683331ecc Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 14 Nov 2014 15:42:23 +0000 Subject: [PATCH 571/710] Do not automatically remove the DISTINCT keyword from "a IN (SELECT DISTINCT ...)" expressions. Fix for [db87229497]. FossilOrigin-Name: 98457a57d642b35917eb9ad8f70065e273aad206 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 1 - test/in5.test | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1d9fe4d78f..a2350554db 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sa\stransaction\sor\ssavepoint\srollback\soccurs,\ssave\sthe\spositions\sof\sall\sopen\sread-cursors\sso\sthat\sthey\scan\sbe\srestored\sfollowing\sthe\srollback\soperation.\s\sCherry-pick\sof\scheck-in\s[dd03a2802f3f27] -D 2014-11-13T13:42:39.738 +C Do\snot\sautomatically\sremove\sthe\sDISTINCT\skeyword\sfrom\s"a\sIN\s(SELECT\sDISTINCT\s...)"\sexpressions.\sFix\sfor\s[db87229497]. +D 2014-11-14T15:42:23.965 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -181,7 +181,7 @@ F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 F src/ctime.c bb434068b5308a857b181c2d204a320ff0d6c638 F src/date.c 57a7f9ba9f6b4d5268f5e411739066a611f99036 F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 -F src/expr.c fc204d08af06437ddaffe5a1b1f1f6f9e1a55d6d +F src/expr.c 1891cb50510a31e96de8a54579e7d3aef60f0094 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee @@ -616,7 +616,7 @@ F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068 -F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 +F test/in5.test 1de657472fa9ac2924be25c2c959ac5ca1aae554 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4 @@ -1205,8 +1205,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 839a6df9f98b90fb593534a62145d9c913540bae -Q +dd03a2802f3f276525f3cef9a93f825dd8606626 -R 0801523eb21d9763755a3afd3e84ae8b +P 402780a9c8df9e7ea898bdca49c1191042fe387a +Q +55e453aadbb676dda07f0fa537d39ce184ef636c +R 0bf0d6ff158ba2352191f711d5e9a856 U drh -Z 995ce8d43edc83cec464fe7495c36085 +Z fc11fd8e7149b148ee2e3386efe4bde7 diff --git a/manifest.uuid b/manifest.uuid index 38b0ca2922..8df489121d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -402780a9c8df9e7ea898bdca49c1191042fe387a \ No newline at end of file +98457a57d642b35917eb9ad8f70065e273aad206 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 1ad9a879a3..881cfd4344 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1836,7 +1836,6 @@ int sqlite3CodeSubselect( assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); pSelect->iLimit = 0; testcase( pSelect->selFlags & SF_Distinct ); - pSelect->selFlags &= ~SF_Distinct; testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ if( sqlite3Select(pParse, pSelect, &dest) ){ sqlite3KeyInfoUnref(pKeyInfo); diff --git a/test/in5.test b/test/in5.test index 8a43b8d44a..67d212589d 100644 --- a/test/in5.test +++ b/test/in5.test @@ -12,6 +12,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix in5 do_test in5-1.1 { execsql { @@ -135,4 +136,51 @@ do_test in5-5.3 { }] } {0} +#------------------------------------------------------------------------- +# At one point SQLite was removing the DISTINCT keyword from expressions +# similar to: +# +# IN (SELECT DISTINCT FROM...) +# +# However, there are a few obscure cases where this is incorrect. For +# example, if the SELECT features a LIMIT clause, or if the collation +# sequence or affinity used by the DISTINCT does not match the one used +# by the IN(...) expression. +# +do_execsql_test 6.1.1 { + CREATE TABLE t1(a COLLATE nocase); + INSERT INTO t1 VALUES('one'); + INSERT INTO t1 VALUES('ONE'); +} +do_execsql_test 6.1.2 { + SELECT count(*) FROM t1 WHERE a COLLATE BINARY IN (SELECT DISTINCT a FROM t1) +} {1} + +do_execsql_test 6.2.1 { + CREATE TABLE t3(a, b); + INSERT INTO t3 VALUES(1, 1); + INSERT INTO t3 VALUES(1, 2); + INSERT INTO t3 VALUES(1, 3); + INSERT INTO t3 VALUES(2, 4); + INSERT INTO t3 VALUES(2, 5); + INSERT INTO t3 VALUES(2, 6); + INSERT INTO t3 VALUES(3, 7); + INSERT INTO t3 VALUES(3, 8); + INSERT INTO t3 VALUES(3, 9); +} +do_execsql_test 6.2.2 { + SELECT count(*) FROM t3 WHERE b IN (SELECT DISTINCT a FROM t3 LIMIT 5); +} {3} +do_execsql_test 6.2.3 { + SELECT count(*) FROM t3 WHERE b IN (SELECT a FROM t3 LIMIT 5); +} {2} + +do_execsql_test 6.3.1 { + CREATE TABLE x1(a); + CREATE TABLE x2(b); + INSERT INTO x1 VALUES(1), (1), (2); + INSERT INTO x2 VALUES(1), (2); + SELECT count(*) FROM x2 WHERE b IN (SELECT DISTINCT a FROM x1 LIMIT 2); +} {2} + finish_test From 70273d0babf9b470539d443e8714a1ad947d197e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 14 Nov 2014 19:34:20 +0000 Subject: [PATCH 572/710] Consider using an automatic-index for a scan even if there exists a possible skip-scan that uses one or more "=" operators. FossilOrigin-Name: 93642a65ef3d53ece322ffd85233b68fc9a86c9d --- manifest | 17 ++++++++++------- manifest.uuid | 2 +- src/where.c | 3 ++- test/autoindex3.test | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1b20e13d73..48ea588135 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sautomatically\sremove\sthe\sDISTINCT\skeyword\sfrom\s"a\sIN\s(SELECT\sDISTINCT\s...)"\sexpressions.\sFix\sfor\s[db87229497]. -D 2014-11-14T15:28:33.929 +C Consider\susing\san\sautomatic-index\sfor\sa\sscan\seven\sif\sthere\sexists\sa\spossible\sskip-scan\sthat\suses\sone\sor\smore\s"="\soperators. +D 2014-11-14T19:34:20.967 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -304,7 +304,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c fa090966140602f03a621f87d82ee69e66ca63b5 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 3862a1173ae2716bde12f1ab3fb649f1d85b05c2 +F src/where.c 4d72a350fd5eec080583f95044f0f394590d13a3 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -347,7 +347,7 @@ F test/auth3.test 5cfa94ed90c6617c42b7ba4b133fd79678b251c7 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 F test/autoindex1.test 6ff78b94f43a59616c06c11c55b12935173506d7 F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de -F test/autoindex3.test 8254f689c3241081fad52b7bea18ba53e07e14a2 +F test/autoindex3.test a3be0d1a53a7d2edff208a5e442312957047e972 F test/autoindex4.test fc807f9efd158bec60f5dfdf34ebe46fb274612d F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 F test/autovacuum_ioerr2.test 8a367b224183ad801e0e24dcb7d1501f45f244b4 @@ -1221,7 +1221,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef03a203351a6002e2b1075139717e4234c816cd -R bb54553354aa14551393c5e5701542c8 +P 55e453aadbb676dda07f0fa537d39ce184ef636c +R 0a82574686b12650ce6ac2a5e463073b +T *branch * experimental-autoindex-fix +T *sym-experimental-autoindex-fix * +T -sym-trunk * U dan -Z f91e19e3f2d1e58cdccd86465c86ab70 +Z 13c044e2b6669432b7a0bb70427e69a9 diff --git a/manifest.uuid b/manifest.uuid index c5dbef8e09..769887276c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55e453aadbb676dda07f0fa537d39ce184ef636c \ No newline at end of file +93642a65ef3d53ece322ffd85233b68fc9a86c9d \ No newline at end of file diff --git a/src/where.c b/src/where.c index c3641c7cc7..2a06e3f479 100644 --- a/src/where.c +++ b/src/where.c @@ -4130,8 +4130,9 @@ static WhereLoop **whereLoopFindLesser( /* Any loop using an appliation-defined index (or PRIMARY KEY or ** UNIQUE constraint) with one or more == constraints is better - ** than an automatic index. */ + ** than an automatic index. Unless it is a skip-scan. */ if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 + && (pTemplate->nSkip)==0 && (pTemplate->wsFlags & WHERE_INDEXED)!=0 && (pTemplate->wsFlags & WHERE_COLUMN_EQ)!=0 && (p->prereq & pTemplate->prereq)==pTemplate->prereq diff --git a/test/autoindex3.test b/test/autoindex3.test index 33053bba67..c99a175c6d 100644 --- a/test/autoindex3.test +++ b/test/autoindex3.test @@ -17,6 +17,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix autoindex3 # The t1b and t2d indexes are not very selective. It used to be that # the autoindex mechanism would create automatic indexes on t1(b) or @@ -54,5 +55,38 @@ do_execsql_test autoindex3-140 { EXPLAIN QUERY PLAN SELECT * FROM t1, t2 WHERE d IN (5,b) AND x=y; } {/AUTO/} +reset_db +do_execsql_test 210 { + CREATE TABLE v(b, d, e); + CREATE TABLE u(a, b, c); + ANALYZE sqlite_master; + INSERT INTO "sqlite_stat1" VALUES('u','uab','40000 400 1'); + INSERT INTO "sqlite_stat1" VALUES('v','vbde','40000 400 1 1'); + INSERT INTO "sqlite_stat1" VALUES('v','ve','40000 21'); + + CREATE INDEX uab on u(a, b); + CREATE INDEX ve on v(e); + CREATE INDEX vbde on v(b,d,e); + + DROP TABLE IF EXISTS sqlite_stat4; + ANALYZE sqlite_master; +} + +# At one point, SQLite was using the inferior plan: +# +# 0|0|1|SEARCH TABLE v USING INDEX ve (e>?) +# 0|1|0|SEARCH TABLE u USING COVERING INDEX uab (ANY(a) AND b=?) +# +# on the basis that the real index "uab" must be better than the automatic +# index. This is not right - a skip-scan is not necessarily better than an +# automatic index scan. +# +do_eqp_test 220 { + select count(*) from u, v where u.b = v.b and v.e > 34; +} { + 0 0 1 {SEARCH TABLE v USING INDEX ve (e>?)} + 0 1 0 {SEARCH TABLE u USING AUTOMATIC COVERING INDEX (b=?)} +} + finish_test From f9df2fbdcd43eb7f223da88ce8cdcf70766bd945 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 15 Nov 2014 19:08:13 +0000 Subject: [PATCH 573/710] Adding the "noskipscan" token to an sqlite_stat1.stat field prevents an index for being used with the skip-scan algorithm. FossilOrigin-Name: 00fe09505792cd0d104b2da9d040f023e30fa871 --- manifest | 23 +++++++++++++---------- manifest.uuid | 2 +- src/analyze.c | 2 ++ src/sqliteInt.h | 1 + src/where.c | 1 + test/skipscan1.test | 19 +++++++++++++++++++ 6 files changed, 37 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 1b20e13d73..1c607eedd4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sautomatically\sremove\sthe\sDISTINCT\skeyword\sfrom\s"a\sIN\s(SELECT\sDISTINCT\s...)"\sexpressions.\sFix\sfor\s[db87229497]. -D 2014-11-14T15:28:33.929 +C Adding\sthe\s"noskipscan"\stoken\sto\san\ssqlite_stat1.stat\sfield\sprevents\san\nindex\sfor\sbeing\sused\swith\sthe\sskip-scan\salgorithm. +D 2014-11-15T19:08:13.305 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c afbcca663c3f3625340b8e30d440cd7a97ded6bc +F src/analyze.c 951fd859852dfbced4a58b73954f168eeb9d0772 F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea @@ -233,7 +233,7 @@ F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd F src/sqlite.h.in 0c5c0df7e4e436dfc5592511325bf4a96f6a638d F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 71b0bf1a7fc55b5cb374f7579fd140e730a6e0f4 +F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -304,7 +304,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c fa090966140602f03a621f87d82ee69e66ca63b5 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 3862a1173ae2716bde12f1ab3fb649f1d85b05c2 +F src/where.c cb89128d24ddb5fe3ca290166d66968f2dc0c3c8 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -855,7 +855,7 @@ F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/show_speedtest1_rtree.tcl 32e6c5f073d7426148a6936a0408f4b5b169aba5 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 -F test/skipscan1.test 7e15e1cc524524e7b2c4595ec85c75501d22f4ff +F test/skipscan1.test 2ddfe5d168462170c4487f534e2a99fb006b2076 F test/skipscan2.test d1d1450952b7275f0b0a3a981f0230532743951a F test/skipscan3.test ec5bab3f81c7038b43450e7b3062e04a198bdbb5 F test/skipscan5.test 67817a4b6857c47e0e33ba3e506da6f23ef68de2 @@ -1221,7 +1221,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef03a203351a6002e2b1075139717e4234c816cd -R bb54553354aa14551393c5e5701542c8 -U dan -Z f91e19e3f2d1e58cdccd86465c86ab70 +P 55e453aadbb676dda07f0fa537d39ce184ef636c +R 59717354988087765927b6c6b6246253 +T *branch * noskipscan-token +T *sym-noskipscan-token * +T -sym-trunk * +U drh +Z 9d226c4fa6bf82a1f1a22a271191f33c diff --git a/manifest.uuid b/manifest.uuid index c5dbef8e09..1ab180da36 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55e453aadbb676dda07f0fa537d39ce184ef636c \ No newline at end of file +00fe09505792cd0d104b2da9d040f023e30fa871 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 597885237c..769a16565a 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1466,6 +1466,8 @@ static void decodeIntArray( pIndex->bUnordered = 1; }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + }else if( sqlite3_strglob("noskipscan*", z)==0 ){ + pIndex->noSkipScan = 1; } #ifdef SQLITE_ENABLE_COSTMULT else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 4d272b06a1..f3d6ce015e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1795,6 +1795,7 @@ struct Index { unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ + unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ diff --git a/src/where.c b/src/where.c index c3641c7cc7..db6cf94dea 100644 --- a/src/where.c +++ b/src/where.c @@ -4587,6 +4587,7 @@ static int whereLoopAddBtreeIndex( assert( 42==sqlite3LogEst(18) ); if( saved_nEq==saved_nSkip && saved_nEq+1nKeyCol + && pProbe->noSkipScan==0 && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ diff --git a/test/skipscan1.test b/test/skipscan1.test index 6b9f1209a5..4f996df972 100644 --- a/test/skipscan1.test +++ b/test/skipscan1.test @@ -273,4 +273,23 @@ do_execsql_test skipscan1-6.3 { EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; } {~/ANY/} +# If the sqlite_stat1 entry includes the "noskipscan" token, then never use +# skipscan with that index. +# +do_execsql_test skipscan1-7.1 { + UPDATE sqlite_stat1 SET stat='500000 125000 1 sz=100'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {/ANY/} +do_execsql_test skipscan1-7.2 { + UPDATE sqlite_stat1 SET stat='500000 125000 1 noskipscan sz=100'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {~/ANY/} +do_execsql_test skipscan1-7.3 { + UPDATE sqlite_stat1 SET stat='500000 125000 1 sz=100 noskipscan'; + ANALYZE sqlite_master; + EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE b=1; +} {~/ANY/} + finish_test From 302f583aa4dc6a77ca9042dca3143a3bf8a134f0 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Nov 2014 15:22:08 +0000 Subject: [PATCH 574/710] Update a couple of test cases to account for the fact that ROLLBACK does not always abort all running SELECT statements. FossilOrigin-Name: eba171e980fa4491dfee9d7e4df50c87a0ebbf87 --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- test/ioerr2.test | 8 +++++++- test/shared_err.test | 8 +++++++- 4 files changed, 23 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index a2350554db..27ecc3c872 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sautomatically\sremove\sthe\sDISTINCT\skeyword\sfrom\s"a\sIN\s(SELECT\sDISTINCT\s...)"\sexpressions.\sFix\sfor\s[db87229497]. -D 2014-11-14T15:42:23.965 +C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sROLLBACK\sdoes\snot\salways\sabort\sall\srunning\sSELECT\sstatements. +D 2014-11-17T15:22:08.993 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -648,7 +648,7 @@ F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc F test/intpkey.test 7506090fc08e028712a8bf47e5f54111947e3844 F test/io.test 3a7abcef18727cc0f2399e04b0e8903eccae50f8 F test/ioerr.test 2a24bd6ed5a8b062e64bfe1f6cf94fb25e92210d -F test/ioerr2.test 9d71166f8466eda510f1af6137bdabaa82b5408d +F test/ioerr2.test 2593563599e2cc6b6b4fcf5878b177bdd5d8df26 F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 @@ -830,7 +830,7 @@ F test/shared7.test a81e99f83e6c51b02ac99c96fb3a2a7b5978c956 F test/shared8.test 00a07bf5e1337ecf72e94542bdefdc330d7a2538 F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21 F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 -F test/shared_err.test 0079c05c97d88cfa03989b7c20a8b266983087aa +F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test d60946b5fde4d85fe06db7331dfe89011f564350 F test/shell2.test c57da3a381c099b02c813ba156298d5c2f5c93a3 @@ -1205,8 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 402780a9c8df9e7ea898bdca49c1191042fe387a -Q +55e453aadbb676dda07f0fa537d39ce184ef636c -R 0bf0d6ff158ba2352191f711d5e9a856 -U drh -Z fc11fd8e7149b148ee2e3386efe4bde7 +P 98457a57d642b35917eb9ad8f70065e273aad206 +R d8acddabccfe794436f7272593be21d9 +U dan +Z 30deb4af13a2947d821f81a6f506118a diff --git a/manifest.uuid b/manifest.uuid index 8df489121d..bf9ab0f15d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98457a57d642b35917eb9ad8f70065e273aad206 \ No newline at end of file +eba171e980fa4491dfee9d7e4df50c87a0ebbf87 \ No newline at end of file diff --git a/test/ioerr2.test b/test/ioerr2.test index 5150ace3ab..c08c3453f3 100644 --- a/test/ioerr2.test +++ b/test/ioerr2.test @@ -112,6 +112,12 @@ foreach bPersist [list 0 1] { } } +# When this test was written, an IO error within the UPDATE statement caused +# a rollback, which tripped all read-cursors, causing the outer SELECT to +# fail with "abort due to ROLLBACK". Now, the loop continues until the UPDATE +# is run successfully. At this point the next IO error occurs within the +# SELECT - throwing the "disk I/O error" that the test case now expects. +# do_test ioerr2-5 { execsql { CREATE TABLE t2 AS SELECT * FROM t1; @@ -130,7 +136,7 @@ do_test ioerr2-5 { } } msg] list $rc $msg -} {1 {abort due to ROLLBACK}} +} {1 {disk I/O error}} ;# used to be "{1 {abort due to ROLLBACK}}" if {$::tcl_platform(platform) == "unix"} { # Cause the call to xAccess used by [pragma temp_store_directory] to diff --git a/test/shared_err.test b/test/shared_err.test index 17add94bb7..96e5ee4540 100644 --- a/test/shared_err.test +++ b/test/shared_err.test @@ -446,9 +446,15 @@ do_malloc_test shared_err-8 -tclprep { } {1} db2 close } + +# When this test case was written, OOM errors in write statements would +# cause transaction rollback, which would trip cursors in other statements, +# aborting them. This no longer happens. +# do_test shared_malloc-8.X { # Test that one or more queries were aborted due to the malloc() failure. - expr $::aborted>=1 + # expr $::aborted>=1 + expr $::aborted==0 } {1} # This test is designed to catch a specific bug that was present during From 13835c41e824eb2b6a7d504c4df2066fe4ad6c52 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Nov 2014 15:32:47 +0000 Subject: [PATCH 575/710] Fix a bug in the sqlite3TripAllCursors() routine that prevents it from reporting errors. It is unknown at this time whether or not this omission can result in any incorrect result in an actual query. FossilOrigin-Name: 42588207ff5451cb785c394633e1ab631fb82f01 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 27ecc3c872..a0d49916be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sROLLBACK\sdoes\snot\salways\sabort\sall\srunning\sSELECT\sstatements. -D 2014-11-17T15:22:08.993 +C Fix\sa\sbug\sin\sthe\ssqlite3TripAllCursors()\sroutine\sthat\sprevents\sit\sfrom\nreporting\serrors.\s\sIt\sis\sunknown\sat\sthis\stime\swhether\sor\snot\sthis\somission\scan\nresult\sin\sany\sincorrect\sresult\sin\san\sactual\squery. +D 2014-11-17T15:32:47.066 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c 8cdfeb0c8a6d8bdad3faefae418eb3dc767051b6 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 2c15850c5c9a26b10cdf92f9a29c74e299dc3674 +F src/btree.c 2087125172421c78fc0c02f953b7d660f2a2598e F src/btree.h a4afc6b06f5a1dd2076d15aa168baec44fc0121b F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 98457a57d642b35917eb9ad8f70065e273aad206 -R d8acddabccfe794436f7272593be21d9 -U dan -Z 30deb4af13a2947d821f81a6f506118a +P eba171e980fa4491dfee9d7e4df50c87a0ebbf87 +R c97ad964463de041aa0a8f758aac4ee1 +U drh +Z ed993b1ef595900db9792d9ac234a3c7 diff --git a/manifest.uuid b/manifest.uuid index bf9ab0f15d..7891ecfbd0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eba171e980fa4491dfee9d7e4df50c87a0ebbf87 \ No newline at end of file +42588207ff5451cb785c394633e1ab631fb82f01 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 41e097af53..111870548b 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3499,7 +3499,7 @@ int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ int i; if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ if( p->eState==CURSOR_VALID ){ - int rc = saveCursorPosition(p); + rc = saveCursorPosition(p); if( rc!=SQLITE_OK ){ (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); break; From 77b1deec400c40676dfca2986dbf659af548c40f Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Nov 2014 17:13:06 +0000 Subject: [PATCH 576/710] When a SELECT statement is terminated by a ROLLBACK TO operation, make the error message be "abort due to ROLLBACK" rather than "callback requested query abort". FossilOrigin-Name: 34fc4a082c192830e48f643549c04a4f91912b8b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 3 ++- test/savepoint7.test | 4 ++-- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index a0d49916be..93ef894b6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\sthe\ssqlite3TripAllCursors()\sroutine\sthat\sprevents\sit\sfrom\nreporting\serrors.\s\sIt\sis\sunknown\sat\sthis\stime\swhether\sor\snot\sthis\somission\scan\nresult\sin\sany\sincorrect\sresult\sin\san\sactual\squery. -D 2014-11-17T15:32:47.066 +C When\sa\sSELECT\sstatement\sis\sterminated\sby\sa\sROLLBACK\sTO\soperation,\smake\sthe\nerror\smessage\sbe\s"abort\sdue\sto\sROLLBACK"\srather\sthan\s\n"callback\srequested\squery\sabort". +D 2014-11-17T17:13:06.964 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c a6b604364c7cbb079c083418e7359d1d665f2ef0 +F src/vdbe.c 3e8f9787d9a5fe24077f7af0baa2d11042471f12 F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -797,7 +797,7 @@ F test/savepoint3.test e328085853b14898d78ceea00dfe7db18bb6a9ec F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 -F test/savepoint7.test 1c8f26b1e2a4221b0214e222ce12a97a59918eb2 +F test/savepoint7.test db3db281486c925095f305aad09fe806e5188ff3 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eba171e980fa4491dfee9d7e4df50c87a0ebbf87 -R c97ad964463de041aa0a8f758aac4ee1 +P 42588207ff5451cb785c394633e1ab631fb82f01 +R dfbcaabb48c2f5c435e2c7179a62bd27 U drh -Z ed993b1ef595900db9792d9ac234a3c7 +Z c435b7b174ccd5699f4e15dc48114455 diff --git a/manifest.uuid b/manifest.uuid index 7891ecfbd0..d4b77b5f9f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42588207ff5451cb785c394633e1ab631fb82f01 \ No newline at end of file +34fc4a082c192830e48f643549c04a4f91912b8b \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 1ad8aab753..9a8db1a638 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2827,7 +2827,8 @@ case OP_Savepoint: { if( p1==SAVEPOINT_ROLLBACK ){ isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; for(ii=0; iinDb; ii++){ - rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT, + rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, + SQLITE_ABORT_ROLLBACK, isSchemaChange==0); if( rc!=SQLITE_OK ) goto abort_due_to_error; } diff --git a/test/savepoint7.test b/test/savepoint7.test index 908ec571f5..d8a02f1f80 100644 --- a/test/savepoint7.test +++ b/test/savepoint7.test @@ -78,7 +78,7 @@ do_test savepoint7-2.1 { } msg] db eval {RELEASE x1} list $rc $msg [db eval {SELECT * FROM t2}] -} {1 {callback requested query abort} {}} +} {1 {abort due to ROLLBACK} {}} do_test savepoint7-2.2 { db eval {DELETE FROM t2;} @@ -93,6 +93,6 @@ do_test savepoint7-2.2 { } } msg] list $rc $msg [db eval {SELECT * FROM t2}] -} {1 {callback requested query abort} {}} +} {1 {abort due to ROLLBACK} {}} finish_test From d816e00d13e0f1bb4704086760e0a394a61e9981 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Nov 2014 19:25:15 +0000 Subject: [PATCH 577/710] Improved comments on the BtCursor.skipNext field. No changes to code. FossilOrigin-Name: e956e7db057d1112badf5e0671cea95201385b44 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btreeInt.h | 10 ++++++++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 93ef894b6e..d851cc2e87 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sa\sSELECT\sstatement\sis\sterminated\sby\sa\sROLLBACK\sTO\soperation,\smake\sthe\nerror\smessage\sbe\s"abort\sdue\sto\sROLLBACK"\srather\sthan\s\n"callback\srequested\squery\sabort". -D 2014-11-17T17:13:06.964 +C Improved\scomments\son\sthe\sBtCursor.skipNext\sfield.\s\sNo\schanges\sto\scode. +D 2014-11-17T19:25:15.092 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,7 +174,7 @@ F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c 2087125172421c78fc0c02f953b7d660f2a2598e F src/btree.h a4afc6b06f5a1dd2076d15aa168baec44fc0121b -F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 +F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 535183afb3c75628b78ce82612931ac7cdf26f14 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 42588207ff5451cb785c394633e1ab631fb82f01 -R dfbcaabb48c2f5c435e2c7179a62bd27 +P 34fc4a082c192830e48f643549c04a4f91912b8b +R b667da582f38b5593e8a936d8e9b63b7 U drh -Z c435b7b174ccd5699f4e15dc48114455 +Z 1e479ba79d4cbfb8037d6ba7f0cc99ff diff --git a/manifest.uuid b/manifest.uuid index d4b77b5f9f..a049e1fe86 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -34fc4a082c192830e48f643549c04a4f91912b8b \ No newline at end of file +e956e7db057d1112badf5e0671cea95201385b44 \ No newline at end of file diff --git a/src/btreeInt.h b/src/btreeInt.h index 2368e6c884..a28a6a297e 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -489,6 +489,11 @@ struct CellInfo { ** ** Fields in this structure are accessed under the BtShared.mutex ** found at self->pBt->mutex. +** +** skipNext meaning: +** eState==SKIPNEXT && skipNext>0: Next sqlite3BtreeNext() is no-op. +** eState==SKIPNEXT && skipNext<0: Next sqlite3BtreePrevious() is no-op. +** eState==FAULT: Cursor fault with skipNext as error code. */ struct BtCursor { Btree *pBtree; /* The Btree to which this cursor belongs */ @@ -501,7 +506,8 @@ struct BtCursor { void *pKey; /* Saved key that was cursor last known position */ Pgno pgnoRoot; /* The root page of this tree */ int nOvflAlloc; /* Allocated size of aOverflow[] array */ - int skipNext; /* Prev() is noop if negative. Next() is noop if positive */ + int skipNext; /* Prev() is noop if negative. Next() is noop if positive. + ** Error code if eState==CURSOR_FAULT */ u8 curFlags; /* zero or more BTCF_* flags defined below */ u8 eState; /* One of the CURSOR_XXX constants (see below) */ u8 hints; /* As configured by CursorSetHints() */ @@ -547,7 +553,7 @@ struct BtCursor { ** on a different connection that shares the BtShared cache with this ** cursor. The error has left the cache in an inconsistent state. ** Do nothing else with this cursor. Any attempt to use the cursor -** should return the error code stored in BtCursor.skip +** should return the error code stored in BtCursor.skipNext */ #define CURSOR_INVALID 0 #define CURSOR_VALID 1 From 2b8669a9da3296ce0ca99da34f9b158ee1eb4d1e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Nov 2014 19:42:48 +0000 Subject: [PATCH 578/710] Avoid calling sqlite3BtreeKeysize() on a b-tree cursor in SKIPNEXT or SKIPPREV state. FossilOrigin-Name: 54e7d3fcb1ab21c03ffef1af93ae029a2901098a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 4 ++++ test/misc8.test | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1b20e13d73..12a819fd2f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\sautomatically\sremove\sthe\sDISTINCT\skeyword\sfrom\s"a\sIN\s(SELECT\sDISTINCT\s...)"\sexpressions.\sFix\sfor\s[db87229497]. -D 2014-11-14T15:28:33.929 +C Avoid\scalling\ssqlite3BtreeKeysize()\son\sa\sb-tree\scursor\sin\sSKIPNEXT\sor\sSKIPPREV\sstate. +D 2014-11-17T19:42:48.262 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c 5e47308836e9bb5fdb4835fdf88eeab071848d3f +F src/vdbe.c 5d3991d723f00ef86263f4d494e105faba5a5abd F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -734,7 +734,7 @@ F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 -F test/misc8.test e838ec20c9c988bc94812fdb89af26409c20931b +F test/misc8.test fc2754d38892f7dac30c22db3616c2764f117d66 F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 F test/mmap1.test 1bfd611b9841eafb44f7d83c0788e146d84a33c9 F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef03a203351a6002e2b1075139717e4234c816cd -R bb54553354aa14551393c5e5701542c8 +P 55e453aadbb676dda07f0fa537d39ce184ef636c +R 6cd25f6196e6f786753b0fb88328cee1 U dan -Z f91e19e3f2d1e58cdccd86465c86ab70 +Z f594bfe54166ed1c0f3ae7a5a057ba04 diff --git a/manifest.uuid b/manifest.uuid index c5dbef8e09..e3e77af8a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -55e453aadbb676dda07f0fa537d39ce184ef636c \ No newline at end of file +54e7d3fcb1ab21c03ffef1af93ae029a2901098a \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index e4cbeb6565..78c5511e5a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4408,6 +4408,10 @@ case OP_Rowid: { /* out2-prerelease */ assert( pC->pCursor!=0 ); rc = sqlite3VdbeCursorRestore(pC); if( rc ) goto abort_due_to_error; + if( pC->nullRow ){ + pOut->flags = MEM_Null; + break; + } rc = sqlite3BtreeKeySize(pC->pCursor, &v); assert( rc==SQLITE_OK ); /* Always so because of CursorRestore() above */ } diff --git a/test/misc8.test b/test/misc8.test index 8c0c126a61..3ff52e56f1 100644 --- a/test/misc8.test +++ b/test/misc8.test @@ -58,4 +58,41 @@ do_catchsql_test misc8-1.7 { } {1 {abort due to ROLLBACK}} +reset_db + +proc dbeval {sql} { db eval $sql } +db func eval dbeval + +do_execsql_test misc8-2.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b INTEGER) WITHOUT ROWID; + CREATE TABLE t2(c INTEGER PRIMARY KEY, d INTEGER, x BLOB); + INSERT INTO t1 VALUES(0,0); + INSERT INTO t1 VALUES(10,10); + INSERT INTO t2 VALUES(1,1,zeroblob(200)); + INSERT INTO t2 VALUES(2,2,zeroblob(200)); + INSERT INTO t2 VALUES(3,3,zeroblob(200)); + INSERT INTO t2 VALUES(4,4,zeroblob(200)); + INSERT INTO t2 VALUES(5,5,zeroblob(200)); + INSERT INTO t2 VALUES(6,6,zeroblob(200)); + INSERT INTO t2 VALUES(7,7,zeroblob(200)); + INSERT INTO t2 VALUES(8,8,zeroblob(200)); + INSERT INTO t2 VALUES(9,9,zeroblob(200)); + INSERT INTO t2 VALUES(10,10,zeroblob(200)); + SELECT a, c, eval( + printf('DELETE FROM t2 WHERE c=%d AND %d>5', a+c, a+c) + ) FROM t1, t2; +} { + 0 1 {} 10 1 {} + 0 2 {} 10 2 {} + 0 3 {} 10 3 {} + 0 4 {} 10 4 {} + 0 5 {} 10 5 {} + 0 6 {} 10 {} {} + 0 7 {} 10 {} {} + 0 8 {} 10 {} {} + 0 9 {} 10 {} {} + 0 10 {} 10 {} {} +} + + finish_test From 756d3b3e9d597a3bd04666f4909f43b4da07965e Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 17 Nov 2014 19:44:44 +0000 Subject: [PATCH 579/710] Avoid calling sqlite3BtreeKeysize() on a b-tree cursor in SKIPNEXT or SKIPPREV state. FossilOrigin-Name: 2f2ecb994889acb783616acb7307f8fed962d213 --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/vdbe.c | 4 ++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d851cc2e87..55c07fe240 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improved\scomments\son\sthe\sBtCursor.skipNext\sfield.\s\sNo\schanges\sto\scode. -D 2014-11-17T19:25:15.092 +C Avoid\scalling\ssqlite3BtreeKeysize()\son\sa\sb-tree\scursor\sin\sSKIPNEXT\sor\sSKIPPREV\sstate. +D 2014-11-17T19:44:44.917 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -289,7 +289,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 4006c01772bd8d8ac4306d523bbcee41d3e392d8 F src/vacuum.c 59f03f92bcff57faa6a8ca256eb29ccddfb0614a -F src/vdbe.c 3e8f9787d9a5fe24077f7af0baa2d11042471f12 +F src/vdbe.c 8bc291aa00646d07dab33047520960ea454c5a2f F src/vdbe.h 09f5b4e3719fa454f252322b1cdab5cf1f361327 F src/vdbeInt.h e2a060a55ee18a6ab973353a5e2ec7ee569bf787 F src/vdbeapi.c 37a6c6ae284a97bcace365f2f0a225680c0499d9 @@ -1205,7 +1205,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 34fc4a082c192830e48f643549c04a4f91912b8b -R b667da582f38b5593e8a936d8e9b63b7 -U drh -Z 1e479ba79d4cbfb8037d6ba7f0cc99ff +P e956e7db057d1112badf5e0671cea95201385b44 +Q +54e7d3fcb1ab21c03ffef1af93ae029a2901098a +R b1492d8f4cbcc74596556f240b662de2 +U dan +Z 45bbf8c9451b95c29c15305e9bbd4586 diff --git a/manifest.uuid b/manifest.uuid index a049e1fe86..bec82c6bb3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e956e7db057d1112badf5e0671cea95201385b44 \ No newline at end of file +2f2ecb994889acb783616acb7307f8fed962d213 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 9a8db1a638..366c7a0166 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4405,6 +4405,10 @@ case OP_Rowid: { /* out2-prerelease */ assert( pC->pCursor!=0 ); rc = sqlite3VdbeCursorRestore(pC); if( rc ) goto abort_due_to_error; + if( pC->nullRow ){ + pOut->flags = MEM_Null; + break; + } rc = sqlite3BtreeKeySize(pC->pCursor, &v); assert( rc==SQLITE_OK ); /* Always so because of CursorRestore() above */ } From c5352b996a590c7a9ab8233545c3cb40d3ce125c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Nov 2014 20:33:07 +0000 Subject: [PATCH 580/710] Remove code from sqlite3BtreeKeySize() made unreachable by the previous check-in. FossilOrigin-Name: 57c4aa988c8eda3cc513c1e5df5804d88bee99a0 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/btree.c | 10 +++------- 3 files changed, 11 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 55c07fe240..d98c6b6486 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scalling\ssqlite3BtreeKeysize()\son\sa\sb-tree\scursor\sin\sSKIPNEXT\sor\sSKIPPREV\sstate. -D 2014-11-17T19:44:44.917 +C Remove\scode\sfrom\ssqlite3BtreeKeySize()\smade\sunreachable\sby\sthe\sprevious\ncheck-in. +D 2014-11-17T20:33:07.735 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/auth.c d8abcde53426275dab6243b441256fcd8ccbebb2 F src/backup.c 8cdfeb0c8a6d8bdad3faefae418eb3dc767051b6 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 2087125172421c78fc0c02f953b7d660f2a2598e +F src/btree.c c961588f01bd95d37b90359220c640f9763a3f58 F src/btree.h a4afc6b06f5a1dd2076d15aa168baec44fc0121b F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 9dc2bd94347b878c89627000c92b0c8d97ec2919 @@ -1205,8 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e956e7db057d1112badf5e0671cea95201385b44 -Q +54e7d3fcb1ab21c03ffef1af93ae029a2901098a -R b1492d8f4cbcc74596556f240b662de2 -U dan -Z 45bbf8c9451b95c29c15305e9bbd4586 +P 2f2ecb994889acb783616acb7307f8fed962d213 +R 3350d2f69701d38a07608c1007a483bc +U drh +Z df1a082a18c57f32c53583132171df67 diff --git a/manifest.uuid b/manifest.uuid index bec82c6bb3..2223a6ffff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2f2ecb994889acb783616acb7307f8fed962d213 \ No newline at end of file +57c4aa988c8eda3cc513c1e5df5804d88bee99a0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 111870548b..7ea66e0d3b 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3883,13 +3883,9 @@ int sqlite3BtreeCursorIsValid(BtCursor *pCur){ */ int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_INVALID || pCur->eState==CURSOR_VALID ); - if( pCur->eState!=CURSOR_VALID ){ - *pSize = 0; - }else{ - getCellInfo(pCur); - *pSize = pCur->info.nKey; - } + assert( pCur->eState==CURSOR_VALID ); + getCellInfo(pCur); + *pSize = pCur->info.nKey; return SQLITE_OK; } From 4429c20b866e038b155d17a347686abc6102d455 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Nov 2014 02:44:32 +0000 Subject: [PATCH 581/710] Add an ALWAYS() to an always-true conditional in the WAL rollback logic. FossilOrigin-Name: c5eae8a60d474131fbfa4d0c2b459005267e8be4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index d98c6b6486..e7e986f7fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\scode\sfrom\ssqlite3BtreeKeySize()\smade\sunreachable\sby\sthe\sprevious\ncheck-in. -D 2014-11-17T20:33:07.735 +C Add\san\sALWAYS()\sto\san\salways-true\sconditional\sin\sthe\sWAL\srollback\slogic. +D 2014-11-18T02:44:32.636 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -299,7 +299,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 975aeffa99acb0991b2f288d30294756bff41438 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c cb0c194303fea276b48d7d4b6d970b5a96bde8de -F src/wal.c 73051f1222321712fa4280c495780ba81d302dad +F src/wal.c 095d41f7114d7a8699207f5313488aa88372d540 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c 2947912f1f3d6a7766fe087fd532a5d688d745b1 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2f2ecb994889acb783616acb7307f8fed962d213 -R 3350d2f69701d38a07608c1007a483bc +P 57c4aa988c8eda3cc513c1e5df5804d88bee99a0 +R f70435983903607c251f8514c7f3013d U drh -Z df1a082a18c57f32c53583132171df67 +Z f0ae269a66dc8c4b111d5661661171ab diff --git a/manifest.uuid b/manifest.uuid index 2223a6ffff..3ceeab2cdb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -57c4aa988c8eda3cc513c1e5df5804d88bee99a0 \ No newline at end of file +c5eae8a60d474131fbfa4d0c2b459005267e8be4 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 24540a2cd5..d134a8b52a 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2506,7 +2506,7 @@ int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *pUndoCtx){ memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); for(iFrame=pWal->hdr.mxFrame+1; - rc==SQLITE_OK && iFrame<=iMax; + ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; iFrame++ ){ /* This call cannot fail. Unless the page for which the page number From bb8f92529401b09dbc6e9df4ba3e38ae3a0789f3 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Nov 2014 12:28:52 +0000 Subject: [PATCH 582/710] Increment the version number to 3.8.7.2 FossilOrigin-Name: 945a9e687fdfee5f7103d85d131024e85d594ac3 --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index f53c1ed56b..6df6debe9d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.7.1 +3.8.7.2 diff --git a/configure b/configure index d253f21a7d..1aedcaf4d3 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.7.1. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.7.2. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.7.1' -PACKAGE_STRING='sqlite 3.8.7.1' +PACKAGE_VERSION='3.8.7.2' +PACKAGE_STRING='sqlite 3.8.7.2' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.7.1 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.7.2 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.7.1:";; + short | recursive ) echo "Configuration of sqlite 3.8.7.2:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.7.1 +sqlite configure 3.8.7.2 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.7.1, which was +It was created by sqlite $as_me 3.8.7.2, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.7.1, which was +This file was extended by sqlite $as_me 3.8.7.2, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.7.1 +sqlite config.status 3.8.7.2 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index e7e986f7fe..00bcf54505 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Add\san\sALWAYS()\sto\san\salways-true\sconditional\sin\sthe\sWAL\srollback\slogic. -D 2014-11-18T02:44:32.636 +C Increment\sthe\sversion\snumber\sto\s3.8.7.2 +D 2014-11-18T12:28:52.482 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in cf57f673d77606ab0f2d9627ca52a9ba1464146a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc e31dee24038965fb6269d6d61073fd6b7e331dec F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION 5cc0baaee7eeee3238f0f7b5871398d17f79d0cd +F VERSION 3978bf46d1599bc324ae171a99c4e8fca7481822 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 56fe985cf0e59cd594f9b929099d0be40260e667 x +F configure 135207dac9b9ff35a91cdb17871322c26fa5de73 x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1205,7 +1205,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 57c4aa988c8eda3cc513c1e5df5804d88bee99a0 -R f70435983903607c251f8514c7f3013d +P c5eae8a60d474131fbfa4d0c2b459005267e8be4 +R 876a331fea8d7e2c7f47a110195c3ecb U drh -Z f0ae269a66dc8c4b111d5661661171ab +Z 48596521afe0080c14cc5d1c64dcfe43 diff --git a/manifest.uuid b/manifest.uuid index 3ceeab2cdb..a62a0bd661 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5eae8a60d474131fbfa4d0c2b459005267e8be4 \ No newline at end of file +945a9e687fdfee5f7103d85d131024e85d594ac3 \ No newline at end of file From 58f95c43adb7e26452e3136f55ee40a12c479d7d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Nov 2014 20:16:27 +0000 Subject: [PATCH 583/710] Update a couple of test cases to account for the fact that ROLLBACK does not always abort all running SELECT statements. FossilOrigin-Name: abccda769a3f6b755c3bf70b5fb31a5e16718ef3 --- manifest | 17 +++++++++-------- manifest.uuid | 2 +- test/ioerr2.test | 8 +++++++- test/shared_err.test | 8 +++++++- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 12a819fd2f..267f0a0da9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\scalling\ssqlite3BtreeKeysize()\son\sa\sb-tree\scursor\sin\sSKIPNEXT\sor\sSKIPPREV\sstate. -D 2014-11-17T19:42:48.262 +C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sROLLBACK\sdoes\snot\salways\sabort\sall\srunning\sSELECT\sstatements. +D 2014-11-18T20:16:27.742 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -659,7 +659,7 @@ F test/interrupt.test dfe9a67a94b0b2d8f70545ba1a6cca10780d71cc F test/intpkey.test 7506090fc08e028712a8bf47e5f54111947e3844 F test/io.test 3a7abcef18727cc0f2399e04b0e8903eccae50f8 F test/ioerr.test 2a24bd6ed5a8b062e64bfe1f6cf94fb25e92210d -F test/ioerr2.test 9d71166f8466eda510f1af6137bdabaa82b5408d +F test/ioerr2.test 2593563599e2cc6b6b4fcf5878b177bdd5d8df26 F test/ioerr3.test d3cec5e1a11ad6d27527d0d38573fbff14c71bdd F test/ioerr4.test f130fe9e71008577b342b8874d52984bd04ede2c F test/ioerr5.test 2edfa4fb0f896f733071303b42224df8bedd9da4 @@ -844,7 +844,7 @@ F test/shared7.test a81e99f83e6c51b02ac99c96fb3a2a7b5978c956 F test/shared8.test 00a07bf5e1337ecf72e94542bdefdc330d7a2538 F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21 F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 -F test/shared_err.test 0079c05c97d88cfa03989b7c20a8b266983087aa +F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test d60946b5fde4d85fe06db7331dfe89011f564350 F test/shell2.test c57da3a381c099b02c813ba156298d5c2f5c93a3 @@ -1221,7 +1221,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 55e453aadbb676dda07f0fa537d39ce184ef636c -R 6cd25f6196e6f786753b0fb88328cee1 -U dan -Z f594bfe54166ed1c0f3ae7a5a057ba04 +P 54e7d3fcb1ab21c03ffef1af93ae029a2901098a +Q +eba171e980fa4491dfee9d7e4df50c87a0ebbf87 +R 07c45ea3d962860999af22ae8c2af8c8 +U drh +Z f4bad3f3683413628206e5a23517cfd8 diff --git a/manifest.uuid b/manifest.uuid index e3e77af8a0..22da5e0eba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54e7d3fcb1ab21c03ffef1af93ae029a2901098a \ No newline at end of file +abccda769a3f6b755c3bf70b5fb31a5e16718ef3 \ No newline at end of file diff --git a/test/ioerr2.test b/test/ioerr2.test index 5150ace3ab..c08c3453f3 100644 --- a/test/ioerr2.test +++ b/test/ioerr2.test @@ -112,6 +112,12 @@ foreach bPersist [list 0 1] { } } +# When this test was written, an IO error within the UPDATE statement caused +# a rollback, which tripped all read-cursors, causing the outer SELECT to +# fail with "abort due to ROLLBACK". Now, the loop continues until the UPDATE +# is run successfully. At this point the next IO error occurs within the +# SELECT - throwing the "disk I/O error" that the test case now expects. +# do_test ioerr2-5 { execsql { CREATE TABLE t2 AS SELECT * FROM t1; @@ -130,7 +136,7 @@ do_test ioerr2-5 { } } msg] list $rc $msg -} {1 {abort due to ROLLBACK}} +} {1 {disk I/O error}} ;# used to be "{1 {abort due to ROLLBACK}}" if {$::tcl_platform(platform) == "unix"} { # Cause the call to xAccess used by [pragma temp_store_directory] to diff --git a/test/shared_err.test b/test/shared_err.test index 17add94bb7..96e5ee4540 100644 --- a/test/shared_err.test +++ b/test/shared_err.test @@ -446,9 +446,15 @@ do_malloc_test shared_err-8 -tclprep { } {1} db2 close } + +# When this test case was written, OOM errors in write statements would +# cause transaction rollback, which would trip cursors in other statements, +# aborting them. This no longer happens. +# do_test shared_malloc-8.X { # Test that one or more queries were aborted due to the malloc() failure. - expr $::aborted>=1 + # expr $::aborted>=1 + expr $::aborted==0 } {1} # This test is designed to catch a specific bug that was present during From bea3b976a95b00968d6f3ca496d5c1dfd3104527 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Nov 2014 20:22:05 +0000 Subject: [PATCH 584/710] Fix a bug in the sqlite3TripAllCursors() routine that prevents it from reporting errors. It is unknown at this time whether or not this omission can result in any incorrect result in an actual query. FossilOrigin-Name: 2896f2640ab3e102ee248d20fb68c497817524eb --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 267f0a0da9..9cb41e8715 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sa\scouple\sof\stest\scases\sto\saccount\sfor\sthe\sfact\sthat\sROLLBACK\sdoes\snot\salways\sabort\sall\srunning\sSELECT\sstatements. -D 2014-11-18T20:16:27.742 +C Fix\sa\sbug\sin\sthe\ssqlite3TripAllCursors()\sroutine\sthat\sprevents\sit\sfrom\nreporting\serrors.\s\sIt\sis\sunknown\sat\sthis\stime\swhether\sor\snot\sthis\somission\scan\nresult\sin\sany\sincorrect\sresult\sin\san\sactual\squery. +D 2014-11-18T20:22:05.030 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c d5d991b518fa5bebc64037dfeb98a48051d864d7 +F src/btree.c 75edb585cc2c66615e0ea01a48807a7bfae4f2fe F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 026d0129724e8f265fdc60d44ec240cf5a4e6179 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1221,8 +1221,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 54e7d3fcb1ab21c03ffef1af93ae029a2901098a -Q +eba171e980fa4491dfee9d7e4df50c87a0ebbf87 -R 07c45ea3d962860999af22ae8c2af8c8 +P abccda769a3f6b755c3bf70b5fb31a5e16718ef3 +Q +42588207ff5451cb785c394633e1ab631fb82f01 +R be8390e45e84a659cdb889947026b04a U drh -Z f4bad3f3683413628206e5a23517cfd8 +Z 45d41fa50f6cb7c97c28d0b284fe4014 diff --git a/manifest.uuid b/manifest.uuid index 22da5e0eba..922bec9a78 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -abccda769a3f6b755c3bf70b5fb31a5e16718ef3 \ No newline at end of file +2896f2640ab3e102ee248d20fb68c497817524eb \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f153a0cc45..7a49fb1a79 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3541,7 +3541,7 @@ int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int writeOnly){ int i; if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ if( p->eState==CURSOR_VALID ){ - int rc = saveCursorPosition(p); + rc = saveCursorPosition(p); if( rc!=SQLITE_OK ){ (void)sqlite3BtreeTripAllCursors(pBtree, rc, 0); break; From 664f85dd59c7995b2651ca9de390607dbfadb419 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Nov 2014 14:05:41 +0000 Subject: [PATCH 585/710] Add an ALWAYS on an always-true branch in wal.c. Fix the ANALYZE command so that it resets the "unordered" and "noskipscan" flags on indices when reloading the sqlite_stat1 table. FossilOrigin-Name: 9ed97a85feee6593faefa2b54cc4cf9a60f515f9 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/analyze.c | 2 ++ src/wal.c | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 272d0369e1..5976aae2e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adding\sthe\s"noskipscan"\stoken\sto\san\ssqlite_stat1.stat\sfield\sprevents\nan\sindex\sfor\sbeing\sused\swith\sthe\sskip-scan\salgorithm. -D 2014-11-18T21:54:31.890 +C Add\san\sALWAYS\son\san\salways-true\sbranch\sin\swal.c.\nFix\sthe\sANALYZE\scommand\sso\sthat\sit\sresets\sthe\s"unordered"\sand\s"noskipscan"\nflags\son\sindices\swhen\sreloading\sthe\ssqlite_stat1\stable. +D 2014-11-19T14:05:41.209 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c 951fd859852dfbced4a58b73954f168eeb9d0772 +F src/analyze.c c59f238a39aacece176f8bb7dfece40deb268ee5 F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea @@ -301,7 +301,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 -F src/wal.c fa090966140602f03a621f87d82ee69e66ca63b5 +F src/wal.c 486e644b3b8aa5ad066f625bc428aa8ff7001405 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c e275cb74731a3351a9da6ba8280bd5054db6192d @@ -1221,8 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c52f7971e90cac1058d6b66c9e334cbc8607def3 00fe09505792cd0d104b2da9d040f023e30fa871 -R 9bae3f4f5e0db167ef1a706a77e76141 -T +closed 00fe09505792cd0d104b2da9d040f023e30fa871 +P 4461bf045d8eecf98478035efcdba3f41c709bc5 +R b83c6fa5464847f6b6472d2244f6f040 U drh -Z fec927bef6596f9a5e5eec2871a731e7 +Z cd93530e8709a199e7d3e18ae995b78d diff --git a/manifest.uuid b/manifest.uuid index fd301fc3c9..5abcc58c68 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4461bf045d8eecf98478035efcdba3f41c709bc5 \ No newline at end of file +9ed97a85feee6593faefa2b54cc4cf9a60f515f9 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 769a16565a..ffb5c22b63 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1438,6 +1438,8 @@ static void decodeIntArray( if( z==0 ) z = ""; #else assert( z!=0 ); + pIndex->bUnordered = 0; + pIndex->noSkipScan = 0; #endif for(i=0; *z && ihdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); for(iFrame=pWal->hdr.mxFrame+1; - rc==SQLITE_OK && iFrame<=iMax; + ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; iFrame++ ){ /* This call cannot fail. Unless the page for which the page number From 42a7b4bac94983bfde72f3779e6ee15bfe847335 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Nov 2014 14:31:12 +0000 Subject: [PATCH 586/710] Completely remove an assert() that had previously been commented out. FossilOrigin-Name: 89b3c1c4555c98c633089b21cdd2a1a3a1e751eb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 1 - 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5976aae2e5..b9bc7630da 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sALWAYS\son\san\salways-true\sbranch\sin\swal.c.\nFix\sthe\sANALYZE\scommand\sso\sthat\sit\sresets\sthe\s"unordered"\sand\s"noskipscan"\nflags\son\sindices\swhen\sreloading\sthe\ssqlite_stat1\stable. -D 2014-11-19T14:05:41.209 +C Completely\sremove\san\sassert()\sthat\shad\spreviously\sbeen\scommented\sout. +D 2014-11-19T14:31:12.601 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c 5563459c06c434bc43131044fcf8164654008ebd +F src/vdbe.c 16914136ea1e8b18366868671ddb019159687f47 F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4461bf045d8eecf98478035efcdba3f41c709bc5 -R b83c6fa5464847f6b6472d2244f6f040 +P 9ed97a85feee6593faefa2b54cc4cf9a60f515f9 +R 612c462b22665e254a8e686035c1cd93 U drh -Z cd93530e8709a199e7d3e18ae995b78d +Z cb045903d6d20b681c939d9aa729532d diff --git a/manifest.uuid b/manifest.uuid index 5abcc58c68..7fd5801d3e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ed97a85feee6593faefa2b54cc4cf9a60f515f9 \ No newline at end of file +89b3c1c4555c98c633089b21cdd2a1a3a1e751eb \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index f883412943..31b42a32bf 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3808,7 +3808,6 @@ case OP_Found: { /* jump, in3 */ ); if( pIdxKey==0 ) goto no_mem; assert( pIn3->flags & MEM_Blob ); - /* assert( (pIn3->flags & MEM_Zero)==0 ); // zeroblobs already expanded */ ExpandBlob(pIn3); sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); } From 113762a284699ca55192eec1b482d695fcb281e6 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Nov 2014 16:36:25 +0000 Subject: [PATCH 587/710] Add new requirements marks associated with the file format documentation. No changes to code. FossilOrigin-Name: 6d00bcca6ed1903fb17275752cab71c14392355b --- manifest | 18 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 57 +++++++++++++++++++++++++++++++++++++++++++++---- src/expr.c | 5 ++++- src/pager.c | 2 +- src/sqlite.h.in | 6 +++--- 6 files changed, 71 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index b9bc7630da..a251e14322 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Completely\sremove\san\sassert()\sthat\shad\spreviously\sbeen\scommented\sout. -D 2014-11-19T14:31:12.601 +C Add\snew\srequirements\smarks\sassociated\swith\sthe\sfile\sformat\sdocumentation.\nNo\schanges\sto\scode. +D 2014-11-19T16:36:25.272 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c b562da29eb370aaac8015026827c2e2fb70ae990 +F src/btree.c d0a7cfc8788ad028c86020637df70c789615196e F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -182,7 +182,7 @@ F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 93594514aae68de117ca4a2a0d6cc63eddf26744 F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 -F src/expr.c a3ff05db5709d628c23890db862e30f3dd9dc428 +F src/expr.c 73de4c0da2eed6b149d40a05c589dfeb2c4a87a1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee @@ -216,7 +216,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a9e500dd963fb1f67d7860e58b5772abe6123862 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 8d97b3633f098fef817656dcbf167ca904511d78 +F src/pager.c b8764f90c135482988268eec93d7f5cdb89d687a F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd -F src/sqlite.h.in 0c5c0df7e4e436dfc5592511325bf4a96f6a638d +F src/sqlite.h.in dfbdcd6e52e7652648d8e7634a7c191fcac8e6ce F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9ed97a85feee6593faefa2b54cc4cf9a60f515f9 -R 612c462b22665e254a8e686035c1cd93 +P 89b3c1c4555c98c633089b21cdd2a1a3a1e751eb +R 92805b92bb878e3838f25a4f3bef8761 U drh -Z cb045903d6d20b681c939d9aa729532d +Z 529027073b2f32ceeacba058565c65ab diff --git a/manifest.uuid b/manifest.uuid index 7fd5801d3e..cb62723bc8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -89b3c1c4555c98c633089b21cdd2a1a3a1e751eb \ No newline at end of file +6d00bcca6ed1903fb17275752cab71c14392355b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9587e567a3..f61078cac6 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1247,10 +1247,15 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ for(iAddr=hdr+1; (pc = get2byte(&aData[iAddr]))>0; iAddr=pc){ int size; /* Size of the free slot */ + /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of + ** increasing offset. */ if( pc>usableSize-4 || pc=nByte ){ int x = size - nByte; @@ -1314,6 +1319,11 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ top = get2byte(&data[hdr+5]); if( gap>top ){ if( top==0 ){ + /* EVIDENCE-OF: R-29356-02391 If the database uses a 65536-byte page size + ** and the reserved space is zero (the usual value for reserved space) + ** then the cell content offset of an empty page wants to be 65536. + ** However, that integer is too large to be stored in a 2-byte unsigned + ** integer, so a value of 0 is used in its place. */ top = 65536; }else{ return SQLITE_CORRUPT_BKPT; @@ -1998,6 +2008,9 @@ int sqlite3BtreeOpen( #ifdef SQLITE_SECURE_DELETE pBt->btsFlags |= BTS_SECURE_DELETE; #endif + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ pBt->pageSize = (zDbHeader[16]<<8) | (zDbHeader[17]<<16); if( pBt->pageSize<512 || pBt->pageSize>SQLITE_MAX_PAGE_SIZE || ((pBt->pageSize-1)&pBt->pageSize)!=0 ){ @@ -2016,6 +2029,9 @@ int sqlite3BtreeOpen( #endif nReserve = 0; }else{ + /* EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ nReserve = zDbHeader[20]; pBt->btsFlags |= BTS_PAGESIZE_FIXED; #ifndef SQLITE_OMIT_AUTOVACUUM @@ -2525,6 +2541,9 @@ static int lockBtree(BtShared *pBt){ u32 usableSize; u8 *page1 = pPage1->aData; rc = SQLITE_NOTADB; + /* EVIDENCE-OF: R-43737-39999 Every valid SQLite database file begins + ** with the following 16 bytes (in hex): 53 51 4c 69 74 65 20 66 6f 72 6d + ** 61 74 20 33 00. */ if( memcmp(page1, zMagicHeader, 16)!=0 ){ goto page1_init_failed; } @@ -2565,15 +2584,21 @@ static int lockBtree(BtShared *pBt){ } #endif - /* The maximum embedded fraction must be exactly 25%. And the minimum - ** embedded fraction must be 12.5% for both leaf-data and non-leaf-data. + /* EVIDENCE-OF: R-15465-20813 The maximum and minimum embedded payload + ** fractions and the leaf payload fraction values must be 64, 32, and 32. + ** ** The original design allowed these amounts to vary, but as of ** version 3.6.0, we require them to be fixed. */ if( memcmp(&page1[21], "\100\040\040",3)!=0 ){ goto page1_init_failed; } + /* EVIDENCE-OF: R-51873-39618 The page size for a database file is + ** determined by the 2-byte integer located at an offset of 16 bytes from + ** the beginning of the database file. */ pageSize = (page1[16]<<8) | (page1[17]<<16); + /* EVIDENCE-OF: R-25008-21688 The size of a page is a power of two + ** between 512 and 65536 inclusive. */ if( ((pageSize-1)&pageSize)!=0 || pageSize>SQLITE_MAX_PAGE_SIZE || pageSize<=256 @@ -2581,6 +2606,13 @@ static int lockBtree(BtShared *pBt){ goto page1_init_failed; } assert( (pageSize & 7)==0 ); + /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte + ** integer at offset 20 is the number of bytes of space at the end of + ** each page to reserve for extensions. + ** + ** EVIDENCE-OF: R-37497-42412 The size of the reserved region is + ** determined by the one-byte unsigned integer found at an offset of 20 + ** into the database file header. */ usableSize = pageSize - page1[20]; if( (u32)pageSize!=pBt->pageSize ){ /* After reading the first page of the database assuming a page size @@ -2601,6 +2633,9 @@ static int lockBtree(BtShared *pBt){ rc = SQLITE_CORRUPT_BKPT; goto page1_init_failed; } + /* EVIDENCE-OF: R-28312-64704 However, the usable size is not allowed to + ** be less than 480. In other words, if the page size is 512, then the + ** reserved space size cannot exceed 32. */ if( usableSize<480 ){ goto page1_init_failed; } @@ -5178,6 +5213,8 @@ static int allocateBtreePage( assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); + /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 + ** stores stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); if( n>=mxPage ){ @@ -5224,8 +5261,14 @@ static int allocateBtreePage( do { pPrevTrunk = pTrunk; if( pPrevTrunk ){ + /* EVIDENCE-OF: R-01506-11053 The first integer on a freelist trunk page + ** is the page number of the next freelist trunk page in the list or + ** zero if this is the last freelist trunk page. */ iTrunk = get4byte(&pPrevTrunk->aData[0]); }else{ + /* EVIDENCE-OF: R-59841-13798 The 4-byte big-endian integer at offset 32 + ** stores the page number of the first page of the freelist, or zero if + ** the freelist is empty. */ iTrunk = get4byte(&pPage1->aData[32]); } testcase( iTrunk==mxPage ); @@ -5240,8 +5283,9 @@ static int allocateBtreePage( } assert( pTrunk!=0 ); assert( pTrunk->aData!=0 ); - - k = get4byte(&pTrunk->aData[4]); /* # of leaves on this trunk page */ + /* EVIDENCE-OF: R-13523-04394 The second integer on a freelist trunk page + ** is the number of leaf page pointers to follow. */ + k = get4byte(&pTrunk->aData[4]); if( k==0 && !searchList ){ /* The trunk has no leaves and the list is not being searched. ** So extract the trunk page itself and use it as the newly @@ -5559,6 +5603,11 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ ** for now. At some point in the future (once everyone has upgraded ** to 3.6.0 or later) we should consider fixing the conditional above ** to read "usableSize/4-2" instead of "usableSize/4-8". + ** + ** EVIDENCE-OF: R-19920-11576 However, newer versions of SQLite still + ** avoid using the last six entries in the freelist trunk page array in + ** order that database files created by newer versions of SQLite can be + ** read by older versions of SQLite. */ rc = sqlite3PagerWrite(pTrunk->pDbPage); if( rc==SQLITE_OK ){ diff --git a/src/expr.c b/src/expr.c index 25f0be400f..c28f196221 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3003,7 +3003,10 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){ #ifndef SQLITE_OMIT_FLOATING_POINT /* If the column has REAL affinity, it may currently be stored as an - ** integer. Use OP_RealAffinity to make sure it is really real. */ + ** integer. Use OP_RealAffinity to make sure it is really real. + ** + ** EVIDENCE-OF: R-60985-57662 SQLite will convert the value back to + ** floating point when extracting it from the record. */ if( pExpr->iColumn>=0 && pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL ){ diff --git a/src/pager.c b/src/pager.c index 997f842d0a..e0ede87f07 100644 --- a/src/pager.c +++ b/src/pager.c @@ -2899,7 +2899,7 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ ** ** For an encrypted database, the situation is more complex: bytes ** 24..39 of the database are white noise. But the probability of - ** white noising equaling 16 bytes of 0xff is vanishingly small so + ** white noise equaling 16 bytes of 0xff is vanishingly small so ** we should still be ok. */ memset(pPager->dbFileVers, 0xff, sizeof(pPager->dbFileVers)); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 4427f39d06..a59599a0c5 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -4164,9 +4164,9 @@ int sqlite3_create_function_v2( ** These constant define integer codes that represent the various ** text encodings supported by SQLite. */ -#define SQLITE_UTF8 1 -#define SQLITE_UTF16LE 2 -#define SQLITE_UTF16BE 3 +#define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ +#define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ +#define SQLITE_UTF16BE 3 /* IMP: R-51971-34154 */ #define SQLITE_UTF16 4 /* Use native byte order */ #define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ From 654858d7f45254f915f0796431699ecc841fdf06 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 02:18:14 +0000 Subject: [PATCH 588/710] Add some requirements marks to the record formatting logic. Comment changes only - the code is unaltered. FossilOrigin-Name: 9a9627e178a67bbfc85366aaea900e674d22fb53 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 9 ++++++++- src/vdbeaux.c | 25 +++++++++++++++++++++++-- 4 files changed, 39 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index a251e14322..065eb19989 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\srequirements\smarks\sassociated\swith\sthe\sfile\sformat\sdocumentation.\nNo\schanges\sto\scode. -D 2014-11-19T16:36:25.272 +C Add\ssome\srequirements\smarks\sto\sthe\srecord\sformatting\slogic.\s\sComment\schanges\nonly\s-\sthe\scode\sis\sunaltered. +D 2014-11-20T02:18:14.206 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,11 +291,11 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c 16914136ea1e8b18366868671ddb019159687f47 +F src/vdbe.c ec1f55acef4864520ca2017b9f0d60c2ac1b8b78 F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 -F src/vdbeaux.c 9b0a251b6dfab349dd6c6efb40062eb7386b26f5 +F src/vdbeaux.c e3ae27459a4eb986d2892de2d7f309d18c9971ef F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 89b3c1c4555c98c633089b21cdd2a1a3a1e751eb -R 92805b92bb878e3838f25a4f3bef8761 +P 6d00bcca6ed1903fb17275752cab71c14392355b +R b487443c841a5e8e1bfed86b80cd7f65 U drh -Z 529027073b2f32ceeacba058565c65ab +Z bbc37d068b0ef41277590cc500b3cc54 diff --git a/manifest.uuid b/manifest.uuid index cb62723bc8..a704e79b38 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d00bcca6ed1903fb17275752cab71c14392355b \ No newline at end of file +9a9627e178a67bbfc85366aaea900e674d22fb53 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 31b42a32bf..822bf80bb8 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -2636,7 +2636,10 @@ case OP_MakeRecord: { nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type); }while( (--pRec)>=pData0 ); - /* Add the initial header varint and total the size */ + /* EVIDENCE-OF: R-22564-11647 The header begins with a single varint + ** which determines the total number of bytes in the header. The varint + ** value is the size of the header in bytes including the size varint + ** itself. */ testcase( nHdr==126 ); testcase( nHdr==127 ); if( nHdr<=126 ){ @@ -2670,7 +2673,11 @@ case OP_MakeRecord: { pRec = pData0; do{ serial_type = pRec->uTemp; + /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more + ** additional varints, one per column. */ i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ + /* EVIDENCE-OF: R-64536-51728 The values for each column in the record + ** immediately follow the header. */ j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ }while( (++pRec)<=pLast ); assert( i==nHdr ); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index d8ee5c8e8c..145bf453ad 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3051,10 +3051,14 @@ static u32 SQLITE_NOINLINE serialGet( u32 y = FOUR_BYTE_UINT(buf+4); x = (x<<32) + y; if( serial_type==6 ){ + /* EVIDENCE-OF: R-29851-52272 Value is a big-endian 64-bit + ** twos-complement integer. */ pMem->u.i = *(i64*)&x; pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); }else{ + /* EVIDENCE-OF: R-57343-49114 Value is a big-endian IEEE 754-2008 64-bit + ** floating point number. */ #if !defined(NDEBUG) && !defined(SQLITE_OMIT_FLOATING_POINT) /* Verify that integers and floating point values use the same ** byte order. Or, that if SQLITE_MIXED_ENDIAN_64BIT_FLOAT is @@ -3082,35 +3086,46 @@ u32 sqlite3VdbeSerialGet( switch( serial_type ){ case 10: /* Reserved for future use */ case 11: /* Reserved for future use */ - case 0: { /* NULL */ + case 0: { /* Null */ + /* EVIDENCE-OF: R-24078-09375 Value is a NULL. */ pMem->flags = MEM_Null; break; } - case 1: { /* 1-byte signed integer */ + case 1: { + /* EVIDENCE-OF: R-44885-25196 Value is an 8-bit twos-complement + ** integer. */ pMem->u.i = ONE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 1; } case 2: { /* 2-byte signed integer */ + /* EVIDENCE-OF: R-49794-35026 Value is a big-endian 16-bit + ** twos-complement integer. */ pMem->u.i = TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 2; } case 3: { /* 3-byte signed integer */ + /* EVIDENCE-OF: R-37839-54301 Value is a big-endian 24-bit + ** twos-complement integer. */ pMem->u.i = THREE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 3; } case 4: { /* 4-byte signed integer */ + /* EVIDENCE-OF: R-01849-26079 Value is a big-endian 32-bit + ** twos-complement integer. */ pMem->u.i = FOUR_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); return 4; } case 5: { /* 6-byte signed integer */ + /* EVIDENCE-OF: R-50385-09674 Value is a big-endian 48-bit + ** twos-complement integer. */ pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); @@ -3124,11 +3139,17 @@ u32 sqlite3VdbeSerialGet( } case 8: /* Integer 0 */ case 9: { /* Integer 1 */ + /* EVIDENCE-OF: R-12976-22893 Value is the integer 0. */ + /* EVIDENCE-OF: R-18143-12121 Value is the integer 1. */ pMem->u.i = serial_type-8; pMem->flags = MEM_Int; return 0; } default: { + /* EVIDENCE-OF: R-14606-31564 Value is a BLOB that is (N-12)/2 bytes in + ** length. + ** EVIDENCE-OF: R-28401-00140 Value is a string in the text encoding and + ** (N-13)/2 bytes in length. */ static const u16 aFlag[] = { MEM_Blob|MEM_Ephem, MEM_Str|MEM_Ephem }; pMem->z = (char *)buf; pMem->n = (serial_type-12)/2; From 1b40e63f9bdf8f51241a0511a7c9bb663fe419ed Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 02:58:10 +0000 Subject: [PATCH 589/710] Fix the encoding of some integers to use the minimum amount of space: -128, -32768, -8388608, -217483648, and -140737488355328. FossilOrigin-Name: 2d7c8da5f16e64eaa7b0c2d66898682ea3d102a0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 4 +--- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 065eb19989..fe5a6d1bb4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssome\srequirements\smarks\sto\sthe\srecord\sformatting\slogic.\s\sComment\schanges\nonly\s-\sthe\scode\sis\sunaltered. -D 2014-11-20T02:18:14.206 +C Fix\sthe\sencoding\sof\ssome\sintegers\sto\suse\sthe\sminimum\samount\sof\sspace:\n-128,\s-32768,\s-8388608,\s-217483648,\sand\s-140737488355328. +D 2014-11-20T02:58:10.344 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -295,7 +295,7 @@ F src/vdbe.c ec1f55acef4864520ca2017b9f0d60c2ac1b8b78 F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 -F src/vdbeaux.c e3ae27459a4eb986d2892de2d7f309d18c9971ef +F src/vdbeaux.c 5ce4f414147a3bc3cbcf00ec57f2606c25791629 F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6d00bcca6ed1903fb17275752cab71c14392355b -R b487443c841a5e8e1bfed86b80cd7f65 +P 9a9627e178a67bbfc85366aaea900e674d22fb53 +R acb3727f628d804856a9646910d389b3 U drh -Z bbc37d068b0ef41277590cc500b3cc54 +Z ae13a50368fe7d7b53b7e4276c8588bd diff --git a/manifest.uuid b/manifest.uuid index a704e79b38..1ae875991f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a9627e178a67bbfc85366aaea900e674d22fb53 \ No newline at end of file +2d7c8da5f16e64eaa7b0c2d66898682ea3d102a0 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 145bf453ad..b8f5bca2c3 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2883,9 +2883,7 @@ u32 sqlite3VdbeSerialType(Mem *pMem, int file_format){ i64 i = pMem->u.i; u64 u; if( i<0 ){ - if( i<(-MAX_6BYTE) ) return 6; - /* Previous test prevents: u = -(-9223372036854775808) */ - u = -i; + u = ~i; }else{ u = i; } From fdab02635c2304acc99b04272235463e1ed1ac51 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 15:30:50 +0000 Subject: [PATCH 590/710] Ensure that when the number of cells on a page drops to zero that the freelist and fragment counter are both cleared. Also add evidence marks corresponding to file-format documentation. FossilOrigin-Name: ef9fbc08b0a047042deeb2d6007d67028fefb9e2 --- manifest | 12 +++--- manifest.uuid | 2 +- src/btree.c | 102 ++++++++++++++++++++++++++++++++++++++++---------- 3 files changed, 89 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index fe5a6d1bb4..7541704f57 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sencoding\sof\ssome\sintegers\sto\suse\sthe\sminimum\samount\sof\sspace:\n-128,\s-32768,\s-8388608,\s-217483648,\sand\s-140737488355328. -D 2014-11-20T02:58:10.344 +C Ensure\sthat\swhen\sthe\snumber\sof\scells\son\sa\spage\sdrops\sto\szero\sthat\sthe\sfreelist\nand\sfragment\scounter\sare\sboth\scleared.\s\sAlso\sadd\sevidence\smarks\scorresponding\nto\sfile-format\sdocumentation. +D 2014-11-20T15:30:50.141 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c d0a7cfc8788ad028c86020637df70c789615196e +F src/btree.c 4db5e06ca2d1a5437be7075251fa702c76179b0e F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9a9627e178a67bbfc85366aaea900e674d22fb53 -R acb3727f628d804856a9646910d389b3 +P 2d7c8da5f16e64eaa7b0c2d66898682ea3d102a0 +R aa709859c03f6b1c53283cb1512cbe99 U drh -Z ae13a50368fe7d7b53b7e4276c8588bd +Z 12bf55f57cbfdde86aed8c230487a6ad diff --git a/manifest.uuid b/manifest.uuid index 1ae875991f..e8ed272102 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2d7c8da5f16e64eaa7b0c2d66898682ea3d102a0 \ No newline at end of file +ef9fbc08b0a047042deeb2d6007d67028fefb9e2 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index f61078cac6..5ae5986cfa 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1139,6 +1139,11 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ ** end of the page and all free space is collected into one ** big FreeBlk that occurs in between the header and cell ** pointer array and the cell content area. +** +** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a +** b-tree page so that there are no freeblocks or fragment bytes, all +** unused bytes are contained in the unallocated space region, and all +** cells are packed tightly at the end of the page. */ static int defragmentPage(MemPage *pPage){ int i; /* Loop counter */ @@ -1262,6 +1267,8 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc, int *pbDefrag){ testcase( x==4 ); testcase( x==3 ); if( x<4 ){ + /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total + ** number of bytes in fragments may not exceed 60. */ if( aData[hdr+7]>=60 ){ if( pbDefrag ) *pbDefrag = 1; return 0; @@ -1316,19 +1323,13 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ assert( pPage->cellOffset == hdr + 12 - 4*pPage->leaf ); gap = pPage->cellOffset + 2*pPage->nCell; assert( gap<=65536 ); - top = get2byte(&data[hdr+5]); - if( gap>top ){ - if( top==0 ){ - /* EVIDENCE-OF: R-29356-02391 If the database uses a 65536-byte page size - ** and the reserved space is zero (the usual value for reserved space) - ** then the cell content offset of an empty page wants to be 65536. - ** However, that integer is too large to be stored in a 2-byte unsigned - ** integer, so a value of 0 is used in its place. */ - top = 65536; - }else{ - return SQLITE_CORRUPT_BKPT; - } - } + /* EVIDENCE-OF: R-29356-02391 If the database uses a 65536-byte page size + ** and the reserved space is zero (the usual value for reserved space) + ** then the cell content offset of an empty page wants to be 65536. + ** However, that integer is too large to be stored in a 2-byte unsigned + ** integer, so a value of 0 is used in its place. */ + top = get2byteNotZero(&data[hdr+5]); + if( gap>top ) return SQLITE_CORRUPT_BKPT; /* If there is enough space between gap and top for one more cell pointer ** array entry offset, and if the freelist is not empty, then search the @@ -1497,18 +1498,32 @@ static int decodeFlags(MemPage *pPage, int flagByte){ pPage->childPtrSize = 4-4*pPage->leaf; pBt = pPage->pBt; if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ + /* EVIDENCE-OF: R-03640-13415 A value of 5 means the page is an interior + ** table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); + /* EVIDENCE-OF: R-20501-61796 A value of 13 means the page is a leaf + ** table b-tree page. */ + assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); pPage->intKey = 1; pPage->intKeyLeaf = pPage->leaf; pPage->noPayload = !pPage->leaf; pPage->maxLocal = pBt->maxLeaf; pPage->minLocal = pBt->minLeaf; }else if( flagByte==PTF_ZERODATA ){ + /* EVIDENCE-OF: R-27225-53936 A value of 2 means the page is an interior + ** index b-tree page. */ + assert( (PTF_ZERODATA)==2 ); + /* EVIDENCE-OF: R-16571-11615 A value of 10 means the page is a leaf + ** index b-tree page. */ + assert( (PTF_ZERODATA|PTF_LEAF)==10 ); pPage->intKey = 0; pPage->intKeyLeaf = 0; pPage->noPayload = 0; pPage->maxLocal = pBt->maxLocal; pPage->minLocal = pBt->minLocal; }else{ + /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is + ** an error. */ return SQLITE_CORRUPT_BKPT; } pPage->max1bytePayload = pBt->max1bytePayload; @@ -1548,21 +1563,33 @@ static int btreeInitPage(MemPage *pPage){ hdr = pPage->hdrOffset; data = pPage->aData; + /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating + ** the b-tree page type. */ if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); pPage->maskPage = (u16)(pBt->pageSize - 1); pPage->nOverflow = 0; usableSize = pBt->usableSize; - pPage->cellOffset = cellOffset = hdr + 12 - 4*pPage->leaf; + pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize; pPage->aDataEnd = &data[usableSize]; pPage->aCellIdx = &data[cellOffset]; + /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates + ** the start of the cell content area. A zero value for this integer is + ** interpreted as 65536. */ top = get2byteNotZero(&data[hdr+5]); + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ pPage->nCell = get2byte(&data[hdr+3]); if( pPage->nCell>MX_CELL(pBt) ){ /* To many cells for a single page. The page must be corrupt */ return SQLITE_CORRUPT_BKPT; } testcase( pPage->nCell==MX_CELL(pBt) ); + /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only + ** possible for a root page of a table that contains no rows) then the + ** offset to the cell content area will equal the page size minus the + ** bytes of reserved space. */ + assert( pPage->nCell>0 || top==usableSize || CORRUPT_DB ); /* A malformed database page might cause us to read past the end ** of page when parsing a cell. @@ -1596,13 +1623,20 @@ static int btreeInitPage(MemPage *pPage){ } #endif - /* Compute the total free space on the page */ + /* Compute the total free space on the page + ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the + ** start of the first freeblock on the page, or is zero if there are no + ** freeblocks. */ pc = get2byte(&data[hdr+1]); - nFree = data[hdr+7] + top; + nFree = data[hdr+7] + top; /* Init nFree to non-freeblock free space */ while( pc>0 ){ u16 next, size; if( pciCellLast ){ - /* Start of free block is off the page */ + /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will + ** always be at least one cell before the first freeblock. + ** + ** Or, the freeblock is off the end of the page + */ return SQLITE_CORRUPT_BKPT; } next = get2byte(&data[pc]); @@ -5959,9 +5993,17 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ return; } pPage->nCell--; - memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); - put2byte(&data[hdr+3], pPage->nCell); - pPage->nFree += 2; + if( pPage->nCell==0 ){ + memset(&data[hdr+1], 0, 4); + data[hdr+7] = 0; + put2byte(&data[hdr+5], pPage->pBt->usableSize); + pPage->nFree = pPage->pBt->usableSize - pPage->hdrOffset + - pPage->childPtrSize - 8; + }else{ + memmove(ptr, ptr+2, 2*(pPage->nCell - idx)); + put2byte(&data[hdr+3], pPage->nCell); + pPage->nFree += 2; + } } /* @@ -8609,8 +8651,14 @@ static int checkTreePage( assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ memset(hit+contentOffset, 0, usableSize-contentOffset); memset(hit, 1, contentOffset); + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ nCell = get2byte(&data[hdr+3]); + /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page + ** immediately follows the b-tree page header. */ cellStart = hdr + 12 - 4*pPage->leaf; + /* EVIDENCE-OF: R-02776-14802 The cell pointer array consists of K 2-byte + ** integer offsets to the cell contents. */ for(i=0; i=pc; j--) hit[j]++; } } + /* EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header + ** is the offset of the first freeblock, or zero if there are no + ** freeblocks on the page. */ i = get2byte(&data[hdr+1]); while( i>0 ){ int size, j; @@ -8633,7 +8684,13 @@ static int checkTreePage( size = get2byte(&data[i+2]); assert( i+size<=usableSize ); /* Enforced by btreeInitPage() */ for(j=i+size-1; j>=i; j--) hit[j]++; + /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a + ** big-endian integer which is the offset in the b-tree page of the next + ** freeblock in the chain, or zero if the freeblock is the last on the + ** chain. */ j = get2byte(&data[i]); + /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of + ** increasing offset. */ assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ assert( j<=usableSize-4 ); /* Enforced by btreeInitPage() */ i = j; @@ -8647,6 +8704,11 @@ static int checkTreePage( break; } } + /* EVIDENCE-OF: R-43263-13491 The total number of bytes in all fragments + ** is stored in the fifth field of the b-tree page header. + ** EVIDENCE-OF: R-07161-27322 The one-byte integer at offset 7 gives the + ** number of fragmented free bytes within the cell content area. + */ if( cnt!=data[hdr+7] ){ checkAppendMsg(pCheck, "Fragmentation of %d bytes reported as %d on page %d", From 5e3b49bc42c1a10222571bf433045bb03c7057e9 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 19:22:26 +0000 Subject: [PATCH 591/710] Add requirements marks on the built-in collating functions. FossilOrigin-Name: 4b608b62ac8d4eafdb76192b3b5db272332a4bfd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 19 +++++++++++++++---- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7541704f57..0e5910fc06 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Ensure\sthat\swhen\sthe\snumber\sof\scells\son\sa\spage\sdrops\sto\szero\sthat\sthe\sfreelist\nand\sfragment\scounter\sare\sboth\scleared.\s\sAlso\sadd\sevidence\smarks\scorresponding\nto\sfile-format\sdocumentation. -D 2014-11-20T15:30:50.141 +C Add\srequirements\smarks\son\sthe\sbuilt-in\scollating\sfunctions. +D 2014-11-20T19:22:26.830 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c d3310d5ed56e246bf1589e47eeaca8be582bd4b8 +F src/main.c 54d0f4896cebc61ae5f831937464953780fe5346 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2d7c8da5f16e64eaa7b0c2d66898682ea3d102a0 -R aa709859c03f6b1c53283cb1512cbe99 +P ef9fbc08b0a047042deeb2d6007d67028fefb9e2 +R 87fc16da633f081b9c3d6138857b30da U drh -Z 12bf55f57cbfdde86aed8c230487a6ad +Z 4f4a01f4cbeff0ca684ca414f4595d57 diff --git a/manifest.uuid b/manifest.uuid index e8ed272102..de14eeaf45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef9fbc08b0a047042deeb2d6007d67028fefb9e2 \ No newline at end of file +4b608b62ac8d4eafdb76192b3b5db272332a4bfd \ No newline at end of file diff --git a/src/main.c b/src/main.c index f223b71f74..1d34b13d6f 100644 --- a/src/main.c +++ b/src/main.c @@ -773,13 +773,20 @@ static int binCollFunc( ){ int rc, n; n = nKey1mallocFailed ){ goto opendb_out; } + /* EVIDENCE-OF: R-08308-17224 The default collating function for all + ** strings is BINARY. + */ db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, "BINARY", 0); assert( db->pDfltColl!=0 ); - /* Also add a UTF-8 case-insensitive collation sequence. */ - createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); - /* Parse the filename/URI argument. */ db->openFlags = flags; rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); From 341eca7f68e2ad7a6102c98342c23ddaf65f3c89 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 23:03:42 +0000 Subject: [PATCH 592/710] Updates to requirements tags on the mutex documentation. FossilOrigin-Name: fcf8b7e4c6c3893e2004a28dc9f0f677907b4ba9 --- manifest | 12 ++++---- manifest.uuid | 2 +- src/sqlite.h.in | 80 +++++++++++++++++++++++-------------------------- 3 files changed, 45 insertions(+), 49 deletions(-) diff --git a/manifest b/manifest index 0e5910fc06..3b9c283125 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\srequirements\smarks\son\sthe\sbuilt-in\scollating\sfunctions. -D 2014-11-20T19:22:26.830 +C Updates\sto\srequirements\stags\son\sthe\smutex\sdocumentation. +D 2014-11-20T23:03:42.154 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd -F src/sqlite.h.in dfbdcd6e52e7652648d8e7634a7c191fcac8e6ce +F src/sqlite.h.in c63db0117aeb749ca02b6016dbbbccbbbd9a141d F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef9fbc08b0a047042deeb2d6007d67028fefb9e2 -R 87fc16da633f081b9c3d6138857b30da +P 4b608b62ac8d4eafdb76192b3b5db272332a4bfd +R b10e3d696841d651117121c42237df0a U drh -Z 4f4a01f4cbeff0ca684ca414f4595d57 +Z 5d29b887364b9011d92605ad266bc7f8 diff --git a/manifest.uuid b/manifest.uuid index de14eeaf45..fb1afe2201 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4b608b62ac8d4eafdb76192b3b5db272332a4bfd \ No newline at end of file +fcf8b7e4c6c3893e2004a28dc9f0f677907b4ba9 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a59599a0c5..f1b917c308 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5900,34 +5900,34 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** The SQLite source code contains multiple implementations ** of these mutex routines. An appropriate implementation -** is selected automatically at compile-time. ^(The following +** is selected automatically at compile-time. The following ** implementations are available in the SQLite core: ** **
      **
    • SQLITE_MUTEX_PTHREADS **
    • SQLITE_MUTEX_W32 **
    • SQLITE_MUTEX_NOOP -**
    )^ +** ** -** ^The SQLITE_MUTEX_NOOP implementation is a set of routines +** The SQLITE_MUTEX_NOOP implementation is a set of routines ** that does no real locking and is appropriate for use in -** a single-threaded application. ^The SQLITE_MUTEX_PTHREADS and +** a single-threaded application. The SQLITE_MUTEX_PTHREADS and ** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix ** and Windows. ** -** ^(If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor +** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor ** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex ** implementation is included with the library. In this case the ** application must supply a custom mutex implementation using the ** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function ** before calling sqlite3_initialize() or any other public sqlite3_ -** function that calls sqlite3_initialize().)^ +** function that calls sqlite3_initialize(). ** ** ^The sqlite3_mutex_alloc() routine allocates a new -** mutex and returns a pointer to it. ^If it returns NULL -** that means that a mutex could not be allocated. ^SQLite -** will unwind its stack and return an error. ^(The argument -** to sqlite3_mutex_alloc() is one of these integer constants: +** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() +** routine returns NULL if it is unable to allocate the requested +** mutex. The argument to sqlite3_mutex_alloc() must one of these +** integer constants: ** **
      **
    • SQLITE_MUTEX_FAST @@ -5940,7 +5940,8 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); **
    • SQLITE_MUTEX_STATIC_PMEM **
    • SQLITE_MUTEX_STATIC_APP1 **
    • SQLITE_MUTEX_STATIC_APP2 -**
    )^ +**
  • SQLITE_MUTEX_STATIC_APP3 +** ** ** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) ** cause sqlite3_mutex_alloc() to create @@ -5948,14 +5949,14 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** is used but not necessarily so when SQLITE_MUTEX_FAST is used. ** The mutex implementation does not need to make a distinction ** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does -** not want to. ^SQLite will only request a recursive mutex in -** cases where it really needs one. ^If a faster non-recursive mutex +** not want to. SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex ** implementation is available on the host platform, the mutex subsystem ** might return such a mutex in response to SQLITE_MUTEX_FAST. ** ** ^The other allowed parameters to sqlite3_mutex_alloc() (anything other ** than SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return -** a pointer to a static preexisting mutex. ^Six static mutexes are +** a pointer to a static preexisting mutex. ^Nine static mutexes are ** used by the current version of SQLite. Future versions of SQLite ** may add additional static mutexes. Static mutexes are for internal ** use by SQLite only. Applications that use SQLite mutexes should @@ -5964,16 +5965,13 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** ** ^Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. ^But for the static +** returns a different mutex on every call. ^For the static ** mutex types, the same mutex is returned on every call that has ** the same type number. ** ** ^The sqlite3_mutex_free() routine deallocates a previously -** allocated dynamic mutex. ^SQLite is careful to deallocate every -** dynamic mutex that it allocates. The dynamic mutexes must not be in -** use when they are deallocated. Attempting to deallocate a static -** mutex results in undefined behavior. ^SQLite never deallocates -** a static mutex. +** allocated dynamic mutex. Attempting to deallocate a static +** mutex results in undefined behavior. ** ** ^The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt ** to enter a mutex. ^If another thread is already within the mutex, @@ -5981,23 +5979,21 @@ int sqlite3_vfs_unregister(sqlite3_vfs*); ** SQLITE_BUSY. ^The sqlite3_mutex_try() interface returns [SQLITE_OK] ** upon successful entry. ^(Mutexes created using ** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread. -** In such cases the, +** In such cases, the ** mutex must be exited an equal number of times before another thread -** can enter.)^ ^(If the same thread tries to enter any other -** kind of mutex more than once, the behavior is undefined. -** SQLite will never exhibit -** such behavior in its own use of mutexes.)^ +** can enter.)^ If the same thread tries to enter any mutex other +** than an SQLITE_MUTEX_RECURSIVE more than once, the behavior is undefined. ** ** ^(Some systems (for example, Windows 95) do not support the operation ** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() -** will always return SQLITE_BUSY. The SQLite core only ever uses -** sqlite3_mutex_try() as an optimization so this is acceptable behavior.)^ +** will always return SQLITE_BUSY. The SQLite core only ever uses +** sqlite3_mutex_try() as an optimization so this is acceptable +** behavior.)^ ** ** ^The sqlite3_mutex_leave() routine exits a mutex that was -** previously entered by the same thread. ^(The behavior +** previously entered by the same thread. The behavior ** is undefined if the mutex is not currently entered by the -** calling thread or is not currently allocated. SQLite will -** never do either.)^ +** calling thread or is not currently allocated. ** ** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or ** sqlite3_mutex_leave() is a NULL pointer, then all three routines @@ -6018,9 +6014,9 @@ void sqlite3_mutex_leave(sqlite3_mutex*); ** used to allocate and use mutexes. ** ** Usually, the default mutex implementations provided by SQLite are -** sufficient, however the user has the option of substituting a custom +** sufficient, however the application has the option of substituting a custom ** implementation for specialized deployments or systems for which SQLite -** does not provide a suitable implementation. In this case, the user +** does not provide a suitable implementation. In this case, the application ** creates and populates an instance of this structure to pass ** to sqlite3_config() along with the [SQLITE_CONFIG_MUTEX] option. ** Additionally, an instance of this structure can be used as an @@ -6061,13 +6057,13 @@ void sqlite3_mutex_leave(sqlite3_mutex*); ** (i.e. it is acceptable to provide an implementation that segfaults if ** it is passed a NULL pointer). ** -** The xMutexInit() method must be threadsafe. ^It must be harmless to +** The xMutexInit() method must be threadsafe. It must be harmless to ** invoke xMutexInit() multiple times within the same process and without ** intervening calls to xMutexEnd(). Second and subsequent calls to ** xMutexInit() must be no-ops. ** -** ^xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] -** and its associates). ^Similarly, xMutexAlloc() must not use SQLite memory +** xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] +** and its associates). Similarly, xMutexAlloc() must not use SQLite memory ** allocation for a static mutex. ^However xMutexAlloc() may use SQLite ** memory allocation for a fast or recursive mutex. ** @@ -6093,29 +6089,29 @@ struct sqlite3_mutex_methods { ** CAPI3REF: Mutex Verification Routines ** ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines -** are intended for use inside assert() statements. ^The SQLite core +** are intended for use inside assert() statements. The SQLite core ** never uses these routines except inside an assert() and applications -** are advised to follow the lead of the core. ^The SQLite core only +** are advised to follow the lead of the core. The SQLite core only ** provides implementations for these routines when it is compiled -** with the SQLITE_DEBUG flag. ^External mutex implementations +** with the SQLITE_DEBUG flag. External mutex implementations ** are only required to provide these routines if SQLITE_DEBUG is ** defined and if NDEBUG is not defined. ** -** ^These routines should return true if the mutex in their argument +** These routines should return true if the mutex in their argument ** is held or not held, respectively, by the calling thread. ** -** ^The implementation is not required to provide versions of these +** The implementation is not required to provide versions of these ** routines that actually work. If the implementation does not provide working ** versions of these routines, it should at least provide stubs that always ** return true so that one does not get spurious assertion failures. ** -** ^If the argument to sqlite3_mutex_held() is a NULL pointer then +** If the argument to sqlite3_mutex_held() is a NULL pointer then ** the routine should return 1. This seems counter-intuitive since ** clearly the mutex cannot be held if it does not exist. But ** the reason the mutex does not exist is because the build is not ** using mutexes. And we do not want the assert() containing the ** call to sqlite3_mutex_held() to fail, so a non-zero return is -** the appropriate thing to do. ^The sqlite3_mutex_notheld() +** the appropriate thing to do. The sqlite3_mutex_notheld() ** interface should also return 1 when given a NULL pointer. */ #ifndef NDEBUG From 2d8233157da0be2fa529b14c4f625f3c15ec38af Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 23:11:30 +0000 Subject: [PATCH 593/710] Fix a benign test error on PRAGMA collation_list introduced by a recent checkin. FossilOrigin-Name: 332cc9591d05508ac9cb56fde2b82e20e0342d1f --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/pragma.test | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3b9c283125..3641e5837e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\srequirements\stags\son\sthe\smutex\sdocumentation. -D 2014-11-20T23:03:42.154 +C Fix\sa\sbenign\stest\serror\son\sPRAGMA\scollation_list\nintroduced\sby\sa\srecent\scheckin. +D 2014-11-20T23:11:30.405 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -777,7 +777,7 @@ F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 F test/permutations.test cef25f5e8499a15846eccd06785f17f4180407ab -F test/pragma.test 19d0241a007bcdd77fc2606ec60fc60357e7fc8b +F test/pragma.test 49ac8a73c0daa574824538fed28727d1259fe735 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4b608b62ac8d4eafdb76192b3b5db272332a4bfd -R b10e3d696841d651117121c42237df0a +P fcf8b7e4c6c3893e2004a28dc9f0f677907b4ba9 +R 97435df5b02a85b2d9795aaf4e4e7c6e U drh -Z 5d29b887364b9011d92605ad266bc7f8 +Z ee8d2a1291f4df2e216b5b6437eb1599 diff --git a/manifest.uuid b/manifest.uuid index fb1afe2201..aa16af69b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fcf8b7e4c6c3893e2004a28dc9f0f677907b4ba9 \ No newline at end of file +332cc9591d05508ac9cb56fde2b82e20e0342d1f \ No newline at end of file diff --git a/test/pragma.test b/test/pragma.test index 539d867366..e660ab0fe7 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -1190,13 +1190,13 @@ ifcapable schema_pragmas { execsql2 { pragma collation_list; } - } {seq 0 name NOCASE seq 1 name RTRIM seq 2 name BINARY} + } {seq 0 name RTRIM seq 1 name NOCASE seq 2 name BINARY} do_test pragma-11.2 { db collate New_Collation blah... execsql { pragma collation_list; } - } {0 New_Collation 1 NOCASE 2 RTRIM 3 BINARY} + } {0 New_Collation 1 RTRIM 2 NOCASE 3 BINARY} } ifcapable schema_pragmas&&tempdb { From 643091f07104b3dd2e3793286ddb1f1b5eb99f87 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Nov 2014 23:21:23 +0000 Subject: [PATCH 594/710] Fix a typo in a requirements mark on the abs() SQL function. FossilOrigin-Name: b1e6c02f8b9a2afaa12ac15a33e3f698c3be27d6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/func.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3641e5837e..3722a0fbac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbenign\stest\serror\son\sPRAGMA\scollation_list\nintroduced\sby\sa\srecent\scheckin. -D 2014-11-20T23:11:30.405 +C Fix\sa\stypo\sin\sa\srequirements\smark\son\sthe\sabs()\sSQL\sfunction. +D 2014-11-20T23:21:23.554 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -185,7 +185,7 @@ F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 F src/expr.c 73de4c0da2eed6b149d40a05c589dfeb2c4a87a1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 -F src/func.c ba47c1671ab3cfdafa6e9d6ee490939ea578adee +F src/func.c 6d3c4ebd72aa7923ce9b110a7dc15f9b8c548430 F src/global.c 6ded36dda9466fc1c9a3c5492ded81d79bf3977d F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 F src/hash.h c8f3c31722cf3277d03713909761e152a5b81094 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fcf8b7e4c6c3893e2004a28dc9f0f677907b4ba9 -R 97435df5b02a85b2d9795aaf4e4e7c6e +P 332cc9591d05508ac9cb56fde2b82e20e0342d1f +R 5693b17271f5aafc5aa0ee222d419f56 U drh -Z ee8d2a1291f4df2e216b5b6437eb1599 +Z 26e3a9a90b782d10bf5cba52cec231a6 diff --git a/manifest.uuid b/manifest.uuid index aa16af69b0..2d35c62379 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -332cc9591d05508ac9cb56fde2b82e20e0342d1f \ No newline at end of file +b1e6c02f8b9a2afaa12ac15a33e3f698c3be27d6 \ No newline at end of file diff --git a/src/func.c b/src/func.c index cf556e2439..a057993413 100644 --- a/src/func.c +++ b/src/func.c @@ -157,8 +157,8 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ default: { /* Because sqlite3_value_double() returns 0.0 if the argument is not ** something that can be converted into a number, we have: - ** IMP: R-57326-31541 Abs(X) return 0.0 if X is a string or blob that - ** cannot be converted to a numeric value. + ** IMP: R-01992-00519 Abs(X) returns 0.0 if X is a string or blob + ** that cannot be converted to a numeric value. */ double rVal = sqlite3_value_double(argv[0]); if( rVal<0 ) rVal = -rVal; From a3898250d4584338fc1f4df6d5af28c2acbd33df Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Nov 2014 12:22:13 +0000 Subject: [PATCH 595/710] Remove a redundant test case (probably a copy/paste error). Add an assert() to where.c to ensure that automatic indexes do not have there output row counts adjusted downward by supplementary constraints. FossilOrigin-Name: eea47933493c85a85247ad383bd148b06f627d04 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 1 + test/autoindex2.test | 44 -------------------------------------------- 4 files changed, 9 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index 3722a0fbac..1536469c11 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sa\srequirements\smark\son\sthe\sabs()\sSQL\sfunction. -D 2014-11-20T23:21:23.554 +C Remove\sa\sredundant\stest\scase\s(probably\sa\scopy/paste\serror).\s\sAdd\san\sassert()\nto\swhere.c\sto\sensure\sthat\sautomatic\sindexes\sdo\snot\shave\sthere\soutput\srow\s\ncounts\sadjusted\sdownward\sby\ssupplementary\sconstraints. +D 2014-11-22T12:22:13.640 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -304,7 +304,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 486e644b3b8aa5ad066f625bc428aa8ff7001405 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c e275cb74731a3351a9da6ba8280bd5054db6192d +F src/where.c 13a9920431358fc4885b31fe13a893d98f813a74 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -346,7 +346,7 @@ F test/auth2.test 264c6af53cad9aba5218c68bbe18036e39007bfa F test/auth3.test 5cfa94ed90c6617c42b7ba4b133fd79678b251c7 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 F test/autoindex1.test 6ff78b94f43a59616c06c11c55b12935173506d7 -F test/autoindex2.test 60d2fc6f38364308ce73a9beb01b47ded38697de +F test/autoindex2.test af7e595c6864cc6ef5fc38d5db579a3e34940cb8 F test/autoindex3.test a3be0d1a53a7d2edff208a5e442312957047e972 F test/autoindex4.test fc807f9efd158bec60f5dfdf34ebe46fb274612d F test/autovacuum.test 941892505d2c0f410a0cb5970dfa1c7c4e5f6e74 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 332cc9591d05508ac9cb56fde2b82e20e0342d1f -R 5693b17271f5aafc5aa0ee222d419f56 +P b1e6c02f8b9a2afaa12ac15a33e3f698c3be27d6 +R e5e0eb3b1af9c334c014cb61507da7b4 U drh -Z 26e3a9a90b782d10bf5cba52cec231a6 +Z 8955433275197ef8c0924c06e567755e diff --git a/manifest.uuid b/manifest.uuid index 2d35c62379..070039fabe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b1e6c02f8b9a2afaa12ac15a33e3f698c3be27d6 \ No newline at end of file +eea47933493c85a85247ad383bd148b06f627d04 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8c9ce38889..6dd47816c3 100644 --- a/src/where.c +++ b/src/where.c @@ -4306,6 +4306,7 @@ static void whereLoopOutputAdjust( int i, j; int nEq = 0; /* Number of = constraints not within likely()/unlikely() */ + assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; diff --git a/test/autoindex2.test b/test/autoindex2.test index f4da416745..c8abdf410a 100644 --- a/test/autoindex2.test +++ b/test/autoindex2.test @@ -224,48 +224,4 @@ do_execsql_test autoindex2-120 { # on t3 and reordering the tables so that t3 was in the outer loop and # implementing the ORDER BY clause using a B-Tree. -do_execsql_test autoindex2-120 { - EXPLAIN QUERY PLAN - SELECT - t1_id, - t1.did, - param2, - param3, - t1.ptime, - t1.trange, - t1.exmass, - t1.mass, - t1.vstatus, - type, - subtype, - t1.deviation, - t1.formula, - dparam1, - reserve1, - reserve2, - param4, - t1.last_operation, - t1.admin_uuid, - t1.previous_value, - t1.job_id, - client_did, - t1.last_t1, - t1.data_t1, - t1.previous_date, - param5, - param6, - mgr_uuid - FROM - t3, - t2, - t1 - WHERE - t1.ptime > 1393520400 - AND param3<>9001 - AND t3.flg7 = 1 - AND t1.did = t2.did - AND t2.uid = t3.uid - ORDER BY t1.ptime desc LIMIT 500; -} {0 0 2 {SEARCH TABLE t1 USING INDEX t1x1 (ptime>?)} 0 1 1 {SEARCH TABLE t2 USING INDEX t2x0 (did=?)} 0 2 0 {SEARCH TABLE t3 USING INDEX t3x0 (uid=?)}} - finish_test From 7a1bca7e74806e2820ca2f9421e660c191b3a05d Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Nov 2014 18:50:44 +0000 Subject: [PATCH 596/710] Deploy heuristics (well-commented) to better estimate how much unindexed terms in the WHERE clause filter the number of output rows from a single table. FossilOrigin-Name: 221659945c3f78d3b6789bfe8fdeb8d3ee1fa038 --- manifest | 14 ++++++------ manifest.uuid | 2 +- src/where.c | 52 +++++++++++++++++++++++++++++++++----------- test/scanstatus.test | 2 +- 4 files changed, 48 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 1536469c11..ab39af5891 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\sredundant\stest\scase\s(probably\sa\scopy/paste\serror).\s\sAdd\san\sassert()\nto\swhere.c\sto\sensure\sthat\sautomatic\sindexes\sdo\snot\shave\sthere\soutput\srow\s\ncounts\sadjusted\sdownward\sby\ssupplementary\sconstraints. -D 2014-11-22T12:22:13.640 +C Deploy\sheuristics\s(well-commented)\sto\sbetter\sestimate\show\smuch\sunindexed\sterms\nin\sthe\sWHERE\sclause\sfilter\sthe\snumber\sof\soutput\srows\sfrom\sa\ssingle\stable. +D 2014-11-22T18:50:44.269 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -304,7 +304,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 486e644b3b8aa5ad066f625bc428aa8ff7001405 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 13a9920431358fc4885b31fe13a893d98f813a74 +F src/where.c 33285024c5d96a17901e1beb04798cd8c15d00ae F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -811,7 +811,7 @@ F test/savepoint4.test c8f8159ade6d2acd9128be61e1230f1c1edc6cc0 F test/savepoint5.test 0735db177e0ebbaedc39812c8d065075d563c4fd F test/savepoint6.test f41279c5e137139fa5c21485773332c7adb98cd7 F test/savepoint7.test db3db281486c925095f305aad09fe806e5188ff3 -F test/scanstatus.test a6dd739bc4d9638e8f5c2493b518057f2b681655 +F test/scanstatus.test 5253c219e331318a437f436268e0e82345700285 F test/schema.test 8f7999be894260f151adf15c2c7540f1c6d6a481 F test/schema2.test 906408621ea881fdb496d878b1822572a34e32c5 F test/schema3.test 1bc1008e1f8cb5654b248c55f27249366eb7ed38 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b1e6c02f8b9a2afaa12ac15a33e3f698c3be27d6 -R e5e0eb3b1af9c334c014cb61507da7b4 +P eea47933493c85a85247ad383bd148b06f627d04 +R c99efa7eb98e96584d3723ffcac72bfb U drh -Z 8955433275197ef8c0924c06e567755e +Z ec1ff7ab563a777168df83432a271a84 diff --git a/manifest.uuid b/manifest.uuid index 070039fabe..e47defa2d3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eea47933493c85a85247ad383bd148b06f627d04 \ No newline at end of file +221659945c3f78d3b6789bfe8fdeb8d3ee1fa038 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6dd47816c3..05663ce472 100644 --- a/src/where.c +++ b/src/where.c @@ -4291,10 +4291,30 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** Adjust the WhereLoop.nOut value downward to account for terms of the ** WHERE clause that reference the loop but which are not used by an ** index. +* +** For every WHERE clause term that is not used by the index +** and which has a truth probability assigned by one of the likelihood(), +** likely(), or unlikely() SQL functions, reduce the estimated number +** of output rows by the probability specified. ** -** In the current implementation, the first extra WHERE clause term reduces -** the number of output rows by a factor of 10 and each additional term -** reduces the number of output rows by sqrt(2). +** TUNING: For every WHERE clause term that is not used by the index +** and which does not have an assigned truth probability, heuristics +** described below are used to try to estimate the truth probability. +** TODO --> Perhaps this is something that could be improved by better +** table statistics. +** +** Heuristic 1: Estimate the truth probability as 6.25%. The 6.25% +** value corresponds to 1 in LogEst notation, so this means decrement +** the WhereLoop.nOut field for every such WHERE clause term. +** +** Heuristic 2: If there exists one or more WHERE clause terms of the +** form "x==EXPR" and EXPR is not a constant 0 or 1, then make sure the +** final output row estimate is no greater than 1/4 of the total number +** of rows in the table. In other words, assume that x==EXPR will filter +** out at least 3 out of 4 rows. If EXPR is -1 or 0 or 1, then maybe the +** "x" column is boolean or else -1 or 0 or 1 is a common default value +** on the "x" column and so in that case only cap the output row estimate +** at 1/2 instead of 1/4. */ static void whereLoopOutputAdjust( WhereClause *pWC, /* The WHERE clause */ @@ -4303,8 +4323,8 @@ static void whereLoopOutputAdjust( ){ WhereTerm *pTerm, *pX; Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); - int i, j; - int nEq = 0; /* Number of = constraints not within likely()/unlikely() */ + int i, j, k; + LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ @@ -4319,20 +4339,26 @@ static void whereLoopOutputAdjust( } if( j<0 ){ if( pTerm->truthProb<=0 ){ + /* If a truth probability is specified using the likelihood() hints, + ** then use the probability provided by the application. */ pLoop->nOut += pTerm->truthProb; }else{ + /* In the absence of explicit truth probabilities, use heuristics to + ** guess a reasonable truth probability. */ pLoop->nOut--; - if( pTerm->eOperator&WO_EQ ) nEq++; + if( pTerm->eOperator&WO_EQ ){ + Expr *pRight = pTerm->pExpr->pRight; + if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ + k = 10; + }else{ + k = 20; + } + if( iReducenOut>nRow-10 ){ - pLoop->nOut = nRow - 10; - } + if( pLoop->nOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce; } /* diff --git a/test/scanstatus.test b/test/scanstatus.test index 7713bae5fc..ed24d97437 100644 --- a/test/scanstatus.test +++ b/test/scanstatus.test @@ -268,7 +268,7 @@ do_scanstatus_test 4.2.2 { nLoop 1 nVisit 1 nEst 1.0 zName sqlite_autoindex_p1_1 zExplain {SEARCH TABLE p1 USING INDEX sqlite_autoindex_p1_1 (x=?)} - nLoop 1 nVisit 3 nEst 524288.0 zName c1 zExplain {SCAN TABLE c1} + nLoop 1 nVisit 3 nEst 262144.0 zName c1 zExplain {SCAN TABLE c1} } #------------------------------------------------------------------------- From ab4624d0058253670ffda8429840d5bf1c3d983d Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Nov 2014 19:52:10 +0000 Subject: [PATCH 597/710] Fix an error in the comments from the previous check-in. FossilOrigin-Name: 9660ce541837ccd8df415641a922274e093056aa --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ab39af5891..cb3e8bd605 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Deploy\sheuristics\s(well-commented)\sto\sbetter\sestimate\show\smuch\sunindexed\sterms\nin\sthe\sWHERE\sclause\sfilter\sthe\snumber\sof\soutput\srows\sfrom\sa\ssingle\stable. -D 2014-11-22T18:50:44.269 +C Fix\san\serror\sin\sthe\scomments\sfrom\sthe\sprevious\scheck-in. +D 2014-11-22T19:52:10.008 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -304,7 +304,7 @@ F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 486e644b3b8aa5ad066f625bc428aa8ff7001405 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 33285024c5d96a17901e1beb04798cd8c15d00ae +F src/where.c a0b16f9d78321cb340a977287d19f826555c7d3b F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eea47933493c85a85247ad383bd148b06f627d04 -R c99efa7eb98e96584d3723ffcac72bfb +P 221659945c3f78d3b6789bfe8fdeb8d3ee1fa038 +R e42483f1b7bc821fcaab0f4c56777390 U drh -Z ec1ff7ab563a777168df83432a271a84 +Z edb3988f151b88d75256a46f70366322 diff --git a/manifest.uuid b/manifest.uuid index e47defa2d3..ba985640bb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -221659945c3f78d3b6789bfe8fdeb8d3ee1fa038 \ No newline at end of file +9660ce541837ccd8df415641a922274e093056aa \ No newline at end of file diff --git a/src/where.c b/src/where.c index 05663ce472..277c8833e8 100644 --- a/src/where.c +++ b/src/where.c @@ -4303,8 +4303,8 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ ** TODO --> Perhaps this is something that could be improved by better ** table statistics. ** -** Heuristic 1: Estimate the truth probability as 6.25%. The 6.25% -** value corresponds to 1 in LogEst notation, so this means decrement +** Heuristic 1: Estimate the truth probability as 93.75%. The 93.75% +** value corresponds to -1 in LogEst notation, so this means decrement ** the WhereLoop.nOut field for every such WHERE clause term. ** ** Heuristic 2: If there exists one or more WHERE clause terms of the From 0da10d32897a5d98ab237cf5efc9dd9ad004600b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Nov 2014 21:37:00 +0000 Subject: [PATCH 598/710] Always reinitialized the Index.bUnordered and Index.noSkipscan flags before rereading the sqlite_stat1 table, even if SQLITE_ENABLE_STAT4 is defined. FossilOrigin-Name: 1e1221fc4823a6bb6fc5d2408732e27aca585de9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 35 ++++++++++++++++++----------------- 3 files changed, 25 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index cb3e8bd605..125cecdf0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\serror\sin\sthe\scomments\sfrom\sthe\sprevious\scheck-in. -D 2014-11-22T19:52:10.008 +C Always\sreinitialized\sthe\sIndex.bUnordered\sand\sIndex.noSkipscan\sflags\sbefore\nrereading\sthe\ssqlite_stat1\stable,\seven\sif\sSQLITE_ENABLE_STAT4\sis\sdefined. +D 2014-11-22T21:37:00.608 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c c59f238a39aacece176f8bb7dfece40deb268ee5 +F src/analyze.c f7d774356ba5a14e7ad4fb637681af16875ad88f F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea @@ -1221,7 +1221,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 221659945c3f78d3b6789bfe8fdeb8d3ee1fa038 -R e42483f1b7bc821fcaab0f4c56777390 +P 9660ce541837ccd8df415641a922274e093056aa +R b51cb063fe9c06786b358d55d58b3f87 U drh -Z edb3988f151b88d75256a46f70366322 +Z 1318c316d07e28b9d1759e5cafc8a50c diff --git a/manifest.uuid b/manifest.uuid index ba985640bb..47b33e7181 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9660ce541837ccd8df415641a922274e093056aa \ No newline at end of file +1e1221fc4823a6bb6fc5d2408732e27aca585de9 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index ffb5c22b63..01c2f12952 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1438,8 +1438,6 @@ static void decodeIntArray( if( z==0 ) z = ""; #else assert( z!=0 ); - pIndex->bUnordered = 0; - pIndex->noSkipScan = 0; #endif for(i=0; *z && ibUnordered = 1; - }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ - pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); - }else if( sqlite3_strglob("noskipscan*", z)==0 ){ - pIndex->noSkipScan = 1; - } + pIndex->bUnordered = 0; + pIndex->noSkipScan = 0; + while( z[0] ){ + if( sqlite3_strglob("unordered*", z)==0 ){ + pIndex->bUnordered = 1; + }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ + pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + }else if( sqlite3_strglob("noskipscan*", z)==0 ){ + pIndex->noSkipScan = 1; + } #ifdef SQLITE_ENABLE_COSTMULT - else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ - pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); - } + else if( sqlite3_strglob("costmult=[0-9]*",z)==0 ){ + pIndex->pTable->costMult = sqlite3LogEst(sqlite3Atoi(z+9)); + } #endif - while( z[0]!=0 && z[0]!=' ' ) z++; - while( z[0]==' ' ) z++; + while( z[0]!=0 && z[0]!=' ' ) z++; + while( z[0]==' ' ) z++; + } } } From 0a79238b3925a2c94539379d7fe996425d7e589b Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 25 Nov 2014 18:59:55 +0000 Subject: [PATCH 599/710] Fix an integer overflow bug in vdbesort.c. FossilOrigin-Name: 623827192532f08b68bc0eb9ed1449e173361f0c --- manifest | 17 +++++++++-------- manifest.uuid | 2 +- src/vdbesort.c | 9 ++++++++- test/bigsort.test | 43 ++++++++++++++++++++++++++++++++++++++++++ test/permutations.test | 1 + 5 files changed, 62 insertions(+), 10 deletions(-) create mode 100644 test/bigsort.test diff --git a/manifest b/manifest index 125cecdf0c..61c47f746e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Always\sreinitialized\sthe\sIndex.bUnordered\sand\sIndex.noSkipscan\sflags\sbefore\nrereading\sthe\ssqlite_stat1\stable,\seven\sif\sSQLITE_ENABLE_STAT4\sis\sdefined. -D 2014-11-22T21:37:00.608 +C Fix\san\sinteger\soverflow\sbug\sin\svdbesort.c. +D 2014-11-25T18:59:55.761 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -298,7 +298,7 @@ F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 F src/vdbeaux.c 5ce4f414147a3bc3cbcf00ec57f2606c25791629 F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f -F src/vdbesort.c 87f3923483113d1c95d84640becb4e4946f27d9a +F src/vdbesort.c 42c166f7ca78cb643c7f4e4bdfa83c59d363d1a6 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 F src/wal.c 486e644b3b8aa5ad066f625bc428aa8ff7001405 @@ -366,6 +366,7 @@ F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747 +F test/bigsort.test 835478d0ce83bd1e5b05c90571dedd9871a09196 F test/bind.test 3c7b320969000c441a70952b0b15938fbb66237c F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 @@ -776,7 +777,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test cef25f5e8499a15846eccd06785f17f4180407ab +F test/permutations.test 4e12d43f4639ea8a0e366d9c64e0009afe2eb544 F test/pragma.test 49ac8a73c0daa574824538fed28727d1259fe735 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1221,7 +1222,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9660ce541837ccd8df415641a922274e093056aa -R b51cb063fe9c06786b358d55d58b3f87 -U drh -Z 1318c316d07e28b9d1759e5cafc8a50c +P 1e1221fc4823a6bb6fc5d2408732e27aca585de9 +R d9420ee27960cebcc5a5d23852d1a5da +U dan +Z 0b92bbab30b87a9f1cf79fdddb96f8d5 diff --git a/manifest.uuid b/manifest.uuid index 47b33e7181..03036e1aff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e1221fc4823a6bb6fc5d2408732e27aca585de9 \ No newline at end of file +623827192532f08b68bc0eb9ed1449e173361f0c \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index df8357a57e..7c736adefe 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -147,6 +147,13 @@ # define SQLITE_DEBUG_SORTER_THREADS 1 #endif +/* +** Hard-coded maximum amount of data to accumulate in memory before flushing +** to a level 0 PMA. The purpose of this limit is to prevent various integer +** overflows. 512MiB. +*/ +#define SQLITE_MAX_MXPMASIZE (1<<29) + /* ** Private objects used by the sorter */ @@ -845,7 +852,7 @@ int sqlite3VdbeSorterInit( pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; mxCache = db->aDb[0].pSchema->cache_size; if( mxCachemxPmaSize = mxCache * pgsz; + pSorter->mxPmaSize = MIN((i64)mxCache*pgsz, SQLITE_MAX_MXPMASIZE); /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary diff --git a/test/bigsort.test b/test/bigsort.test new file mode 100644 index 0000000000..259adc3747 --- /dev/null +++ b/test/bigsort.test @@ -0,0 +1,43 @@ +# 2014 November 26 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix bigsort + +#-------------------------------------------------------------------- +# At one point there was an overflow problem if the product of the +# cache-size and page-size was larger than 2^31. Causing an infinite +# loop if the product was also an integer multiple of 2^32, or +# inefficiency otherwise. +# +do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a, b); + BEGIN; + WITH data(x,y) AS ( + SELECT 1, zeroblob(10000) + UNION ALL + SELECT x+1, y FROM data WHERE x < 300000 + ) + INSERT INTO t1 SELECT * FROM data; + COMMIT; +} +do_execsql_test 1.1 { + PRAGMA cache_size = 4194304; + CREATE INDEX i1 ON t1(a, b); +} + + +finish_test + + diff --git a/test/permutations.test b/test/permutations.test index 54239cc759..ff61bf644b 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -113,6 +113,7 @@ set allquicktests [test_set $alltests -exclude { vtab_err.test walslow.test walcrash.test walcrash3.test walthread.test rtree3.test indexfault.test securedel2.test sort3.test sort4.test fts4growth.test fts4growth2.test + bigsort.test }] if {[info exists ::env(QUICKTEST_INCLUDE)]} { set allquicktests [concat $allquicktests $::env(QUICKTEST_INCLUDE)] From 5ab63775ddb052353c3d1c0774865be3f201bf67 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Nov 2014 03:46:04 +0000 Subject: [PATCH 600/710] Fix a problem in the new b-tree balancer that was causing corruption of the fragmentation count. FossilOrigin-Name: f242394e079dd185aad90f2aee902a5edf27e150 --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/btree.c | 13 +++++++++++-- test/btree01.test | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 test/btree01.test diff --git a/manifest b/manifest index 61c47f746e..21c1471b6b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sinteger\soverflow\sbug\sin\svdbesort.c. -D 2014-11-25T18:59:55.761 +C Fix\sa\sproblem\sin\sthe\snew\sb-tree\sbalancer\sthat\swas\scausing\scorruption\sof\nthe\sfragmentation\scount. +D 2014-11-27T03:46:04.203 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 4db5e06ca2d1a5437be7075251fa702c76179b0e +F src/btree.c c82a514c0622e47141cca05617f5fa17ecbc909f F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -379,6 +379,7 @@ F test/boundary3.tcl 23361e108a125dca9c4080c2feb884fe54d69243 F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b +F test/btree01.test 717fcf43f66da534671bef795c993e2f9b3b2e3e F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de @@ -1222,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1e1221fc4823a6bb6fc5d2408732e27aca585de9 -R d9420ee27960cebcc5a5d23852d1a5da -U dan -Z 0b92bbab30b87a9f1cf79fdddb96f8d5 +P 623827192532f08b68bc0eb9ed1449e173361f0c +R 93dc522ced281537ee4ee3dde5b73f25 +U drh +Z 0ffb28de24b6b45b6e628c5a31b22be7 diff --git a/manifest.uuid b/manifest.uuid index 03036e1aff..a6fb19304d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -623827192532f08b68bc0eb9ed1449e173361f0c \ No newline at end of file +f242394e079dd185aad90f2aee902a5edf27e150 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 5ae5986cfa..49ac6208a9 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6257,6 +6257,14 @@ static int pageFreeArray( } /* +** apCell[] and szCell[] contains pointers to and sizes of all cells in the +** pages being balanced. The current page, pPg, has pPg->nCell cells starting +** with apCell[iOld]. After balancing, this page should hold nNew cells +** starting at apCell[iNew]. +** +** This routine makes the necessary adjustments to pPg so that it contains +** the correct cells after being balanced. +** ** The pPg->nFree field is invalid when this function returns. It is the ** responsibility of the caller to set it correctly. */ @@ -6297,12 +6305,13 @@ static void editPage( ); } - pData = &aData[get2byte(&aData[hdr+5])]; + pData = &aData[get2byteNotZero(&aData[hdr+5])]; if( pDataaCellIdx; memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); if( pageInsertArray( diff --git a/test/btree01.test b/test/btree01.test new file mode 100644 index 0000000000..09358ff76f --- /dev/null +++ b/test/btree01.test @@ -0,0 +1,34 @@ +# 2014-11-27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains test cases for b-tree logic. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix btree01 + +# The refactoring on the b-tree balance() routine in check-in +# http://www.sqlite.org/src/info/face33bea1ba3a (2014-10-27) +# caused the integrity_check on the following SQL to fail. +# +do_execsql_test btree01-1.1 { + PRAGMA page_size=65536; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB); + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(3000); + UPDATE t1 SET b=zeroblob(64000) WHERE a=2; + PRAGMA integrity_check; +} {ok} + +finish_test From 3f2d22e1a3a8f49a47d131df38b319f3575dffa8 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Nov 2014 04:23:19 +0000 Subject: [PATCH 601/710] More test cases for the balancer. FossilOrigin-Name: 358ea818f7ea5aa55bafaf4057e9fc7a5fd77c11 --- manifest | 12 +++--- manifest.uuid | 2 +- test/btree01.test | 98 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 21c1471b6b..06e08e13af 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\snew\sb-tree\sbalancer\sthat\swas\scausing\scorruption\sof\nthe\sfragmentation\scount. -D 2014-11-27T03:46:04.203 +C More\stest\scases\sfor\sthe\sbalancer. +D 2014-11-27T04:23:19.652 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -379,7 +379,7 @@ F test/boundary3.tcl 23361e108a125dca9c4080c2feb884fe54d69243 F test/boundary3.test 56ef82096b4329aca2be74fa1e2b0f762ea0eb45 F test/boundary4.tcl 0bb4b1a94f4fc5ae59b79b9a2b7a140c405e2983 F test/boundary4.test 89e02fa66397b8a325d5eb102b5806f961f8ec4b -F test/btree01.test 717fcf43f66da534671bef795c993e2f9b3b2e3e +F test/btree01.test e08b3613540145b353f20c81cb18ead54ff12e0f F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 623827192532f08b68bc0eb9ed1449e173361f0c -R 93dc522ced281537ee4ee3dde5b73f25 +P f242394e079dd185aad90f2aee902a5edf27e150 +R dee46347027ae53917346a5c99ff404e U drh -Z 0ffb28de24b6b45b6e628c5a31b22be7 +Z 45538c84f085c10577fd84190398d06f diff --git a/manifest.uuid b/manifest.uuid index a6fb19304d..19339500a5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f242394e079dd185aad90f2aee902a5edf27e150 \ No newline at end of file +358ea818f7ea5aa55bafaf4057e9fc7a5fd77c11 \ No newline at end of file diff --git a/test/btree01.test b/test/btree01.test index 09358ff76f..25f2c6897b 100644 --- a/test/btree01.test +++ b/test/btree01.test @@ -31,4 +31,102 @@ do_execsql_test btree01-1.1 { PRAGMA integrity_check; } {ok} +# The previous test is sufficient to prevent a regression. But we +# add a number of additional tests to stress the balancer in similar +# ways, looking for related problems. +# +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.2.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(3000); + UPDATE t1 SET b=zeroblob(64000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.3.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(2000); + UPDATE t1 SET b=zeroblob(64000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.4.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(6499) WHERE (a%3)==0; + UPDATE t1 SET b=zeroblob(6499) WHERE (a%3)==1; + UPDATE t1 SET b=zeroblob(6499) WHERE (a%3)==2; + UPDATE t1 SET b=zeroblob(64000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.5.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6542) FROM c; + UPDATE t1 SET b=zeroblob(2331); + UPDATE t1 SET b=zeroblob(65496) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.6.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6542) FROM c; + UPDATE t1 SET b=zeroblob(2332); + UPDATE t1 SET b=zeroblob(65496) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=30} {incr i} { + do_test btree01-1.7.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<30) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(1); + UPDATE t1 SET b=zeroblob(65000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} +for {set i 1} {$i<=31} {incr i} { + do_test btree01-1.8.$i { + db eval { + DELETE FROM t1; + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<31) + INSERT INTO t1(a,b) SELECT i, zeroblob(6500) FROM c; + UPDATE t1 SET b=zeroblob(4000); + UPDATE t1 SET b=zeroblob(65000) WHERE a=$::i; + PRAGMA integrity_check; + } + } {ok} +} + finish_test From d89b834f54b4026944df0668dbc5fe7803033a8f Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 27 Nov 2014 11:36:36 +0000 Subject: [PATCH 602/710] Fix a buffer overread during compilation of CREATE VIRTUAL TABLE statements that featured an explicit database name but no virtual table arguments. For example, "CREATE VIRTUAL TABLE main.ft USING fts4". FossilOrigin-Name: f095cde579e7417306e11b5c1d2dd90b6bb547d5 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/test1.c | 17 ++++++++++++++++- src/vtab.c | 7 ++++++- test/vtab1.test | 42 ++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 06e08e13af..273e715897 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C More\stest\scases\sfor\sthe\sbalancer. -D 2014-11-27T04:23:19.652 +C Fix\sa\sbuffer\soverread\sduring\scompilation\sof\sCREATE\sVIRTUAL\sTABLE\sstatements\sthat\sfeatured\san\sexplicit\sdatabase\sname\sbut\sno\svirtual\stable\sarguments.\sFor\sexample,\s"CREATE\sVIRTUAL\sTABLE\smain.ft\sUSING\sfts4". +D 2014-11-27T11:36:36.295 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -238,7 +238,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 -F src/test1.c 6b0469b8e06c77b1de1d3e4a3834cf26edea9cc7 +F src/test1.c a0bce4f47da65b76c80e5f8bf9a5ef174603866a F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -300,7 +300,7 @@ F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 42c166f7ca78cb643c7f4e4bdfa83c59d363d1a6 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 -F src/vtab.c 2a30791bbd7926b589401bd09c3abb33de563793 +F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 F src/wal.c 486e644b3b8aa5ad066f625bc428aa8ff7001405 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 @@ -1091,7 +1091,7 @@ F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test f311691d696a5cc27e3c1b875cec1b0866b4ccd9 -F test/vtab1.test b631d147b198cfd7903ab5fed028eb2a3d321dc6 +F test/vtab1.test 1cef14310144718812351a61c5cfb4ba8494a171 F test/vtab2.test 7bcffc050da5c68f4f312e49e443063e2d391c0d F test/vtab3.test b45f47d20f225ccc9c28dc915d92740c2dee311e F test/vtab4.test 942f8b8280b3ea8a41dae20e7822d065ca1cb275 @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f242394e079dd185aad90f2aee902a5edf27e150 -R dee46347027ae53917346a5c99ff404e -U drh -Z 45538c84f085c10577fd84190398d06f +P 358ea818f7ea5aa55bafaf4057e9fc7a5fd77c11 +R a4b8cae126976a974aa80f7f44ce5b7f +U dan +Z 67582253c41fc622f4b8d67b4605ed6a diff --git a/manifest.uuid b/manifest.uuid index 19339500a5..e316358ddb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -358ea818f7ea5aa55bafaf4057e9fc7a5fd77c11 \ No newline at end of file +f095cde579e7417306e11b5c1d2dd90b6bb547d5 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index ca3b54a513..1c43861547 100644 --- a/src/test1.c +++ b/src/test1.c @@ -3638,6 +3638,7 @@ static int test_prepare_v2( ){ sqlite3 *db; const char *zSql; + char *zCopy = 0; /* malloc() copy of zSql */ int bytes; const char *zTail = 0; sqlite3_stmt *pStmt = 0; @@ -3653,7 +3654,21 @@ static int test_prepare_v2( zSql = Tcl_GetString(objv[2]); if( Tcl_GetIntFromObj(interp, objv[3], &bytes) ) return TCL_ERROR; - rc = sqlite3_prepare_v2(db, zSql, bytes, &pStmt, objc>=5 ? &zTail : 0); + /* Instead of using zSql directly, make a copy into a buffer obtained + ** directly from malloc(). The idea is to make it easier for valgrind + ** to spot buffer overreads. */ + if( bytes>=0 ){ + zCopy = malloc(bytes); + memcpy(zCopy, zSql, bytes); + }else{ + int n = strlen(zSql) + 1; + zCopy = malloc(n); + memcpy(zCopy, zSql, n); + } + rc = sqlite3_prepare_v2(db, zCopy, bytes, &pStmt, objc>=5 ? &zTail : 0); + free(zCopy); + zTail = &zSql[(zTail - zCopy)]; + assert(rc==SQLITE_OK || pStmt==0); Tcl_ResetResult(interp); if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; diff --git a/src/vtab.c b/src/vtab.c index 334de9aacb..00d0882b04 100644 --- a/src/vtab.c +++ b/src/vtab.c @@ -332,7 +332,12 @@ void sqlite3VtabBeginParse( addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); addModuleArgument(db, pTable, 0); addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); - pParse->sNameToken.n = (int)(&pModuleName->z[pModuleName->n] - pName1->z); + assert( (pParse->sNameToken.z==pName2->z && pName2->z!=0) + || (pParse->sNameToken.z==pName1->z && pName2->z==0) + ); + pParse->sNameToken.n = (int)( + &pModuleName->z[pModuleName->n] - pParse->sNameToken.z + ); #ifndef SQLITE_OMIT_AUTHORIZATION /* Creating a virtual table invokes the authorization callback twice. diff --git a/test/vtab1.test b/test/vtab1.test index 0542ee6fdd..2929b1e54d 100644 --- a/test/vtab1.test +++ b/test/vtab1.test @@ -1395,4 +1395,46 @@ do_execsql_test 21.3 { SELECT * FROM t9v WHERE a=b; } {2 2 2} +#------------------------------------------------------------------------- +# At one point executing a CREATE VIRTUAL TABLE statement that specified +# a database name but no virtual table arguments was causing an internal +# buffer overread. Valgrind would report errors while running the following +# tests. Specifically: +# +# CREATE VIRTUAL TABLE t1 USING fts4; -- Ok - no db name. +# CREATE VIRTUAL TABLE main.t1 USING fts4(x); -- Ok - has vtab arguments. +# CREATE VIRTUAL TABLE main.t1 USING fts4; -- Had the problem. +# +ifcapable fts3 { + forcedelete test.db2 + set nm [string repeat abcdefghij 100] + do_execsql_test 22.1 { + ATTACH 'test.db2' AS $nm + } + + execsql "SELECT * FROM sqlite_master" + do_execsql_test 22.2 "CREATE VIRTUAL TABLE ${nm}.t1 USING fts4" + + do_test 22.3.1 { + set sql "CREATE VIRTUAL TABLE ${nm}.t2 USING fts4" + set stmt [sqlite3_prepare_v2 db $sql -1 dummy] + sqlite3_step $stmt + } {SQLITE_DONE} + + do_test 22.3.2 { + sqlite3_finalize $stmt + } {SQLITE_OK} + + do_test 22.4.1 { + set sql "CREATE VIRTUAL TABLE ${nm}.t3 USING fts4" + set n [string length $sql] + set stmt [sqlite3_prepare db "${sql}xyz" $n dummy] + sqlite3_step $stmt + } {SQLITE_DONE} + + do_test 22.4.2 { + sqlite3_finalize $stmt + } {SQLITE_OK} +} + finish_test From cb7fe0ab9cab1f902ea2ff32243c266cef078151 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Nov 2014 11:54:44 +0000 Subject: [PATCH 603/710] Add the -end option to the command-line shell, which forces it to exit after reading prior command-line options (presumably including one or more -cmd options) and without reading standard input. FossilOrigin-Name: b59397b1f1e32c478b5fa96659cd4300177d39f7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 3 +++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 273e715897..b2e178bb04 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbuffer\soverread\sduring\scompilation\sof\sCREATE\sVIRTUAL\sTABLE\sstatements\sthat\sfeatured\san\sexplicit\sdatabase\sname\sbut\sno\svirtual\stable\sarguments.\sFor\sexample,\s"CREATE\sVIRTUAL\sTABLE\smain.ft\sUSING\sfts4". -D 2014-11-27T11:36:36.295 +C Add\sthe\s-end\soption\sto\sthe\scommand-line\sshell,\swhich\sforces\sit\sto\sexit\safter\nreading\sprior\scommand-line\soptions\s(presumably\sincluding\sone\sor\smore\s-cmd\noptions)\sand\swithout\sreading\sstandard\sinput. +D 2014-11-28T11:54:44.417 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c bc28d5992109717c87804e2eb1a08a7c8cc7a2fd +F src/shell.c f49ecad5afcae4c7fa2754a1446848582b105509 F src/sqlite.h.in c63db0117aeb749ca02b6016dbbbccbbbd9a141d F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 358ea818f7ea5aa55bafaf4057e9fc7a5fd77c11 -R a4b8cae126976a974aa80f7f44ce5b7f -U dan -Z 67582253c41fc622f4b8d67b4605ed6a +P f095cde579e7417306e11b5c1d2dd90b6bb547d5 +R 705c89a250127228fc15e4e2150ad4c0 +U drh +Z ed3d85af1529a8fcb757b6e7459b553e diff --git a/manifest.uuid b/manifest.uuid index e316358ddb..fa03660517 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f095cde579e7417306e11b5c1d2dd90b6bb547d5 \ No newline at end of file +b59397b1f1e32c478b5fa96659cd4300177d39f7 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index bbd37c7a04..46b4c0f10f 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3944,6 +3944,7 @@ static const char zOptions[] = " -cmd COMMAND run \"COMMAND\" before reading stdin\n" " -csv set output mode to 'csv'\n" " -echo print commands before execution\n" + " -end Halt. Useful after one or more -cmd\n" " -init FILENAME read/process named file\n" " -[no]header turn headers on or off\n" #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) @@ -4288,6 +4289,8 @@ int main(int argc, char **argv){ if( bail_on_error ) return rc; } } + }else if( strcmp(z, "-end")==0 ){ + return 0; }else{ fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); fprintf(stderr,"Use -help for a list of options.\n"); From ac5649a98529c751cac09c72a7a03a2c4f8b14a8 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Nov 2014 13:35:03 +0000 Subject: [PATCH 604/710] Remove the ill-designed "-end" option from the command-line shell. Instead, allow multiple SQL or dot-commands as command-line arguments. Any -cmd commands are processed first, followed by other command-line arguments, for backwards compatibility. FossilOrigin-Name: 24fa2e9832daaa5d68ee28a00c56c55f97a4da9e --- manifest | 16 +++++----- manifest.uuid | 2 +- src/shell.c | 80 +++++++++++++++++++++++++++++------------------- test/shell1.test | 31 ++++++++++--------- test/shell2.test | 6 ++-- 5 files changed, 77 insertions(+), 58 deletions(-) diff --git a/manifest b/manifest index b2e178bb04..527a5caae6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s-end\soption\sto\sthe\scommand-line\sshell,\swhich\sforces\sit\sto\sexit\safter\nreading\sprior\scommand-line\soptions\s(presumably\sincluding\sone\sor\smore\s-cmd\noptions)\sand\swithout\sreading\sstandard\sinput. -D 2014-11-28T11:54:44.417 +C Remove\sthe\sill-designed\s"-end"\soption\sfrom\sthe\scommand-line\sshell.\s\sInstead,\nallow\smultiple\sSQL\sor\sdot-commands\sas\scommand-line\sarguments.\s\sAny\s-cmd\ncommands\sare\sprocessed\sfirst,\sfollowed\sby\sother\scommand-line\sarguments,\sfor\nbackwards\scompatibility. +D 2014-11-28T13:35:03.566 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b -F src/shell.c f49ecad5afcae4c7fa2754a1446848582b105509 +F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf F src/sqlite.h.in c63db0117aeb749ca02b6016dbbbccbbbd9a141d F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d @@ -848,8 +848,8 @@ F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21 F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 -F test/shell1.test d60946b5fde4d85fe06db7331dfe89011f564350 -F test/shell2.test c57da3a381c099b02c813ba156298d5c2f5c93a3 +F test/shell1.test ab6025d941f9c84c5b83412c6b4d8b57f78dfa3a +F test/shell2.test 12b8bf901b0e3a8ac58cf5c0c63a0a388d4d1862 F test/shell3.test 5e8545ec72c4413a0e8d4c6be56496e3c257ca29 F test/shell4.test 8a9c08976291e6c6c808b4d718f4a8b299f339f5 F test/shell5.test 15a419cc1df21c892ed64f5596ae7a501f2816f2 @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f095cde579e7417306e11b5c1d2dd90b6bb547d5 -R 705c89a250127228fc15e4e2150ad4c0 +P b59397b1f1e32c478b5fa96659cd4300177d39f7 +R 1ff83ec33f372c19416fe00ba6e9161c U drh -Z ed3d85af1529a8fcb757b6e7459b553e +Z 816c9b1778ea4dbed9ea56de4b3769ca diff --git a/manifest.uuid b/manifest.uuid index fa03660517..c82d09b6ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b59397b1f1e32c478b5fa96659cd4300177d39f7 \ No newline at end of file +24fa2e9832daaa5d68ee28a00c56c55f97a4da9e \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 46b4c0f10f..94d20d3549 100644 --- a/src/shell.c +++ b/src/shell.c @@ -3944,7 +3944,6 @@ static const char zOptions[] = " -cmd COMMAND run \"COMMAND\" before reading stdin\n" " -csv set output mode to 'csv'\n" " -echo print commands before execution\n" - " -end Halt. Useful after one or more -cmd\n" " -init FILENAME read/process named file\n" " -[no]header turn headers on or off\n" #if defined(SQLITE_ENABLE_MEMSYS3) || defined(SQLITE_ENABLE_MEMSYS5) @@ -4039,10 +4038,12 @@ int main(int argc, char **argv){ char *zErrMsg = 0; ShellState data; const char *zInitFile = 0; - char *zFirstCmd = 0; int i; int rc = 0; int warnInmemoryDb = 0; + int readStdin = 1; + int nCmd = 0; + char **azCmd = 0; #if USE_SYSTEM_SQLITE+0!=1 if( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)!=0 ){ @@ -4062,6 +4063,18 @@ int main(int argc, char **argv){ signal(SIGINT, interrupt_handler); #endif +#ifdef SQLITE_SHELL_DBNAME_PROC + { + /* If the SQLITE_SHELL_DBNAME_PROC macro is defined, then it is the name + ** of a C-function that will provide the name of the database file. Use + ** this compile-time option to embed this shell program in larger + ** applications. */ + extern void SQLITE_SHELL_DBNAME_PROC(const char**); + SQLITE_SHELL_DBNAME_PROC(&data.zDbFilename); + warnInmemoryDb = 0; + } +#endif + /* Do an initial pass through the command-line argument to locate ** the name of the database file, the name of the initialization file, ** the size of the alternative malloc heap, @@ -4073,15 +4086,18 @@ int main(int argc, char **argv){ if( z[0]!='-' ){ if( data.zDbFilename==0 ){ data.zDbFilename = z; - continue; + }else{ + /* Excesss arguments are interpreted as SQL (or dot-commands) and + ** mean that nothing is read from stdin */ + readStdin = 0; + nCmd++; + azCmd = realloc(azCmd, sizeof(azCmd[0])*nCmd); + if( azCmd==0 ){ + fprintf(stderr, "out of memory\n"); + exit(1); + } + azCmd[nCmd-1] = z; } - if( zFirstCmd==0 ){ - zFirstCmd = z; - continue; - } - fprintf(stderr,"%s: Error: too many options: \"%s\"\n", Argv0, argv[i]); - fprintf(stderr,"Use -help for a list of options.\n"); - return 1; } if( z[1]=='-' ) z++; if( strcmp(z,"-separator")==0 @@ -4171,11 +4187,6 @@ int main(int argc, char **argv){ #else fprintf(stderr,"%s: Error: no database filename specified\n", Argv0); return 1; -#endif -#ifdef SQLITE_SHELL_DBNAME_PROC - { extern void SQLITE_SHELL_DBNAME_PROC(const char**); - SQLITE_SHELL_DBNAME_PROC(&data.zDbFilename); - warnInmemoryDb = 0; } #endif } data.out = stdout; @@ -4273,6 +4284,10 @@ int main(int argc, char **argv){ }else if( strcmp(z,"-help")==0 ){ usage(1); }else if( strcmp(z,"-cmd")==0 ){ + /* Run commands that follow -cmd first and separately from commands + ** that simply appear on the command-line. This seems goofy. It would + ** be better if all commands ran in the order that they appear. But + ** we retain the goofy behavior for historical compatibility. */ if( i==argc-1 ) break; z = cmdline_option_value(argc,argv,++i); if( z[0]=='.' ){ @@ -4289,8 +4304,6 @@ int main(int argc, char **argv){ if( bail_on_error ) return rc; } } - }else if( strcmp(z, "-end")==0 ){ - return 0; }else{ fprintf(stderr,"%s: Error: unknown option: %s\n", Argv0, z); fprintf(stderr,"Use -help for a list of options.\n"); @@ -4298,23 +4311,28 @@ int main(int argc, char **argv){ } } - if( zFirstCmd ){ - /* Run just the command that follows the database name + if( !readStdin ){ + /* Run all arguments that do not begin with '-' as if they were separate + ** command-line inputs, except for the argToSkip argument which contains + ** the database filename. */ - if( zFirstCmd[0]=='.' ){ - rc = do_meta_command(zFirstCmd, &data); - if( rc==2 ) rc = 0; - }else{ - open_db(&data, 0); - rc = shell_exec(data.db, zFirstCmd, shell_callback, &data, &zErrMsg); - if( zErrMsg!=0 ){ - fprintf(stderr,"Error: %s\n", zErrMsg); - return rc!=0 ? rc : 1; - }else if( rc!=0 ){ - fprintf(stderr,"Error: unable to process SQL \"%s\"\n", zFirstCmd); - return rc; + for(i=0; i Date: Tue, 2 Dec 2014 13:46:53 +0000 Subject: [PATCH 605/710] Work around overzealous NULL pointer checking in memcpy() and memset() for some systems. FossilOrigin-Name: 0d04f380e1bd17104b3cf76b64d0cfc79a726606 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 527a5caae6..ad4a3aef34 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sill-designed\s"-end"\soption\sfrom\sthe\scommand-line\sshell.\s\sInstead,\nallow\smultiple\sSQL\sor\sdot-commands\sas\scommand-line\sarguments.\s\sAny\s-cmd\ncommands\sare\sprocessed\sfirst,\sfollowed\sby\sother\scommand-line\sarguments,\sfor\nbackwards\scompatibility. -D 2014-11-28T13:35:03.566 +C Work\saround\soverzealous\sNULL\spointer\schecking\sin\smemcpy()\sand\smemset()\sfor\nsome\ssystems. +D 2014-12-02T13:46:53.642 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -295,7 +295,7 @@ F src/vdbe.c ec1f55acef4864520ca2017b9f0d60c2ac1b8b78 F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 -F src/vdbeaux.c 5ce4f414147a3bc3cbcf00ec57f2606c25791629 +F src/vdbeaux.c 6f7f39c3fcf0f5923758df8561bb5d843908a553 F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 42c166f7ca78cb643c7f4e4bdfa83c59d363d1a6 @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b59397b1f1e32c478b5fa96659cd4300177d39f7 -R 1ff83ec33f372c19416fe00ba6e9161c +P 24fa2e9832daaa5d68ee28a00c56c55f97a4da9e +R 2462364d5d687298c5bd05c143924351 U drh -Z 816c9b1778ea4dbed9ea56de4b3769ca +Z 6348ca916079ca99c53e597a8af8c112 diff --git a/manifest.uuid b/manifest.uuid index c82d09b6ab..6af54cb5bb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -24fa2e9832daaa5d68ee28a00c56c55f97a4da9e \ No newline at end of file +0d04f380e1bd17104b3cf76b64d0cfc79a726606 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index b8f5bca2c3..7900bd52ac 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -1742,7 +1742,7 @@ void sqlite3VdbeMakeReady( p->aVar[n].db = db; } } - if( p->azVar ){ + if( p->azVar && pParse->nzVar>0 ){ p->nzVar = pParse->nzVar; memcpy(p->azVar, pParse->azVar, p->nzVar*sizeof(p->azVar[0])); memset(pParse->azVar, 0, pParse->nzVar*sizeof(pParse->azVar[0])); From 1fd2d7d4715695d09bf7120c6a335e5806e8418d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 2 Dec 2014 16:16:47 +0000 Subject: [PATCH 606/710] Convert two unreachable branches into assert() statements. FossilOrigin-Name: 61b31e771430f490fc2c4cef55046debc4a5f4f5 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ad4a3aef34..7f9f9524d0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Work\saround\soverzealous\sNULL\spointer\schecking\sin\smemcpy()\sand\smemset()\sfor\nsome\ssystems. -D 2014-12-02T13:46:53.642 +C Convert\stwo\sunreachable\sbranches\sinto\sassert()\sstatements. +D 2014-12-02T16:16:47.791 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c c82a514c0622e47141cca05617f5fa17ecbc909f +F src/btree.c 44b58cd798a32579816ce274e415de262df9843d F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 24fa2e9832daaa5d68ee28a00c56c55f97a4da9e -R 2462364d5d687298c5bd05c143924351 +P 0d04f380e1bd17104b3cf76b64d0cfc79a726606 +R 22214da224af1bd07c2ba4ce527797c4 U drh -Z 6348ca916079ca99c53e597a8af8c112 +Z 4469ca4ec037115204af98a9b9e41068 diff --git a/manifest.uuid b/manifest.uuid index 6af54cb5bb..fe8f0fdad5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d04f380e1bd17104b3cf76b64d0cfc79a726606 \ No newline at end of file +61b31e771430f490fc2c4cef55046debc4a5f4f5 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 49ac6208a9..796b44403d 100644 --- a/src/btree.c +++ b/src/btree.c @@ -1356,7 +1356,7 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ defragment_page: - testcase( pPage->nCell==0 ); + assert( pPage->nCell>0 || CORRUPT_DB ); rc = defragmentPage(pPage); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); @@ -6416,7 +6416,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( pPage->nOverflow==1 ); /* This error condition is now caught prior to reaching this function */ - if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; + if( NEVER(pPage->nCell==0) ) return SQLITE_CORRUPT_BKPT; /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell From f26a1549ac363019e032f7d328d435e1c118b5fd Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Dec 2014 19:04:54 +0000 Subject: [PATCH 607/710] Add the SQLITE_CHECKPOINT_TRUNCATE option. FossilOrigin-Name: 8e20a43419e46b6b9d1f60ec7ea420bbfb3ef358 --- manifest | 29 +++++++++++--------- manifest.uuid | 2 +- src/main.c | 9 ++++--- src/pragma.c | 4 ++- src/sqlite.h.in | 21 +++++++++------ src/test1.c | 3 ++- src/vdbe.c | 1 + src/wal.c | 70 +++++++++++++++++++++++++++++++++++-------------- test/wal5.test | 43 +++++++++++++++++++++++++++++- 9 files changed, 133 insertions(+), 49 deletions(-) diff --git a/manifest b/manifest index 7f9f9524d0..35fb1bb478 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\stwo\sunreachable\sbranches\sinto\sassert()\sstatements. -D 2014-12-02T16:16:47.791 +C Add\sthe\sSQLITE_CHECKPOINT_TRUNCATE\soption. +D 2014-12-02T19:04:54.595 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 54d0f4896cebc61ae5f831937464953780fe5346 +F src/main.c 34b895b5ebfc73cd690dcd9ac6d0eecb28ce729f F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -222,7 +222,7 @@ F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 F src/pcache1.c facbdd3ecc09c8f750089d941305694301328e98 -F src/pragma.c 3f3e959390a10c0131676f0e307acce372777e0f +F src/pragma.c d54cdd40b63d608f2d95b7482c710690e3593a73 F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in c63db0117aeb749ca02b6016dbbbccbbbd9a141d +F src/sqlite.h.in 400bac7dd1294cb902b5eb85feed1689a2623ced F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 @@ -238,7 +238,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 -F src/test1.c a0bce4f47da65b76c80e5f8bf9a5ef174603866a +F src/test1.c f5d7ecd3dd663b11f35269fd91f7090db0570903 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -291,7 +291,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c ec1f55acef4864520ca2017b9f0d60c2ac1b8b78 +F src/vdbe.c 60217f3b8ab984b2e2bb9e3965276dd29e5efe5d F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -301,7 +301,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 42c166f7ca78cb643c7f4e4bdfa83c59d363d1a6 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 -F src/wal.c 486e644b3b8aa5ad066f625bc428aa8ff7001405 +F src/wal.c f09818db7ba6e31d7a681eb99f801a7722c731d9 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c a0b16f9d78321cb340a977287d19f826555c7d3b @@ -1113,7 +1113,7 @@ F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b +F test/wal5.test 174cc1512e304a7dfa28ac30527e28ea02fc37df F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd @@ -1223,7 +1223,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0d04f380e1bd17104b3cf76b64d0cfc79a726606 -R 22214da224af1bd07c2ba4ce527797c4 -U drh -Z 4469ca4ec037115204af98a9b9e41068 +P 61b31e771430f490fc2c4cef55046debc4a5f4f5 +R 837049ab8d5153c1422981d3f48166e4 +T *branch * checkpoint-truncate +T *sym-checkpoint-truncate * +T -sym-trunk * +U dan +Z ce82007701b4a39f21164e74e2380c4f diff --git a/manifest.uuid b/manifest.uuid index fe8f0fdad5..9aff9e9fbc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -61b31e771430f490fc2c4cef55046debc4a5f4f5 \ No newline at end of file +8e20a43419e46b6b9d1f60ec7ea420bbfb3ef358 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 1d34b13d6f..9ac3fde733 100644 --- a/src/main.c +++ b/src/main.c @@ -1936,10 +1936,11 @@ int sqlite3_wal_checkpoint_v2( if( pnLog ) *pnLog = -1; if( pnCkpt ) *pnCkpt = -1; - assert( SQLITE_CHECKPOINT_FULL>SQLITE_CHECKPOINT_PASSIVE ); - assert( SQLITE_CHECKPOINT_FULLSQLITE_CHECKPOINT_RESTART ){ + assert( SQLITE_CHECKPOINT_PASSIVE==0 ); + assert( SQLITE_CHECKPOINT_FULL==1 ); + assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); + if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ return SQLITE_MISUSE; } diff --git a/src/pragma.c b/src/pragma.c index 543f265ba9..ab9a283629 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2195,7 +2195,7 @@ void sqlite3Pragma( #ifndef SQLITE_OMIT_WAL /* - ** PRAGMA [database.]wal_checkpoint = passive|full|restart + ** PRAGMA [database.]wal_checkpoint = passive|full|restart|truncate ** ** Checkpoint the database. */ @@ -2207,6 +2207,8 @@ void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_FULL; }else if( sqlite3StrICmp(zRight, "restart")==0 ){ eMode = SQLITE_CHECKPOINT_RESTART; + }else if( sqlite3StrICmp(zRight, "truncate")==0 ){ + eMode = SQLITE_CHECKPOINT_TRUNCATE; } } sqlite3VdbeSetNumCols(v, 3); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index f1b917c308..807c9ee62c 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7291,6 +7291,10 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** that the next client to write to the database file restarts the log file ** from the beginning. This call blocks database writers while it is running, ** but not database readers. +** +**
    SQLITE_CHECKPOINT_TRUNCATE
    +** This mode works the same way as SQLITE_CHECKPOINT_RESTART except that, +** if successful, it also truncates the log file to zero bytes in size. ** ** ** If pnLog is not NULL, then *pnLog is set to the total number of frames in @@ -7306,11 +7310,11 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a ** busy-handler configured, it will not be invoked in this case. ** -** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive -** "writer" lock on the database file. If the writer lock cannot be obtained -** immediately, and a busy-handler is configured, it is invoked and the writer -** lock retried until either the busy-handler returns 0 or the lock is -** successfully obtained. The busy-handler is also invoked while waiting for +** The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** exclusive "writer" lock on the database file. If the writer lock cannot be +** obtained immediately, and a busy-handler is configured, it is invoked and +** the writer lock retried until either the busy-handler returns 0 or the lock +** is successfully obtained. The busy-handler is also invoked while waiting for ** database readers as described above. If the busy-handler returns 0 before ** the writer lock is obtained or while waiting for database readers, the ** checkpoint operation proceeds from that point in the same way as @@ -7349,9 +7353,10 @@ int sqlite3_wal_checkpoint_v2( ** documentation for additional information about the meaning and use of ** each of these values. */ -#define SQLITE_CHECKPOINT_PASSIVE 0 -#define SQLITE_CHECKPOINT_FULL 1 -#define SQLITE_CHECKPOINT_RESTART 2 +#define SQLITE_CHECKPOINT_PASSIVE 0 +#define SQLITE_CHECKPOINT_FULL 1 +#define SQLITE_CHECKPOINT_RESTART 2 +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* ** CAPI3REF: Virtual Table Interface Configuration diff --git a/src/test1.c b/src/test1.c index 1c43861547..be4ad92c11 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5690,10 +5690,11 @@ static int test_wal_checkpoint_v2( int nCkpt = -555; Tcl_Obj *pRet; - const char * aMode[] = { "passive", "full", "restart", 0 }; + const char * aMode[] = { "passive", "full", "restart", "truncate", 0 }; assert( SQLITE_CHECKPOINT_PASSIVE==0 ); assert( SQLITE_CHECKPOINT_FULL==1 ); assert( SQLITE_CHECKPOINT_RESTART==2 ); + assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); if( objc!=3 && objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "DB MODE ?NAME?"); diff --git a/src/vdbe.c b/src/vdbe.c index 822bf80bb8..7f661b42b1 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5724,6 +5724,7 @@ case OP_Checkpoint: { assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE || pOp->p2==SQLITE_CHECKPOINT_FULL || pOp->p2==SQLITE_CHECKPOINT_RESTART + || pOp->p2==SQLITE_CHECKPOINT_TRUNCATE ); rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &aRes[1], &aRes[2]); if( rc==SQLITE_BUSY ){ diff --git a/src/wal.c b/src/wal.c index 3033444b41..849b776743 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1623,6 +1623,34 @@ static int walPagesize(Wal *pWal){ return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16); } +/* +** The following is guaranteed when this function is called: +** +** a) the WRITER lock is held, +** b) the entire log file has been checkpointed, and +** c) any existing readers are reading exclusively from the database +** file - there are no readers that may attempt to read a frame from +** the log file. +** +** This function updates the shared-memory structures so that the next +** client to write to the database (which may be this one) does so by +** writing frames into the start of the log file. +*/ +static void walRestartHdr(Wal *pWal){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + int i; /* Loop counter */ + u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ + pWal->nCkpt++; + pWal->hdr.mxFrame = 0; + sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); + sqlite3_randomness(4, &aSalt[1]); + walIndexWriteHdr(pWal); + pInfo->nBackfill = 0; + pInfo->aReadMark[1] = 0; + for(i=2; iaReadMark[i] = READMARK_NOT_USED; + assert( pInfo->aReadMark[0]==0 ); +} + /* ** Copy as much content as we can from the WAL back into the database file ** in response to an sqlite3_wal_checkpoint() request or the equivalent. @@ -1775,19 +1803,34 @@ static int walCheckpoint( rc = SQLITE_OK; } - /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal - ** file has been copied into the database file, then block until all - ** readers have finished using the wal file. This ensures that the next - ** process to write to the database restarts the wal file. + /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the + ** entire wal file has been copied into the database file, then block + ** until all readers have finished using the wal file. This ensures that + ** the next process to write to the database restarts the wal file. */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); if( pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; - }else if( eMode==SQLITE_CHECKPOINT_RESTART ){ + }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ assert( mxSafeFrame==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ + if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ + /* If this is a TRUNCATE checkpoint, also truncate the wal file + ** to zero bytes in size on disk. + ** + ** In theory, it might be safe to do this without updating the + ** wal-index header in shared memory, as all subsequent reader or + ** writer clients should see that the entire log file has been + ** checkpointed and behave accordingly. This seems unsafe though, + ** as it would leave the system in a state where the contents of + ** the wal-index header do not match the contents of the + ** file-system. To avoid this, update the wal-index header to + ** indicate that the log file contains zero valid frames. */ + walRestartHdr(pWal); + rc = sqlite3OsTruncate(pWal->pWalFd, 0); + } walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); } } @@ -2573,7 +2616,6 @@ int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ return rc; } - /* ** This function is called just before writing a set of frames to the log ** file (see sqlite3WalFrames()). It checks to see if, instead of appending @@ -2606,20 +2648,8 @@ static int walRestartLog(Wal *pWal){ ** In theory it would be Ok to update the cache of the header only ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() - ** to handle if this transaction is rolled back. - */ - int i; /* Loop counter */ - u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ - - pWal->nCkpt++; - pWal->hdr.mxFrame = 0; - sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); - aSalt[1] = salt1; - walIndexWriteHdr(pWal); - pInfo->nBackfill = 0; - pInfo->aReadMark[1] = 0; - for(i=2; iaReadMark[i] = READMARK_NOT_USED; - assert( pInfo->aReadMark[0]==0 ); + ** to handle if this transaction is rolled back. */ + walRestartHdr(pWal); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); }else if( rc!=SQLITE_BUSY ){ return rc; diff --git a/test/wal5.test b/test/wal5.test index 68750f1479..09c7d76040 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -55,7 +55,8 @@ foreach {testprefix do_wal_checkpoint} { if {[lsearch {-mode -db} $key]<0} { error "unknown switch: $key" } } - if {$a(-mode)!="restart" && $a(-mode)!="full"} { set a(-mode) passive } + set vals {restart full truncate} + if {[lsearch -exact $vals $a(-mode)]<0} { set a(-mode) passive } set cmd [list sqlite3_wal_checkpoint_v2 $dbhandle $a(-mode)] if {[info exists a(-db)]} { lappend sql $a(-db) } @@ -278,6 +279,11 @@ foreach {testprefix do_wal_checkpoint} { 9 RESTART 2 {1 4 3} 2 10 RESTART 3 {1 4 4} 3 + 11 TRUNCATE - {0 0 0} 3 + 12 TRUNCATE 1 {1 3 3} 1 + 13 TRUNCATE 2 {1 4 3} 2 + 14 TRUNCATE 3 {1 4 4} 3 + } { do_multiclient_test tn { setup_and_attach_aux @@ -348,6 +354,41 @@ foreach {testprefix do_wal_checkpoint} { do_test 3.$tn.6 { code3 { do_wal_checkpoint db3 } } {0 0 0} } + + # Test SQLITE_CHECKPOINT_TRUNCATE. + # + do_multiclient_test tn { + + code1 $do_wal_checkpoint + code2 $do_wal_checkpoint + code3 $do_wal_checkpoint + + do_test 3.$tn.1 { + sql1 { + PRAGMA page_size = 1024; + PRAGMA journal_mode = WAL; + PRAGMA synchronous = normal; + CREATE TABLE t1(x, y); + CREATE INDEX i1 ON t1(x, y); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + } + file size test.db-wal + } [wal_file_size 8 1024] + + do_test 3.$tn.2 { do_wal_checkpoint db -mode truncate } {0 0 0} + do_test 3.$tn.3 { file size test.db-wal } 0 + + do_test 3.$tn.4 { + sql2 { SELECT * FROM t1 } + } {1 2 3 4} + + do_test 3.$tn.5 { + sql2 { INSERT INTO t1 VALUES('a', 'b') } + file size test.db-wal + } [wal_file_size 2 1024] + + } } From 0fe8c1b96724c954628594fcbd44b284f9636d13 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 2 Dec 2014 19:35:09 +0000 Subject: [PATCH 608/710] When attempting to restart a wal file, make any required calls to sqlite3_randomness() before waiting on or checking for wal file readers. This restores the behaviour exhibited by the trunk. FossilOrigin-Name: 6ee08769f0ffbb3d620c66b89180ece7782dc820 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/wal.c | 14 ++++++++++---- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 35fb1bb478..0caddab954 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_CHECKPOINT_TRUNCATE\soption. -D 2014-12-02T19:04:54.595 +C When\sattempting\sto\srestart\sa\swal\sfile,\smake\sany\srequired\scalls\sto\ssqlite3_randomness()\sbefore\swaiting\son\sor\schecking\sfor\swal\sfile\sreaders.\sThis\srestores\sthe\sbehaviour\sexhibited\sby\sthe\strunk. +D 2014-12-02T19:35:09.278 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -301,7 +301,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 42c166f7ca78cb643c7f4e4bdfa83c59d363d1a6 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 -F src/wal.c f09818db7ba6e31d7a681eb99f801a7722c731d9 +F src/wal.c e396f31038aa01b9de95584b703692716275aedc F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c a0b16f9d78321cb340a977287d19f826555c7d3b @@ -1223,10 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 61b31e771430f490fc2c4cef55046debc4a5f4f5 -R 837049ab8d5153c1422981d3f48166e4 -T *branch * checkpoint-truncate -T *sym-checkpoint-truncate * -T -sym-trunk * +P 8e20a43419e46b6b9d1f60ec7ea420bbfb3ef358 +R 54481572fc28e1041792401f62b71701 U dan -Z ce82007701b4a39f21164e74e2380c4f +Z a7c4df9879a4ca8a80f8224dd87c78e4 diff --git a/manifest.uuid b/manifest.uuid index 9aff9e9fbc..4930bb9b15 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8e20a43419e46b6b9d1f60ec7ea420bbfb3ef358 \ No newline at end of file +6ee08769f0ffbb3d620c66b89180ece7782dc820 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 849b776743..b88ece88b8 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1635,15 +1635,19 @@ static int walPagesize(Wal *pWal){ ** This function updates the shared-memory structures so that the next ** client to write to the database (which may be this one) does so by ** writing frames into the start of the log file. +** +** The value of parameter salt1 is used as the aSalt[1] value in the +** new wal-index header. It should be passed a pseudo-random value (i.e. +** one obtained from sqlite3_randomness()). */ -static void walRestartHdr(Wal *pWal){ +static void walRestartHdr(Wal *pWal, u32 salt1){ volatile WalCkptInfo *pInfo = walCkptInfo(pWal); int i; /* Loop counter */ u32 *aSalt = pWal->hdr.aSalt; /* Big-endian salt values */ pWal->nCkpt++; pWal->hdr.mxFrame = 0; sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); - sqlite3_randomness(4, &aSalt[1]); + memcpy(&pWal->hdr.aSalt[1], &salt1, 4); walIndexWriteHdr(pWal); pInfo->nBackfill = 0; pInfo->aReadMark[1] = 0; @@ -1813,6 +1817,8 @@ static int walCheckpoint( if( pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ + u32 salt1; + sqlite3_randomness(4, &salt1); assert( mxSafeFrame==pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ @@ -1828,7 +1834,7 @@ static int walCheckpoint( ** the wal-index header do not match the contents of the ** file-system. To avoid this, update the wal-index header to ** indicate that the log file contains zero valid frames. */ - walRestartHdr(pWal); + walRestartHdr(pWal, salt1); rc = sqlite3OsTruncate(pWal->pWalFd, 0); } walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); @@ -2649,7 +2655,7 @@ static int walRestartLog(Wal *pWal){ ** at this point. But updating the actual wal-index header is also ** safe and means there is no special case for sqlite3WalUndo() ** to handle if this transaction is rolled back. */ - walRestartHdr(pWal); + walRestartHdr(pWal, salt1); walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1); }else if( rc!=SQLITE_BUSY ){ return rc; From 2d2e7bfeff0dcac767d0bd69a7d11df303d261fa Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Dec 2014 15:50:09 +0000 Subject: [PATCH 609/710] Updates to the documentation for sqlite3_wal_checkpoint_v2() and related interfaces, including adding many requirements marks. FossilOrigin-Name: 1e212d9899387344fd62c7b6fbcc59ea885b6d7c --- manifest | 13 +++--- manifest.uuid | 2 +- src/sqlite.h.in | 111 ++++++++++++++++++++++++++---------------------- 3 files changed, 67 insertions(+), 59 deletions(-) diff --git a/manifest b/manifest index e04f96ce65..97c3ddfce2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\sSQLITE_CHECKPOINT_TRUNCATE. -D 2014-12-02T20:51:52.630 +C Updates\sto\sthe\sdocumentation\sfor\ssqlite3_wal_checkpoint_v2()\sand\srelated\ninterfaces,\sincluding\sadding\smany\srequirements\smarks. +D 2014-12-03T15:50:09.223 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in 400bac7dd1294cb902b5eb85feed1689a2623ced +F src/sqlite.h.in 7d33b94a32b013832b53ad86be543168f87f8eb7 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 @@ -1223,8 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 61b31e771430f490fc2c4cef55046debc4a5f4f5 6ee08769f0ffbb3d620c66b89180ece7782dc820 -R 54481572fc28e1041792401f62b71701 -T +closed 6ee08769f0ffbb3d620c66b89180ece7782dc820 +P edda2b9e7a15ed486de81b10dd9bacd39c571d3f +R d436425f6a4681c70264f79cf1bfa41d U drh -Z 2b54bfeee1d348300e5d5565430bbc39 +Z 881f6486595c3b0d62652277de190ac9 diff --git a/manifest.uuid b/manifest.uuid index e1c0f3e973..74bdbd7193 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -edda2b9e7a15ed486de81b10dd9bacd39c571d3f \ No newline at end of file +1e212d9899387344fd62c7b6fbcc59ea885b6d7c \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 807c9ee62c..74c59f9257 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7249,7 +7249,7 @@ int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); ** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a ** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. ** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL -** or RESET checkpoint. +** or RESET or TRUNCATE checkpoint. ** ** ^The [wal_checkpoint pragma] can be used to invoke this interface ** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the @@ -7263,79 +7263,87 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Checkpoint a database ** -** Run a checkpoint operation on WAL database zDb attached to database -** handle db. The specific operation is determined by the value of the -** eMode parameter: +** ^(The sqlite3_wal_checkpoint_v2(D,X,M,L,C) interface runs a checkpoint operation +** on database X of [database connection] D in mode M. Status information is +** written back into integers pointed to by L and C.)^ ^(The M parameter must be +** a valid [SQLITE_CHECKPOINT_PASSIVE|checkpoint mode]:)^ ** **
    **
    SQLITE_CHECKPOINT_PASSIVE
    -** Checkpoint as many frames as possible without waiting for any database -** readers or writers to finish. Sync the db file if all frames in the log -** are checkpointed. This mode is the same as calling -** sqlite3_wal_checkpoint(). The [sqlite3_busy_handler|busy-handler callback] -** is never invoked. +** ^Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish, then sync the database file if all frames +** in the log were checkpointed. ^This mode is the same as calling +** [sqlite3_wal_checkpoint()]. ^The [sqlite3_busy_handler|busy-handler callback] +** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. ^On the other hand, +** passive mode might leave the checkpoint unfinished if there are concurrent +** readers or writers. ** **
    SQLITE_CHECKPOINT_FULL
    -** This mode blocks (it invokes the +** ^This mode blocks (it invokes the ** [sqlite3_busy_handler|busy-handler callback]) until there is no ** database writer and all readers are reading from the most recent database -** snapshot. It then checkpoints all frames in the log file and syncs the -** database file. This call blocks database writers while it is running, -** but not database readers. +** snapshot. ^It then checkpoints all frames in the log file and syncs the +** database file. ^This mode blocks new database writers while it is pending, +** but new database readers are allowed to continue unimpeded. ** **
    SQLITE_CHECKPOINT_RESTART
    -** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after -** checkpointing the log file it blocks (calls the +** ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition +** that after checkpointing the log file it blocks (calls the ** [sqlite3_busy_handler|busy-handler callback]) -** until all readers are reading from the database file only. This ensures -** that the next client to write to the database file restarts the log file -** from the beginning. This call blocks database writers while it is running, -** but not database readers. +** until all readers are reading from the database file only. ^This ensures +** that the next writer will restart the log file from the beginning. +** ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new +** database writer attempts while it is pending, but does not impede readers. ** **
    SQLITE_CHECKPOINT_TRUNCATE
    -** This mode works the same way as SQLITE_CHECKPOINT_RESTART except that, -** if successful, it also truncates the log file to zero bytes in size. +** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the addition +** that it also truncates the log file to zero bytes just prior to a successful +** return. **
    ** -** If pnLog is not NULL, then *pnLog is set to the total number of frames in -** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to -** the total number of checkpointed frames (including any that were already -** checkpointed when this function is called). *pnLog and *pnCkpt may be -** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK. -** If no values are available because of an error, they are both set to -1 -** before returning to communicate this to the caller. +** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file before returning or to -1 if the checkpoint could not run because +** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not NULL, +** then *pnCkpt is set to the total number of checkpointed frames (including any +** that were already checkpointed before the function was called) or to -1 if the +** checkpoint could not run due to an error or because the database is not in +** WAL mode. ** -** All calls obtain an exclusive "checkpoint" lock on the database file. If +** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If ** any other process is running a checkpoint operation at the same time, the -** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a +** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a ** busy-handler configured, it will not be invoked in this case. ** -** The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the -** exclusive "writer" lock on the database file. If the writer lock cannot be +** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** exclusive "writer" lock on the database file. ^If the writer lock cannot be ** obtained immediately, and a busy-handler is configured, it is invoked and ** the writer lock retried until either the busy-handler returns 0 or the lock -** is successfully obtained. The busy-handler is also invoked while waiting for -** database readers as described above. If the busy-handler returns 0 before +** is successfully obtained. ^The busy-handler is also invoked while waiting for +** database readers as described above. ^If the busy-handler returns 0 before ** the writer lock is obtained or while waiting for database readers, the ** checkpoint operation proceeds from that point in the same way as ** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible -** without blocking any further. SQLITE_BUSY is returned in this case. +** without blocking any further. ^SQLITE_BUSY is returned in this case. ** -** If parameter zDb is NULL or points to a zero length string, then the -** specified operation is attempted on all WAL databases. In this case the -** values written to output parameters *pnLog and *pnCkpt are undefined. If +** ^If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases [attached] to +** [database connection] db. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. ^If ** an SQLITE_BUSY error is encountered when processing one or more of the ** attached WAL databases, the operation is still attempted on any remaining -** attached databases and SQLITE_BUSY is returned to the caller. If any other +** attached databases and SQLITE_BUSY is returned at the end. ^If any other ** error occurs while processing an attached database, processing is abandoned -** and the error code returned to the caller immediately. If no error +** and the error code is returned to the caller immediately. ^If no error ** (SQLITE_BUSY or otherwise) is encountered while processing the attached ** databases, SQLITE_OK is returned. ** -** If database zDb is the name of an attached database that is not in WAL -** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. If +** ^If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. ^If ** zDb is not NULL (or a zero length string) and is not the name of any ** attached database, SQLITE_ERROR is returned to the caller. +** +** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface +** from SQL. */ int sqlite3_wal_checkpoint_v2( sqlite3 *db, /* Database handle */ @@ -7346,17 +7354,18 @@ int sqlite3_wal_checkpoint_v2( ); /* -** CAPI3REF: Checkpoint operation parameters +** CAPI3REF: Checkpoint Mode Values +** KEYWORDS: {checkpoint mode} ** -** These constants can be used as the 3rd parameter to -** [sqlite3_wal_checkpoint_v2()]. See the [sqlite3_wal_checkpoint_v2()] -** documentation for additional information about the meaning and use of -** each of these values. +** These constants define all valid values for the "checkpoint mode" passed +** as the third parameter to the [sqlite3_wal_checkpoint_v2()] interface. +** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the +** meaning of each of these checkpoint modes. */ -#define SQLITE_CHECKPOINT_PASSIVE 0 -#define SQLITE_CHECKPOINT_FULL 1 -#define SQLITE_CHECKPOINT_RESTART 2 -#define SQLITE_CHECKPOINT_TRUNCATE 3 +#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible without blocking */ +#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers to finish, then checkpoint */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but afterwards block for readers */ +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate the WAL file */ /* ** CAPI3REF: Virtual Table Interface Configuration From 5b875311a3f99aae9f489bd30ffe9d9a2295d981 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Dec 2014 16:30:27 +0000 Subject: [PATCH 610/710] Futher tweaks to the sqlite3_wal_checkpoint_v2() documentation. FossilOrigin-Name: 7d284d047b0677fb4532df5aae06f5bad997f5e9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 17 ++++++++++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 97c3ddfce2..e54424a2c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\sthe\sdocumentation\sfor\ssqlite3_wal_checkpoint_v2()\sand\srelated\ninterfaces,\sincluding\sadding\smany\srequirements\smarks. -D 2014-12-03T15:50:09.223 +C Futher\stweaks\sto\sthe\ssqlite3_wal_checkpoint_v2()\sdocumentation. +D 2014-12-03T16:30:27.252 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in 7d33b94a32b013832b53ad86be543168f87f8eb7 +F src/sqlite.h.in 8c881e8d225167a878ed7d78a776b372304823a9 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P edda2b9e7a15ed486de81b10dd9bacd39c571d3f -R d436425f6a4681c70264f79cf1bfa41d +P 1e212d9899387344fd62c7b6fbcc59ea885b6d7c +R e9765e182534ca03b14c9167cc7abee7 U drh -Z 881f6486595c3b0d62652277de190ac9 +Z 0ecdc98d536d1a5e17e6fc5f0fd27d98 diff --git a/manifest.uuid b/manifest.uuid index 74bdbd7193..b754092001 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e212d9899387344fd62c7b6fbcc59ea885b6d7c \ No newline at end of file +7d284d047b0677fb4532df5aae06f5bad997f5e9 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 74c59f9257..09ba5038dd 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7302,12 +7302,14 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** ** ** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in -** the log file before returning or to -1 if the checkpoint could not run because +** the log file or to -1 if the checkpoint could not run because ** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not NULL, -** then *pnCkpt is set to the total number of checkpointed frames (including any -** that were already checkpointed before the function was called) or to -1 if the -** checkpoint could not run due to an error or because the database is not in -** WAL mode. +** then *pnCkpt is set to the total number of checkpointed frames in the log file +** (including any that were already checkpointed before the function was called) +** or to -1 if the checkpoint could not run due to an error or because the +** database is not in WAL mode. ^Note that upon successful completion of +** an SQLITE_CHECKPOINT_TRUNCATE, the log file will have been truncated to +** zero bytes and so both *pnLog and *pnCkpt will be set to zero. ** ** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If ** any other process is running a checkpoint operation at the same time, the @@ -7342,6 +7344,11 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); ** zDb is not NULL (or a zero length string) and is not the name of any ** attached database, SQLITE_ERROR is returned to the caller. ** +** ^Unless it returns SQLITE_MISUSE, +** the sqlite3_wal_checkpoint_v2() interface +** sets the error information that is queried by +** [sqlite3_errcode()] and [sqlite3_errmsg()]. +** ** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface ** from SQL. */ From bb9a378df37dca03893bca2eff8a8182d87eb7db Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Dec 2014 18:32:47 +0000 Subject: [PATCH 611/710] Simplify the sqlite3_wal_checkpoint() documentation. Add some source code evidence marks. FossilOrigin-Name: 026c44ff2c092b14faa19985dd46873aeb8727dc --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 6 +++++- src/sqlite.h.in | 30 ++++++++++++++---------------- 4 files changed, 27 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index e54424a2c5..350040c926 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Futher\stweaks\sto\sthe\ssqlite3_wal_checkpoint_v2()\sdocumentation. -D 2014-12-03T16:30:27.252 +C Simplify\sthe\ssqlite3_wal_checkpoint()\sdocumentation.\s\sAdd\ssome\ssource\scode\nevidence\smarks. +D 2014-12-03T18:32:47.200 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 34b895b5ebfc73cd690dcd9ac6d0eecb28ce729f +F src/main.c 84cc0fc4861bd2b06c286b38bf03bce604a0a66a F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in 8c881e8d225167a878ed7d78a776b372304823a9 +F src/sqlite.h.in dd3721b533e9608677203917aa18bb477aede13d F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1e212d9899387344fd62c7b6fbcc59ea885b6d7c -R e9765e182534ca03b14c9167cc7abee7 +P 7d284d047b0677fb4532df5aae06f5bad997f5e9 +R 0867d5ea698016229ebc4122ede6b104 U drh -Z 0ecdc98d536d1a5e17e6fc5f0fd27d98 +Z a6f1638a8e5b63a53571b9743845c380 diff --git a/manifest.uuid b/manifest.uuid index b754092001..0243d4de58 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7d284d047b0677fb4532df5aae06f5bad997f5e9 \ No newline at end of file +026c44ff2c092b14faa19985dd46873aeb8727dc \ No newline at end of file diff --git a/src/main.c b/src/main.c index 9ac3fde733..f7a0b1f7fd 100644 --- a/src/main.c +++ b/src/main.c @@ -1941,6 +1941,8 @@ int sqlite3_wal_checkpoint_v2( assert( SQLITE_CHECKPOINT_RESTART==2 ); assert( SQLITE_CHECKPOINT_TRUNCATE==3 ); if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ + /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint + ** mode: */ return SQLITE_MISUSE; } @@ -1968,7 +1970,9 @@ int sqlite3_wal_checkpoint_v2( ** checkpointed. */ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ - return sqlite3_wal_checkpoint_v2(db, zDb, SQLITE_CHECKPOINT_PASSIVE, 0, 0); + /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to + ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ + return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); } #ifndef SQLITE_OMIT_WAL diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 09ba5038dd..6c44ffefd6 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7241,22 +7241,21 @@ int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); /* ** CAPI3REF: Checkpoint a database ** -** ^The [sqlite3_wal_checkpoint(D,X)] interface causes database named X -** on [database connection] D to be [checkpointed]. ^If X is NULL or an -** empty string, then a checkpoint is run on all databases of -** connection D. ^If the database connection D is not in -** [WAL | write-ahead log mode] then this interface is a harmless no-op. -** ^The [sqlite3_wal_checkpoint(D,X)] interface initiates a -** [sqlite3_wal_checkpoint_v2|PASSIVE] checkpoint. -** Use the [sqlite3_wal_checkpoint_v2()] interface to get a FULL -** or RESET or TRUNCATE checkpoint. +** ^(The sqlite3_wal_checkpoint(D,X) is equivalent to +** [sqlite3_wal_checkpoint_v2](D,X,[SQLITE_CHECKPOINT_PASSIVE],0,0).)^ ** -** ^The [wal_checkpoint pragma] can be used to invoke this interface -** from SQL. ^The [sqlite3_wal_autocheckpoint()] interface and the -** [wal_autocheckpoint pragma] can be used to cause this interface to be -** run whenever the WAL reaches a certain size threshold. +** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the +** [write-ahead log] for database X on [database connection] D to be +** transferred into the database file and for the write-ahead log to +** be reset. See the [checkpointing] documentation for addition +** information. ** -** See also: [sqlite3_wal_checkpoint_v2()] +** This interface used to be the only way to cause a checkpoint to +** occur. But then the newer and more powerful [sqlite3_wal_checkpoint_v2()] +** interface was added. This interface is retained for backwards +** compatibility and as a convenience for applications that need to manually +** start a callback but which do not need the full power (and corresponding +** complication) of [sqlite3_wal_checkpoint_v2()]. */ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); @@ -7272,8 +7271,7 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); **
    SQLITE_CHECKPOINT_PASSIVE
    ** ^Checkpoint as many frames as possible without waiting for any database ** readers or writers to finish, then sync the database file if all frames -** in the log were checkpointed. ^This mode is the same as calling -** [sqlite3_wal_checkpoint()]. ^The [sqlite3_busy_handler|busy-handler callback] +** in the log were checkpointed. ^The [sqlite3_busy_handler|busy-handler callback] ** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. ^On the other hand, ** passive mode might leave the checkpoint unfinished if there are concurrent ** readers or writers. From 86e166a77823c437c051540895c33380cc3d6229 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Dec 2014 19:08:00 +0000 Subject: [PATCH 612/710] Fix over-length comment lines in sqlite.h.in. No changes to code. FossilOrigin-Name: cbd357fd8c25c9c6043063710a3e2a89ff8b4575 --- manifest | 12 +++--- manifest.uuid | 2 +- src/sqlite.h.in | 107 +++++++++++++++++++++++++----------------------- 3 files changed, 63 insertions(+), 58 deletions(-) diff --git a/manifest b/manifest index 350040c926..7aebe335e2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\ssqlite3_wal_checkpoint()\sdocumentation.\s\sAdd\ssome\ssource\scode\nevidence\smarks. -D 2014-12-03T18:32:47.200 +C Fix\sover-length\scomment\slines\sin\ssqlite.h.in.\s\sNo\schanges\sto\scode. +D 2014-12-03T19:08:00.822 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in dd3721b533e9608677203917aa18bb477aede13d +F src/sqlite.h.in f16487cd0b8f363faf08ad6009d5222e56d78b66 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7d284d047b0677fb4532df5aae06f5bad997f5e9 -R 0867d5ea698016229ebc4122ede6b104 +P 026c44ff2c092b14faa19985dd46873aeb8727dc +R 4ed5b70763e0743b0e322aa2df0b85da U drh -Z a6f1638a8e5b63a53571b9743845c380 +Z 6507e8b7c8c2131b9414f714dc2b9292 diff --git a/manifest.uuid b/manifest.uuid index 0243d4de58..21ea5c4e23 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -026c44ff2c092b14faa19985dd46873aeb8727dc \ No newline at end of file +cbd357fd8c25c9c6043063710a3e2a89ff8b4575 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 6c44ffefd6..6a04972559 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1519,8 +1519,8 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_MEMSTATUS]]
    SQLITE_CONFIG_MEMSTATUS
    **
    ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, ** interpreted as a boolean, which enables or disables the collection of -** memory allocation statistics. ^(When memory allocation statistics are disabled, the -** following SQLite interfaces become non-operational: +** memory allocation statistics. ^(When memory allocation statistics are +** disabled, the following SQLite interfaces become non-operational: **
      **
    • [sqlite3_memory_used()] **
    • [sqlite3_memory_highwater()] @@ -1561,7 +1561,8 @@ struct sqlite3_mem_methods { ** This configuration should not be used if an application-define page ** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2] ** configuration option. -** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to 8-byte aligned +** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to +** 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page ** (a power of two between 512 and 32768) plus some extra bytes for each @@ -1581,7 +1582,8 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_HEAP]]
      SQLITE_CONFIG_HEAP
      **
      ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer ** that SQLite will use for all of its dynamic memory allocation needs -** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and [SQLITE_CONFIG_PAGECACHE]. +** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and +** [SQLITE_CONFIG_PAGECACHE]. ** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled ** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns ** [SQLITE_ERROR] if invoked otherwise. @@ -1601,9 +1603,9 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_MUTEX]]
      SQLITE_CONFIG_MUTEX
      **
      ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a ** pointer to an instance of the [sqlite3_mutex_methods] structure. -** The argument specifies alternative low-level mutex routines to be used in place -** the mutex routines built into SQLite.)^ ^SQLite makes a copy of the -** content of the [sqlite3_mutex_methods] structure before the call to +** The argument specifies alternative low-level mutex routines to be used +** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** the content of the [sqlite3_mutex_methods] structure before the call to ** [sqlite3_config()] returns. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then ** the entire mutexing subsystem is omitted from the build and hence calls to @@ -1641,8 +1643,8 @@ struct sqlite3_mem_methods { ** ** [[SQLITE_CONFIG_GETPCACHE2]]
      SQLITE_CONFIG_GETPCACHE2
      **
      ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which -** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of the current -** page cache implementation into that object.)^
      +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** the current page cache implementation into that object.)^
    ** ** [[SQLITE_CONFIG_LOG]]
    SQLITE_CONFIG_LOG
    **
    The SQLITE_CONFIG_LOG option is used to configure the SQLite @@ -1667,8 +1669,9 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_URI]]
    SQLITE_CONFIG_URI **
    ^(The SQLITE_CONFIG_URI option takes a single argument of type int. ** If non-zero, then URI handling is globally enabled. If the parameter is zero, -** then URI handling is globally disabled.)^ ^If URI handling is globally enabled, -** all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], [sqlite3_open16()] or +** then URI handling is globally disabled.)^ ^If URI handling is globally +** enabled, all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], +** [sqlite3_open16()] or ** specified as part of [ATTACH] commands are interpreted as URIs, regardless ** of whether or not the [SQLITE_OPEN_URI] flag is set when the database ** connection is opened. ^If it is globally disabled, filenames are @@ -1730,8 +1733,8 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] **
    SQLITE_CONFIG_WIN32_HEAPSIZE **
    ^The SQLITE_CONFIG_WIN32_HEAPSIZE option is only available if SQLite is -** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro defined. -** ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro +** defined. ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value ** that specifies the maximum size of the created heap. ** ** @@ -1739,8 +1742,8 @@ struct sqlite3_mem_methods { **
    SQLITE_CONFIG_PCACHE_HDRSZ **
    ^The SQLITE_CONFIG_PCACHE_HDRSZ option takes a single parameter which ** is a pointer to an integer and writes into that integer the number of extra -** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. The amount of -** extra space required can change depending on the compiler, +** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. +** The amount of extra space required can change depending on the compiler, ** target platform, and SQLite version. ** */ @@ -2044,6 +2047,7 @@ int sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors +** KEYWORDS: {busy-handler callback} {busy handler} ** ** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X ** that might be invoked with argument P whenever @@ -4515,7 +4519,8 @@ typedef void (*sqlite3_destructor_type)(void*); ** the [sqlite3_context] pointer, the results are undefined. */ void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); -void sqlite3_result_blob64(sqlite3_context*,const void*,sqlite3_uint64,void(*)(void*)); +void sqlite3_result_blob64(sqlite3_context*,const void*, + sqlite3_uint64,void(*)(void*)); void sqlite3_result_double(sqlite3_context*, double); void sqlite3_result_error(sqlite3_context*, const char*, int); void sqlite3_result_error16(sqlite3_context*, const void*, int); @@ -7262,19 +7267,19 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Checkpoint a database ** -** ^(The sqlite3_wal_checkpoint_v2(D,X,M,L,C) interface runs a checkpoint operation -** on database X of [database connection] D in mode M. Status information is -** written back into integers pointed to by L and C.)^ ^(The M parameter must be -** a valid [SQLITE_CHECKPOINT_PASSIVE|checkpoint mode]:)^ +** ^(The sqlite3_wal_checkpoint_v2(D,X,M,L,C) interface runs a checkpoint +** operation on database X of [database connection] D in mode M. Status +** information is written back into integers pointed to by L and C.)^ +** ^(The M parameter must be a valid [checkpoint mode]:)^ ** **
    **
    SQLITE_CHECKPOINT_PASSIVE
    ** ^Checkpoint as many frames as possible without waiting for any database ** readers or writers to finish, then sync the database file if all frames -** in the log were checkpointed. ^The [sqlite3_busy_handler|busy-handler callback] -** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. ^On the other hand, -** passive mode might leave the checkpoint unfinished if there are concurrent -** readers or writers. +** in the log were checkpointed. ^The [busy-handler callback] +** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. +** ^On the other hand, passive mode might leave the checkpoint unfinished +** if there are concurrent readers or writers. ** **
    SQLITE_CHECKPOINT_FULL
    ** ^This mode blocks (it invokes the @@ -7287,27 +7292,27 @@ int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); **
    SQLITE_CHECKPOINT_RESTART
    ** ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition ** that after checkpointing the log file it blocks (calls the -** [sqlite3_busy_handler|busy-handler callback]) +** [busy-handler callback]) ** until all readers are reading from the database file only. ^This ensures ** that the next writer will restart the log file from the beginning. ** ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new ** database writer attempts while it is pending, but does not impede readers. ** **
    SQLITE_CHECKPOINT_TRUNCATE
    -** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the addition -** that it also truncates the log file to zero bytes just prior to a successful -** return. +** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the +** addition that it also truncates the log file to zero bytes just prior +** to a successful return. **
    ** ** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in ** the log file or to -1 if the checkpoint could not run because -** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not NULL, -** then *pnCkpt is set to the total number of checkpointed frames in the log file -** (including any that were already checkpointed before the function was called) -** or to -1 if the checkpoint could not run due to an error or because the -** database is not in WAL mode. ^Note that upon successful completion of -** an SQLITE_CHECKPOINT_TRUNCATE, the log file will have been truncated to -** zero bytes and so both *pnLog and *pnCkpt will be set to zero. +** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not +** NULL,then *pnCkpt is set to the total number of checkpointed frames in the +** log file (including any that were already checkpointed before the function +** was called) or to -1 if the checkpoint could not run due to an error or +** because the database is not in WAL mode. ^Note that upon successful +** completion of an SQLITE_CHECKPOINT_TRUNCATE, the log file will have been +** truncated to zero bytes and so both *pnLog and *pnCkpt will be set to zero. ** ** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If ** any other process is running a checkpoint operation at the same time, the @@ -7367,10 +7372,10 @@ int sqlite3_wal_checkpoint_v2( ** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the ** meaning of each of these checkpoint modes. */ -#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible without blocking */ -#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers to finish, then checkpoint */ -#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but afterwards block for readers */ -#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate the WAL file */ +#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ +#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* ** CAPI3REF: Virtual Table Interface Configuration @@ -7469,12 +7474,12 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** **
    ** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    -**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set to the -** total number of times that the X-th loop has run.
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be +** set to the total number of times that the X-th loop has run.
    ** ** [[SQLITE_SCANSTAT_NVISIT]]
    SQLITE_SCANSTAT_NVISIT
    -**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set to the -** total number of rows examined by all iterations of the X-th loop.
    +**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set +** to the total number of rows examined by all iterations of the X-th loop.
    ** ** [[SQLITE_SCANSTAT_EST]]
    SQLITE_SCANSTAT_EST
    **
    ^The "double" variable pointed to by the T parameter will be set to the @@ -7485,14 +7490,14 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** be the NLOOP value for the current loop. ** ** [[SQLITE_SCANSTAT_NAME]]
    SQLITE_SCANSTAT_NAME
    -**
    ^The "const char *" variable pointed to by the T parameter will be set to -** a zero-terminated UTF-8 string containing the name of the index or table used -** for the X-th loop. +**
    ^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the name of the index or table +** used for the X-th loop. ** ** [[SQLITE_SCANSTAT_EXPLAIN]]
    SQLITE_SCANSTAT_EXPLAIN
    -**
    ^The "const char *" variable pointed to by the T parameter will be set to -** a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] description -** for the X-th loop. +**
    ^The "const char *" variable pointed to by the T parameter will be set +** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] +** description for the X-th loop. ** ** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    **
    ^The "int" variable pointed to by the T parameter will be set to the @@ -7515,8 +7520,8 @@ int sqlite3_vtab_on_conflict(sqlite3 *); ** Return status data for a single loop within query pStmt. ** ** The "iScanStatusOp" parameter determines which status information to return. -** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior of -** this interface is undefined. +** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior +** of this interface is undefined. ** ^The requested measurement is written into a variable pointed to by ** the "pOut" parameter. ** Parameter "idx" identifies the specific loop to retrieve statistics for. From dd90d7ee819b0a33d832ba3bc9bda8880b38b94a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 3 Dec 2014 19:25:41 +0000 Subject: [PATCH 613/710] Add evidence marks and assert()s used as evidence for checkpoint requirements. FossilOrigin-Name: b2da8afc7657266fbe8e683c6e50fe18216cbcf3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/pager.c | 3 ++- src/wal.c | 45 ++++++++++++++++++++++++++++++--------------- 4 files changed, 40 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 7aebe335e2..d4db2a78e1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sover-length\scomment\slines\sin\ssqlite.h.in.\s\sNo\schanges\sto\scode. -D 2014-12-03T19:08:00.822 +C Add\sevidence\smarks\sand\sassert()s\sused\sas\sevidence\sfor\scheckpoint\srequirements. +D 2014-12-03T19:25:41.425 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -216,7 +216,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c a9e500dd963fb1f67d7860e58b5772abe6123862 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c b8764f90c135482988268eec93d7f5cdb89d687a +F src/pager.c 7a5c5bc0e29b9b16834f5558a9d5d22bbae59a08 F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 @@ -301,7 +301,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 42c166f7ca78cb643c7f4e4bdfa83c59d363d1a6 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 -F src/wal.c e396f31038aa01b9de95584b703692716275aedc +F src/wal.c fc62ec9dc7dfa08dbe08801332230cf28130a159 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c a0b16f9d78321cb340a977287d19f826555c7d3b @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 026c44ff2c092b14faa19985dd46873aeb8727dc -R 4ed5b70763e0743b0e322aa2df0b85da +P cbd357fd8c25c9c6043063710a3e2a89ff8b4575 +R bf75abced12e09ba377b1add0a3ea61a U drh -Z 6507e8b7c8c2131b9414f714dc2b9292 +Z 0545aa7eaccc2c8413582324bb7aef19 diff --git a/manifest.uuid b/manifest.uuid index 21ea5c4e23..e429b5d1e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cbd357fd8c25c9c6043063710a3e2a89ff8b4575 \ No newline at end of file +b2da8afc7657266fbe8e683c6e50fe18216cbcf3 \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index e0ede87f07..a2ae9cc410 100644 --- a/src/pager.c +++ b/src/pager.c @@ -7075,7 +7075,8 @@ int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; if( pPager->pWal ){ rc = sqlite3WalCheckpoint(pPager->pWal, eMode, - pPager->xBusyHandler, pPager->pBusyHandlerArg, + (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), + pPager->pBusyHandlerArg, pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt ); diff --git a/src/wal.c b/src/wal.c index b88ece88b8..8ba5ce823f 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1689,7 +1689,7 @@ static void walRestartHdr(Wal *pWal, u32 salt1){ static int walCheckpoint( Wal *pWal, /* Wal connection */ int eMode, /* One of PASSIVE, FULL or RESTART */ - int (*xBusyCall)(void*), /* Function to call when busy */ + int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags for OsSync() (or 0) */ u8 *zBuf /* Temporary buffer to use */ @@ -1703,7 +1703,6 @@ static int walCheckpoint( u32 mxPage; /* Max database page to write */ int i; /* Loop counter */ volatile WalCkptInfo *pInfo; /* The checkpoint status information */ - int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */ szPage = walPagesize(pWal); testcase( szPage<=32768 ); @@ -1718,7 +1717,9 @@ static int walCheckpoint( } assert( pIter ); - if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall; + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); /* Compute in mxSafeFrame the index of the last frame of the WAL that is ** safe to write into the database. Frames beyond mxSafeFrame might @@ -2943,7 +2944,7 @@ int sqlite3WalFrames( */ int sqlite3WalCheckpoint( Wal *pWal, /* Wal connection */ - int eMode, /* PASSIVE, FULL or RESTART */ + int eMode, /* PASSIVE, FULL, RESTART, or TRUNCATE */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ int sync_flags, /* Flags to sync db file with (or 0) */ @@ -2955,29 +2956,42 @@ int sqlite3WalCheckpoint( int rc; /* Return code */ int isChanged = 0; /* True if a new wal-index header is loaded */ int eMode2 = eMode; /* Mode to pass to walCheckpoint() */ + int (*xBusy2)(void*) = xBusy; /* Busy handler for eMode2 */ assert( pWal->ckptLock==0 ); assert( pWal->writeLock==0 ); + /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ + assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); + if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); + + /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive + ** "checkpoint" lock on the database file. */ rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); if( rc ){ - /* Usually this is SQLITE_BUSY meaning that another thread or process - ** is already running a checkpoint, or maybe a recovery. But it might - ** also be SQLITE_IOERR. */ + /* EVIDENCE-OF: R-10421-19736 If any other process is running a + ** checkpoint operation at the same time, the lock cannot be obtained and + ** SQLITE_BUSY is returned. + ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + ** it will not be invoked in this case. + */ + testcase( rc==SQLITE_BUSY ); + testcase( xBusy!=0 ); return rc; } pWal->ckptLock = 1; - /* If this is a blocking-checkpoint, then obtain the write-lock as well - ** to prevent any writers from running while the checkpoint is underway. - ** This has to be done before the call to walIndexReadHdr() below. + /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and + ** TRUNCATE modes also obtain the exclusive "writer" lock on the database + ** file. ** - ** If the writer lock cannot be obtained, then a passive checkpoint is - ** run instead. Since the checkpointer is not holding the writer lock, - ** there is no point in blocking waiting for any readers. Assuming no - ** other error occurs, this function will return SQLITE_BUSY to the caller. + ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + ** immediately, and a busy-handler is configured, it is invoked and the + ** writer lock retried until either the busy-handler returns 0 or the + ** lock is successfully obtained. */ if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); @@ -2985,6 +2999,7 @@ int sqlite3WalCheckpoint( pWal->writeLock = 1; }else if( rc==SQLITE_BUSY ){ eMode2 = SQLITE_CHECKPOINT_PASSIVE; + xBusy2 = 0; rc = SQLITE_OK; } } @@ -3002,7 +3017,7 @@ int sqlite3WalCheckpoint( if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ rc = SQLITE_CORRUPT_BKPT; }else{ - rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags, zBuf); + rc = walCheckpoint(pWal, eMode2, xBusy2, pBusyArg, sync_flags, zBuf); } /* If no error occurred, set the output variables. */ From a25165fa6852e71b3346e6d9457a50bd8332e0e8 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Dec 2014 04:50:59 +0000 Subject: [PATCH 614/710] Add an implementation mark and fix a comment describing the OP_Checkpoint opcode. FossilOrigin-Name: 7475b90c558157802b036162376fceacc1b9ab4a --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbe.c | 4 ++-- src/wal.c | 6 ++++-- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index d4db2a78e1..07b9de4c3d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sevidence\smarks\sand\sassert()s\sused\sas\sevidence\sfor\scheckpoint\srequirements. -D 2014-12-03T19:25:41.425 +C Add\san\simplementation\smark\sand\sfix\sa\scomment\sdescribing\sthe\sOP_Checkpoint\nopcode. +D 2014-12-04T04:50:59.682 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c F src/util.c 3b627daa45c7308c1e36e3dbaa3f9ce7e5c7fa73 F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c -F src/vdbe.c 60217f3b8ab984b2e2bb9e3965276dd29e5efe5d +F src/vdbe.c 1a9e671c9cfc259e4d2affc71f7df4a4c00a842c F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 @@ -301,7 +301,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c 42c166f7ca78cb643c7f4e4bdfa83c59d363d1a6 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 -F src/wal.c fc62ec9dc7dfa08dbe08801332230cf28130a159 +F src/wal.c 847692349eb6e1fb8543dbc97e69ddbfa4cc7ea7 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c a0b16f9d78321cb340a977287d19f826555c7d3b @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cbd357fd8c25c9c6043063710a3e2a89ff8b4575 -R bf75abced12e09ba377b1add0a3ea61a +P b2da8afc7657266fbe8e683c6e50fe18216cbcf3 +R f07d202428c907151b75d8e97783127c U drh -Z 0545aa7eaccc2c8413582324bb7aef19 +Z fc5c85b33aac137327215d4078c65229 diff --git a/manifest.uuid b/manifest.uuid index e429b5d1e1..8653d4ff34 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b2da8afc7657266fbe8e683c6e50fe18216cbcf3 \ No newline at end of file +7475b90c558157802b036162376fceacc1b9ab4a \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 7f661b42b1..3dac74dfd4 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -5705,8 +5705,8 @@ case OP_AggFinal: { /* Opcode: Checkpoint P1 P2 P3 * * ** ** Checkpoint database P1. This is a no-op if P1 is not currently in -** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL -** or RESTART. Write 1 or 0 into mem[P3] if the checkpoint returns +** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL, +** RESTART, or TRUNCATE. Write 1 or 0 into mem[P3] if the checkpoint returns ** SQLITE_BUSY or not, respectively. Write the number of pages in the ** WAL after the checkpoint into mem[P3+1] and the number of pages ** in the WAL that have been checkpointed after the checkpoint diff --git a/src/wal.c b/src/wal.c index 8ba5ce823f..2b80c7a95e 100644 --- a/src/wal.c +++ b/src/wal.c @@ -1824,8 +1824,10 @@ static int walCheckpoint( rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1); if( rc==SQLITE_OK ){ if( eMode==SQLITE_CHECKPOINT_TRUNCATE ){ - /* If this is a TRUNCATE checkpoint, also truncate the wal file - ** to zero bytes in size on disk. + /* IMPLEMENTATION-OF: R-44699-57140 This mode works the same way as + ** SQLITE_CHECKPOINT_RESTART with the addition that it also + ** truncates the log file to zero bytes just prior to a + ** successful return. ** ** In theory, it might be safe to do this without updating the ** wal-index header in shared memory, as all subsequent reader or From 063970a15f317e156d4e2ee59059502fa326ee3b Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Dec 2014 14:01:39 +0000 Subject: [PATCH 615/710] Fix comment typos reported on the mailing list. No changes to code. FossilOrigin-Name: 93a71c9f058b58d11187293a1e8d5324d0911880 --- ext/fts3/fts3_porter.c | 2 +- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/fts3/fts3_porter.c b/ext/fts3/fts3_porter.c index db175acdbb..8fb4c25daa 100644 --- a/ext/fts3/fts3_porter.c +++ b/ext/fts3/fts3_porter.c @@ -183,7 +183,7 @@ static int isVowel(const char *z){ ** by a consonant. ** ** In this routine z[] is in reverse order. So we are really looking -** for an instance of of a consonant followed by a vowel. +** for an instance of a consonant followed by a vowel. */ static int m_gt_0(const char *z){ while( isVowel(z) ){ z++; } diff --git a/manifest b/manifest index 07b9de4c3d..0404ece72e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\simplementation\smark\sand\sfix\sa\scomment\sdescribing\sthe\sOP_Checkpoint\nopcode. -D 2014-12-04T04:50:59.682 +C Fix\scomment\stypos\sreported\son\sthe\smailing\slist.\s\sNo\schanges\sto\scode. +D 2014-12-04T14:01:39.082 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -86,7 +86,7 @@ F ext/fts3/fts3_expr.c 40123785eaa3ebd4c45c9b23407cc44ac0c49905 F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 F ext/fts3/fts3_hash.h 39cf6874dc239d6b4e30479b1975fe5b22a3caaf F ext/fts3/fts3_icu.c e319e108661147bcca8dd511cd562f33a1ba81b5 -F ext/fts3/fts3_porter.c 7f8b4bf5af7c0f20f73b8e87e14fa9298f52e290 +F ext/fts3/fts3_porter.c 3565faf04b626cddf85f03825e86056a4562c009 F ext/fts3/fts3_snippet.c 51beb5c1498176fd9caccaf1c75b55cb803a985a F ext/fts3/fts3_term.c a521f75132f9a495bdca1bdd45949b3191c52763 F ext/fts3/fts3_test.c 8a3a78c4458b2d7c631fcf4b152a5cd656fa7038 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in f16487cd0b8f363faf08ad6009d5222e56d78b66 +F src/sqlite.h.in acc4d70043f508311c6b9df277f7afe43506863c F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b2da8afc7657266fbe8e683c6e50fe18216cbcf3 -R f07d202428c907151b75d8e97783127c +P 7475b90c558157802b036162376fceacc1b9ab4a +R 397f3e2a10a22093a2e147bbeaeacc16 U drh -Z fc5c85b33aac137327215d4078c65229 +Z d46deb1796c364e4a21ca2d8fe0413fe diff --git a/manifest.uuid b/manifest.uuid index 8653d4ff34..53519b35c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7475b90c558157802b036162376fceacc1b9ab4a \ No newline at end of file +93a71c9f058b58d11187293a1e8d5324d0911880 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 6a04972559..e6470042b5 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1216,7 +1216,7 @@ struct sqlite3_vfs { ** ** ** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as -** was given no the corresponding lock. +** was given on the corresponding lock. ** ** The xShmLock method can transition between unlocked and SHARED or ** between unlocked and EXCLUSIVE. It cannot transition between SHARED From d8922052b27a3d2d62ccf3cf0ef1ea9b7272124d Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Dec 2014 15:02:03 +0000 Subject: [PATCH 616/710] Clarification of the meaning of the second parameter to the busy-handler callback. No changes to code. FossilOrigin-Name: 1e2bc484f8e7766a493bfeab04d82e50f37217b8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 0404ece72e..806771a618 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scomment\stypos\sreported\son\sthe\smailing\slist.\s\sNo\schanges\sto\scode. -D 2014-12-04T14:01:39.082 +C Clarification\sof\sthe\smeaning\sof\sthe\ssecond\sparameter\sto\sthe\sbusy-handler\ncallback.\s\sNo\schanges\sto\scode. +D 2014-12-04T15:02:03.058 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in acc4d70043f508311c6b9df277f7afe43506863c +F src/sqlite.h.in 6ec654324cb490ea3d8a7be28b8c7d37fe4ad282 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7475b90c558157802b036162376fceacc1b9ab4a -R 397f3e2a10a22093a2e147bbeaeacc16 +P 93a71c9f058b58d11187293a1e8d5324d0911880 +R 00727a6ee41feaad0f3193ccd0896958 U drh -Z d46deb1796c364e4a21ca2d8fe0413fe +Z c85dc24a7d66e411a436a86d655a083a diff --git a/manifest.uuid b/manifest.uuid index 53519b35c8..c993290f19 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -93a71c9f058b58d11187293a1e8d5324d0911880 \ No newline at end of file +1e2bc484f8e7766a493bfeab04d82e50f37217b8 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index e6470042b5..e889b2670f 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2064,7 +2064,7 @@ int sqlite3_complete16(const void *sql); ** ^The first argument to the busy handler is a copy of the void* pointer which ** is the third argument to sqlite3_busy_handler(). ^The second argument to ** the busy handler callback is the number of times that the busy handler has -** been invoked for the same locking event. ^If the +** been invoked previously for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to ** access the database and [SQLITE_BUSY] is returned ** to the application. From 72673a24e2d38c1699d911cfca0203f5e545f78c Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Dec 2014 16:27:17 +0000 Subject: [PATCH 617/710] If a table is the right operand of a LEFT JOIN, then any column of that table can be NULL even if that column as a NOT NULL constraint. Fix for ticket [6f2222d550f5b0ee7ed]. FossilOrigin-Name: 6f6fcbe4736b9468a495c684d5eebc8bfe5c566a --- manifest | 18 ++++++++--------- manifest.uuid | 2 +- src/expr.c | 3 ++- src/resolve.c | 4 ++++ src/sqliteInt.h | 3 ++- test/join5.test | 54 +++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 72 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 806771a618..4f3131ec66 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clarification\sof\sthe\smeaning\sof\sthe\ssecond\sparameter\sto\sthe\sbusy-handler\ncallback.\s\sNo\schanges\sto\scode. -D 2014-12-04T15:02:03.058 +C If\sa\stable\sis\sthe\sright\soperand\sof\sa\sLEFT\sJOIN,\sthen\sany\scolumn\sof\sthat\ntable\scan\sbe\sNULL\seven\sif\sthat\scolumn\sas\sa\sNOT\sNULL\sconstraint.\nFix\sfor\sticket\s[6f2222d550f5b0ee7ed]. +D 2014-12-04T16:27:17.993 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -182,7 +182,7 @@ F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 93594514aae68de117ca4a2a0d6cc63eddf26744 F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 -F src/expr.c 73de4c0da2eed6b149d40a05c589dfeb2c4a87a1 +F src/expr.c 00da3072f362b06f39ce4052baa1d4ce2bb36d1c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 F src/func.c 6d3c4ebd72aa7923ce9b110a7dc15f9b8c548430 @@ -226,14 +226,14 @@ F src/pragma.c d54cdd40b63d608f2d95b7482c710690e3593a73 F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 -F src/resolve.c 4965007d6497b6a4d7a6d98751cc39712885f952 +F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c 428165951748151e87a15295b7357221433e311b F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf F src/sqlite.h.in 6ec654324cb490ea3d8a7be28b8c7d37fe4ad282 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h c9e95b8fa9aee30d46387735c5be73fa58886e38 +F src/sqliteInt.h 28049b803b74a7f73242a8226915ea00ebb1309f F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -670,7 +670,7 @@ F test/join.test 52d4d49f86d0cf46926672878c4eaf0da399104a F test/join2.test f2171c265e57ee298a27e57e7051d22962f9f324 F test/join3.test 6f0c774ff1ba0489e6c88a3e77b9d3528fb4fda0 F test/join4.test 1a352e4e267114444c29266ce79e941af5885916 -F test/join5.test 86675fc2919269aa923c84dd00ee4249b97990fe +F test/join5.test 5df23eba184f159ed9705a954957e765a10c141d F test/join6.test cfe6503791ceb0cbb509966740286ec423cbf10b F test/journal1.test 69abc726c51b4a0409189f9a85191205297c0577 F test/journal2.test ae06f566c28552c313ded3fee79a6c69e6d049b1 @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 93a71c9f058b58d11187293a1e8d5324d0911880 -R 00727a6ee41feaad0f3193ccd0896958 +P 1e2bc484f8e7766a493bfeab04d82e50f37217b8 +R 6261b2c66938591ba7d193856851eb02 U drh -Z c85dc24a7d66e411a436a86d655a083a +Z d631884fbd9f9cb654155b89b69f5e28 diff --git a/manifest.uuid b/manifest.uuid index c993290f19..abaad9a04b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e2bc484f8e7766a493bfeab04d82e50f37217b8 \ No newline at end of file +6f6fcbe4736b9468a495c684d5eebc8bfe5c566a \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index c28f196221..817975ab3a 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1414,7 +1414,8 @@ int sqlite3ExprCanBeNull(const Expr *p){ return 0; case TK_COLUMN: assert( p->pTab!=0 ); - return p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0; + return ExprHasProperty(p, EP_CanBeNull) || + (p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0); default: return 1; } diff --git a/src/resolve.c b/src/resolve.c index 8fb580b3a1..d4bd548c93 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -320,6 +320,10 @@ static int lookupName( if( pMatch ){ pExpr->iTable = pMatch->iCursor; pExpr->pTab = pMatch->pTab; + assert( (pMatch->jointype & JT_RIGHT)==0 ); /* RIGHT JOIN not (yet) supported */ + if( (pMatch->jointype & JT_LEFT)!=0 ){ + ExprSetProperty(pExpr, EP_CanBeNull); + } pSchema = pExpr->pTab->pSchema; } } /* if( pSrcList ) */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f3d6ce015e..3498517f25 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2015,7 +2015,7 @@ struct Expr { /* ** The following are the meanings of bits in the Expr.flags field. */ -#define EP_FromJoin 0x000001 /* Originated in ON or USING clause of a join */ +#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ #define EP_Agg 0x000002 /* Contains one or more aggregate functions */ #define EP_Resolved 0x000004 /* IDs have been resolved to COLUMNs */ #define EP_Error 0x000008 /* Expression contains one or more errors */ @@ -2035,6 +2035,7 @@ struct Expr { #define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ #define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ #define EP_Constant 0x080000 /* Node is a constant */ +#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ /* ** These macros can be used to test, set, or clear bits in the diff --git a/test/join5.test b/test/join5.test index 45d8a31596..b0b0df4f9d 100644 --- a/test/join5.test +++ b/test/join5.test @@ -106,5 +106,59 @@ do_test join5-2.12 { execsql {SELECT * FROM xy LEFT JOIN ab ON NULL WHERE NULL} } {} +# Ticket https://www.sqlite.org/src/tktview/6f2222d550f5b0ee7ed37601 +# Incorrect output on a LEFT JOIN. +# +do_execsql_test join5-3.1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + DROP TABLE IF EXISTS t3; + CREATE TABLE x1(a); + INSERT INTO x1 VALUES(1); + CREATE TABLE x2(b NOT NULL); + CREATE TABLE x3(c, d); + INSERT INTO x3 VALUES('a', NULL); + INSERT INTO x3 VALUES('b', NULL); + INSERT INTO x3 VALUES('c', NULL); + SELECT * FROM x1 LEFT JOIN x2 LEFT JOIN x3 ON x3.d = x2.b; +} {1 {} {} {}} +do_execsql_test join5-3.2 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + DROP TABLE IF EXISTS t3; + DROP TABLE IF EXISTS t4; + DROP TABLE IF EXISTS t5; + CREATE TABLE t1(x text NOT NULL, y text); + CREATE TABLE t2(u text NOT NULL, x text NOT NULL); + CREATE TABLE t3(w text NOT NULL, v text); + CREATE TABLE t4(w text NOT NULL, z text NOT NULL); + CREATE TABLE t5(z text NOT NULL, m text); + INSERT INTO t1 VALUES('f6d7661f-4efe-4c90-87b5-858e61cd178b',NULL); + INSERT INTO t1 VALUES('f6ea82c3-2cad-45ce-ae8f-3ddca4fb2f48',NULL); + INSERT INTO t1 VALUES('f6f47499-ecb4-474b-9a02-35be73c235e5',NULL); + INSERT INTO t1 VALUES('56f47499-ecb4-474b-9a02-35be73c235e5',NULL); + INSERT INTO t3 VALUES('007f2033-cb20-494c-b135-a1e4eb66130c', + 'f6d7661f-4efe-4c90-87b5-858e61cd178b'); + SELECT * + FROM t3 + INNER JOIN t1 ON t1.x= t3.v AND t1.y IS NULL + LEFT JOIN t4 ON t4.w = t3.w + LEFT JOIN t5 ON t5.z = t4.z + LEFT JOIN t2 ON t2.u = t5.m + LEFT JOIN t1 xyz ON xyz.y = t2.x; +} {007f2033-cb20-494c-b135-a1e4eb66130c f6d7661f-4efe-4c90-87b5-858e61cd178b f6d7661f-4efe-4c90-87b5-858e61cd178b {} {} {} {} {} {} {} {} {}} +do_execsql_test join5-3.3 { + DROP TABLE IF EXISTS x1; + DROP TABLE IF EXISTS x2; + DROP TABLE IF EXISTS x3; + CREATE TABLE x1(a); + INSERT INTO x1 VALUES(1); + CREATE TABLE x2(b NOT NULL); + CREATE TABLE x3(c, d); + INSERT INTO x3 VALUES('a', NULL); + INSERT INTO x3 VALUES('b', NULL); + INSERT INTO x3 VALUES('c', NULL); + SELECT * FROM x1 LEFT JOIN x2 JOIN x3 WHERE x3.d = x2.b; +} {} finish_test From 1d324880370ae70e7c9af187f59c3f3c351b0ef4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Dec 2014 20:24:50 +0000 Subject: [PATCH 618/710] Performance enhancement for single-table queries with many OR-connected WHERE clause terms and multiple indexes with the same left-most columns. FossilOrigin-Name: 1461d543ac8a3e4a54405067893146c74576bb4e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 5 ++--- test/whereJ.test | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4f3131ec66..ddc85c1040 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sa\stable\sis\sthe\sright\soperand\sof\sa\sLEFT\sJOIN,\sthen\sany\scolumn\sof\sthat\ntable\scan\sbe\sNULL\seven\sif\sthat\scolumn\sas\sa\sNOT\sNULL\sconstraint.\nFix\sfor\sticket\s[6f2222d550f5b0ee7ed]. -D 2014-12-04T16:27:17.993 +C Performance\senhancement\sfor\ssingle-table\squeries\swith\smany\sOR-connected\nWHERE\sclause\sterms\sand\smultiple\sindexes\swith\sthe\ssame\sleft-most\scolumns. +D 2014-12-04T20:24:50.938 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -304,7 +304,7 @@ F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 F src/wal.c 847692349eb6e1fb8543dbc97e69ddbfa4cc7ea7 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c a0b16f9d78321cb340a977287d19f826555c7d3b +F src/where.c d67fe69dd1bb32ad3d488a8c5bc803a29814f357 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1154,7 +1154,7 @@ F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 F test/whereG.test 69f5ec4b15760a8c860f80e2d55525669390aab3 F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2 F test/whereI.test 1d89199697919d4930be05a71e7fe620f114e622 -F test/whereJ.test 63599653dfefe4e74ebb358db753417fe0aa8a49 +F test/whereJ.test 55a3221706a7ab706293f17cc8f96da563bf0767 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1e2bc484f8e7766a493bfeab04d82e50f37217b8 -R 6261b2c66938591ba7d193856851eb02 +P 6f6fcbe4736b9468a495c684d5eebc8bfe5c566a +R 5428d48e129f2db3029319e68ae21b70 U drh -Z d631884fbd9f9cb654155b89b69f5e28 +Z 008a44be7133b3164f9b1ac313df9791 diff --git a/manifest.uuid b/manifest.uuid index abaad9a04b..b913e108c5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f6fcbe4736b9468a495c684d5eebc8bfe5c566a \ No newline at end of file +1461d543ac8a3e4a54405067893146c74576bb4e \ No newline at end of file diff --git a/src/where.c b/src/where.c index 277c8833e8..bda2a801ea 100644 --- a/src/where.c +++ b/src/where.c @@ -3591,10 +3591,9 @@ static Bitmask codeOneLoopStart( Expr *pExpr = pWC->a[iTerm].pExpr; if( &pWC->a[iTerm] == pTerm ) continue; if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; - testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); - testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); - if( pWC->a[iTerm].wtFlags & (TERM_ORINFO|TERM_VIRTUAL) ) continue; + if( (pWC->a[iTerm].wtFlags & TERM_VIRTUAL)!=0 ) continue; if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); pExpr = sqlite3ExprDup(db, pExpr, 0); pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); } diff --git a/test/whereJ.test b/test/whereJ.test index 8431c3a4b0..48924d0fcf 100644 --- a/test/whereJ.test +++ b/test/whereJ.test @@ -640,4 +640,39 @@ do_execsql_test 4.2 { } {/.*SCAN TABLE cx.*SEARCH TABLE px.*SEARCH TABLE le.*/} +# The following test is derived from a performance problem reported from +# the field. Notice the multiple indexes with the same initial tables, +# and the unusual WHERE clause terms. +# +do_test 5.1 { + set res [db eval { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a,b,c,d,e,f,g,h); + CREATE INDEX t1abc ON t1(a,b,c); + CREATE INDEX t1abe ON t1(a,b,e); + CREATE INDEX t1abf ON t1(a,b,f); + ANALYZE; + DROP TABLE IF EXISTS sqlite_stat4; + DROP TABLE IF EXISTS sqlite_stat3; + DELETE FROM sqlite_stat1; + INSERT INTO sqlite_stat1(tbl,idx,stat) + VALUES('t1','t1abc','2000000 8000 1600 800'), + ('t1','t1abe','2000000 8000 1600 150'), + ('t1','t1abf','2000000 8000 1600 150'); + ANALYZE sqlite_master; + + EXPLAIN QUERY PLAN + SELECT * FROM t1 + WHERE (a=1 OR a=2) + AND (b=3 OR b=4) + AND (d>=5 AND d<=5) + AND ((e>=7 AND e<=7) OR (f>=8 AND f<=8)) + AND g>0; + }] +} {~/ANY/} +do_test 5.2 {set res} {/USING INDEX t1abe/} +do_test 5.3 {set res} {/USING INDEX t1abf/} + + + finish_test From dea7d70d1b9516956a7550d0ffde1b4560fc77ca Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 4 Dec 2014 21:54:58 +0000 Subject: [PATCH 619/710] Make sure that a DISTINCT query with an ORDER BY works correctly even if it uses a descending index. Fix for ticket [c5ea805691bfc4204b1cb9e]. FossilOrigin-Name: 0d3aef97ebddf422b8bdcbc5878970c6129e3f54 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 3 +-- test/distinct.test | 30 ++++++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index ddc85c1040..e3691737da 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\senhancement\sfor\ssingle-table\squeries\swith\smany\sOR-connected\nWHERE\sclause\sterms\sand\smultiple\sindexes\swith\sthe\ssame\sleft-most\scolumns. -D 2014-12-04T20:24:50.938 +C Make\ssure\sthat\sa\sDISTINCT\squery\swith\san\sORDER\sBY\sworks\scorrectly\seven\sif\nit\suses\sa\sdescending\sindex.\s\sFix\sfor\sticket\s[c5ea805691bfc4204b1cb9e]. +D 2014-12-04T21:54:58.307 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -228,7 +228,7 @@ F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e -F src/select.c 428165951748151e87a15295b7357221433e311b +F src/select.c f377fb8a5c73c10678ea74f3400f7913943e3d75 F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf F src/sqlite.h.in 6ec654324cb490ea3d8a7be28b8c7d37fe4ad282 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad @@ -454,7 +454,7 @@ F test/descidx1.test 6d03b44c8538fe0eb4924e19fba10cdd8f3c9240 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 +F test/distinct.test 175d49ee783febaf368192dfe7f5afbc68910230 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_blobbytes.test 9bea1d3e2b20f3010b04abba58f6ba172301f49f F test/e_blobclose.test df756753f571bc30e42e3a6cba2807576e49e716 @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6f6fcbe4736b9468a495c684d5eebc8bfe5c566a -R 5428d48e129f2db3029319e68ae21b70 +P 1461d543ac8a3e4a54405067893146c74576bb4e +R fc5b0159564faa200a156d134ab758df U drh -Z 008a44be7133b3164f9b1ac313df9791 +Z 018f1b0eb45096a13f52fc7d70ef2bc6 diff --git a/manifest.uuid b/manifest.uuid index b913e108c5..7416b1cda3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1461d543ac8a3e4a54405067893146c74576bb4e \ No newline at end of file +0d3aef97ebddf422b8bdcbc5878970c6129e3f54 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 3b422f1100..070ac00410 100644 --- a/src/select.c +++ b/src/select.c @@ -4829,7 +4829,7 @@ int sqlite3Select( ** ** is transformed to: ** - ** SELECT xyz FROM ... GROUP BY xyz + ** SELECT xyz FROM ... GROUP BY xyz ORDER BY xyz ** ** The second form is preferred as a single index (or temp-table) may be ** used for both the ORDER BY and DISTINCT processing. As originally @@ -4842,7 +4842,6 @@ int sqlite3Select( p->selFlags &= ~SF_Distinct; p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; - sSort.pOrderBy = 0; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ diff --git a/test/distinct.test b/test/distinct.test index 78c2c1df36..2fb90dc3e3 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -222,4 +222,34 @@ do_execsql_test 4.1 { SELECT quote(x) FROM t2 ORDER BY 1; } {'xyzzy' X'0000000000'} +#---------------------------------------------------------------------------- +# Ticket [c5ea805691bfc4204b1cb9e9aa0103bd48bc7d34] (2014-12-04) +# Make sure that DISTINCT works together with ORDER BY and descending +# indexes. +# +do_execsql_test 5.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + INSERT INTO t1(x) VALUES(3),(1),(5),(2),(6),(4),(5),(1),(3); + CREATE INDEX t1x ON t1(x DESC); + SELECT DISTINCT x FROM t1 ORDER BY x ASC; +} {1 2 3 4 5 6} +do_execsql_test 5.2 { + SELECT DISTINCT x FROM t1 ORDER BY x DESC; +} {6 5 4 3 2 1} +do_execsql_test 5.3 { + SELECT DISTINCT x FROM t1 ORDER BY x; +} {1 2 3 4 5 6} +do_execsql_test 5.4 { + DROP INDEX t1x; + CREATE INDEX t1x ON t1(x ASC); + SELECT DISTINCT x FROM t1 ORDER BY x ASC; +} {1 2 3 4 5 6} +do_execsql_test 5.5 { + SELECT DISTINCT x FROM t1 ORDER BY x DESC; +} {6 5 4 3 2 1} +do_execsql_test 5.6 { + SELECT DISTINCT x FROM t1 ORDER BY x; +} {1 2 3 4 5 6} + finish_test From 5de7d966bd1bd61d4a197585db449bd111f60cab Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 5 Dec 2014 00:17:39 +0000 Subject: [PATCH 620/710] Fix the autoconf and MSVC makefiles, which have been broken for nearly a month. :-( FossilOrigin-Name: 520c2b838da8b230487c8c66f3ba8e5daa1ca886 --- Makefile.in | 1 + Makefile.msc | 1 + manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Makefile.in b/Makefile.in index 646cb39756..4929ce313c 100644 --- a/Makefile.in +++ b/Makefile.in @@ -359,6 +359,7 @@ TESTSRC = \ $(TOP)/src/test_autoext.c \ $(TOP)/src/test_async.c \ $(TOP)/src/test_backup.c \ + $(TOP)/src/test_blob.c \ $(TOP)/src/test_btree.c \ $(TOP)/src/test_config.c \ $(TOP)/src/test_demovfs.c \ diff --git a/Makefile.msc b/Makefile.msc index 4aaa2894a7..64175a0679 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -828,6 +828,7 @@ TESTSRC = \ $(TOP)\src\test_autoext.c \ $(TOP)\src\test_async.c \ $(TOP)\src\test_backup.c \ + $(TOP)\src\test_blob.c \ $(TOP)\src\test_btree.c \ $(TOP)\src\test_config.c \ $(TOP)\src\test_demovfs.c \ diff --git a/manifest b/manifest index e3691737da..042ccce636 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Make\ssure\sthat\sa\sDISTINCT\squery\swith\san\sORDER\sBY\sworks\scorrectly\seven\sif\nit\suses\sa\sdescending\sindex.\s\sFix\sfor\sticket\s[c5ea805691bfc4204b1cb9e]. -D 2014-12-04T21:54:58.307 +C Fix\sthe\sautoconf\sand\sMSVC\smakefiles,\swhich\shave\sbeen\sbroken\sfor\snearly\sa\nmonth.\s\s:-( +D 2014-12-05T00:17:39.977 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in a226317fdf3f4c895fb3cfedc355b4d0868ce1fb +F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 788f1288633a0c3c3cbbe0f3e4827d033f7ba530 +F Makefile.msc 10720782f88648bf2b5dcedf4c1524b067d43e47 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION d846487aff892625eb8e75960234e7285f0462fe @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1461d543ac8a3e4a54405067893146c74576bb4e -R fc5b0159564faa200a156d134ab758df +P 0d3aef97ebddf422b8bdcbc5878970c6129e3f54 +R cd249245e23c4920ae8ada9787d27cef U drh -Z 018f1b0eb45096a13f52fc7d70ef2bc6 +Z 911d29bfeef3460d645482250c6ca2bc diff --git a/manifest.uuid b/manifest.uuid index 7416b1cda3..3768cb56eb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0d3aef97ebddf422b8bdcbc5878970c6129e3f54 \ No newline at end of file +520c2b838da8b230487c8c66f3ba8e5daa1ca886 \ No newline at end of file From 2c3abeb8c3c01c8f0143557a3bebddf5d5d26405 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 5 Dec 2014 00:32:09 +0000 Subject: [PATCH 621/710] Fix compiler warnings. FossilOrigin-Name: e9955c0e14d13ba1411f013acb4979958dae2516 --- ext/misc/eval.c | 19 +++++++++++-------- manifest | 14 +++++++------- manifest.uuid | 2 +- src/test1.c | 2 +- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/ext/misc/eval.c b/ext/misc/eval.c index a5e297ad38..71b6b69f20 100644 --- a/ext/misc/eval.c +++ b/ext/misc/eval.c @@ -21,11 +21,11 @@ SQLITE_EXTENSION_INIT1 ** Structure used to accumulate the output */ struct EvalResult { - char *z; /* Accumulated output */ - const char *zSep; /* Separator */ - int szSep; /* Size of the separator string */ - int nAlloc; /* Number of bytes allocated for z[] */ - int nUsed; /* Number of bytes of z[] actually used */ + char *z; /* Accumulated output */ + const char *zSep; /* Separator */ + int szSep; /* Size of the separator string */ + sqlite3_int64 nAlloc; /* Number of bytes allocated for z[] */ + sqlite3_int64 nUsed; /* Number of bytes of z[] actually used */ }; /* @@ -37,10 +37,13 @@ static int callback(void *pCtx, int argc, char **argv, char **colnames){ for(i=0; inUsed+p->szSep+1 > p->nAlloc ){ + if( (sqlite3_int64)sz+p->nUsed+p->szSep+1 > p->nAlloc ){ char *zNew; p->nAlloc = p->nAlloc*2 + sz + p->szSep + 1; - zNew = sqlite3_realloc(p->z, p->nAlloc); + /* Using sqlite3_realloc64() would be better, but it is a recent + ** addition and will cause a segfault if loaded by an older version + ** of SQLite. */ + zNew = p->nAlloc<=0x7fffffff ? sqlite3_realloc(p->z, (int)p->nAlloc) : 0; if( zNew==0 ){ sqlite3_free(p->z); memset(p, 0, sizeof(*p)); @@ -93,7 +96,7 @@ static void sqlEvalFunc( sqlite3_result_error_nomem(context); sqlite3_free(x.z); }else{ - sqlite3_result_text(context, x.z, x.nUsed, sqlite3_free); + sqlite3_result_text(context, x.z, (int)x.nUsed, sqlite3_free); } } diff --git a/manifest b/manifest index 042ccce636..375f1f89b0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sautoconf\sand\sMSVC\smakefiles,\swhich\shave\sbeen\sbroken\sfor\snearly\sa\nmonth.\s\s:-( -D 2014-12-05T00:17:39.977 +C Fix\scompiler\swarnings. +D 2014-12-05T00:32:09.905 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -109,7 +109,7 @@ F ext/icu/sqliteicu.h 728867a802baa5a96de7495e9689a8e01715ef37 F ext/misc/amatch.c 678056a4bfcd83c4e82dea81d37543cd1d6dbee1 F ext/misc/closure.c 636024302cde41b2bf0c542f81c40c624cfb7012 F ext/misc/compress.c 76e45655f4046e756064ab10c62e18f2eb846b9f -F ext/misc/eval.c 04e630bde869aa1fec6b993d40591f963be2f868 +F ext/misc/eval.c f971962e92ebb8b0a4e6b62949463ee454d88fa2 F ext/misc/fileio.c d4171c815d6543a9edef8308aab2951413cd8d0f F ext/misc/fuzzer.c 136533c53cfce0957f0b48fa11dba27e21c5c01d F ext/misc/ieee754.c b0362167289170627659e84173f5d2e8fee8566e @@ -238,7 +238,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 -F src/test1.c f5d7ecd3dd663b11f35269fd91f7090db0570903 +F src/test1.c c24d7f67252348d756773a6dbe23a61b4552f709 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -1223,7 +1223,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0d3aef97ebddf422b8bdcbc5878970c6129e3f54 -R cd249245e23c4920ae8ada9787d27cef +P 520c2b838da8b230487c8c66f3ba8e5daa1ca886 +R 2f1363efb358b758f4fe53b98d762c1d U drh -Z 911d29bfeef3460d645482250c6ca2bc +Z 8ac94f0b609313b69c75d4df8f71997e diff --git a/manifest.uuid b/manifest.uuid index 3768cb56eb..0fa8785978 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -520c2b838da8b230487c8c66f3ba8e5daa1ca886 \ No newline at end of file +e9955c0e14d13ba1411f013acb4979958dae2516 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index be4ad92c11..7839b30494 100644 --- a/src/test1.c +++ b/src/test1.c @@ -3661,7 +3661,7 @@ static int test_prepare_v2( zCopy = malloc(bytes); memcpy(zCopy, zSql, bytes); }else{ - int n = strlen(zSql) + 1; + int n = (int)strlen(zSql) + 1; zCopy = malloc(n); memcpy(zCopy, zSql, n); } From ffc8f3e201396f08db148a9f0e21d48c32273232 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 5 Dec 2014 05:38:02 +0000 Subject: [PATCH 622/710] When closing a (shared-cache) database connection, be sure to clear out all KeyInfo objects cached on Index objects. Proposed fix for ticket [e4a18565a36884b00edf]. FossilOrigin-Name: 651ed97de13234be60a1138a98b06d308449a791 --- manifest | 15 +++++++++------ manifest.uuid | 2 +- src/main.c | 8 +++++++- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 375f1f89b0..6a37a13e06 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\scompiler\swarnings. -D 2014-12-05T00:32:09.905 +C When\sclosing\sa\s(shared-cache)\sdatabase\sconnection,\sbe\ssure\sto\sclear\sout\sall\nKeyInfo\sobjects\scached\son\sIndex\sobjects.\nProposed\sfix\sfor\sticket\s[e4a18565a36884b00edf]. +D 2014-12-05T05:38:02.138 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 84cc0fc4861bd2b06c286b38bf03bce604a0a66a +F src/main.c fa9f0e511a64caf66b8e7bb835c55172e7d61527 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -1223,7 +1223,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 520c2b838da8b230487c8c66f3ba8e5daa1ca886 -R 2f1363efb358b758f4fe53b98d762c1d +P e9955c0e14d13ba1411f013acb4979958dae2516 +R 6bcc6e9801f68a90df02ecfc8338dee3 +T *branch * fix-stale-keyinfo-cache +T *sym-fix-stale-keyinfo-cache * +T -sym-trunk * U drh -Z 8ac94f0b609313b69c75d4df8f71997e +Z 0a051974f060a3e4f394579c98ab9920 diff --git a/manifest.uuid b/manifest.uuid index 0fa8785978..cf22e9381d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e9955c0e14d13ba1411f013acb4979958dae2516 \ No newline at end of file +651ed97de13234be60a1138a98b06d308449a791 \ No newline at end of file diff --git a/src/main.c b/src/main.c index f7a0b1f7fd..39c65baabc 100644 --- a/src/main.c +++ b/src/main.c @@ -1034,7 +1034,13 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ if( pDb->pBt ){ sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; - if( j!=1 ){ + if( j!=1 && pDb->pSchema ){ + /* Must clear the KeyInfo cache. See ticket [e4a18565a36884b00edf] */ + for(i=sqliteHashFirst(&pDb->pSchema->idxHash); i; i=sqliteHashNext(i)){ + Index *pIdx = sqliteHashData(i); + sqlite3KeyInfoUnref(pIdx->pKeyInfo); + pIdx->pKeyInfo = 0; + } pDb->pSchema = 0; } } From a895a4d22cedbf9bbfb7288069dc5347139ee36b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 5 Dec 2014 14:07:53 +0000 Subject: [PATCH 623/710] Test case demonstrating the problem described by ticket [e4a18565a36884b00edf]. FossilOrigin-Name: ffea3e905adc108d2dc37f5d6da2024f0389f176 --- manifest | 14 +++++------- manifest.uuid | 2 +- test/sharedB.test | 57 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 test/sharedB.test diff --git a/manifest b/manifest index 6a37a13e06..1c55723bc4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sclosing\sa\s(shared-cache)\sdatabase\sconnection,\sbe\ssure\sto\sclear\sout\sall\nKeyInfo\sobjects\scached\son\sIndex\sobjects.\nProposed\sfix\sfor\sticket\s[e4a18565a36884b00edf]. -D 2014-12-05T05:38:02.138 +C Test\scase\sdemonstrating\sthe\sproblem\sdescribed\sby\nticket\s[e4a18565a36884b00edf]. +D 2014-12-05T14:07:53.311 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -846,6 +846,7 @@ F test/shared7.test a81e99f83e6c51b02ac99c96fb3a2a7b5978c956 F test/shared8.test 00a07bf5e1337ecf72e94542bdefdc330d7a2538 F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21 F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 +F test/sharedB.test 91c8244e0f2a5f0bda731981d62208b6b54bfff2 F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test ab6025d941f9c84c5b83412c6b4d8b57f78dfa3a @@ -1223,10 +1224,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e9955c0e14d13ba1411f013acb4979958dae2516 -R 6bcc6e9801f68a90df02ecfc8338dee3 -T *branch * fix-stale-keyinfo-cache -T *sym-fix-stale-keyinfo-cache * -T -sym-trunk * +P 651ed97de13234be60a1138a98b06d308449a791 +R 9f67caf91540cf2461a4c618c84ef365 U drh -Z 0a051974f060a3e4f394579c98ab9920 +Z 2496f3d4e53b37395b3a158db036ecc7 diff --git a/manifest.uuid b/manifest.uuid index cf22e9381d..69d4dd6ea0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -651ed97de13234be60a1138a98b06d308449a791 \ No newline at end of file +ffea3e905adc108d2dc37f5d6da2024f0389f176 \ No newline at end of file diff --git a/test/sharedB.test b/test/sharedB.test new file mode 100644 index 0000000000..beb99578f4 --- /dev/null +++ b/test/sharedB.test @@ -0,0 +1,57 @@ +# 2013 May 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Open two database connections on the same database in shared cache +# mode. Hold one open while repeatedly closing, reopening, and using +# the second. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +if {[run_thread_tests]==0} { finish_test ; return } +db close +set ::testprefix sharedB + +set ::enable_shared_cache [sqlite3_enable_shared_cache 1] + +#------------------------------------------------------------------------- +# +do_test 1.1 { + sqlite3 db1 test.db + sqlite3 db2 test.db + + db1 eval { + CREATE TABLE t1(x,y TEXT COLLATE nocase); + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<100) + INSERT INTO t1(x,y) SELECT i, printf('x%03dy',i) FROM c; + CREATE INDEX t1yx ON t1(y,x); + } + db2 eval { + SELECT x FROM t1 WHERE y='X014Y'; + } +} {14} + +for {set j 1} {$j<=100} {incr j} { + do_test 1.2.$j { + db2 close + sqlite3 db2 test.db + db2 eval { + SELECT x FROM t1 WHERE y='X014Y'; + } + } {14} +} + +db1 close +db2 close +sqlite3_enable_shared_cache $::enable_shared_cache +finish_test From 351064b4522e9fb7f224697f3abb9adceed169fa Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 5 Dec 2014 14:34:30 +0000 Subject: [PATCH 624/710] Improved comment on the sharedB.test test script. FossilOrigin-Name: 71f589e3f82a07513425e212072f32748c0732d4 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/sharedB.test | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 1c55723bc4..48f0747ab8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Test\scase\sdemonstrating\sthe\sproblem\sdescribed\sby\nticket\s[e4a18565a36884b00edf]. -D 2014-12-05T14:07:53.311 +C Improved\scomment\son\sthe\ssharedB.test\stest\sscript. +D 2014-12-05T14:34:30.550 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -846,7 +846,7 @@ F test/shared7.test a81e99f83e6c51b02ac99c96fb3a2a7b5978c956 F test/shared8.test 00a07bf5e1337ecf72e94542bdefdc330d7a2538 F test/shared9.test 5f2a8f79b4d6c7d107a01ffa1ed05ae7e6333e21 F test/sharedA.test 0cdf1a76dfa00e6beee66af5b534b1e8df2720f5 -F test/sharedB.test 91c8244e0f2a5f0bda731981d62208b6b54bfff2 +F test/sharedB.test 16cc7178e20965d75278f410943109b77b2e645e F test/shared_err.test 2f2aee20db294b9924e81f6ccbe60f19e21e8506 F test/sharedlock.test 5ede3c37439067c43b0198f580fd374ebf15d304 F test/shell1.test ab6025d941f9c84c5b83412c6b4d8b57f78dfa3a @@ -1224,7 +1224,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 651ed97de13234be60a1138a98b06d308449a791 -R 9f67caf91540cf2461a4c618c84ef365 +P ffea3e905adc108d2dc37f5d6da2024f0389f176 +R 2f7d3cb7a2e1e65c7403b8d51d471007 U drh -Z 2496f3d4e53b37395b3a158db036ecc7 +Z 951231dae63f2db1845bdae375b332da diff --git a/manifest.uuid b/manifest.uuid index 69d4dd6ea0..d3a7ef5638 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ffea3e905adc108d2dc37f5d6da2024f0389f176 \ No newline at end of file +71f589e3f82a07513425e212072f32748c0732d4 \ No newline at end of file diff --git a/test/sharedB.test b/test/sharedB.test index beb99578f4..1b15755076 100644 --- a/test/sharedB.test +++ b/test/sharedB.test @@ -1,4 +1,4 @@ -# 2013 May 14 +# 2014-12-05 # # The author disclaims copyright to this source code. In place of # a legal notice, here is a blessing: @@ -13,6 +13,9 @@ # mode. Hold one open while repeatedly closing, reopening, and using # the second. # +# This test is designed to demostrate that the fix for ticket +# [e4a18565a36884b00edf66541f38c693827968ab] works. +# set testdir [file dirname $argv0] From ebdb81dd455a3c3965b3159129d96fbcc49adb13 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 5 Dec 2014 15:31:33 +0000 Subject: [PATCH 625/710] The KeyInfo cache must be cleared before closing the btree, not after. Revised fix for ticket [e4a18565a36884b00edf]. FossilOrigin-Name: 7ed3346e8c10dbf52fd44ab69900699d4f7ad3fd --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/main.c | 8 +++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 14a2b23306..fd3e0c5784 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sclosing\sa\s(shared-cache)\sdatabase\sconnection,\sbe\ssure\sto\sclear\sout\nall\sKeyInfo\sobjects\scached\son\sIndex\sobjects.\s\nFix\sfor\sticket\s[e4a18565a36884b00edf]. -D 2014-12-05T14:44:57.022 +C The\sKeyInfo\scache\smust\sbe\scleared\sbefore\sclosing\sthe\sbtree,\snot\safter.\nRevised\sfix\sfor\sticket\s[e4a18565a36884b00edf]. +D 2014-12-05T15:31:33.761 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c fa9f0e511a64caf66b8e7bb835c55172e7d61527 +F src/main.c e50203613fb77e2f28deb51425ee52b3879e85f8 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -1224,7 +1224,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e9955c0e14d13ba1411f013acb4979958dae2516 71f589e3f82a07513425e212072f32748c0732d4 -R 2f7d3cb7a2e1e65c7403b8d51d471007 +P adca7688de20ff40d8ddf2107dfaf92af3873b83 +Q +48529508cfe2f9445a6940222fe5ff7cc92aef78 +R 943a66e258c43484e6287cfb624784df U drh -Z 971c54886e3dd2c38037eed368c93472 +Z 9be13842ba3000b4a28c2a5894f2ade5 diff --git a/manifest.uuid b/manifest.uuid index adefef81dc..34d0aae3a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -adca7688de20ff40d8ddf2107dfaf92af3873b83 \ No newline at end of file +7ed3346e8c10dbf52fd44ab69900699d4f7ad3fd \ No newline at end of file diff --git a/src/main.c b/src/main.c index 39c65baabc..34093b2495 100644 --- a/src/main.c +++ b/src/main.c @@ -1032,15 +1032,17 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ for(j=0; jnDb; j++){ struct Db *pDb = &db->aDb[j]; if( pDb->pBt ){ - sqlite3BtreeClose(pDb->pBt); - pDb->pBt = 0; - if( j!=1 && pDb->pSchema ){ + if( pDb->pSchema ){ /* Must clear the KeyInfo cache. See ticket [e4a18565a36884b00edf] */ for(i=sqliteHashFirst(&pDb->pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3KeyInfoUnref(pIdx->pKeyInfo); pIdx->pKeyInfo = 0; } + } + sqlite3BtreeClose(pDb->pBt); + pDb->pBt = 0; + if( j!=1 ){ pDb->pSchema = 0; } } From fe32daaeaaffb3306d885b39d33774bc432607ba Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 5 Dec 2014 19:50:58 +0000 Subject: [PATCH 626/710] Make sure the WhereTerm objects are fully zeroed when they are allocated. FossilOrigin-Name: fdb667335c2250239a66143aec4235325dec8274 --- manifest | 13 ++++++------- manifest.uuid | 2 +- src/where.c | 1 + 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index fd3e0c5784..dc82a51063 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\sKeyInfo\scache\smust\sbe\scleared\sbefore\sclosing\sthe\sbtree,\snot\safter.\nRevised\sfix\sfor\sticket\s[e4a18565a36884b00edf]. -D 2014-12-05T15:31:33.761 +C Make\ssure\sthe\sWhereTerm\sobjects\sare\sfully\szeroed\swhen\sthey\sare\sallocated. +D 2014-12-05T19:50:58.294 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -304,7 +304,7 @@ F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 F src/wal.c 847692349eb6e1fb8543dbc97e69ddbfa4cc7ea7 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c d67fe69dd1bb32ad3d488a8c5bc803a29814f357 +F src/where.c 8877b55e6803eaa856254875bca8d12fff3681cf F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1224,8 +1224,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P adca7688de20ff40d8ddf2107dfaf92af3873b83 -Q +48529508cfe2f9445a6940222fe5ff7cc92aef78 -R 943a66e258c43484e6287cfb624784df +P 7ed3346e8c10dbf52fd44ab69900699d4f7ad3fd +R 4b33ce7227bf5d27f48a35be658eb882 U drh -Z 9be13842ba3000b4a28c2a5894f2ade5 +Z 1360efaa2c5f7305d23abadc41fbb137 diff --git a/manifest.uuid b/manifest.uuid index 34d0aae3a8..437ab02c1f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ed3346e8c10dbf52fd44ab69900699d4f7ad3fd \ No newline at end of file +fdb667335c2250239a66143aec4235325dec8274 \ No newline at end of file diff --git a/src/where.c b/src/where.c index bda2a801ea..8ec6018bce 100644 --- a/src/where.c +++ b/src/where.c @@ -222,6 +222,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u8 wtFlags){ sqlite3DbFree(db, pOld); } pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); + memset(&pWC->a[pWC->nTerm], 0, sizeof(pWC->a[0])*(pWC->nSlot-pWC->nTerm)); } pTerm = &pWC->a[idx = pWC->nTerm++]; if( p && ExprHasProperty(p, EP_Unlikely) ){ From 2928d327914836a31599e71b836cd58cbddb7922 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 5 Dec 2014 20:46:19 +0000 Subject: [PATCH 627/710] Add new test file e_walckpt.test. Still some tests to come. FossilOrigin-Name: e4db3db3a65ecfd4069a40d436aa7a5512d61a30 --- manifest | 15 +-- manifest.uuid | 2 +- src/test1.c | 10 +- test/e_walckpt.test | 290 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 305 insertions(+), 12 deletions(-) create mode 100644 test/e_walckpt.test diff --git a/manifest b/manifest index dc82a51063..12cf545560 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sWhereTerm\sobjects\sare\sfully\szeroed\swhen\sthey\sare\sallocated. -D 2014-12-05T19:50:58.294 +C Add\snew\stest\sfile\se_walckpt.test.\sStill\ssome\stests\sto\scome. +D 2014-12-05T20:46:19.108 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -238,7 +238,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 -F src/test1.c c24d7f67252348d756773a6dbe23a61b4552f709 +F src/test1.c ebb8cd3c94a2ac8851b7b0b1349284e73a8b4c7a F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -478,6 +478,7 @@ F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a +F test/e_walckpt.test de5a8d86c5b95569309c6da796dbea870c22e003 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 @@ -1224,7 +1225,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7ed3346e8c10dbf52fd44ab69900699d4f7ad3fd -R 4b33ce7227bf5d27f48a35be658eb882 -U drh -Z 1360efaa2c5f7305d23abadc41fbb137 +P fdb667335c2250239a66143aec4235325dec8274 +R 88cb63a8d2ff8042b23c5ce33c03986a +U dan +Z d1e87cf655be5398ea39ae6fc02beae8 diff --git a/manifest.uuid b/manifest.uuid index 437ab02c1f..786ddbf8ea 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fdb667335c2250239a66143aec4235325dec8274 \ No newline at end of file +e4db3db3a65ecfd4069a40d436aa7a5512d61a30 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 7839b30494..14ccacd863 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5704,15 +5704,17 @@ static int test_wal_checkpoint_v2( if( objc==4 ){ zDb = Tcl_GetString(objv[3]); } - if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) - || Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0, &eMode) - ){ + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) || ( + TCL_OK!=Tcl_GetIntFromObj(0, objv[2], &eMode) + && TCL_OK!=Tcl_GetIndexFromObj(interp, objv[2], aMode, "mode", 0, &eMode) + )){ return TCL_ERROR; } rc = sqlite3_wal_checkpoint_v2(db, zDb, eMode, &nLog, &nCkpt); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ - Tcl_SetResult(interp, (char *)sqlite3_errmsg(db), TCL_VOLATILE); + const char *zErrCode = sqlite3ErrName(rc); + Tcl_AppendResult(interp, zErrCode, " - ", (char *)sqlite3_errmsg(db), 0); return TCL_ERROR; } diff --git a/test/e_walckpt.test b/test/e_walckpt.test new file mode 100644 index 0000000000..8b0aae798a --- /dev/null +++ b/test/e_walckpt.test @@ -0,0 +1,290 @@ +# 2014 December 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/lock_common.tcl +source $testdir/wal_common.tcl +set testprefix e_walckpt + +# The following two commands are used to determine if any of the files +# "test.db", "test.db2" and "test.db3" are modified by a test case. +# +# The [save_db_hashes] command saves a hash of the current contents of +# all three files in global variables. The [compare_db_hashes] compares +# the current contents with the saved hashes and returns a list of the +# files that have changed. +# +proc save_db_hashes {} { + global H + foreach f {test.db test.db2 test.db3} { + set H($f) 0 + catch { set H($f) [md5file $f] } + } +} +proc compare_db_hashes {} { + global H + set ret [list] + foreach f {test.db test.db2 test.db3} { + set expect 0 + catch { set expect [md5file $f] } + if {$H($f) != $expect} { lappend ret $f } + } + set ret +} + + +# The following tests are run 3 times, each using a different method of +# invoking a checkpoint: +# +# 1) Using sqlite3_wal_checkpoint_v2() +# 2) Using "PRAGMA wal_checkpoint" +# 3) Using sqlite3_wal_checkpoint() in place of checkpoint_v2(PASSIVE) +# +# Cases (2) and (3) are to show that the following statements are +# correct, respectively: +# +# EVIDENCE-OF: R-36706-10507 The PRAGMA wal_checkpoint command can be +# used to invoke this interface from SQL. +# +# EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is +# equivalent to +# sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). +# +foreach {tn script} { + 1 { + proc checkpoint {db mode args} { + eval sqlite3_wal_checkpoint_v2 [list $db] [list $mode] $args + } + } + + 2 { + proc checkpoint {db mode args} { + set sql "PRAGMA wal_checkpoint" + if {[llength $args] && [lindex $args 0]!=""} { + set sql "PRAGMA [lindex $args 0].wal_checkpoint" + } + set rc [catch { $db eval $sql } msg] + if {$rc} { + regsub {database} $msg {database:} msg + error "[sqlite3_errcode $db] - $msg" + } + set msg + } + } + + 3 { + proc checkpoint {db mode args} { + if {$mode == "passive"} { + set rc [eval sqlite3_wal_checkpoint [list $db] $args] + if {$rc != "SQLITE_OK"} { + error "$rc - [sqlite3_errmsg $db]" + } + } else { + eval sqlite3_wal_checkpoint_v2 [list $db] [list $mode] $args + } + } + } + +} { + + eval $script + + reset_db + forcedelete test.db2 test.db3 test.db4 + execsql { + ATTACH 'test.db2' AS aux; + ATTACH 'test.db3' AS aux2; + ATTACH 'test.db4' AS aux3; + CREATE TABLE t1(x); + CREATE TABLE aux.t2(x); + CREATE TABLE aux2.t3(x); + CREATE TABLE aux3.t4(x); + PRAGMA main.journal_mode = WAL; + PRAGMA aux.journal_mode = WAL; + PRAGMA aux2.journal_mode = WAL; + /* Leave aux4 in rollback mode */ + } + + # EVIDENCE-OF: R-49787-09095 The sqlite3_wal_checkpoint_v2(D,X,M,L,C) + # interface runs a checkpoint operation on database X of database + # connection D in mode M. Status information is written back into + # integers pointed to by L and C. + # + # Tests 1, 2 and 3 below verify the "on database X" part of the + # above. Other parts of this requirement are tested below. + # + # EVIDENCE-OF: R-00653-06026 If parameter zDb is NULL or points to a + # zero length string, then the specified operation is attempted on all + # WAL databases attached to database connection db. + # + # Tests 4 and 5 below test this. + # + foreach {tn2 zDb dblist} { + 1 main test.db + 2 aux test.db2 + 3 aux2 test.db3 + 4 "" {test.db test.db2 test.db3} + 5 - {test.db test.db2 test.db3} + 6 temp {} + } { + do_test $tn.1.$tn2 { + execsql { + INSERT INTO t1 VALUES(1); + INSERT INTO t2 VALUES(2); + INSERT INTO t3 VALUES(3); + } + save_db_hashes + + if {$zDb == "-"} { + checkpoint db passive + } else { + checkpoint db passive $zDb + } + + compare_db_hashes + } $dblist + } + + # EVIDENCE-OF: R-38207-48996 If zDb is not NULL (or a zero length + # string) and is not the name of any attached database, SQLITE_ERROR is + # returned to the caller. + do_test $tn.2.1 { + list [catch { checkpoint db passive notadb } msg] $msg + } {1 {SQLITE_ERROR - unknown database: notadb}} + + # EVIDENCE-OF: R-14303-42483 If database zDb is the name of an attached + # database that is not in WAL mode, SQLITE_OK is returned and both + # *pnLog and *pnCkpt set to -1. + # + if {$tn==3} { + # With sqlite3_wal_checkpoint() the two output variables cannot be + # tested. So just test that no error is returned when attempting to + # checkpoint a db in rollback mode. + do_test $tn.2.2.a { checkpoint db passive aux3 } {} + } else { + do_test $tn.2.2.b { checkpoint db passive aux3 } {0 -1 -1} + } + + # EVIDENCE-OF: R-62028-47212 All calls obtain an exclusive "checkpoint" + # lock on the database file. + db close + testvfs tvfs + tvfs filter xShmLock + tvfs script filelock + proc filelock {method file handle details} { + # Test for an exclusive checkpoint lock. A checkpoint lock locks a + # single byte starting at offset 1. + if {$details == "1 1 lock exclusive"} { set ::seen_checkpoint_lock 1 } + } + sqlite3 db test.db -vfs tvfs + do_test $tn.3.1 { + execsql { INSERT INTO t1 VALUES('xyz') } + unset -nocomplain ::seen_checkpoint_lock + checkpoint db passive + set ::seen_checkpoint_lock + } {1} + db close + tvfs delete + reset_db + + + + + #----------------------------------------------------------------------- + # EVIDENCE-OF: R-10421-19736 If any other process is running a + # checkpoint operation at the same time, the lock cannot be obtained and + # SQLITE_BUSY is returned. + # + # EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + # it will not be invoked in this case. + # + testvfs tvfs + tvfs filter xWrite + sqlite3 db test.db -vfs tvfs + sqlite3 db2 test.db -vfs tvfs + + do_test $tn.3.2.1 { + db2 eval { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x, y); + INSERT INTO t1 VALUES(1,2); + INSERT INTO t1 VALUES(3,4); + INSERT INTO t1 VALUES(5,6); + } + file size test.db-wal + } [wal_file_size 5 1024] + + + # Connection [db] runs a checkpoint. During this checkpoint, each + # time it calls xWrite() to write a page into the database file, we + # attempt to start a checkpoint using [db2]. According to the + # first requirement being tested, this should return SQLITE_BUSY. According + # to the second, the busy-handler belonging to [db2] should not be + # invoked. + # + set ::write_count 0 + set ::write_errors [list] + proc busy_callback {args} { + lappend ::write_errors "busy handler called!" + } + proc write_callback {args} { + set rc [catch {checkpoint db2 passive} msg] + if {0==[regexp "database is locked" $msg] && $msg!="1 -1 -1"} { + lappend ::write_errors "$rc $msg" + } + incr ::write_count + } + db2 busy busy_callback + tvfs script write_callback + + do_test $tn.3.2.2 { + db eval {SELECT * FROM sqlite_master} + checkpoint db full + set ::write_count + } {2} + + do_test $tn.3.2.3 { + set ::write_errors + } {} + + db close + db2 close + tvfs delete + +} + +#----------------------------------------------------------------------- +# EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint +# mode: +# +# Valid checkpoint modes are 0, 1, 2 and 3. +# +sqlite3 db test.db +foreach {tn mode res} { + 0 -1001 {1 {SQLITE_MISUSE - not an error}} + 1 -1 {1 {SQLITE_MISUSE - not an error}} + 2 0 {0 {0 -1 -1}} + 3 1 {0 {0 -1 -1}} + 4 2 {0 {0 -1 -1}} + 5 3 {0 {0 -1 -1}} + 6 4 {1 {SQLITE_MISUSE - not an error}} + 7 114 {1 {SQLITE_MISUSE - not an error}} + 8 1000000 {1 {SQLITE_MISUSE - not an error}} +} { + do_test 4.$tn { + list [catch "sqlite3_wal_checkpoint_v2 db $mode" msg] $msg + } $res +} + + +finish_test From 5cca94ea710b63d30213c9fad4fbde5cd751ae5e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 5 Dec 2014 21:04:26 +0000 Subject: [PATCH 628/710] Fix a buffer overread that might occur in analyze.c if SQLITE_ENABLE_STAT4 was defined. FossilOrigin-Name: c1ae1268b9023a771fda98f26bf451c6066fe70b --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/analyze.c | 2 +- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 12cf545560..2c008fe79f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\se_walckpt.test.\sStill\ssome\stests\sto\scome. -D 2014-12-05T20:46:19.108 +C Fix\sa\sbuffer\soverread\sthat\smight\soccur\sin\sanalyze.c\sif\sSQLITE_ENABLE_STAT4\swas\sdefined. +D 2014-12-05T21:04:26.713 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -167,7 +167,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb -F src/analyze.c f7d774356ba5a14e7ad4fb637681af16875ad88f +F src/analyze.c 7a2986e6ea8247e5f21aca3d0b584598f58d84fe F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea @@ -1225,7 +1225,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fdb667335c2250239a66143aec4235325dec8274 -R 88cb63a8d2ff8042b23c5ce33c03986a +P e4db3db3a65ecfd4069a40d436aa7a5512d61a30 +Q +194c90db637ad4197a54be83a665feb2a9c96014 +R 4fea4497b1906b35d5aec52d63c738f3 U dan -Z d1e87cf655be5398ea39ae6fc02beae8 +Z 35f7cd6f840ef1a069fd6944a80e996c diff --git a/manifest.uuid b/manifest.uuid index 786ddbf8ea..9514250541 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e4db3db3a65ecfd4069a40d436aa7a5512d61a30 \ No newline at end of file +c1ae1268b9023a771fda98f26bf451c6066fe70b \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 01c2f12952..e483807116 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1596,7 +1596,7 @@ static void initAvgEq(Index *pIdx){ i64 nSum100 = 0; /* Number of terms contributing to sumEq */ i64 nDist100; /* Number of distinct values in index */ - if( pIdx->aiRowEst==0 || pIdx->aiRowEst[iCol+1]==0 ){ + if( !pIdx->aiRowEst || iCol>=pIdx->nKeyCol || pIdx->aiRowEst[iCol+1]==0 ){ nRow = pFinal->anLt[iCol]; nDist100 = (i64)100 * pFinal->anDLt[iCol]; nSample--; From cd934c3d2c41156770ffba1ec5e2f4c69205ba0d Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 5 Dec 2014 21:18:19 +0000 Subject: [PATCH 629/710] Avoid a potential NULL pointer deference in the sqlite3_stmt_scanstatus() logic. FossilOrigin-Name: 42d44adc13d52b8dd571c9375eb48298123d5a60 --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 2c008fe79f..e449f815e3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbuffer\soverread\sthat\smight\soccur\sin\sanalyze.c\sif\sSQLITE_ENABLE_STAT4\swas\sdefined. -D 2014-12-05T21:04:26.713 +C Avoid\sa\spotential\sNULL\spointer\sdeference\sin\sthe\ssqlite3_stmt_scanstatus()\nlogic. +D 2014-12-05T21:18:19.223 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -304,7 +304,7 @@ F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 F src/wal.c 847692349eb6e1fb8543dbc97e69ddbfa4cc7ea7 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c 8877b55e6803eaa856254875bca8d12fff3681cf +F src/where.c e914fdb9159bb36af4a673193bbda08aaf9e5a73 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1225,8 +1225,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e4db3db3a65ecfd4069a40d436aa7a5512d61a30 -Q +194c90db637ad4197a54be83a665feb2a9c96014 -R 4fea4497b1906b35d5aec52d63c738f3 -U dan -Z 35f7cd6f840ef1a069fd6944a80e996c +P c1ae1268b9023a771fda98f26bf451c6066fe70b +R f53c4ff709cb4a78a86ac1697287f29f +U drh +Z 2440e7c1515e4798b405fcb66c6b9d26 diff --git a/manifest.uuid b/manifest.uuid index 9514250541..0933f85c23 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1ae1268b9023a771fda98f26bf451c6066fe70b \ No newline at end of file +42d44adc13d52b8dd571c9375eb48298123d5a60 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8ec6018bce..ee42534f34 100644 --- a/src/where.c +++ b/src/where.c @@ -2948,7 +2948,7 @@ static void addScanStatus( ){ const char *zObj = 0; WhereLoop *pLoop = pLvl->pWLoop; - if( (pLoop->wsFlags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ zObj = pLoop->u.btree.pIndex->zName; }else{ zObj = pSrclist->a[pLvl->iFrom].zName; From d52d52be80a209486772376a8962084bd0fe7cea Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 6 Dec 2014 02:05:44 +0000 Subject: [PATCH 630/710] Fix a (harmless) uninitialized variable reference in b-tree balancing for auto-vacuumed tables with overflow pages. FossilOrigin-Name: dd1dd4451f468599f7a0c2f7b5ee6125db3bb152 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e449f815e3..63c34c8bc0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sa\spotential\sNULL\spointer\sdeference\sin\sthe\ssqlite3_stmt_scanstatus()\nlogic. -D 2014-12-05T21:18:19.223 +C Fix\sa\s(harmless)\suninitialized\svariable\sreference\sin\sb-tree\sbalancing\sfor\nauto-vacuumed\stables\swith\soverflow\spages. +D 2014-12-06T02:05:44.178 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 44b58cd798a32579816ce274e415de262df9843d +F src/btree.c b17571670289b07a9de2eeb07a0b6f8d3b0e5368 F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1225,7 +1225,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c1ae1268b9023a771fda98f26bf451c6066fe70b -R f53c4ff709cb4a78a86ac1697287f29f +P 42d44adc13d52b8dd571c9375eb48298123d5a60 +R e3e823c5583914bbca49308565af7dd0 U drh -Z 2440e7c1515e4798b405fcb66c6b9d26 +Z 667f4fff17e87dbc2be9d4c807ff4dab diff --git a/manifest.uuid b/manifest.uuid index 0933f85c23..d0ed6a9979 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -42d44adc13d52b8dd571c9375eb48298123d5a60 \ No newline at end of file +dd1dd4451f468599f7a0c2f7b5ee6125db3bb152 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 796b44403d..671f0fb2fa 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7094,7 +7094,11 @@ static int balance_nonroot( ** if sibling page iOld had the same page number as pNew, and if ** pCell really was a part of sibling page iOld (not a divider or ** overflow cell), we can skip updating the pointer map entries. */ - if( pNew->pgno!=aPgno[iOld] || pCell=&aOld[usableSize] ){ + if( iOld>=nNew + || pNew->pgno!=aPgno[iOld] + || pCell=&aOld[usableSize] + ){ if( !leafCorrection ){ ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); } From 8f1eb8a1cb3dd5c61b28470ec66e908193dc03a4 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 6 Dec 2014 14:56:49 +0000 Subject: [PATCH 631/710] Avoid accessing a single uninitialized byte when moving a rare 3-byte cell from an internal page to a leaf. This was not actually causing a problem, just a valgrind warning. FossilOrigin-Name: 6aeece19a235344be2537e66a3fe08b1febfb5a0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 6 +++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 63c34c8bc0..2427334a17 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\s(harmless)\suninitialized\svariable\sreference\sin\sb-tree\sbalancing\sfor\nauto-vacuumed\stables\swith\soverflow\spages. -D 2014-12-06T02:05:44.178 +C Avoid\saccessing\sa\ssingle\suninitialized\sbyte\swhen\smoving\sa\srare\s3-byte\scell\sfrom\san\sinternal\spage\sto\sa\sleaf.\sThis\swas\snot\sactually\scausing\sa\sproblem,\sjust\sa\svalgrind\swarning. +D 2014-12-06T14:56:49.956 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c b17571670289b07a9de2eeb07a0b6f8d3b0e5368 +F src/btree.c 9023963463b0b1876aea1abc6d208d9ffa0228ae F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1225,7 +1225,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 42d44adc13d52b8dd571c9375eb48298123d5a60 -R e3e823c5583914bbca49308565af7dd0 -U drh -Z 667f4fff17e87dbc2be9d4c807ff4dab +P dd1dd4451f468599f7a0c2f7b5ee6125db3bb152 +R fd0e43f358120e3c011052c6be23c038 +U dan +Z 22968e22e0931f64edb1c44af2aed6b9 diff --git a/manifest.uuid b/manifest.uuid index d0ed6a9979..7af5128636 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dd1dd4451f468599f7a0c2f7b5ee6125db3bb152 \ No newline at end of file +6aeece19a235344be2537e66a3fe08b1febfb5a0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 671f0fb2fa..e9cd89928e 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6859,7 +6859,11 @@ static int balance_nonroot( }else{ assert( leafCorrection==4 ); if( szCell[nCell]<4 ){ - /* Do not allow any cells smaller than 4 bytes. */ + /* Do not allow any cells smaller than 4 bytes. If a smaller cell + ** does exist, pad it with 0x00 bytes. */ + assert( szCell[nCell]==3 ); + assert( apCell[nCell]==&pTemp[iSpace1-3] ); + pTemp[iSpace1++] = 0x00; szCell[nCell] = 4; } } From b88c7b550c3fd50b6f9c8837735696f7f66a944b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 8 Dec 2014 18:08:45 +0000 Subject: [PATCH 632/710] Add a missing mutex around calls to clear the KeyInfo cache when closing a database connection. FossilOrigin-Name: 7047ce32a234484b8ba15311e6560aa74ff692c9 --- manifest | 15 ++++++++------- manifest.uuid | 2 +- src/main.c | 2 ++ 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 2427334a17..6c9b02693b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\saccessing\sa\ssingle\suninitialized\sbyte\swhen\smoving\sa\srare\s3-byte\scell\sfrom\san\sinternal\spage\sto\sa\sleaf.\sThis\swas\snot\sactually\scausing\sa\sproblem,\sjust\sa\svalgrind\swarning. -D 2014-12-06T14:56:49.956 +C Add\sa\smissing\smutex\saround\scalls\sto\sclear\sthe\sKeyInfo\scache\swhen\sclosing\na\sdatabase\sconnection. +D 2014-12-08T18:08:45.076 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c e50203613fb77e2f28deb51425ee52b3879e85f8 +F src/main.c c04c232ad484ca82b5757e564d9c4da6fdeaae55 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -1225,7 +1225,8 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P dd1dd4451f468599f7a0c2f7b5ee6125db3bb152 -R fd0e43f358120e3c011052c6be23c038 -U dan -Z 22968e22e0931f64edb1c44af2aed6b9 +P 6aeece19a235344be2537e66a3fe08b1febfb5a0 +Q +3ddc7e4c7778a6708856776471ded65f78825487 +R 1221a1e2e1d9363391aee656884338b7 +U drh +Z 22afcb22dd16de39e36b3c393f026fe6 diff --git a/manifest.uuid b/manifest.uuid index 7af5128636..708a241f10 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6aeece19a235344be2537e66a3fe08b1febfb5a0 \ No newline at end of file +7047ce32a234484b8ba15311e6560aa74ff692c9 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 34093b2495..23239bc94a 100644 --- a/src/main.c +++ b/src/main.c @@ -1034,11 +1034,13 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ if( pDb->pBt ){ if( pDb->pSchema ){ /* Must clear the KeyInfo cache. See ticket [e4a18565a36884b00edf] */ + sqlite3BtreeEnter(pDb->pBt); for(i=sqliteHashFirst(&pDb->pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3KeyInfoUnref(pIdx->pKeyInfo); pIdx->pKeyInfo = 0; } + sqlite3BtreeLeave(pDb->pBt); } sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; From 0235a0331f2d3726c1479e7f14b91391bdc61c7d Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 8 Dec 2014 20:20:16 +0000 Subject: [PATCH 633/710] Add missing mutex calls around a call to sqlite3SchemaGet() within sqlite3_open(). FossilOrigin-Name: 45415899545767888d36dcc0bafaf0ef415d94c2 --- manifest | 18 +++++------ manifest.uuid | 2 +- src/main.c | 2 ++ test/threadtest3.c | 5 ++++ test/tt3_index.c | 74 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 test/tt3_index.c diff --git a/manifest b/manifest index 6c9b02693b..c02b815662 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smissing\smutex\saround\scalls\sto\sclear\sthe\sKeyInfo\scache\swhen\sclosing\na\sdatabase\sconnection. -D 2014-12-08T18:08:45.076 +C Add\smissing\smutex\scalls\saround\sa\scall\sto\ssqlite3SchemaGet()\swithin\ssqlite3_open(). +D 2014-12-08T20:20:16.943 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c c04c232ad484ca82b5757e564d9c4da6fdeaae55 +F src/main.c 27c102143048802af53a8e55919ea3f6d9072951 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -911,7 +911,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c 0ed13e09690f6204d7455fac3b0e8ece490f6eef +F test/threadtest3.c fca8d360b470405ae3ed431b5cb4cdf031f85a74 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 @@ -1075,6 +1075,7 @@ F test/triggerC.test a68980c5955d62ee24be6f97129d824f199f9a4c F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 355e9c5cbaed5cd039a60baad1fb2197caeb8e52 F test/tt3_checkpoint.c 415eccce672d681b297485fc20f44cdf0eac93af +F test/tt3_index.c 2e7f3151a0ae522f031e8e2761ca2bda63f4d221 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a @@ -1225,8 +1226,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6aeece19a235344be2537e66a3fe08b1febfb5a0 -Q +3ddc7e4c7778a6708856776471ded65f78825487 -R 1221a1e2e1d9363391aee656884338b7 -U drh -Z 22afcb22dd16de39e36b3c393f026fe6 +P 7047ce32a234484b8ba15311e6560aa74ff692c9 +R 4e003866b87e6ab46d2d338deacb76ac +U dan +Z bc6db0f22fd7a4f485084a7efedf6bfd diff --git a/manifest.uuid b/manifest.uuid index 708a241f10..0ebf53643d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7047ce32a234484b8ba15311e6560aa74ff692c9 \ No newline at end of file +45415899545767888d36dcc0bafaf0ef415d94c2 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 23239bc94a..3cbfe1d36d 100644 --- a/src/main.c +++ b/src/main.c @@ -2790,7 +2790,9 @@ static int openDatabase( sqlite3Error(db, rc); goto opendb_out; } + sqlite3BtreeEnter(db->aDb[0].pBt); db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + sqlite3BtreeLeave(db->aDb[0].pBt); db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); /* The default safety_level for the main database is 'full'; for the temp diff --git a/test/threadtest3.c b/test/threadtest3.c index cb7e2fa41b..084ca022a9 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -1394,7 +1394,10 @@ static void dynamic_triggers(int nMs){ print_and_free_err(&err); } + + #include "tt3_checkpoint.c" +#include "tt3_index.c" int main(int argc, char **argv){ struct ThreadTest { @@ -1414,6 +1417,8 @@ int main(int argc, char **argv){ { checkpoint_starvation_1, "checkpoint_starvation_1", 10000 }, { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 }, + + { create_drop_index_1, "create_drop_index_1", 10000 }, }; int i; diff --git a/test/tt3_index.c b/test/tt3_index.c new file mode 100644 index 0000000000..b79768c6c7 --- /dev/null +++ b/test/tt3_index.c @@ -0,0 +1,74 @@ +/* +** 2014 December 9 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** create_drop_index_1 +*/ + + +static char *create_drop_index_thread(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + + while( !timetostop(&err) ){ + opendb(&err, &db, "test.db", 0); + + sql_script(&err, &db, + + "DROP INDEX IF EXISTS i1;" + "DROP INDEX IF EXISTS i2;" + "DROP INDEX IF EXISTS i3;" + "DROP INDEX IF EXISTS i4;" + + "CREATE INDEX IF NOT EXISTS i1 ON t1(a);" + "CREATE INDEX IF NOT EXISTS i2 ON t1(b);" + "CREATE INDEX IF NOT EXISTS i3 ON t1(c);" + "CREATE INDEX IF NOT EXISTS i4 ON t1(d);" + + "SELECT * FROM t1 ORDER BY a;" + "SELECT * FROM t1 ORDER BY b;" + "SELECT * FROM t1 ORDER BY c;" + "SELECT * FROM t1 ORDER BY d;" + ); + + closedb(&err, &db); + } + + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + +static void create_drop_index_1(int nMs){ + Error err = {0}; + Sqlite db = {0}; + Threadset threads = {0}; + + opendb(&err, &db, "test.db", 1); + sql_script(&err, &db, + "CREATE TABLE t1(a, b, c, d);" + "WITH data(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM data WHERE x<100) " + "INSERT INTO t1 SELECT x,x,x,x FROM data;" + ); + closedb(&err, &db); + + setstoptime(&err, nMs); + + sqlite3_enable_shared_cache(1); + launch_thread(&err, &threads, create_drop_index_thread, 0); + launch_thread(&err, &threads, create_drop_index_thread, 0); + launch_thread(&err, &threads, create_drop_index_thread, 0); + launch_thread(&err, &threads, create_drop_index_thread, 0); + launch_thread(&err, &threads, create_drop_index_thread, 0); + sqlite3_enable_shared_cache(0); + + join_all_threads(&err, &threads); + print_and_free_err(&err); +} From 785a38f066312184bd947d05b76b576adf7afa50 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 8 Dec 2014 20:29:23 +0000 Subject: [PATCH 634/710] Add extra tests to e_walckpt.test. FossilOrigin-Name: d6832aa24c8d93b4532a651b86605bd0a0d0aa78 --- manifest | 12 +-- manifest.uuid | 2 +- test/e_walckpt.test | 202 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 207 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c02b815662..ec93f98b98 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\smissing\smutex\scalls\saround\sa\scall\sto\ssqlite3SchemaGet()\swithin\ssqlite3_open(). -D 2014-12-08T20:20:16.943 +C Add\sextra\stests\sto\se_walckpt.test. +D 2014-12-08T20:29:23.119 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -478,7 +478,7 @@ F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a -F test/e_walckpt.test de5a8d86c5b95569309c6da796dbea870c22e003 +F test/e_walckpt.test 18de8fca6b74f29bf7d24a2e267eec749b8fec50 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 @@ -1226,7 +1226,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7047ce32a234484b8ba15311e6560aa74ff692c9 -R 4e003866b87e6ab46d2d338deacb76ac +P 45415899545767888d36dcc0bafaf0ef415d94c2 +R 4d0b2e4f612d2fa1f8086c766c614446 U dan -Z bc6db0f22fd7a4f485084a7efedf6bfd +Z 31f22aa5ae36276bb2e8d99e02610d8f diff --git a/manifest.uuid b/manifest.uuid index 0ebf53643d..804aec562f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -45415899545767888d36dcc0bafaf0ef415d94c2 \ No newline at end of file +d6832aa24c8d93b4532a651b86605bd0a0d0aa78 \ No newline at end of file diff --git a/test/e_walckpt.test b/test/e_walckpt.test index 8b0aae798a..f0edc46bee 100644 --- a/test/e_walckpt.test +++ b/test/e_walckpt.test @@ -69,9 +69,9 @@ foreach {tn script} { 2 { proc checkpoint {db mode args} { - set sql "PRAGMA wal_checkpoint" + set sql "PRAGMA wal_checkpoint = $mode" if {[llength $args] && [lindex $args 0]!=""} { - set sql "PRAGMA [lindex $args 0].wal_checkpoint" + set sql "PRAGMA [lindex $args 0].wal_checkpoint = $mode" } set rc [catch { $db eval $sql } msg] if {$rc} { @@ -261,6 +261,204 @@ foreach {tn script} { db2 close tvfs delete + proc busy_handler {mode busy_handler_mode n} { + incr ::busy_handler_counter + switch -- $busy_handler_mode { + 1 { + # Do nothing. Do not block. + return 1 + } + + 2 { + # Close first the reader, then later the writer. + if {$n==5} { catch {db2 eval commit} } + if {$n==10} { catch {db3 eval commit} } + return 0 + } + + 3 { + # Close first the writer, then later the reader. + if {$n==5} { catch {db2 eval commit} } + if {$n==10} { catch {db3 eval commit} } + return 0 + } + } + } + + foreach {mode busy_handler_mode} { + passive 1 + full 1 + full 2 + full 3 + } { + + set ::sync_counter 0 + + proc tvfs_callback {method args} { + set tail [file tail [lindex $args 0]] + if {$method == "xSync" && $tail == "test.db"} { + incr ::sync_counter + } + + if {$method == "xWrite" && $tail=="test.db"} { + if {$::write_ok < 0} { + set ::write_ok [expr ![catch {db5 eval { BEGIN IMMEDIATE }}]] + catch { db5 eval ROLLBACK } + } + if {$::read_ok < 0} { + set ::read_ok [expr ![catch {db5 eval { SELECT * FROM t1 }}]] + } + } + } + + catch { db close } + forcedelete test.db + testvfs tvfs + sqlite3 db test.db -vfs tvfs + #tvfs filter xSync + tvfs script tvfs_callback + + do_execsql_test $tn.4.$mode.0 { + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + } {wal} + + # Open a reader on the current database snapshot. + do_test $tn.4.$mode.1 { + sqlite3 db2 test.db -vfs tvfs + execsql { + BEGIN; + SELECT * FROM t1 UNION ALL SELECT * FROM t2; + } db2 + } {1 2 3 4 5 6} + + # Open a writer. Write a transaction. Then begin, but do not commit, + # a second transaction. + do_test $tn.4.$mode.2 { + sqlite3 db3 test.db -vfs tvfs + execsql { + INSERT INTO t2 VALUES(7, 8); + BEGIN; + INSERT INTO t2 VALUES(9, 10); + SELECT * FROM t1 UNION ALL SELECT * FROM t2; + } db3 + } {1 2 3 4 5 6 7 8 9 10} + + sqlite3 db5 test.db -vfs tvfs + + # Register a busy-handler with connection [db]. + # + db busy [list busy_handler $mode $busy_handler_mode] + set ::sync_counter 0 + set ::busy_handler_counter 0 + set ::read_ok -1 + set ::write_ok -1 + + do_test $tn.4.$mode.3 { + checkpoint db $mode main + set {} {} + } {} + + if { $mode=="passive" } { + # EVIDENCE-OF: R-16333-64433 Checkpoint as many frames as possible + # without waiting for any database readers or writers to finish, then + # sync the database file if all frames in the log were checkpointed. + # + # "As many frames as possible" means all but the last two transactions + # (the two that write to table t2, of which the scond is unfinished). + # So copying the db file only we see the t1 change, but not the t2 + # modifications. + # + # The busy handler is not invoked (see below) and the db reader and + # writer are still active - so the checkpointer did not wait for either + # readers or writers. As a result the checkpoint was not finished and + # so the db file is not synced. + # + # EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked + # in the SQLITE_CHECKPOINT_PASSIVE mode. + # + # It's not. Test case "$tn.4.$mode.6". + # + do_test $tn.4.$mode.4 { + forcecopy test.db abc.db + sqlite3 db4 abc.db + db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } + } {1 2 3 4 5 6} + do_test $tn.4.$mode.5 { set ::sync_counter } 0 + do_test $tn.4.$mode.6 { set ::busy_handler_counter } 0 + db4 close + + db2 eval COMMIT + db3 eval COMMIT + + # EVIDENCE-OF: R-65499-53765 On the other hand, passive mode might leave + # the checkpoint unfinished if there are concurrent readers or writers. + # + # The reader and writer have now dropped their locks. And so a + # checkpoint now is able to checkpoint more frames. Showing that the + # attempt above was left "unfinished". + # + # Also, because the checkpoint finishes this time, the db is synced. + # Which is part of R-16333-64433 above. + # + do_test $tn.4.$mode.7 { + checkpoint db $mode main + forcecopy test.db abc.db + sqlite3 db4 abc.db + db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } + } {1 2 3 4 5 6 7 8 9 10} + do_test $tn.4.$mode.6 { set ::sync_counter } 1 + do_test $tn.4.$mode.7 { set ::busy_handler_counter } 0 + db4 close + } + + if { $mode=="full" } { + if {$busy_handler_mode==2 || $busy_handler_mode==3} { + # EVIDENCE-OF: R-59171-47567 This mode blocks (it invokes the + # busy-handler callback) until there is no database writer and all + # readers are reading from the most recent database snapshot. + # + # Show that both the reader and writer have finished: + # + do_test $tn.4.$mode.7 { + list [catchsql COMMIT db2] [catchsql COMMIT db3] + } [list \ + {1 {cannot commit - no transaction is active}} \ + {1 {cannot commit - no transaction is active}} \ + ] + + # EVIDENCE-OF: R-29177-48281 It then checkpoints all frames in the log + # file and syncs the database file. + # + do_test $tn.4.$mode.8 { + forcecopy test.db abc.db + sqlite3 db4 abc.db + db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } + } {1 2 3 4 5 6 7 8 9 10} + do_test $tn.4.$mode.9 { set ::sync_counter } 1 + db4 close + + # EVIDENCE-OF: R-51867-44713 This mode blocks new database writers + # while it is pending, but new database readers are allowed to continue + # unimpeded. + do_test $tn.4.$mode.10 { + list $::write_ok $::read_ok + } {0 1} + + } + } + + db2 close + db3 close + db5 close + } + + db close + tvfs delete } #----------------------------------------------------------------------- From 0fbd7350a174f4739999e3d605b65e7ebfed25b7 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 9 Dec 2014 04:26:56 +0000 Subject: [PATCH 635/710] Fix harmless compiler warning. FossilOrigin-Name: e97b7a8b4df784d148fefb9554da0f511e351d9f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/btree.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index ec93f98b98..32ad7f85e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests\sto\se_walckpt.test. -D 2014-12-08T20:29:23.119 +C Fix\sharmless\scompiler\swarning. +D 2014-12-09T04:26:56.984 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 9023963463b0b1876aea1abc6d208d9ffa0228ae +F src/btree.c ea6692ce58bfba55b12c75d2947fec0906d1ef7a F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -1226,7 +1226,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 45415899545767888d36dcc0bafaf0ef415d94c2 -R 4d0b2e4f612d2fa1f8086c766c614446 -U dan -Z 31f22aa5ae36276bb2e8d99e02610d8f +P d6832aa24c8d93b4532a651b86605bd0a0d0aa78 +R 61f069fccea470b5ca359a3a5c90ead2 +U mistachkin +Z b0538f03992f2ac4a5c753853d936f4f diff --git a/manifest.uuid b/manifest.uuid index 804aec562f..f722851738 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6832aa24c8d93b4532a651b86605bd0a0d0aa78 \ No newline at end of file +e97b7a8b4df784d148fefb9554da0f511e351d9f \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e9cd89928e..4283d00fd3 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6785,7 +6785,7 @@ static int balance_nonroot( /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer ** that is more than 6 times the database page size. */ - assert( szScratch<=6*pBt->pageSize ); + assert( szScratch<=6*(int)pBt->pageSize ); apCell = sqlite3ScratchMalloc( szScratch ); if( apCell==0 ){ rc = SQLITE_NOMEM; From b11c3f22b8d392d4db9b8d28ce66b29d96491fbb Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Dec 2014 14:42:49 +0000 Subject: [PATCH 636/710] Increase the default minimum PMA size for multi-threaded sorting from 10x the page size to 250x the page size. Provide the SQLITE_SORTER_PMASZ compile-time option to change this default. FossilOrigin-Name: b05340fe3cd5f1676a55023228dc8d1a92de5936 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbesort.c | 8 +++++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 32ad7f85e5..0542dbf99d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning. -D 2014-12-09T04:26:56.984 +C Increase\sthe\sdefault\sminimum\sPMA\ssize\sfor\smulti-threaded\ssorting\sfrom\s10x\nthe\spage\ssize\sto\s250x\sthe\spage\ssize.\s\sProvide\sthe\sSQLITE_SORTER_PMASZ\ncompile-time\soption\sto\schange\sthis\sdefault. +D 2014-12-09T14:42:49.215 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -298,7 +298,7 @@ F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 F src/vdbeaux.c 6f7f39c3fcf0f5923758df8561bb5d843908a553 F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f -F src/vdbesort.c 42c166f7ca78cb643c7f4e4bdfa83c59d363d1a6 +F src/vdbesort.c db015e20a77b25eca4d1e125815f4998a3ca1354 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 F src/wal.c 847692349eb6e1fb8543dbc97e69ddbfa4cc7ea7 @@ -1226,7 +1226,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d6832aa24c8d93b4532a651b86605bd0a0d0aa78 -R 61f069fccea470b5ca359a3a5c90ead2 -U mistachkin -Z b0538f03992f2ac4a5c753853d936f4f +P e97b7a8b4df784d148fefb9554da0f511e351d9f +R bd26f6a52391b4ad1637a19c173d9967 +U drh +Z c16d1b57c275f469d491c230a3c4e6f8 diff --git a/manifest.uuid b/manifest.uuid index f722851738..b381d7ac7e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e97b7a8b4df784d148fefb9554da0f511e351d9f \ No newline at end of file +b05340fe3cd5f1676a55023228dc8d1a92de5936 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 7c736adefe..29ec40c0d1 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -450,7 +450,9 @@ struct SorterRecord { /* The minimum PMA size is set to this value multiplied by the database ** page size in bytes. */ -#define SORTER_MIN_WORKING 10 +#ifndef SQLITE_SORTER_PMASZ +# define SQLITE_SORTER_PMASZ 250 +#endif /* Maximum number of PMAs that a single MergeEngine can merge */ #define SORTER_MAX_MERGE_COUNT 16 @@ -849,9 +851,9 @@ int sqlite3VdbeSorterInit( } if( !sqlite3TempInMemory(db) ){ - pSorter->mnPmaSize = SORTER_MIN_WORKING * pgsz; + pSorter->mnPmaSize = SQLITE_SORTER_PMASZ * pgsz; mxCache = db->aDb[0].pSchema->cache_size; - if( mxCachemxPmaSize = MIN((i64)mxCache*pgsz, SQLITE_MAX_MXPMASIZE); /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of From f5471925c9345d0e3ef456f03130b5fc65c276b5 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Dec 2014 15:12:11 +0000 Subject: [PATCH 637/710] Add the SQLITE_REVERSE_UNORDERED_SELECTS compile-time option that causes the "PRAGMA reverse_unordered_selects" setting to be on by default. FossilOrigin-Name: 75a803e694d2c4e67579f45d54a0aaf120411a6e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 0542dbf99d..1a0343a1f1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sdefault\sminimum\sPMA\ssize\sfor\smulti-threaded\ssorting\sfrom\s10x\nthe\spage\ssize\sto\s250x\sthe\spage\ssize.\s\sProvide\sthe\sSQLITE_SORTER_PMASZ\ncompile-time\soption\sto\schange\sthis\sdefault. -D 2014-12-09T14:42:49.215 +C Add\sthe\sSQLITE_REVERSE_UNORDERED_SELECTS\scompile-time\soption\sthat\scauses\sthe\n"PRAGMA\sreverse_unordered_selects"\ssetting\sto\sbe\son\sby\sdefault. +D 2014-12-09T15:12:11.575 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 27c102143048802af53a8e55919ea3f6d9072951 +F src/main.c 378923ed4cb27ca7486062d07b1e67973c8beca7 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -1226,7 +1226,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e97b7a8b4df784d148fefb9554da0f511e351d9f -R bd26f6a52391b4ad1637a19c173d9967 +P b05340fe3cd5f1676a55023228dc8d1a92de5936 +R 30f0205c6ecccf655d5a50e721435aae U drh -Z c16d1b57c275f469d491c230a3c4e6f8 +Z b3e21bd27e95301a6024ba1e3af7b9e7 diff --git a/manifest.uuid b/manifest.uuid index b381d7ac7e..c8f4fe7627 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b05340fe3cd5f1676a55023228dc8d1a92de5936 \ No newline at end of file +75a803e694d2c4e67579f45d54a0aaf120411a6e \ No newline at end of file diff --git a/src/main.c b/src/main.c index 3cbfe1d36d..7ca5217dfa 100644 --- a/src/main.c +++ b/src/main.c @@ -2742,6 +2742,9 @@ static int openDatabase( #endif #if defined(SQLITE_DEFAULT_FOREIGN_KEYS) && SQLITE_DEFAULT_FOREIGN_KEYS | SQLITE_ForeignKeys +#endif +#if defined(SQLITE_REVERSE_UNORDERED_SELECTS) + | SQLITE_ReverseOrder #endif ; sqlite3HashInit(&db->aCollSeq); From 8c3026ecf33b34f42ea99b0e365cd39e51b3c969 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Dec 2014 19:07:29 +0000 Subject: [PATCH 638/710] Make the sqlite3_table_column_metadata() interface available by default and without requiring the SQLITE_ENABLE_COLUMN_METADATA compile-time option. Other sqlite3_column_* interfaces that have a run-time penalty even if they are unused still require the SQLITE_ENABLE_COLUMN_METADATA option at compile-time. FossilOrigin-Name: 4f7549ff924b8ed8f90fc447cd4be11421453984 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/loadext.c | 1 - src/main.c | 2 -- src/sqlite.h.in | 3 --- src/test1.c | 4 ---- test/colmeta.test | 5 ----- 7 files changed, 11 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index 1a0343a1f1..a3d0a725cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sSQLITE_REVERSE_UNORDERED_SELECTS\scompile-time\soption\sthat\scauses\sthe\n"PRAGMA\sreverse_unordered_selects"\ssetting\sto\sbe\son\sby\sdefault. -D 2014-12-09T15:12:11.575 +C Make\sthe\ssqlite3_table_column_metadata()\sinterface\savailable\sby\sdefault\sand\nwithout\srequiring\sthe\sSQLITE_ENABLE_COLUMN_METADATA\scompile-time\soption.\nOther\ssqlite3_column_*\sinterfaces\sthat\shave\sa\srun-time\spenalty\seven\sif\sthey\nare\sunused\sstill\srequire\sthe\sSQLITE_ENABLE_COLUMN_METADATA\soption\sat\ncompile-time. +D 2014-12-09T19:07:29.805 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -194,8 +194,8 @@ F src/insert.c 5b9243a33726008cc4132897d2be371db12a13be F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 -F src/loadext.c de741e66e5ddc1598d904d7289239696e40ed994 -F src/main.c 378923ed4cb27ca7486062d07b1e67973c8beca7 +F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660 +F src/main.c ad4c0ed3b4731274aabb91b9158405a48dfaabad F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -230,7 +230,7 @@ F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c f377fb8a5c73c10678ea74f3400f7913943e3d75 F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in 6ec654324cb490ea3d8a7be28b8c7d37fe4ad282 +F src/sqlite.h.in a68ac3ba964e17c446457944b41ad50313a11b21 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 28049b803b74a7f73242a8226915ea00ebb1309f @@ -238,7 +238,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 -F src/test1.c ebb8cd3c94a2ac8851b7b0b1349284e73a8b4c7a +F src/test1.c fa655a378b11bf811f0c15c56eb1d232118224cc F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -404,7 +404,7 @@ F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test df26649cfcbddf109c04122b340301616d3a88f6 F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 -F test/colmeta.test 087c42997754b8c648819832241daf724f813322 +F test/colmeta.test 63f92e28d87de59430b1a1a9bb43ad7f8d19335e F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09 @@ -1226,7 +1226,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b05340fe3cd5f1676a55023228dc8d1a92de5936 -R 30f0205c6ecccf655d5a50e721435aae +P 75a803e694d2c4e67579f45d54a0aaf120411a6e +R 9dd53b5b296f2bafeeb64e18470c82ee U drh -Z b3e21bd27e95301a6024ba1e3af7b9e7 +Z b9a2aab60506af9678a60cdb34c9d56c diff --git a/manifest.uuid b/manifest.uuid index c8f4fe7627..c4f34c586d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -75a803e694d2c4e67579f45d54a0aaf120411a6e \ No newline at end of file +4f7549ff924b8ed8f90fc447cd4be11421453984 \ No newline at end of file diff --git a/src/loadext.c b/src/loadext.c index 2a2afd8654..7b39ff1671 100644 --- a/src/loadext.c +++ b/src/loadext.c @@ -34,7 +34,6 @@ # define sqlite3_column_table_name16 0 # define sqlite3_column_origin_name 0 # define sqlite3_column_origin_name16 0 -# define sqlite3_table_column_metadata 0 #endif #ifdef SQLITE_OMIT_AUTHORIZATION diff --git a/src/main.c b/src/main.c index 7ca5217dfa..6d10c15389 100644 --- a/src/main.c +++ b/src/main.c @@ -3149,7 +3149,6 @@ void sqlite3_thread_cleanup(void){ ** Return meta information about a specific column of a database table. ** See comment in sqlite3.h (sqlite.h.in) for details. */ -#ifdef SQLITE_ENABLE_COLUMN_METADATA int sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ const char *zDbName, /* Database name or NULL */ @@ -3256,7 +3255,6 @@ error_out: sqlite3_mutex_leave(db->mutex); return rc; } -#endif /* ** Sleep for a little while. Return the amount of time slept. diff --git a/src/sqlite.h.in b/src/sqlite.h.in index e889b2670f..66a917e0cd 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5207,9 +5207,6 @@ SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); ** error occurs during this process, or if the requested table or column ** cannot be found, an [error code] is returned and an error message left ** in the [database connection] (to be retrieved using sqlite3_errmsg()).)^ -** -** ^This API is only available if the library was compiled with the -** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol defined. */ int sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ diff --git a/src/test1.c b/src/test1.c index 14ccacd863..9f97c7371f 100644 --- a/src/test1.c +++ b/src/test1.c @@ -1569,7 +1569,6 @@ static int test_libversion_number( ** Usage: sqlite3_table_column_metadata DB dbname tblname colname ** */ -#ifdef SQLITE_ENABLE_COLUMN_METADATA static int test_table_column_metadata( ClientData clientData, /* Pointer to sqlite3_enable_XXX function */ Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ @@ -1618,7 +1617,6 @@ static int test_table_column_metadata( return TCL_OK; } -#endif #ifndef SQLITE_OMIT_INCRBLOB @@ -6778,9 +6776,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_shared_cache_report", sqlite3BtreeSharedCacheReport, 0}, #endif { "sqlite3_libversion_number", test_libversion_number, 0 }, -#ifdef SQLITE_ENABLE_COLUMN_METADATA { "sqlite3_table_column_metadata", test_table_column_metadata, 0 }, -#endif #ifndef SQLITE_OMIT_INCRBLOB { "sqlite3_blob_reopen", test_blob_reopen, 0 }, #endif diff --git a/test/colmeta.test b/test/colmeta.test index 3939f8228a..ec5891f058 100644 --- a/test/colmeta.test +++ b/test/colmeta.test @@ -17,11 +17,6 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !columnmetadata { - finish_test - return -} - # Set up a schema in the main and temp test databases. do_test colmeta-0 { execsql { From 80cdfd199b1527881947d48d3d8a73025920dd22 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Dec 2014 19:16:41 +0000 Subject: [PATCH 639/710] Lower the default SQLITE_SORTER_PMASZ value back to 10, where it has been for the past couple of releases. Applications that need a larger value can set one. FossilOrigin-Name: 1ba8911c18f2fe34c20dc42f25a8f3c1c798fa7a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbesort.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index a3d0a725cf..b0f6c74e0a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\ssqlite3_table_column_metadata()\sinterface\savailable\sby\sdefault\sand\nwithout\srequiring\sthe\sSQLITE_ENABLE_COLUMN_METADATA\scompile-time\soption.\nOther\ssqlite3_column_*\sinterfaces\sthat\shave\sa\srun-time\spenalty\seven\sif\sthey\nare\sunused\sstill\srequire\sthe\sSQLITE_ENABLE_COLUMN_METADATA\soption\sat\ncompile-time. -D 2014-12-09T19:07:29.805 +C Lower\sthe\sdefault\sSQLITE_SORTER_PMASZ\svalue\sback\sto\s10,\swhere\sit\shas\sbeen\sfor\nthe\spast\scouple\sof\sreleases.\s\sApplications\sthat\sneed\sa\slarger\svalue\scan\sset\none. +D 2014-12-09T19:16:41.997 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -298,7 +298,7 @@ F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 F src/vdbeaux.c 6f7f39c3fcf0f5923758df8561bb5d843908a553 F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f -F src/vdbesort.c db015e20a77b25eca4d1e125815f4998a3ca1354 +F src/vdbesort.c c150803a3e98fbc68bd07772cbbd4328a0a7212d F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 F src/wal.c 847692349eb6e1fb8543dbc97e69ddbfa4cc7ea7 @@ -1226,7 +1226,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 75a803e694d2c4e67579f45d54a0aaf120411a6e -R 9dd53b5b296f2bafeeb64e18470c82ee +P 4f7549ff924b8ed8f90fc447cd4be11421453984 +R 07540d2951e499e9ba9f784727d9cc12 U drh -Z b9a2aab60506af9678a60cdb34c9d56c +Z 7254b0c65f796e3de7bb7b37671c0413 diff --git a/manifest.uuid b/manifest.uuid index c4f34c586d..f4de208a5d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4f7549ff924b8ed8f90fc447cd4be11421453984 \ No newline at end of file +1ba8911c18f2fe34c20dc42f25a8f3c1c798fa7a \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index 29ec40c0d1..78ecc1efb9 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -451,7 +451,7 @@ struct SorterRecord { /* The minimum PMA size is set to this value multiplied by the database ** page size in bytes. */ #ifndef SQLITE_SORTER_PMASZ -# define SQLITE_SORTER_PMASZ 250 +# define SQLITE_SORTER_PMASZ 10 #endif /* Maximum number of PMAs that a single MergeEngine can merge */ From 9778bd7292dbf9b4386fab47f16e178c97f20c02 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 9 Dec 2014 20:13:40 +0000 Subject: [PATCH 640/710] Add extra tests to e_walckpt.test. FossilOrigin-Name: 84f9581019961efa31297f8be48427b17bcca857 --- manifest | 16 +-- manifest.uuid | 2 +- src/test1.c | 1 + test/e_walckpt.test | 318 ++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 301 insertions(+), 36 deletions(-) diff --git a/manifest b/manifest index b0f6c74e0a..0baa94d673 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Lower\sthe\sdefault\sSQLITE_SORTER_PMASZ\svalue\sback\sto\s10,\swhere\sit\shas\sbeen\sfor\nthe\spast\scouple\sof\sreleases.\s\sApplications\sthat\sneed\sa\slarger\svalue\scan\sset\none. -D 2014-12-09T19:16:41.997 +C Add\sextra\stests\sto\se_walckpt.test. +D 2014-12-09T20:13:40.856 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -238,7 +238,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 -F src/test1.c fa655a378b11bf811f0c15c56eb1d232118224cc +F src/test1.c f60b9e973cf813fdabb2b67ccbb4a08e9f1d81be F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -478,7 +478,7 @@ F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a -F test/e_walckpt.test 18de8fca6b74f29bf7d24a2e267eec749b8fec50 +F test/e_walckpt.test 3116a98fa0dd9b2c9e493de7c59730adfe436746 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 @@ -1226,7 +1226,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4f7549ff924b8ed8f90fc447cd4be11421453984 -R 07540d2951e499e9ba9f784727d9cc12 -U drh -Z 7254b0c65f796e3de7bb7b37671c0413 +P 1ba8911c18f2fe34c20dc42f25a8f3c1c798fa7a +R 3c53b0c88f36488767cc876fe1e6e0b0 +U dan +Z bbcee16517facfe70298aa47dfb20c31 diff --git a/manifest.uuid b/manifest.uuid index f4de208a5d..587f7e93dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1ba8911c18f2fe34c20dc42f25a8f3c1c798fa7a \ No newline at end of file +84f9581019961efa31297f8be48427b17bcca857 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 9f97c7371f..009b88a543 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5712,6 +5712,7 @@ static int test_wal_checkpoint_v2( rc = sqlite3_wal_checkpoint_v2(db, zDb, eMode, &nLog, &nCkpt); if( rc!=SQLITE_OK && rc!=SQLITE_BUSY ){ const char *zErrCode = sqlite3ErrName(rc); + Tcl_ResetResult(interp); Tcl_AppendResult(interp, zErrCode, " - ", (char *)sqlite3_errmsg(db), 0); return TCL_ERROR; } diff --git a/test/e_walckpt.test b/test/e_walckpt.test index f0edc46bee..e6b6566bed 100644 --- a/test/e_walckpt.test +++ b/test/e_walckpt.test @@ -42,6 +42,38 @@ proc compare_db_hashes {} { set ret } +#------------------------------------------------------------------------- +# All calls to the [sqlite3_wal_checkpoint_v2] command made within this +# file use this wrapper. It's sole purpose is to throw an error if the +# following requirement is violated: +# +# EVIDENCE-OF: R-60567-47780 Unless it returns SQLITE_MISUSE, the +# sqlite3_wal_checkpoint_v2() interface sets the error information that +# is queried by sqlite3_errcode() and sqlite3_errmsg(). +# +proc wal_checkpoint_v2 {db args} { + set rc [catch { + uplevel sqlite3_wal_checkpoint_v2 $db $args + } msg] + + set errcode "SQLITE_OK" + if {$rc} { + set errcode [lindex [split $msg " "] 0] + } elseif { [lindex $msg 0] } { + set errcode "SQLITE_BUSY" + } + + if {$errcode != "SQLITE_MISUSE" && [sqlite3_errcode $db] != $errcode} { + error "sqlite3_errcode mismatch! (1) $errcode!=[sqlite3_errcode $db]" + } + + if {$rc==0} { + return $msg + } else { + error $msg + } +} + # The following tests are run 3 times, each using a different method of # invoking a checkpoint: @@ -63,7 +95,7 @@ proc compare_db_hashes {} { foreach {tn script} { 1 { proc checkpoint {db mode args} { - eval sqlite3_wal_checkpoint_v2 [list $db] [list $mode] $args + eval wal_checkpoint_v2 [list $db] [list $mode] $args } } @@ -90,7 +122,7 @@ foreach {tn script} { error "$rc - [sqlite3_errmsg $db]" } } else { - eval sqlite3_wal_checkpoint_v2 [list $db] [list $mode] $args + eval wal_checkpoint_v2 [list $db] [list $mode] $args } } } @@ -270,36 +302,46 @@ foreach {tn script} { } 2 { - # Close first the reader, then later the writer. + # Close first the reader, then later the writer. Give up before + # closing the [db6] reader. if {$n==5} { catch {db2 eval commit} } if {$n==10} { catch {db3 eval commit} } + if {$n==15} { return 1 } return 0 } 3 { - # Close first the writer, then later the reader. + # Close first the writer, then later the reader. And finally the + # [db6] reader. if {$n==5} { catch {db2 eval commit} } if {$n==10} { catch {db3 eval commit} } + if {$n==15} { catch {db6 eval commit} } return 0 } } } foreach {mode busy_handler_mode} { - passive 1 - full 1 - full 2 - full 3 + passive 1 + full 1 full 2 full 3 + restart 1 restart 2 restart 3 + truncate 1 truncate 2 truncate 3 } { + set tp "$tn.$mode.$busy_handler_mode" set ::sync_counter 0 + # Set up a callback function for xSync and xWrite calls made during + # the checkpoint. + # + set ::checkpoint_ongoing 0 proc tvfs_callback {method args} { + if {$::checkpoint_ongoing==0} return + set tail [file tail [lindex $args 0]] if {$method == "xSync" && $tail == "test.db"} { incr ::sync_counter } - if {$method == "xWrite" && $tail=="test.db"} { if {$::write_ok < 0} { set ::write_ok [expr ![catch {db5 eval { BEGIN IMMEDIATE }}]] @@ -308,6 +350,14 @@ foreach {tn script} { if {$::read_ok < 0} { set ::read_ok [expr ![catch {db5 eval { SELECT * FROM t1 }}]] } + + # If one has not already been opened, open a read-transaction using + # connection [db6] + catch { db6 eval { BEGIN ; SELECT * FROM sqlite_master } } msg + } + if {$method == "xShmLock" } { + set details [lindex $args 2] + if {$details == "0 1 lock exclusive"} { set ::seen_writer_lock 1 } } } @@ -318,7 +368,7 @@ foreach {tn script} { #tvfs filter xSync tvfs script tvfs_callback - do_execsql_test $tn.4.$mode.0 { + do_execsql_test $tp.0 { CREATE TABLE t1(a, b); CREATE TABLE t2(a, b); PRAGMA journal_mode = wal; @@ -328,7 +378,7 @@ foreach {tn script} { } {wal} # Open a reader on the current database snapshot. - do_test $tn.4.$mode.1 { + do_test $tp.1 { sqlite3 db2 test.db -vfs tvfs execsql { BEGIN; @@ -338,7 +388,7 @@ foreach {tn script} { # Open a writer. Write a transaction. Then begin, but do not commit, # a second transaction. - do_test $tn.4.$mode.2 { + do_test $tp.2 { sqlite3 db3 test.db -vfs tvfs execsql { INSERT INTO t2 VALUES(7, 8); @@ -349,6 +399,7 @@ foreach {tn script} { } {1 2 3 4 5 6 7 8 9 10} sqlite3 db5 test.db -vfs tvfs + sqlite3 db6 test.db -vfs tvfs # Register a busy-handler with connection [db]. # @@ -357,11 +408,15 @@ foreach {tn script} { set ::busy_handler_counter 0 set ::read_ok -1 set ::write_ok -1 + set ::seen_writer_lock 0 - do_test $tn.4.$mode.3 { + set ::checkpoint_ongoing 1 + do_test $tp.3 { checkpoint db $mode main set {} {} } {} + set ::checkpoint_ongoing 0 + set ::did_restart_blocking [expr {[catch {db6 eval commit}]}] if { $mode=="passive" } { # EVIDENCE-OF: R-16333-64433 Checkpoint as many frames as possible @@ -381,15 +436,15 @@ foreach {tn script} { # EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked # in the SQLITE_CHECKPOINT_PASSIVE mode. # - # It's not. Test case "$tn.4.$mode.6". + # It's not. Test case "$tp.6". # - do_test $tn.4.$mode.4 { + do_test $tp.4 { forcecopy test.db abc.db sqlite3 db4 abc.db db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } } {1 2 3 4 5 6} - do_test $tn.4.$mode.5 { set ::sync_counter } 0 - do_test $tn.4.$mode.6 { set ::busy_handler_counter } 0 + do_test $tp.5 { set ::sync_counter } 0 + do_test $tp.6 { set ::busy_handler_counter } 0 db4 close db2 eval COMMIT @@ -405,26 +460,53 @@ foreach {tn script} { # Also, because the checkpoint finishes this time, the db is synced. # Which is part of R-16333-64433 above. # - do_test $tn.4.$mode.7 { + set ::checkpoint_ongoing 1 + do_test $tp.7 { checkpoint db $mode main forcecopy test.db abc.db sqlite3 db4 abc.db db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } } {1 2 3 4 5 6 7 8 9 10} - do_test $tn.4.$mode.6 { set ::sync_counter } 1 - do_test $tn.4.$mode.7 { set ::busy_handler_counter } 0 + set ::checkpoint_ongoing 0 + do_test $tp.7 { set ::sync_counter } 1 + do_test $tp.8 { set ::busy_handler_counter } 0 db4 close } - if { $mode=="full" } { + if { $mode=="full" || $mode=="restart" || $mode=="truncate" } { + + # EVIDENCE-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and + # TRUNCATE modes also obtain the exclusive "writer" lock on the + # database file. + # + # Or at least attempts to obtain. + # + do_test $tp.9 { + set ::seen_writer_lock + } {1} + if {$busy_handler_mode==2 || $busy_handler_mode==3} { # EVIDENCE-OF: R-59171-47567 This mode blocks (it invokes the # busy-handler callback) until there is no database writer and all # readers are reading from the most recent database snapshot. # - # Show that both the reader and writer have finished: + # The test below shows that both the reader and writer have + # finished: # - do_test $tn.4.$mode.7 { + # Also restated by the following two. That both busy_handler_mode + # values 2 and 3 work show that both of the following are true - as + # they release the reader and writer transactions in different + # orders. + # + # EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + # immediately, and a busy-handler is configured, it is invoked and the + # writer lock retried until either the busy-handler returns 0 or the + # lock is successfully obtained. + # + # EVIDENCE-OF: R-48107-00250 The busy-handler is also invoked while + # waiting for database readers as described above. + # + do_test $tp.7 { list [catchsql COMMIT db2] [catchsql COMMIT db3] } [list \ {1 {cannot commit - no transaction is active}} \ @@ -434,27 +516,79 @@ foreach {tn script} { # EVIDENCE-OF: R-29177-48281 It then checkpoints all frames in the log # file and syncs the database file. # - do_test $tn.4.$mode.8 { + do_test $tp.8 { forcecopy test.db abc.db sqlite3 db4 abc.db db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } } {1 2 3 4 5 6 7 8 9 10} - do_test $tn.4.$mode.9 { set ::sync_counter } 1 + do_test $tp.9 { set ::sync_counter } 1 db4 close # EVIDENCE-OF: R-51867-44713 This mode blocks new database writers # while it is pending, but new database readers are allowed to continue # unimpeded. - do_test $tn.4.$mode.10 { + # + # EVIDENCE-OF: R-47276-58266 Like SQLITE_CHECKPOINT_FULL, this mode + # blocks new database writer attempts while it is pending, but does not + # impede readers. + # + # The first of the above two refers to "full" mode. The second + # to "restart". + # + do_test $tp.10.1 { list $::write_ok $::read_ok } {0 1} + # EVIDENCE-OF: R-12410-31217 This mode works the same way as + # SQLITE_CHECKPOINT_FULL with the addition that after checkpointing the + # log file it blocks (calls the busy-handler callback) until all + # readers are reading from the database file only. + # + # The stuff above passed, so the first part of this requirement + # is met. The second part is tested below. If the checkpoint mode + # was "restart" or "truncate", then the busy-handler will have + # been called to block on wal-file readers. + # + do_test $tp.11 { + set ::did_restart_blocking + } [expr {($mode=="restart"||$mode=="truncate")&&$busy_handler_mode==3}] + + # EVIDENCE-OF: R-44699-57140 This mode works the same way as + # SQLITE_CHECKPOINT_RESTART with the addition that it also truncates + # the log file to zero bytes just prior to a successful return. + if {$mode=="truncate" && $busy_handler_mode==3} { + do_test $tp.12 { + file size test.db-wal + } 0 + } + } elseif {$busy_handler_mode==1} { + + # EVIDENCE-OF: R-34519-06271 SQLITE_BUSY is returned in this case. + if {$tn!=2} { + # ($tn==2) is the loop that uses "PRAGMA wal_checkpoint" + do_test $tp.13 { sqlite3_errcode db } {SQLITE_BUSY} + } + + # EVIDENCE-OF: R-49155-63541 If the busy-handler returns 0 before the + # writer lock is obtained or while waiting for database readers, the + # checkpoint operation proceeds from that point in the same way as + # SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible + # without blocking any further. + do_test $tp.14 { + forcecopy test.db abc.db + sqlite3 db4 abc.db + db4 eval { SELECT * FROM t1 UNION ALL SELECT * FROM t2 } + } {1 2 3 4 5 6} + do_test $tp.15 { set ::sync_counter } 0 + do_test $tp.16 { set ::busy_handler_counter } 1 + db4 close } } db2 close db3 close db5 close + db6 close } db close @@ -480,9 +614,139 @@ foreach {tn mode res} { 8 1000000 {1 {SQLITE_MISUSE - not an error}} } { do_test 4.$tn { - list [catch "sqlite3_wal_checkpoint_v2 db $mode" msg] $msg + list [catch "wal_checkpoint_v2 db $mode" msg] $msg } $res } +db close + +foreach tn {1 2 3} { + forcedelete test.db test.db2 test.db3 + testvfs tvfs + + sqlite3 db test.db -vfs tvfs + execsql { + ATTACH 'test.db2' AS aux2; + ATTACH 'test.db3' AS aux3; + PRAGMA main.journal_mode = WAL; + PRAGMA aux2.journal_mode = WAL; + PRAGMA aux3.journal_mode = WAL; + + CREATE TABLE main.t1(x,y); + CREATE TABLE aux2.t2(x,y); + CREATE TABLE aux3.t3(x,y); + + INSERT INTO t1 VALUES('a', 'b'); + INSERT INTO t2 VALUES('a', 'b'); + INSERT INTO t3 VALUES('a', 'b'); + } + sqlite3 db2 test.db2 -vfs tvfs + + switch -- $tn { + 1 { + # EVIDENCE-OF: R-41299-52117 If no error (SQLITE_BUSY or otherwise) is + # encountered while processing the attached databases, SQLITE_OK is + # returned. + do_test 5.$tn.1 { + lindex [wal_checkpoint_v2 db truncate] 0 + } {0} ;# 0 -> SQLITE_OK + do_test 5.$tn.2 { + list [expr [file size test.db-wal]==0] \ + [expr [file size test.db2-wal]==0] \ + [expr [file size test.db3-wal]==0] + } {1 1 1} + } + + 2 { + # EVIDENCE-OF: R-38578-34175 If an SQLITE_BUSY error is encountered when + # processing one or more of the attached WAL databases, the operation is + # still attempted on any remaining attached databases and SQLITE_BUSY is + # returned at the end. + db2 eval { BEGIN; INSERT INTO t2 VALUES('d', 'e'); } + do_test 5.$tn.1 { + lindex [wal_checkpoint_v2 db truncate] 0 + } {1} ;# 1 -> SQLITE_BUSY + do_test 5.$tn.2 { + list [expr [file size test.db-wal]==0] \ + [expr [file size test.db2-wal]==0] \ + [expr [file size test.db3-wal]==0] + } {1 0 1} + db2 eval ROLLBACK + } + + 3 { + # EVIDENCE-OF: R-38049-07913 If any other error occurs while processing + # an attached database, processing is abandoned and the error code is + # returned to the caller immediately. + tvfs filter xWrite + tvfs script inject_ioerr + proc inject_ioerr {method file args} { + if {[file tail $file]=="test.db2"} { + return "SQLITE_IOERR" + } + return 0 + } + do_test 5.$tn.1 { + list [catch { wal_checkpoint_v2 db truncate } msg] $msg + } {1 {SQLITE_IOERR - disk I/O error}} + do_test 5.$tn.2 { + list [expr [file size test.db-wal]==0] \ + [expr [file size test.db2-wal]==0] \ + [expr [file size test.db3-wal]==0] + } {1 0 0} + tvfs script "" + } + } + + db close + db2 close +} + +reset_db +sqlite3 db2 test.db + +do_test 6.1 { + execsql { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + } + file size test.db-wal +} [wal_file_size 3 1024] + +do_test 6.2 { + db2 eval { BEGIN; SELECT * FROM t1; } + db eval { INSERT INTO t1 VALUES(3, 4) } + file size test.db-wal +} [wal_file_size 4 1024] + +# At this point the log file contains 4 frames. 3 of which it should +# be possible to checkpoint. +# +# EVIDENCE-OF: R-16642-42503 If pnLog is not NULL, then *pnLog is set to +# the total number of frames in the log file or to -1 if the checkpoint +# could not run because of an error or because the database is not in +# WAL mode. +# +# EVIDENCE-OF: R-10514-25250 If pnCkpt is not NULL,then *pnCkpt is set +# to the total number of checkpointed frames in the log file (including +# any that were already checkpointed before the function was called) or +# to -1 if the checkpoint could not run due to an error or because the +# database is not in WAL mode. +# +do_test 6.4 { + lrange [wal_checkpoint_v2 db passive] 1 2 +} {4 3} + +# EVIDENCE-OF: R-37257-17813 Note that upon successful completion of an +# SQLITE_CHECKPOINT_TRUNCATE, the log file will have been truncated to +# zero bytes and so both *pnLog and *pnCkpt will be set to zero. +# +do_test 6.5 { + db2 eval COMMIT + wal_checkpoint_v2 db truncate +} {0 0 0} + finish_test + From 45d1b2063995f504a0722ac0fe5b65e7443f345d Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 9 Dec 2014 22:24:42 +0000 Subject: [PATCH 641/710] Fix the sqlite3_table_column_metadata() routine so that it gives the correct answer for the "rowid" column in a WITHOUT ROWID table. Enhance it so that it can be used to check for the existence of a table by setting the column name parameter to NULL. The routine is now included in the build by default, even without the SQLITE_ENABLE_COLUMN_METADATA compile-time option. FossilOrigin-Name: cf9be419a16156a9814e1378bb49b780de977343 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/main.c | 16 +++++++++------- src/sqlite.h.in | 33 ++++++++++++++++++++------------- src/test1.c | 4 ++-- test/colmeta.test | 35 ++++++++++++++++++++++++++++------- 6 files changed, 70 insertions(+), 40 deletions(-) diff --git a/manifest b/manifest index 0baa94d673..e4a81bbd8d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests\sto\se_walckpt.test. -D 2014-12-09T20:13:40.856 +C Fix\sthe\ssqlite3_table_column_metadata()\sroutine\sso\sthat\sit\sgives\sthe\scorrect\nanswer\sfor\sthe\s"rowid"\scolumn\sin\sa\sWITHOUT\sROWID\stable.\s\sEnhance\sit\sso\sthat\nit\scan\sbe\sused\sto\scheck\sfor\sthe\sexistence\sof\sa\stable\sby\ssetting\sthe\scolumn\nname\sparameter\sto\sNULL.\s\sThe\sroutine\sis\snow\sincluded\sin\sthe\sbuild\sby\ndefault,\seven\swithout\sthe\sSQLITE_ENABLE_COLUMN_METADATA\scompile-time\soption. +D 2014-12-09T22:24:42.981 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660 -F src/main.c ad4c0ed3b4731274aabb91b9158405a48dfaabad +F src/main.c 4bfb07de96118350a63103380819ff8cbbefc5cd F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -230,7 +230,7 @@ F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c f377fb8a5c73c10678ea74f3400f7913943e3d75 F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in a68ac3ba964e17c446457944b41ad50313a11b21 +F src/sqlite.h.in cc237b8aa62348685b36d06f1b6beb10dfea39ae F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 28049b803b74a7f73242a8226915ea00ebb1309f @@ -238,7 +238,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 -F src/test1.c f60b9e973cf813fdabb2b67ccbb4a08e9f1d81be +F src/test1.c 7e806af12d7915445e46407d32b275d8ca9db4e7 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -404,7 +404,7 @@ F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868 F test/collate8.test df26649cfcbddf109c04122b340301616d3a88f6 F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6 -F test/colmeta.test 63f92e28d87de59430b1a1a9bb43ad7f8d19335e +F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1 F test/colname.test 08948a4809d22817e0e5de89c7c0a8bd90cb551b F test/conflict.test 841bcf7cabbfca39c577eb8411ea8601843b46a8 F test/conflict2.test 0d3af4fb534fa1bd020c79960bb56e4d52655f09 @@ -1226,7 +1226,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1ba8911c18f2fe34c20dc42f25a8f3c1c798fa7a -R 3c53b0c88f36488767cc876fe1e6e0b0 -U dan -Z bbcee16517facfe70298aa47dfb20c31 +P 84f9581019961efa31297f8be48427b17bcca857 +R 0b640911f896cbfee28a6d7f7893118c +U drh +Z 0a34c44c2738c668de181b5c6607c9d7 diff --git a/manifest.uuid b/manifest.uuid index 587f7e93dc..fbd6f6fb7b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84f9581019961efa31297f8be48427b17bcca857 \ No newline at end of file +cf9be419a16156a9814e1378bb49b780de977343 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 6d10c15389..0727c0d75a 100644 --- a/src/main.c +++ b/src/main.c @@ -3188,11 +3188,8 @@ int sqlite3_table_column_metadata( } /* Find the column for which info is requested */ - if( sqlite3IsRowid(zColumnName) ){ - iCol = pTab->iPKey; - if( iCol>=0 ){ - pCol = &pTab->aCol[iCol]; - } + if( zColumnName==0 ){ + /* Query for existance of table only */ }else{ for(iCol=0; iColnCol; iCol++){ pCol = &pTab->aCol[iCol]; @@ -3201,8 +3198,13 @@ int sqlite3_table_column_metadata( } } if( iCol==pTab->nCol ){ - pTab = 0; - goto error_out; + if( HasRowid(pTab) && sqlite3IsRowid(zColumnName) ){ + iCol = pTab->iPKey; + pCol = iCol>=0 ? &pTab->aCol[iCol] : 0; + }else{ + pTab = 0; + goto error_out; + } } } diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 66a917e0cd..b3ff061151 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5152,9 +5152,17 @@ SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); /* ** CAPI3REF: Extract Metadata About A Column Of A Table ** -** ^This routine returns metadata about a specific column of a specific -** database table accessible using the [database connection] handle -** passed as the first function argument. +** ^The sqlite3_table_column_metadata(X,D,T,C,....) routine returns +** information about column C of table T in database D +** on [database connection] X. ^The sqlite3_table_column_metadata() +** interface returns SQLITE_OK and fills in the non-NULL pointers in +** the final five arguments with appropriate vaules if the specified +** column exists. ^The sqlite3_table_column_metadata() interface returns +** SQLITE_ERROR and if the specified column does not exist. +** If the column-name parameter to sqlite3_table_column_metadata() is a +** NULL pointer, then the routine simply checks for the existance of the +** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it +** does not. ** ** ^The column is identified by the second, third and fourth parameters to ** this function. ^The second parameter is either the name of the database @@ -5164,8 +5172,7 @@ SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); ** resolve unqualified table references. ** ** ^The third and fourth parameters to this function are the table and column -** name of the desired column, respectively. Neither of these parameters -** may be NULL. +** name of the desired column, respectively. ** ** ^Metadata is returned by writing to the memory locations passed as the 5th ** and subsequent parameters to this function. ^Any of these arguments may be @@ -5184,16 +5191,17 @@ SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); ** )^ ** ** ^The memory pointed to by the character pointers returned for the -** declaration type and collation sequence is valid only until the next +** declaration type and collation sequence is valid until the next ** call to any SQLite API function. ** ** ^If the specified table is actually a view, an [error code] is returned. ** -** ^If the specified column is "rowid", "oid" or "_rowid_" and an +** ^If the specified column is "rowid", "oid" or "_rowid_" and the table +** is not a [WITHOUT ROWID] table and an ** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output ** parameters are set for the explicitly declared column. ^(If there is no -** explicitly declared [INTEGER PRIMARY KEY] column, then the output -** parameters are set as follows: +** [INTEGER PRIMARY KEY] column, then the outputs +** for the [rowid] are set as follows: ** ** 
     **     data type: "INTEGER"
@@ -5203,10 +5211,9 @@ SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N);
 **     auto increment: 0
 **
    )^ ** -** ^(This function may load one or more schemas from database files. If an -** error occurs during this process, or if the requested table or column -** cannot be found, an [error code] is returned and an error message left -** in the [database connection] (to be retrieved using sqlite3_errmsg()).)^ +** ^This function causes all database schemas to be read from disk and +** parsed, if that has not already been done, and returns an error if +** any errors are encountered while loading the schema. */ int sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ diff --git a/src/test1.c b/src/test1.c index 009b88a543..851249461c 100644 --- a/src/test1.c +++ b/src/test1.c @@ -1588,14 +1588,14 @@ static int test_table_column_metadata( int primarykey; int autoincrement; - if( objc!=5 ){ + if( objc!=5 && objc!=4 ){ Tcl_WrongNumArgs(interp, 1, objv, "DB dbname tblname colname"); return TCL_ERROR; } if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) ) return TCL_ERROR; zDb = Tcl_GetString(objv[2]); zTbl = Tcl_GetString(objv[3]); - zCol = Tcl_GetString(objv[4]); + zCol = objc==5 ? Tcl_GetString(objv[4]) : 0; if( strlen(zDb)==0 ) zDb = 0; diff --git a/test/colmeta.test b/test/colmeta.test index ec5891f058..21b0e0f38a 100644 --- a/test/colmeta.test +++ b/test/colmeta.test @@ -23,6 +23,8 @@ do_test colmeta-0 { CREATE TABLE abc(a, b, c); CREATE TABLE abc2(a PRIMARY KEY COLLATE NOCASE, b VARCHAR(32), c); CREATE TABLE abc3(a NOT NULL, b INTEGER PRIMARY KEY, c); + CREATE TABLE abc5(w,x,y,z,PRIMARY KEY(x,z)) WITHOUT ROWID; + CREATE TABLE abc6(rowid TEXT COLLATE rtrim, oid REAL, _rowid_ BLOB); } ifcapable autoinc { execsql { @@ -52,19 +54,27 @@ set tests { 13 {main abc rowid} {0 {INTEGER BINARY 0 1 0}} 14 {main abc3 rowid} {0 {INTEGER BINARY 0 1 0}} 16 {main abc d} {1 {no such table column: abc.d}} + 20 {main abc5 w} {0 {{} BINARY 0 0 0}} + 21 {main abc5 x} {0 {{} BINARY 1 1 0}} + 22 {main abc5 y} {0 {{} BINARY 0 0 0}} + 23 {main abc5 z} {0 {{} BINARY 1 1 0}} + 24 {main abc5 rowid} {1 {no such table column: abc5.rowid}} + 30 {main abc6 rowid} {0 {TEXT rtrim 0 0 0}} + 31 {main abc6 oid} {0 {REAL BINARY 0 0 0}} + 32 {main abc6 _rowid_} {0 {BLOB BINARY 0 0 0}} } -ifcapable view { +ifcapable autoinc { set tests [concat $tests { - 8 {{} abc4 b} {0 {INTEGER BINARY 0 1 1}} - 15 {main abc4 rowid} {0 {INTEGER BINARY 0 1 1}} + 100 {{} abc4 b} {0 {INTEGER BINARY 0 1 1}} + 101 {main abc4 rowid} {0 {INTEGER BINARY 0 1 1}} }] } ifcapable view { set tests [concat $tests { - 9 {{} v1 a} {1 {no such table column: v1.a}} - 10 {main v1 b} {1 {no such table column: v1.b}} - 11 {main v1 badname} {1 {no such table column: v1.badname}} - 12 {main v1 rowid} {1 {no such table column: v1.rowid}} + 200 {{} v1 a} {1 {no such table column: v1.a}} + 201 {main v1 b} {1 {no such table column: v1.b}} + 202 {main v1 badname} {1 {no such table column: v1.badname}} + 203 {main v1 rowid} {1 {no such table column: v1.rowid}} }] } @@ -86,4 +96,15 @@ foreach {tn params results} $tests { } $results } +# Calling sqlite3_table_column_metadata with a NULL column name merely +# checks for the existance of the table. +# +do_test colmeta-300 { + catch {sqlite3_table_column_metadata $::DB main xyzzy} res +} {1} +do_test colmeta-301 { + catch {sqlite3_table_column_metadata $::DB main abc} res +} {0} + + finish_test From 6f7febffa29cad09d5f9f49450e009cf5f9040f7 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 10 Dec 2014 04:58:43 +0000 Subject: [PATCH 642/710] Fix typos in the documentation for sqlite3_table_column_metadata(). No changes to code. FossilOrigin-Name: 3528f8dd39acace8eeb7337994c8617313f4b04b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index e4a81bbd8d..07a4a6fd33 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\ssqlite3_table_column_metadata()\sroutine\sso\sthat\sit\sgives\sthe\scorrect\nanswer\sfor\sthe\s"rowid"\scolumn\sin\sa\sWITHOUT\sROWID\stable.\s\sEnhance\sit\sso\sthat\nit\scan\sbe\sused\sto\scheck\sfor\sthe\sexistence\sof\sa\stable\sby\ssetting\sthe\scolumn\nname\sparameter\sto\sNULL.\s\sThe\sroutine\sis\snow\sincluded\sin\sthe\sbuild\sby\ndefault,\seven\swithout\sthe\sSQLITE_ENABLE_COLUMN_METADATA\scompile-time\soption. -D 2014-12-09T22:24:42.981 +C Fix\stypos\sin\sthe\sdocumentation\sfor\ssqlite3_table_column_metadata().\nNo\schanges\sto\scode. +D 2014-12-10T04:58:43.490 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c f377fb8a5c73c10678ea74f3400f7913943e3d75 F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in cc237b8aa62348685b36d06f1b6beb10dfea39ae +F src/sqlite.h.in 6c1ca2ee6949a2232029ecd5391dcf3468bf4114 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 28049b803b74a7f73242a8226915ea00ebb1309f @@ -1226,7 +1226,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 84f9581019961efa31297f8be48427b17bcca857 -R 0b640911f896cbfee28a6d7f7893118c +P cf9be419a16156a9814e1378bb49b780de977343 +R 080e42720ecc05dbe5a6fab44e9bfd04 U drh -Z 0a34c44c2738c668de181b5c6607c9d7 +Z 55e0d7a4f974fc615b3f61db59cc799b diff --git a/manifest.uuid b/manifest.uuid index fbd6f6fb7b..9199a8d464 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cf9be419a16156a9814e1378bb49b780de977343 \ No newline at end of file +3528f8dd39acace8eeb7337994c8617313f4b04b \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index b3ff061151..dd07976a1d 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -5152,22 +5152,22 @@ SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); /* ** CAPI3REF: Extract Metadata About A Column Of A Table ** -** ^The sqlite3_table_column_metadata(X,D,T,C,....) routine returns +** ^(The sqlite3_table_column_metadata(X,D,T,C,....) routine returns ** information about column C of table T in database D -** on [database connection] X. ^The sqlite3_table_column_metadata() +** on [database connection] X.)^ ^The sqlite3_table_column_metadata() ** interface returns SQLITE_OK and fills in the non-NULL pointers in -** the final five arguments with appropriate vaules if the specified +** the final five arguments with appropriate values if the specified ** column exists. ^The sqlite3_table_column_metadata() interface returns ** SQLITE_ERROR and if the specified column does not exist. -** If the column-name parameter to sqlite3_table_column_metadata() is a -** NULL pointer, then the routine simply checks for the existance of the +** ^If the column-name parameter to sqlite3_table_column_metadata() is a +** NULL pointer, then this routine simply checks for the existance of the ** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it ** does not. ** ** ^The column is identified by the second, third and fourth parameters to -** this function. ^The second parameter is either the name of the database +** this function. ^(The second parameter is either the name of the database ** (i.e. "main", "temp", or an attached database) containing the specified -** table or NULL. ^If it is NULL, then all attached databases are searched +** table or NULL.)^ ^If it is NULL, then all attached databases are searched ** for the table using the same algorithm used by the database engine to ** resolve unqualified table references. ** From 684ec98fe3f0a11aa825a8747a6724225ee30a24 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 10 Dec 2014 17:34:48 +0000 Subject: [PATCH 643/710] Revise mutex handling by the sqlite3_win32_reset_heap() function. FossilOrigin-Name: eacb3b7baa910e84f984b8e45695a2a2f5a4c861 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 07a4a6fd33..18deb8120f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypos\sin\sthe\sdocumentation\sfor\ssqlite3_table_column_metadata().\nNo\schanges\sto\scode. -D 2014-12-10T04:58:43.490 +C Revise\smutex\shandling\sby\sthe\ssqlite3_win32_reset_heap()\sfunction. +D 2014-12-10T17:34:48.091 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 -F src/os_win.c a9e500dd963fb1f67d7860e58b5772abe6123862 +F src/os_win.c ecb04a0dad2fa6fa659931a9d8f0f3aca33f908a F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 7a5c5bc0e29b9b16834f5558a9d5d22bbae59a08 F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b @@ -1226,7 +1226,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cf9be419a16156a9814e1378bb49b780de977343 -R 080e42720ecc05dbe5a6fab44e9bfd04 -U drh -Z 55e0d7a4f974fc615b3f61db59cc799b +P 3528f8dd39acace8eeb7337994c8617313f4b04b +R fe43d40d066c10ff075f9f4225f2f387 +U mistachkin +Z cb29644ec15937d54f010f859dadafbb diff --git a/manifest.uuid b/manifest.uuid index 9199a8d464..47c85f8870 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3528f8dd39acace8eeb7337994c8617313f4b04b \ No newline at end of file +eacb3b7baa910e84f984b8e45695a2a2f5a4c861 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 2a7681c73f..80ce97a6cd 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1203,8 +1203,8 @@ int sqlite3_win32_reset_heap(){ int rc; MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ MUTEX_LOGIC( sqlite3_mutex *pMem; ) /* The memsys static mutex */ - MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) - MUTEX_LOGIC( pMem = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); ) + MUTEX_LOGIC( pMaster = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MASTER); ) + MUTEX_LOGIC( pMem = sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_MEM); ) sqlite3_mutex_enter(pMaster); sqlite3_mutex_enter(pMem); winMemAssertMagic(); From 6e45e0c8d7cb8cc14ea80896cad8b269bc990c95 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 10 Dec 2014 20:29:49 +0000 Subject: [PATCH 644/710] Add new test file e_walhook.test. FossilOrigin-Name: 2eb6d3e4fbe388ef28e4b7b846e9e8a4361517a4 --- manifest | 17 ++-- manifest.uuid | 2 +- src/sqlite.h.in | 8 +- src/tclsqlite.c | 1 + test/e_walhook.test | 199 ++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 213 insertions(+), 14 deletions(-) create mode 100644 test/e_walhook.test diff --git a/manifest b/manifest index 18deb8120f..f7c2015248 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Revise\smutex\shandling\sby\sthe\ssqlite3_win32_reset_heap()\sfunction. -D 2014-12-10T17:34:48.091 +C Add\snew\stest\sfile\se_walhook.test. +D 2014-12-10T20:29:49.614 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,14 +230,14 @@ F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c f377fb8a5c73c10678ea74f3400f7913943e3d75 F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in 6c1ca2ee6949a2232029ecd5391dcf3468bf4114 +F src/sqlite.h.in 2e699aabd1df6042aff4265cd93bdc8812629f11 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 28049b803b74a7f73242a8226915ea00ebb1309f F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc -F src/tclsqlite.c 0a874655dd39a9875e39c5d3c464db662171d228 +F src/tclsqlite.c c6a21c64da1490e14d53cdc2062d1e2e57942622 F src/test1.c 7e806af12d7915445e46407d32b275d8ca9db4e7 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c @@ -479,6 +479,7 @@ F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a F test/e_walckpt.test 3116a98fa0dd9b2c9e493de7c59730adfe436746 +F test/e_walhook.test 9a859599e9a0f354ffcc7c716f7ec699e41f617c F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 @@ -1226,7 +1227,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3528f8dd39acace8eeb7337994c8617313f4b04b -R fe43d40d066c10ff075f9f4225f2f387 -U mistachkin -Z cb29644ec15937d54f010f859dadafbb +P eacb3b7baa910e84f984b8e45695a2a2f5a4c861 +R de56ad0a085434d1b275f3b398c245cd +U dan +Z 076528022e1550b90a78f1e189e33666 diff --git a/manifest.uuid b/manifest.uuid index 47c85f8870..989cb2cfa4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eacb3b7baa910e84f984b8e45695a2a2f5a4c861 \ No newline at end of file +2eb6d3e4fbe388ef28e4b7b846e9e8a4361517a4 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index dd07976a1d..a4ca793969 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -7180,12 +7180,10 @@ void sqlite3_log(int iErrCode, const char *zFormat, ...); ** CAPI3REF: Write-Ahead Log Commit Hook ** ** ^The [sqlite3_wal_hook()] function is used to register a callback that -** will be invoked each time a database connection commits data to a -** [write-ahead log] (i.e. whenever a transaction is committed in -** [journal_mode | journal_mode=WAL mode]). +** is invoked each time data is committed to a database in wal mode. ** -** ^The callback is invoked by SQLite after the commit has taken place and -** the associated write-lock on the database released, so the implementation +** ^(The callback is invoked by SQLite after the commit has taken place and +** the associated write-lock on the database released)^, so the implementation ** may read, write or [checkpoint] the database as required. ** ** ^The first parameter passed to the callback function when it is invoked diff --git a/src/tclsqlite.c b/src/tclsqlite.c index 4fb78a5d12..32de527301 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -635,6 +635,7 @@ static int DbWalHandler( Tcl_Interp *interp = pDb->interp; assert(pDb->pWalHook); + assert( db==pDb->db ); p = Tcl_DuplicateObj(pDb->pWalHook); Tcl_IncrRefCount(p); Tcl_ListObjAppendElement(interp, p, Tcl_NewStringObj(zDb, -1)); diff --git a/test/e_walhook.test b/test/e_walhook.test new file mode 100644 index 0000000000..e12d3b57e9 --- /dev/null +++ b/test/e_walhook.test @@ -0,0 +1,199 @@ +# 2014 December 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/wal_common.tcl +set testprefix e_walhook + + +# EVIDENCE-OF: R-00752-43975 The sqlite3_wal_hook() function is used to +# register a callback that is invoked each time data is committed to a +# database in wal mode. +# +# 1.1: shows that the wal-hook is not invoked in rollback mode. +# 1.2: but is invoked in wal mode. +# +set ::wal_hook_count 0 +proc my_wal_hook {args} { + incr ::wal_hook_count + return 0 +} + +do_test 1.1.1 { + db wal_hook my_wal_hook + execsql { + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1); + } + set ::wal_hook_count +} 0 +do_test 1.1.2 { + execsql { PRAGMA journal_mode = wal } + set ::wal_hook_count +} 0 + +do_test 1.3 { + execsql { INSERT INTO t1 VALUES(2) } + set wal_hook_count +} 1 + +do_test 1.4 { + execsql { + BEGIN; + INSERT INTO t1 VALUES(3); + INSERT INTO t1 VALUES(4); + COMMIT; + } + set wal_hook_count +} 2 + +# EVIDENCE-OF: R-65366-15139 The callback is invoked by SQLite after the +# commit has taken place and the associated write-lock on the database +# released +# +set ::read_ok 0 +proc my_wal_hook {args} { + sqlite3 db2 test.db + if {[db2 eval { SELECT * FROM t1 }] == "1 2 3 4 5"} { + set ::read_ok 1 + } + db2 close +} +do_test 2.1 { + execsql { INSERT INTO t1 VALUES(5) } + set ::read_ok +} 1 + +# EVIDENCE-OF: R-44294-52863 The third parameter is the name of the +# database that was written to - either "main" or the name of an +# ATTACH-ed database. +# +# EVIDENCE-OF: R-18913-19355 The fourth parameter is the number of pages +# currently in the write-ahead log file, including those that were just +# committed. +# +set ::wal_hook_args [list] +proc my_wal_hook {dbname nEntry} { + set ::wal_hook_args [list $dbname $nEntry] +} +forcedelete test.db2 +do_test 3.0 { + execsql { + ATTACH 'test.db2' AS aux; + CREATE TABLE aux.t2(x); + PRAGMA aux.journal_mode = wal; + } +} {wal} + +# Database "aux" +do_test 3.1.1 { + set wal_hook_args [list] + execsql { INSERT INTO t2 VALUES('a') } +} {} +do_test 3.1.2 { + set wal_hook_args +} [list aux [wal_frame_count test.db2-wal 1024]] + +# Database "main" +do_test 3.2.1 { + set wal_hook_args [list] + execsql { INSERT INTO t1 VALUES(6) } +} {} +do_test 3.1.2 { + set wal_hook_args +} [list main [wal_frame_count test.db-wal 1024]] + +# EVIDENCE-OF: R-14034-00929 If an error code is returned, that error +# will propagate back up through the SQLite code base to cause the +# statement that provoked the callback to report an error, though the +# commit will have still occurred. +# +proc my_wal_hook {args} { return 1 ;# SQLITE_ERROR } +do_catchsql_test 4.1 { + INSERT INTO t1 VALUES(7) +} {1 {SQL logic error or missing database}} + +proc my_wal_hook {args} { return 5 ;# SQLITE_BUSY } +do_catchsql_test 4.2 { + INSERT INTO t1 VALUES(8) +} {1 {database is locked}} + +proc my_wal_hook {args} { return 14 ;# SQLITE_CANTOPEN } +do_catchsql_test 4.3 { + INSERT INTO t1 VALUES(9) +} {1 {unable to open database file}} + +do_execsql_test 4.4 { + SELECT * FROM t1 +} {1 2 3 4 5 6 7 8 9} + +# EVIDENCE-OF: R-10466-53920 Calling sqlite3_wal_hook() replaces any +# previously registered write-ahead log callback. +set ::old_wal_hook 0 +proc my_old_wal_hook {args} { + incr ::old_wal_hook + return 0 +} +db wal_hook my_old_wal_hook +do_test 5.1 { + execsql { INSERT INTO t1 VALUES(10) } + set ::old_wal_hook +} {1} + +# Replace old_wal_hook. Observe that it is not invoked after it has +# been replaced. +proc my_new_wal_hook {args} { return 0 } +db wal_hook my_new_wal_hook +do_test 5.2 { + execsql { INSERT INTO t1 VALUES(11) } + set ::old_wal_hook +} {1} + + + +# EVIDENCE-OF: R-42842-27162 Note that the sqlite3_wal_autocheckpoint() +# interface and the wal_autocheckpoint pragma both invoke +# sqlite3_wal_hook() and will those overwrite any prior +# sqlite3_wal_hook() settings. +# +set ::old_wal_hook 0 +proc my_old_wal_hook {args} { incr ::old_wal_hook ; return 0 } +do_test 6.1.1 { + execsql { INSERT INTO t1 VALUES(12) } + set ::old_wal_hook +} {1} +do_test 6.1.2 { + execsql { PRAGA wal_autocheckpoint = 1000 } + execsql { INSERT INTO t1 VALUES(12) } + set ::old_wal_hook +} {1} + +# EVIDENCE-OF: R-52629-38967 The first parameter passed to the callback +# function when it is invoked is a copy of the third parameter passed to +# sqlite3_wal_hook() when registering the callback. +# +# This is tricky to test using the tcl interface. However, the +# mechanism used to invoke the tcl script registered as a wal-hook +# depends on the context pointer being correctly passed through. And +# since multiple different wal-hook scripts have been successfully +# invoked by this test script, consider this tested. +# +# EVIDENCE-OF: R-23378-42536 The second is a copy of the database +# handle. +# +# There is an assert() in the C wal-hook used by tclsqlite.c to +# prove this. And that hook has been invoked multiple times when +# running this script. So consider this requirement tested as well. +# + +finish_test From 1f03b86523a0560f92eb3d8b2950a6a81ddc95c5 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 10 Dec 2014 20:57:20 +0000 Subject: [PATCH 645/710] Fix a typo causing a test error in e_walhook.test. FossilOrigin-Name: d9f916ba09f1a61684b4d59548ab6cf71cdb6a37 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/e_walhook.test | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f7c2015248..528f1250c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\se_walhook.test. -D 2014-12-10T20:29:49.614 +C Fix\sa\stypo\scausing\sa\stest\serror\sin\se_walhook.test. +D 2014-12-10T20:57:20.700 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -479,7 +479,7 @@ F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a F test/e_walckpt.test 3116a98fa0dd9b2c9e493de7c59730adfe436746 -F test/e_walhook.test 9a859599e9a0f354ffcc7c716f7ec699e41f617c +F test/e_walhook.test da3ea8b3483d1af72190337bda50155a91a4b664 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 F test/enc3.test 90683ad0e6ea587b9d5542ca93568af9a9858c40 @@ -1227,7 +1227,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P eacb3b7baa910e84f984b8e45695a2a2f5a4c861 -R de56ad0a085434d1b275f3b398c245cd +P 2eb6d3e4fbe388ef28e4b7b846e9e8a4361517a4 +R 6969ca3627ab2efde502167b1f0ba7a8 U dan -Z 076528022e1550b90a78f1e189e33666 +Z dbe3e99b743e5bbd88e69201e8af7d3a diff --git a/manifest.uuid b/manifest.uuid index 989cb2cfa4..b790577929 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2eb6d3e4fbe388ef28e4b7b846e9e8a4361517a4 \ No newline at end of file +d9f916ba09f1a61684b4d59548ab6cf71cdb6a37 \ No newline at end of file diff --git a/test/e_walhook.test b/test/e_walhook.test index e12d3b57e9..c8c8819493 100644 --- a/test/e_walhook.test +++ b/test/e_walhook.test @@ -168,12 +168,13 @@ do_test 5.2 { # set ::old_wal_hook 0 proc my_old_wal_hook {args} { incr ::old_wal_hook ; return 0 } +db wal_hook my_old_wal_hook do_test 6.1.1 { execsql { INSERT INTO t1 VALUES(12) } set ::old_wal_hook } {1} do_test 6.1.2 { - execsql { PRAGA wal_autocheckpoint = 1000 } + execsql { PRAGMA wal_autocheckpoint = 1000 } execsql { INSERT INTO t1 VALUES(12) } set ::old_wal_hook } {1} From 0a3520c0f4fa504e92646cf3c182ad2a9e98d0f6 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Dec 2014 15:27:04 +0000 Subject: [PATCH 646/710] Fix a typo in the documentation for sqlite3_threadsafe(). FossilOrigin-Name: 258e747bb7e3a2bc46f932cc2b06c2689d43aeb0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 528f1250c5..17ec6c7d04 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\scausing\sa\stest\serror\sin\se_walhook.test. -D 2014-12-10T20:57:20.700 +C Fix\sa\stypo\sin\sthe\sdocumentation\sfor\ssqlite3_threadsafe(). +D 2014-12-11T15:27:04.851 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c f377fb8a5c73c10678ea74f3400f7913943e3d75 F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in 2e699aabd1df6042aff4265cd93bdc8812629f11 +F src/sqlite.h.in 116dc731361549ee3fc79dcebace11b57d24dcfd F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h 28049b803b74a7f73242a8226915ea00ebb1309f @@ -1227,7 +1227,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2eb6d3e4fbe388ef28e4b7b846e9e8a4361517a4 -R 6969ca3627ab2efde502167b1f0ba7a8 -U dan -Z dbe3e99b743e5bbd88e69201e8af7d3a +P d9f916ba09f1a61684b4d59548ab6cf71cdb6a37 +R be485ee74d8699a155e9699a0ceccb5f +U drh +Z 8b7cdafef867d0fe5e1164d6f32f45ce diff --git a/manifest.uuid b/manifest.uuid index b790577929..de0c011b0b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9f916ba09f1a61684b4d59548ab6cf71cdb6a37 \ No newline at end of file +258e747bb7e3a2bc46f932cc2b06c2689d43aeb0 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index a4ca793969..701e6de2e3 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -196,7 +196,7 @@ const char *sqlite3_compileoption_get(int N); ** SQLITE_THREADSAFE=1 or =2 then mutexes are enabled by default but ** can be fully or partially disabled using a call to [sqlite3_config()] ** with the verbs [SQLITE_CONFIG_SINGLETHREAD], [SQLITE_CONFIG_MULTITHREAD], -** or [SQLITE_CONFIG_MUTEX]. ^(The return value of the +** or [SQLITE_CONFIG_SERIALIZED]. ^(The return value of the ** sqlite3_threadsafe() function shows only the compile-time setting of ** thread safety, not any run-time changes to that setting made by ** sqlite3_config(). In other words, the return value from sqlite3_threadsafe() From 857536623a7cf447c028ca1d9fd251fdbcf9ea0b Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 11 Dec 2014 16:38:18 +0000 Subject: [PATCH 647/710] Fix a race condition to do with very large index keys in shared-cache mode. FossilOrigin-Name: fc157dd7f18c94b7ae5f155e1b4a5d7714b7da8c --- manifest | 21 ++++----- manifest.uuid | 2 +- src/btree.c | 8 ++-- src/vdbeapi.c | 1 - test/threadtest3.c | 2 + test/tt3_index.c | 2 +- test/tt3_lookaside1.c | 99 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 119 insertions(+), 16 deletions(-) create mode 100644 test/tt3_lookaside1.c diff --git a/manifest b/manifest index 17ec6c7d04..69fe030103 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\sdocumentation\sfor\ssqlite3_threadsafe(). -D 2014-12-11T15:27:04.851 +C Fix\sa\srace\scondition\sto\sdo\swith\svery\slarge\sindex\skeys\sin\sshared-cache\smode. +D 2014-12-11T16:38:18.086 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c ea6692ce58bfba55b12c75d2947fec0906d1ef7a +F src/btree.c ab4b60fcf9920d862ff4d96efb1d605e4e7701a0 F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 @@ -294,7 +294,7 @@ F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c F src/vdbe.c 1a9e671c9cfc259e4d2affc71f7df4a4c00a842c F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 -F src/vdbeapi.c 07acb615d1e4170e71fc1b0d087f3c53a1ad8e83 +F src/vdbeapi.c f8dd2d33a30938188fc292d524e88a91f2e65887 F src/vdbeaux.c 6f7f39c3fcf0f5923758df8561bb5d843908a553 F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f @@ -912,7 +912,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c fca8d360b470405ae3ed431b5cb4cdf031f85a74 +F test/threadtest3.c 2b6e07e915c383c250a5b531cf6ef163a3047d7e F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 @@ -1076,7 +1076,8 @@ F test/triggerC.test a68980c5955d62ee24be6f97129d824f199f9a4c F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 355e9c5cbaed5cd039a60baad1fb2197caeb8e52 F test/tt3_checkpoint.c 415eccce672d681b297485fc20f44cdf0eac93af -F test/tt3_index.c 2e7f3151a0ae522f031e8e2761ca2bda63f4d221 +F test/tt3_index.c 14f4b0cc3f4c05353e25778cf531c9d6e3b0d27f +F test/tt3_lookaside1.c 0b5b79ba37f21a1eb849cd4a54eed367f4d4aaaf F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a @@ -1227,7 +1228,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d9f916ba09f1a61684b4d59548ab6cf71cdb6a37 -R be485ee74d8699a155e9699a0ceccb5f -U drh -Z 8b7cdafef867d0fe5e1164d6f32f45ce +P 258e747bb7e3a2bc46f932cc2b06c2689d43aeb0 +R 016d1b9c16d412ff926cdf599ae97f64 +U dan +Z 8da7cca75b292da60a4687ff246cc6b3 diff --git a/manifest.uuid b/manifest.uuid index de0c011b0b..0de4da3602 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -258e747bb7e3a2bc46f932cc2b06c2689d43aeb0 \ No newline at end of file +fc157dd7f18c94b7ae5f155e1b4a5d7714b7da8c \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 4283d00fd3..a73c831219 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3913,7 +3913,7 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){ releasePage(pCur->apPage[i]); } unlockBtreeIfUnused(pBt); - sqlite3DbFree(pBtree->db, pCur->aOverflow); + sqlite3_free(pCur->aOverflow); /* sqlite3_free(pCur); */ sqlite3BtreeLeave(pBtree); } @@ -4208,6 +4208,7 @@ static int accessPayload( offset -= pCur->info.nLocal; } + if( rc==SQLITE_OK && amt>0 ){ const u32 ovflSize = pBt->usableSize - 4; /* Bytes content per ovfl page */ Pgno nextPage; @@ -4225,8 +4226,8 @@ static int accessPayload( if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; if( nOvfl>pCur->nOvflAlloc ){ - Pgno *aNew = (Pgno*)sqlite3DbRealloc( - pCur->pBtree->db, pCur->aOverflow, nOvfl*2*sizeof(Pgno) + Pgno *aNew = (Pgno*)sqlite3Realloc( + pCur->aOverflow, nOvfl*2*sizeof(Pgno) ); if( aNew==0 ){ rc = SQLITE_NOMEM; @@ -4273,6 +4274,7 @@ static int accessPayload( */ assert( eOp!=2 ); assert( pCur->curFlags & BTCF_ValidOvfl ); + assert( pCur->pBtree->db==pBt->db ); if( pCur->aOverflow[iIdx+1] ){ nextPage = pCur->aOverflow[iIdx+1]; }else{ diff --git a/src/vdbeapi.c b/src/vdbeapi.c index 5744c28632..c611688193 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -580,7 +580,6 @@ int sqlite3_step(sqlite3_stmt *pStmt){ ** sqlite3_errmsg() and sqlite3_errcode(). */ const char *zErr = (const char *)sqlite3_value_text(db->pErr); - assert( zErr!=0 || db->mallocFailed ); sqlite3DbFree(db, v->zErrMsg); if( !db->mallocFailed ){ v->zErrMsg = sqlite3DbStrDup(db, zErr); diff --git a/test/threadtest3.c b/test/threadtest3.c index 084ca022a9..afa4197ee2 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -1398,6 +1398,7 @@ static void dynamic_triggers(int nMs){ #include "tt3_checkpoint.c" #include "tt3_index.c" +#include "tt3_lookaside1.c" int main(int argc, char **argv){ struct ThreadTest { @@ -1419,6 +1420,7 @@ int main(int argc, char **argv){ { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 }, { create_drop_index_1, "create_drop_index_1", 10000 }, + { lookaside1, "lookaside1", 10000 }, }; int i; diff --git a/test/tt3_index.c b/test/tt3_index.c index b79768c6c7..4bad64aabc 100644 --- a/test/tt3_index.c +++ b/test/tt3_index.c @@ -67,8 +67,8 @@ static void create_drop_index_1(int nMs){ launch_thread(&err, &threads, create_drop_index_thread, 0); launch_thread(&err, &threads, create_drop_index_thread, 0); launch_thread(&err, &threads, create_drop_index_thread, 0); - sqlite3_enable_shared_cache(0); join_all_threads(&err, &threads); + sqlite3_enable_shared_cache(0); print_and_free_err(&err); } diff --git a/test/tt3_lookaside1.c b/test/tt3_lookaside1.c new file mode 100644 index 0000000000..486de808cd --- /dev/null +++ b/test/tt3_lookaside1.c @@ -0,0 +1,99 @@ +/* +** 2014 December 9 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** lookaside1 +*/ + +/* +** The test in this file attempts to expose a specific race condition +** that is suspected to exist at time of writing. +*/ + +static char *lookaside1_thread_reader(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + + opendb(&err, &db, "test.db", 0); + + while( !timetostop(&err) ){ + sqlite3_stmt *pStmt = 0; + int rc; + + sqlite3_prepare_v2(db.db, "SELECT 1 FROM t1", -1, &pStmt, 0); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + execsql(&err, &db, "SELECT length(x||y||z) FROM t2"); + } + rc = sqlite3_finalize(pStmt); + if( err.rc==SQLITE_OK && rc!=SQLITE_OK ){ + sqlite_error(&err, &db, "finalize"); + } + } + + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + +static char *lookaside1_thread_writer(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + + opendb(&err, &db, "test.db", 0); + + do{ + sql_script(&err, &db, + "BEGIN;" + "UPDATE t3 SET i=i+1 WHERE x=1;" + "ROLLBACK;" + ); + }while( !timetostop(&err) ); + + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + + +static void lookaside1(int nMs){ + Error err = {0}; + Sqlite db = {0}; + Threadset threads = {0}; + + opendb(&err, &db, "test.db", 1); + sql_script(&err, &db, + "CREATE TABLE t1(x PRIMARY KEY) WITHOUT ROWID;" + "WITH data(x,y) AS (" + " SELECT 1, quote(randomblob(750)) UNION ALL " + " SELECT x*2, y||y FROM data WHERE x<5) " + "INSERT INTO t1 SELECT y FROM data;" + + "CREATE TABLE t3(x PRIMARY KEY,i) WITHOUT ROWID;" + "INSERT INTO t3 VALUES(1, 1);" + + "CREATE TABLE t2(x,y,z);" + "INSERT INTO t2 VALUES(randomblob(50), randomblob(50), randomblob(50));" + ); + closedb(&err, &db); + + setstoptime(&err, nMs); + + sqlite3_enable_shared_cache(1); + launch_thread(&err, &threads, lookaside1_thread_reader, 0); + launch_thread(&err, &threads, lookaside1_thread_reader, 0); + launch_thread(&err, &threads, lookaside1_thread_reader, 0); + launch_thread(&err, &threads, lookaside1_thread_reader, 0); + launch_thread(&err, &threads, lookaside1_thread_reader, 0); + launch_thread(&err, &threads, lookaside1_thread_writer, 0); + join_all_threads(&err, &threads); + sqlite3_enable_shared_cache(0); + print_and_free_err(&err); +} From 1e57430e6326c0cb3b07de81fcae2baf4f5c0d15 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 11 Dec 2014 19:29:42 +0000 Subject: [PATCH 648/710] Add the threadtest4.c test program. Not yet working. FossilOrigin-Name: ec3a74469ca2f0f3fb7d82a05fdac7500354e78f --- manifest | 14 +- manifest.uuid | 2 +- test/threadtest4.c | 435 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 445 insertions(+), 6 deletions(-) create mode 100644 test/threadtest4.c diff --git a/manifest b/manifest index 17ec6c7d04..a949b5b360 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sthe\sdocumentation\sfor\ssqlite3_threadsafe(). -D 2014-12-11T15:27:04.851 +C Add\sthe\sthreadtest4.c\stest\sprogram.\s\sNot\syet\sworking. +D 2014-12-11T19:29:42.156 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -913,6 +913,7 @@ F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 F test/threadtest3.c fca8d360b470405ae3ed431b5cb4cdf031f85a74 +F test/threadtest4.c e2bcee459bbd27fa2d55751e9900a4a5a4675cd5 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 @@ -1227,7 +1228,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d9f916ba09f1a61684b4d59548ab6cf71cdb6a37 -R be485ee74d8699a155e9699a0ceccb5f +P 258e747bb7e3a2bc46f932cc2b06c2689d43aeb0 +R 4bb41891ae2017245bf160d9a28a8371 +T *branch * threadtest4 +T *sym-threadtest4 * +T -sym-trunk * U drh -Z 8b7cdafef867d0fe5e1164d6f32f45ce +Z 474616aece6be366bfcf36cb343f9bea diff --git a/manifest.uuid b/manifest.uuid index de0c011b0b..ea9eb0df3e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -258e747bb7e3a2bc46f932cc2b06c2689d43aeb0 \ No newline at end of file +ec3a74469ca2f0f3fb7d82a05fdac7500354e78f \ No newline at end of file diff --git a/test/threadtest4.c b/test/threadtest4.c new file mode 100644 index 0000000000..e4cd644a4f --- /dev/null +++ b/test/threadtest4.c @@ -0,0 +1,435 @@ +/* +** 2014-12-11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file implements a simple standalone program used to stress the +** SQLite library when accessing the same set of databases simultaneously +** from multiple threads in shared-cache mode. +** +** This test program runs on unix-like systems only. It uses pthreads. +** To compile: +** +** gcc -o tt4 -I. threadtest4.c sqlite3.c -ldl -lpthread +** +** To run: +** +** ./tt4 10 +** +** The argument is the number of threads. +*/ +#include "sqlite3.h" +#include +#include +#include +#include +#include +#include +#include + +/* +** An instance of the following structure is passed into each worker +** thread. +*/ +typedef struct WorkerInfo WorkerInfo; +struct WorkerInfo { + int tid; /* Thread ID */ + unsigned wkrFlags; /* Flags */ + sqlite3 *mainDb; /* Database connection of the main thread */ + sqlite3 *db; /* Database connection of this thread */ + int nErr; /* Number of errors seen by this thread */ + int nTest; /* Number of tests run by this thread */ + char *zMsg; /* Message returned by this thread */ + pthread_t id; /* Thread id */ + pthread_mutex_t *pWrMutex; /* Hold this mutex while writing */ +}; + +/* +** Allowed values for WorkerInfo.wkrFlags +*/ +#define TT4_SERIALIZED 0x0000001 /* The --serialized option is used */ +#define TT4_WAL 0x0000002 /* WAL mode in use */ +#define TT4_TRACE 0x0000004 /* Trace activity */ + + +/* +** Report an OOM error and die if the argument is NULL +*/ +static void check_oom(void *x){ + if( x==0 ){ + fprintf(stderr, "out of memory\n"); + exit(1); + } +} + +/* +** Allocate memory. If the allocation fails, print an error message and +** kill the process. +*/ +static void *safe_malloc(int sz){ + void *x = sqlite3_malloc(sz>0?sz:1); + check_oom(x); + return x; +} + +/* +** Print a trace message for a worker +*/ +static void worker_trace(WorkerInfo *p, const char *zFormat, ...){ + va_list ap; + char *zMsg; + if( (p->wkrFlags & TT4_TRACE)==0 ) return; + va_start(ap, zFormat); + zMsg = sqlite3_vmprintf(zFormat, ap); + check_oom(zMsg); + va_end(ap); + fprintf(stderr, "TRACE(%02d): %s\n", p->tid, zMsg); + sqlite3_free(zMsg); +} + +/* +** Prepare a single SQL query +*/ +static sqlite3_stmt *prep_sql(sqlite3 *db, const char *zFormat, ...){ + va_list ap; + char *zSql; + int rc; + sqlite3_stmt *pStmt = 0; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + check_oom(zSql); + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "SQL error (%d,%d): %s\nWhile preparing: [%s]\n", + rc, sqlite3_extended_errcode(db), sqlite3_errmsg(db), zSql); + exit(1); + } + sqlite3_free(zSql); + return pStmt; +} + +/* +** Run a SQL statements. Panic if unable. +*/ +static void run_sql(WorkerInfo *p, const char *zFormat, ...){ + va_list ap; + char *zSql; + int rc; + sqlite3_stmt *pStmt = 0; + int nRetry = 0; + + va_start(ap, zFormat); + zSql = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + check_oom(zSql); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "SQL error (%d,%d): %s\nWhile preparing: [%s]\n", + rc, sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zSql); + exit(1); + } + worker_trace(p, "running [%s]", zSql); + while( (rc = sqlite3_step(pStmt))!=SQLITE_DONE ){ + if( (rc&0xff)==SQLITE_BUSY || (rc&0xff)==SQLITE_LOCKED ){ + sqlite3_reset(pStmt); + nRetry++; + if( nRetry<10 ){ + worker_trace(p, "retry %d for [%s]", nRetry, zSql); + sched_yield(); + continue; + }else{ + fprintf(stderr, "Deadlock in thread %d while running [%s]\n", + p->tid, zSql); + exit(1); + } + } + if( rc!=SQLITE_ROW ){ + fprintf(stderr, "SQL error (%d,%d): %s\nWhile running [%s]\n", + rc, sqlite3_extended_errcode(p->db), sqlite3_errmsg(p->db), zSql); + exit(1); + } + } + sqlite3_free(zSql); + sqlite3_finalize(pStmt); +} + + +/* +** Open the database connection for WorkerInfo. The order in which +** the files are opened is a function of the tid value. +*/ +static void worker_open_connection(WorkerInfo *p, int iCnt){ + char *zFile; + int x; + int rc; + static const unsigned char aOrder[6][3] = { + { 1, 2, 3}, + { 1, 3, 2}, + { 2, 1, 3}, + { 2, 3, 1}, + { 3, 1, 2}, + { 3, 2, 1} + }; + x = (p->tid + iCnt) % 6; + zFile = sqlite3_mprintf("tt4-test%d.db", aOrder[x][0]); + check_oom(zFile); + worker_trace(p, "open %s", zFile); + rc = sqlite3_open_v2(zFile, &p->db, + SQLITE_OPEN_READWRITE|SQLITE_OPEN_SHAREDCACHE, 0); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "sqlite_open_v2(%s) failed on thread %d\n", + zFile, p->tid); + exit(1); + } + sqlite3_free(zFile); + run_sql(p, "PRAGMA read_uncommitted=ON;"); + sqlite3_busy_timeout(p->db, 10000); + run_sql(p, "PRAGMA synchronous=OFF;"); + run_sql(p, "ATTACH 'tt4-test%d.db' AS aux1", aOrder[x][1]); + run_sql(p, "ATTACH 'tt4-test%d.db' AS aux2", aOrder[x][2]); +} + +/* +** Close the worker database connection +*/ +static void worker_close_connection(WorkerInfo *p){ + if( p->db ){ + worker_trace(p, "close"); + sqlite3_close(p->db); + p->db = 0; + } +} + +/* +** Delete all content in the three databases associated with a +** single thread. Make this happen all in a single transaction if +** inTrans is true, or separately for each database if inTrans is +** false. +*/ +static void worker_delete_all_content(WorkerInfo *p, int inTrans){ + if( inTrans ){ + pthread_mutex_lock(p->pWrMutex); + run_sql(p, "BEGIN"); + run_sql(p, "DELETE FROM t1 WHERE tid=%d", p->tid); + run_sql(p, "DELETE FROM t2 WHERE tid=%d", p->tid); + run_sql(p, "DELETE FROM t3 WHERE tid=%d", p->tid); + run_sql(p, "COMMIT"); + pthread_mutex_unlock(p->pWrMutex); + p->nTest++; + }else{ + pthread_mutex_lock(p->pWrMutex); + run_sql(p, "DELETE FROM t1 WHERE tid=%d", p->tid); + pthread_mutex_unlock(p->pWrMutex); + p->nTest++; + pthread_mutex_lock(p->pWrMutex); + run_sql(p, "DELETE FROM t2 WHERE tid=%d", p->tid); + pthread_mutex_unlock(p->pWrMutex); + p->nTest++; + pthread_mutex_lock(p->pWrMutex); + run_sql(p, "DELETE FROM t3 WHERE tid=%d", p->tid); + pthread_mutex_unlock(p->pWrMutex); + p->nTest++; + } +} + +/* +** Create rows mn through mx in table iTab for the given worker +*/ +static void worker_add_content(WorkerInfo *p, int mn, int mx, int iTab){ + char *zTabDef; + switch( iTab ){ + case 1: zTabDef = "t1(tid,sp,a,b,c)"; break; + case 2: zTabDef = "t2(tid,sp,d,e,f)"; break; + case 3: zTabDef = "t3(tid,sp,x,y,z)"; break; + } + pthread_mutex_lock(p->pWrMutex); + run_sql(p, + "WITH RECURSIVE\n" + " c(i) AS (VALUES(%d) UNION ALL SELECT i+1 FROM c WHERE i<%d)\n" + "INSERT INTO %s SELECT %d, zeroblob(3000), i, printf('%%d',i), i FROM c;", + mn, mx, zTabDef, p->tid + ); + pthread_mutex_unlock(p->pWrMutex); + p->nTest++; +} + +/* +** Set an error message on a worker +*/ +static void worker_error(WorkerInfo *p, const char *zFormat, ...){ + va_list ap; + p->nErr++; + sqlite3_free(p->zMsg); + va_start(ap, zFormat); + p->zMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Each thread runs the following function. +*/ +static void *worker_thread(void *pArg){ + WorkerInfo *p = (WorkerInfo*)pArg; + int iOuter; + int i; + int rc; + sqlite3_stmt *pStmt; + + printf("worker %d startup\n", p->tid); fflush(stdout); + for(iOuter=1; iOuter<=4; iOuter++){ + worker_open_connection(p, iOuter); + for(i=0; i<4; i++){ + worker_add_content(p, i*100+1, (i+1)*100, (p->tid+iOuter)%3 + 1); + worker_add_content(p, i*100+1, (i+1)*100, (p->tid+iOuter+1)%3 + 1); + worker_add_content(p, i*100+1, (i+1)*100, (p->tid+iOuter+2)%3 + 1); + } + + pStmt = prep_sql(p->db, "SELECT count(a) FROM t1 WHERE tid=%d", p->tid); + worker_trace(p, "query [%s]", sqlite3_sql(pStmt)); + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + worker_error(p, "Failed to step: %s", sqlite3_sql(pStmt)); + }else if( sqlite3_column_int(pStmt, 0)!=400 ){ + worker_error(p, "Wrong result: %d", sqlite3_column_int(pStmt,0)); + } + if( p->nErr ) break; + + worker_delete_all_content(p, (p->tid+iOuter)%2); + worker_close_connection(p); + p->db = 0; + } + worker_close_connection(p); + printf("worker %d finished\n", p->tid); fflush(stdout); + return 0; +} + +int main(int argc, char **argv){ + int nWorker = 0; /* Number of worker threads */ + int i; /* Loop counter */ + WorkerInfo *aInfo; /* Information for each worker */ + unsigned wkrFlags = 0; /* Default worker flags */ + int nErr = 0; /* Number of errors */ + int nTest = 0; /* Number of tests */ + int rc; /* Return code */ + sqlite3 *db = 0; /* Main database connection */ + pthread_mutex_t wrMutex; /* The write serialization mutex */ + WorkerInfo infoTop; /* WorkerInfo for the main thread */ + WorkerInfo *p; /* Pointer to infoTop */ + + sqlite3_config(SQLITE_CONFIG_MULTITHREAD); + for(i=1; i='1' && z[0]<='9' && nWorker==0 ){ + nWorker = atoi(z); + if( nWorker<2 ){ + fprintf(stderr, "minimum of 2 threads\n"); + exit(1); + } + }else{ + fprintf(stderr, "extra command-line argument: \"%s\"\n", argv[i]); + exit(1); + } + } + if( nWorker==0 ){ + fprintf(stderr, + "usage: %s ?OPTIONS? N\n" + "N is the number of threads and must be at least 2.\n" + "Options:\n" + " --serialized\n" + " --multithread\n" + ,argv[0] + ); + exit(1); + } + if( !sqlite3_threadsafe() ){ + fprintf(stderr, "requires a threadsafe build of SQLite\n"); + exit(1); + } + sqlite3_initialize(); + sqlite3_enable_shared_cache(1); + pthread_mutex_init(&wrMutex, 0); + + /* Initialize the test database files */ + (void)unlink("tt4-test1.db"); + (void)unlink("tt4-test2.db"); + (void)unlink("tt4-test3.db"); + rc = sqlite3_open("tt4-test1.db", &db); + if( rc!=SQLITE_OK ){ + fprintf(stderr, "Unable to open test database: tt4-test2.db\n"); + exit(1); + } + memset(&infoTop, 0, sizeof(infoTop)); + infoTop.db = db; + infoTop.wkrFlags = wkrFlags; + p = &infoTop; + if( wkrFlags & TT4_WAL ){ + run_sql(p, "PRAGMA journal_mode=WAL"); + } + run_sql(p, "PRAGMA synchronous=OFF"); + run_sql(p, "CREATE TABLE IF NOT EXISTS t1(tid INTEGER, sp, a, b, c)"); + run_sql(p, "CREATE INDEX t1tid ON t1(tid)"); + run_sql(p, "CREATE INDEX t1ab ON t1(a,b)"); + run_sql(p, "ATTACH 'tt4-test2.db' AS 'test2'"); + run_sql(p, "CREATE TABLE IF NOT EXISTS test2.t2(tid INTEGER, sp, d, e, f)"); + run_sql(p, "CREATE INDEX test2.t2tid ON t2(tid)"); + run_sql(p, "CREATE INDEX test2.t2de ON t2(d,e)"); + run_sql(p, "ATTACH 'tt4-test3.db' AS 'test3'"); + run_sql(p, "CREATE TABLE IF NOT EXISTS test3.t3(tid INTEGER, sp, x, y, z)"); + run_sql(p, "CREATE INDEX test3.t3tid ON t3(tid)"); + run_sql(p, "CREATE INDEX test3.t3xy ON t3(x,y)"); + aInfo = safe_malloc( sizeof(*aInfo)*nWorker ); + memset(aInfo, 0, sizeof(*aInfo)*nWorker); + for(i=0; i Date: Fri, 12 Dec 2014 00:20:37 +0000 Subject: [PATCH 649/710] Fix a bug in the threadtest4.c program. Remove the keyinfo cache as it provides minimal performance improvements, and then only at SQL preparation time, not at runtime, and it has problems with data races in shared-cache mode. We might later add the keyinfo cache back but only enable it when shared-cache mode is off. FossilOrigin-Name: b7489f9451628c68f1dfc1d457fc161a0921c631 --- manifest | 23 ++++++++++----------- manifest.uuid | 2 +- src/build.c | 50 +++++++++++++++++++--------------------------- src/main.c | 37 ---------------------------------- src/sqliteInt.h | 1 - src/where.c | 1 - test/threadtest4.c | 1 + 7 files changed, 32 insertions(+), 83 deletions(-) diff --git a/manifest b/manifest index a949b5b360..9e5cb882e3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sthreadtest4.c\stest\sprogram.\s\sNot\syet\sworking. -D 2014-12-11T19:29:42.156 +C Fix\sa\sbug\sin\sthe\sthreadtest4.c\sprogram.\s\sRemove\sthe\skeyinfo\scache\sas\sit\sprovides\nminimal\sperformance\simprovements,\sand\sthen\sonly\sat\sSQL\spreparation\stime,\snot\nat\sruntime,\sand\sit\shas\sproblems\swith\sdata\sraces\sin\sshared-cache\smode.\s\sWe\smight\nlater\sadd\sthe\skeyinfo\scache\sback\sbut\sonly\senable\sit\swhen\sshared-cache\smode\nis\soff. +D 2014-12-12T00:20:37.023 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c ea6692ce58bfba55b12c75d2947fec0906d1ef7a F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 -F src/build.c 67bb05b1077e0cdaccb2e36bfcbe7a5df9ed31e8 +F src/build.c 162d84e4833b03f9d07192ef06057b0226f6e543 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660 -F src/main.c 4bfb07de96118350a63103380819ff8cbbefc5cd +F src/main.c 1f40f66165a6609203a5ff7ecb0292b90b302130 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -233,7 +233,7 @@ F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf F src/sqlite.h.in 116dc731361549ee3fc79dcebace11b57d24dcfd F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 28049b803b74a7f73242a8226915ea00ebb1309f +F src/sqliteInt.h 073d54f7a631b978b66d50d255c84549fb9e5429 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -304,7 +304,7 @@ F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 F src/wal.c 847692349eb6e1fb8543dbc97e69ddbfa4cc7ea7 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 -F src/where.c e914fdb9159bb36af4a673193bbda08aaf9e5a73 +F src/where.c d46de821bc604a4fd36fa3928c086950e91aafb1 F src/whereInt.h d3633e9b592103241b74b0ec76185f3e5b8b62e0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -913,7 +913,7 @@ F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 F test/threadtest3.c fca8d360b470405ae3ed431b5cb4cdf031f85a74 -F test/threadtest4.c e2bcee459bbd27fa2d55751e9900a4a5a4675cd5 +F test/threadtest4.c 38cb574939d5e0c8bd3baa5eb45def2ac6da4db4 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 @@ -1228,10 +1228,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 258e747bb7e3a2bc46f932cc2b06c2689d43aeb0 -R 4bb41891ae2017245bf160d9a28a8371 -T *branch * threadtest4 -T *sym-threadtest4 * -T -sym-trunk * +P ec3a74469ca2f0f3fb7d82a05fdac7500354e78f +R e481d4adf19d880dc699635431f7c379 U drh -Z 474616aece6be366bfcf36cb343f9bea +Z 4073b2f4a4cd5608e79ca3cad9ca9b28 diff --git a/manifest.uuid b/manifest.uuid index ea9eb0df3e..7461209f05 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec3a74469ca2f0f3fb7d82a05fdac7500354e78f \ No newline at end of file +b7489f9451628c68f1dfc1d457fc161a0921c631 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 0b4affc664..db954647cc 100644 --- a/src/build.c +++ b/src/build.c @@ -435,7 +435,6 @@ static void freeIndex(sqlite3 *db, Index *p){ #ifndef SQLITE_OMIT_ANALYZE sqlite3DeleteIndexSamples(db, p); #endif - if( db==0 || db->pnBytesFreed==0 ) sqlite3KeyInfoUnref(p->pKeyInfo); sqlite3ExprDelete(db, p->pPartIdxWhere); sqlite3DbFree(db, p->zColAff); if( p->isResized ) sqlite3DbFree(db, p->azColl); @@ -4190,40 +4189,31 @@ void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ ** when it has finished using it. */ KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ + int i; + int nCol = pIdx->nColumn; + int nKey = pIdx->nKeyCol; + KeyInfo *pKey; if( pParse->nErr ) return 0; -#ifndef SQLITE_OMIT_SHARED_CACHE - if( pIdx->pKeyInfo && pIdx->pKeyInfo->db!=pParse->db ){ - sqlite3KeyInfoUnref(pIdx->pKeyInfo); - pIdx->pKeyInfo = 0; + if( pIdx->uniqNotNull ){ + pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey); + }else{ + pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0); } -#endif - if( pIdx->pKeyInfo==0 ){ - int i; - int nCol = pIdx->nColumn; - int nKey = pIdx->nKeyCol; - KeyInfo *pKey; - if( pIdx->uniqNotNull ){ - pKey = sqlite3KeyInfoAlloc(pParse->db, nKey, nCol-nKey); - }else{ - pKey = sqlite3KeyInfoAlloc(pParse->db, nCol, 0); + if( pKey ){ + assert( sqlite3KeyInfoIsWriteable(pKey) ); + for(i=0; iazColl[i]; + assert( zColl!=0 ); + pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 : + sqlite3LocateCollSeq(pParse, zColl); + pKey->aSortOrder[i] = pIdx->aSortOrder[i]; } - if( pKey ){ - assert( sqlite3KeyInfoIsWriteable(pKey) ); - for(i=0; iazColl[i]; - assert( zColl!=0 ); - pKey->aColl[i] = strcmp(zColl,"BINARY")==0 ? 0 : - sqlite3LocateCollSeq(pParse, zColl); - pKey->aSortOrder[i] = pIdx->aSortOrder[i]; - } - if( pParse->nErr ){ - sqlite3KeyInfoUnref(pKey); - }else{ - pIdx->pKeyInfo = pKey; - } + if( pParse->nErr ){ + sqlite3KeyInfoUnref(pKey); + pKey = 0; } } - return sqlite3KeyInfoRef(pIdx->pKeyInfo); + return pKey; } #ifndef SQLITE_OMIT_CTE diff --git a/src/main.c b/src/main.c index 0727c0d75a..5dfa29279f 100644 --- a/src/main.c +++ b/src/main.c @@ -1032,16 +1032,6 @@ void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ for(j=0; jnDb; j++){ struct Db *pDb = &db->aDb[j]; if( pDb->pBt ){ - if( pDb->pSchema ){ - /* Must clear the KeyInfo cache. See ticket [e4a18565a36884b00edf] */ - sqlite3BtreeEnter(pDb->pBt); - for(i=sqliteHashFirst(&pDb->pSchema->idxHash); i; i=sqliteHashNext(i)){ - Index *pIdx = sqliteHashData(i); - sqlite3KeyInfoUnref(pIdx->pKeyInfo); - pIdx->pKeyInfo = 0; - } - sqlite3BtreeLeave(pDb->pBt); - } sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; if( j!=1 ){ @@ -2169,32 +2159,6 @@ const char *sqlite3_errstr(int rc){ return sqlite3ErrStr(rc); } -/* -** Invalidate all cached KeyInfo objects for database connection "db" -*/ -static void invalidateCachedKeyInfo(sqlite3 *db){ - Db *pDb; /* A single database */ - int iDb; /* The database index number */ - HashElem *k; /* For looping over tables in pDb */ - Table *pTab; /* A table in the database */ - Index *pIdx; /* Each index */ - - for(iDb=0, pDb=db->aDb; iDbnDb; iDb++, pDb++){ - if( pDb->pBt==0 ) continue; - sqlite3BtreeEnter(pDb->pBt); - for(k=sqliteHashFirst(&pDb->pSchema->tblHash); k; k=sqliteHashNext(k)){ - pTab = (Table*)sqliteHashData(k); - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->pKeyInfo && pIdx->pKeyInfo->db==db ){ - sqlite3KeyInfoUnref(pIdx->pKeyInfo); - pIdx->pKeyInfo = 0; - } - } - } - sqlite3BtreeLeave(pDb->pBt); - } -} - /* ** Create a new collating function for database "db". The name is zName ** and the encoding is enc. @@ -2238,7 +2202,6 @@ static int createCollation( return SQLITE_BUSY; } sqlite3ExpirePreparedStatements(db); - invalidateCachedKeyInfo(db); /* If collation sequence pColl was created directly by a call to ** sqlite3_create_collation, and not generated by synthCollSeq(), diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3498517f25..80f9597557 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1784,7 +1784,6 @@ struct Index { u8 *aSortOrder; /* for each column: True==DESC, False==ASC */ char **azColl; /* Array of collation sequence names for index */ Expr *pPartIdxWhere; /* WHERE clause for partial indices */ - KeyInfo *pKeyInfo; /* A KeyInfo object suitable for this index */ int tnum; /* DB Page containing root of this index */ LogEst szIdxRow; /* Estimated average row size in bytes */ u16 nKeyCol; /* Number of columns forming the key */ diff --git a/src/where.c b/src/where.c index ee42534f34..183a8cb667 100644 --- a/src/where.c +++ b/src/where.c @@ -3941,7 +3941,6 @@ static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ p->u.vtab.idxStr = 0; }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ sqlite3DbFree(db, p->u.btree.pIndex->zColAff); - sqlite3KeyInfoUnref(p->u.btree.pIndex->pKeyInfo); sqlite3DbFree(db, p->u.btree.pIndex); p->u.btree.pIndex = 0; } diff --git a/test/threadtest4.c b/test/threadtest4.c index e4cd644a4f..79ba094b1a 100644 --- a/test/threadtest4.c +++ b/test/threadtest4.c @@ -301,6 +301,7 @@ static void *worker_thread(void *pArg){ worker_error(p, "Wrong result: %d", sqlite3_column_int(pStmt,0)); } if( p->nErr ) break; + sqlite3_finalize(pStmt); worker_delete_all_content(p, (p->tid+iOuter)%2); worker_close_connection(p); From 5942b01611b9d44d7c64898b24a414d1dfbe5821 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 12 Dec 2014 00:40:58 +0000 Subject: [PATCH 650/710] Make sure the Btree mutex is held when setting the locking mode and the secure delete flag when attaching a shared-cache database. FossilOrigin-Name: 6bef7ede2bbf0a51729e1943b0b0c895cb57c718 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/attach.c | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3dc59d55a4..022b15647e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\scell\soverflow\spage\snumber\scache\sthread\srace\sfix\sfrom\strunk. -D 2014-12-12T00:26:59.974 +C Make\ssure\sthe\sBtree\smutex\sis\sheld\swhen\ssetting\sthe\slocking\smode\sand\sthe\nsecure\sdelete\sflag\swhen\sattaching\sa\sshared-cache\sdatabase. +D 2014-12-12T00:40:58.882 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -168,7 +168,7 @@ F sqlite3.1 fc7ad8990fc8409983309bb80de8c811a7506786 F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c ba266a779bc7ce10e52e59e7d3dc79fa342e8fdb F src/analyze.c 7a2986e6ea8247e5f21aca3d0b584598f58d84fe -F src/attach.c f4e94df2d1826feda65eb0939f7f6f5f923a0ad9 +F src/attach.c 7f6b3fafa2290b407e4a94dcf1afda7ec0fe394b F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb @@ -1229,7 +1229,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b7489f9451628c68f1dfc1d457fc161a0921c631 fc157dd7f18c94b7ae5f155e1b4a5d7714b7da8c -R e0ac31855803c332d28311de41dcc42f +P cefad47ec2ad58d7ecd58bab9a261e4d5816cd69 +R 43ea10bc03cfab6b8cae2a465b94aee2 U drh -Z 01b0e5c37f35211bc16bf2591a3974d7 +Z 01d198dabe70d93c23f2c971bf866fd9 diff --git a/manifest.uuid b/manifest.uuid index b6c1d34b27..2d4407626b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cefad47ec2ad58d7ecd58bab9a261e4d5816cd69 \ No newline at end of file +6bef7ede2bbf0a51729e1943b0b0c895cb57c718 \ No newline at end of file diff --git a/src/attach.c b/src/attach.c index cf52bb24b1..de8742938b 100644 --- a/src/attach.c +++ b/src/attach.c @@ -150,6 +150,7 @@ static void attachFunc( "attached databases must use the same text encoding as main database"); rc = SQLITE_ERROR; } + sqlite3BtreeEnter(aNew->pBt); pPager = sqlite3BtreePager(aNew->pBt); sqlite3PagerLockingMode(pPager, db->dfltLockMode); sqlite3BtreeSecureDelete(aNew->pBt, @@ -157,6 +158,7 @@ static void attachFunc( #ifndef SQLITE_OMIT_PAGER_PRAGMAS sqlite3BtreeSetPagerFlags(aNew->pBt, 3 | (db->flags & PAGER_FLAGS_MASK)); #endif + sqlite3BtreeLeave(aNew->pBt); } aNew->safety_level = 3; aNew->zName = sqlite3DbStrDup(db, zName); From ef15c6e9e65cec6f9a847b8780265ca6d4124286 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 12 Dec 2014 01:27:17 +0000 Subject: [PATCH 651/710] Add new tests to the threadtest4.c program. Fix a long-standing data race in WAL mode for shared-cache. FossilOrigin-Name: d8d3e6d04cbb9e3033ad8613e3dbd4ad0b01765a --- manifest | 15 +++++++-------- manifest.uuid | 2 +- src/vdbeapi.c | 5 ++++- test/threadtest4.c | 27 +++++++++++++++++++++++---- 4 files changed, 35 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 446beca49b..fb9bed5983 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sKeyInfo\scache\s(for\snow\s-\sperhaps\swe\swill\sadd\sit\sback\sin\slater\s-\sor\nmaybe\snot\ssince\sit\sprovides\snegligible\sbenefit\sbut\sadds\sa\slot\sof\scomplexity\nand\sthread-safety\srisk).\s\sAdd\sa\smutex\sto\sATTACH\sto\sdeal\swith\sa\sdata\srace. -D 2014-12-12T00:52:10.892 +C Add\snew\stests\sto\sthe\sthreadtest4.c\sprogram.\s\sFix\sa\slong-standing\sdata\srace\nin\sWAL\smode\sfor\sshared-cache. +D 2014-12-12T01:27:17.213 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -294,7 +294,7 @@ F src/vacuum.c 9b30ec729337dd012ed88d4c292922c8ef9cf00c F src/vdbe.c 1a9e671c9cfc259e4d2affc71f7df4a4c00a842c F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 -F src/vdbeapi.c f8dd2d33a30938188fc292d524e88a91f2e65887 +F src/vdbeapi.c 4bc511a46b9839392ae0e90844a71dc96d9dbd71 F src/vdbeaux.c 6f7f39c3fcf0f5923758df8561bb5d843908a553 F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f @@ -913,7 +913,7 @@ F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 F test/threadtest3.c 2b6e07e915c383c250a5b531cf6ef163a3047d7e -F test/threadtest4.c 38cb574939d5e0c8bd3baa5eb45def2ac6da4db4 +F test/threadtest4.c 1678c340387c19ae28b18e4d8f71d4a989297e46 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 @@ -1229,8 +1229,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P fc157dd7f18c94b7ae5f155e1b4a5d7714b7da8c 6bef7ede2bbf0a51729e1943b0b0c895cb57c718 -R 43ea10bc03cfab6b8cae2a465b94aee2 -T +closed 6bef7ede2bbf0a51729e1943b0b0c895cb57c718 +P 03c443eaf24413d6faaa91a33575d9dfd3528b5c +R 7d057661252570fbf3877a000bcb607e U drh -Z 6886c071ca5a249d6b983c22ce222f5d +Z 24d2ddd2be9b77d0c401fa6e4cbdad23 diff --git a/manifest.uuid b/manifest.uuid index e881507359..82e8f8a854 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -03c443eaf24413d6faaa91a33575d9dfd3528b5c \ No newline at end of file +d8d3e6d04cbb9e3033ad8613e3dbd4ad0b01765a \ No newline at end of file diff --git a/src/vdbeapi.c b/src/vdbeapi.c index c611688193..21c537d776 100644 --- a/src/vdbeapi.c +++ b/src/vdbeapi.c @@ -400,7 +400,10 @@ static int doWalCallbacks(sqlite3 *db){ for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( pBt ){ - int nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); + int nEntry; + sqlite3BtreeEnter(pBt); + nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); + sqlite3BtreeLeave(pBt); if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){ rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry); } diff --git a/test/threadtest4.c b/test/threadtest4.c index 79ba094b1a..859841c738 100644 --- a/test/threadtest4.c +++ b/test/threadtest4.c @@ -16,13 +16,17 @@ ** This test program runs on unix-like systems only. It uses pthreads. ** To compile: ** -** gcc -o tt4 -I. threadtest4.c sqlite3.c -ldl -lpthread +** gcc -g -Wall -I. threadtest4.c sqlite3.c -ldl -lpthread ** ** To run: ** -** ./tt4 10 +** ./a.out 10 ** -** The argument is the number of threads. +** The argument is the number of threads. There are also options, such +** as -wal and -multithread and -serialized. +** +** Consider also compiling with clang instead of gcc and adding the +** -fsanitize=thread option. */ #include "sqlite3.h" #include @@ -40,6 +44,7 @@ typedef struct WorkerInfo WorkerInfo; struct WorkerInfo { int tid; /* Thread ID */ + int nWorker; /* Total number of workers */ unsigned wkrFlags; /* Flags */ sqlite3 *mainDb; /* Database connection of the main thread */ sqlite3 *db; /* Database connection of this thread */ @@ -284,7 +289,7 @@ static void *worker_thread(void *pArg){ sqlite3_stmt *pStmt; printf("worker %d startup\n", p->tid); fflush(stdout); - for(iOuter=1; iOuter<=4; iOuter++){ + for(iOuter=1; iOuter<=p->nWorker; iOuter++){ worker_open_connection(p, iOuter); for(i=0; i<4; i++){ worker_add_content(p, i*100+1, (i+1)*100, (p->tid+iOuter)%3 + 1); @@ -303,6 +308,17 @@ static void *worker_thread(void *pArg){ if( p->nErr ) break; sqlite3_finalize(pStmt); + if( ((iOuter+p->tid)%3)==0 ){ + sqlite3_db_release_memory(p->db); + p->nTest++; + } + + if( iOuter==p->tid ){ + pthread_mutex_lock(p->pWrMutex); + run_sql(p, "VACUUM"); + pthread_mutex_unlock(p->pWrMutex); + } + worker_delete_all_content(p, (p->tid+iOuter)%2); worker_close_connection(p); p->db = 0; @@ -362,6 +378,8 @@ int main(int argc, char **argv){ "Options:\n" " --serialized\n" " --multithread\n" + " --wal\n" + " --trace\n" ,argv[0] ); exit(1); @@ -406,6 +424,7 @@ int main(int argc, char **argv){ memset(aInfo, 0, sizeof(*aInfo)*nWorker); for(i=0; i Date: Fri, 12 Dec 2014 16:39:38 +0000 Subject: [PATCH 652/710] Add extra tests to threadtest3. FossilOrigin-Name: f6bf86f907cbff31bed3cbfc922c10c973575498 --- main.mk | 12 +++- manifest | 20 +++--- manifest.uuid | 2 +- test/threadtest3.c | 6 +- test/tt3_index.c | 22 +++---- test/tt3_stress.c | 161 +++++++++++++++++++++++++++++++++++++++++++++ test/tt3_vacuum.c | 90 +++++++++++++++++++++++++ 7 files changed, 288 insertions(+), 25 deletions(-) create mode 100644 test/tt3_stress.c create mode 100644 test/tt3_vacuum.c diff --git a/main.mk b/main.mk index cac996864d..e2213bc62c 100644 --- a/main.mk +++ b/main.mk @@ -625,9 +625,15 @@ test: testfixture$(EXE) sqlite3$(EXE) # threadtest runs a few thread-safety tests that are implemented in C. This # target is invoked by the releasetest.tcl script. # -threadtest3$(EXE): sqlite3.o $(TOP)/test/threadtest3.c $(TOP)/test/tt3_checkpoint.c - $(TCCX) -O2 sqlite3.o $(TOP)/test/threadtest3.c \ - -o threadtest3$(EXE) $(THREADLIB) +THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ + $(TOP)/test/tt3_checkpoint.c \ + $(TOP)/test/tt3_index.c \ + $(TOP)/test/tt3_vacuum.c \ + $(TOP)/test/tt3_stress.c \ + $(TOP)/test/tt3_lookaside1.c + +threadtest3$(EXE): sqlite3.o $(THREADTEST3_SRC) + $(TCCX) $(TOP)/test/threadtest3.c sqlite3.o -o $@ $(THREADLIB) threadtest: threadtest3$(EXE) ./threadtest3$(EXE) diff --git a/manifest b/manifest index fb9bed5983..599bf4fb39 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stests\sto\sthe\sthreadtest4.c\sprogram.\s\sFix\sa\slong-standing\sdata\srace\nin\sWAL\smode\sfor\sshared-cache. -D 2014-12-12T01:27:17.213 +C Add\sextra\stests\sto\sthreadtest3. +D 2014-12-12T16:39:38.824 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -152,7 +152,7 @@ F ext/userauth/userauth.c 5fa3bdb492f481bbc1709fc83c91ebd13460c69e F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8 F magic.txt 8273bf49ba3b0c8559cb2774495390c31fd61c60 -F main.mk 084976077a4aa3bd985154b5423e7aed88e4a2e9 +F main.mk 9f8c54fe62b60e0a24a2e65cfc8d2add063dda07 F mkopcodec.awk c2ff431854d702cdd2d779c9c0d1f58fa16fa4ea F mkopcodeh.awk c6b3fa301db6ef7ac916b14c60868aeaec1337b5 F mkso.sh fd21c06b063bb16a5d25deea1752c2da6ac3ed83 @@ -912,7 +912,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c 2b6e07e915c383c250a5b531cf6ef163a3047d7e +F test/threadtest3.c bef2bde18b4e638b6cf4b119aa2076123ffdc425 F test/threadtest4.c 1678c340387c19ae28b18e4d8f71d4a989297e46 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 @@ -1077,8 +1077,10 @@ F test/triggerC.test a68980c5955d62ee24be6f97129d824f199f9a4c F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 355e9c5cbaed5cd039a60baad1fb2197caeb8e52 F test/tt3_checkpoint.c 415eccce672d681b297485fc20f44cdf0eac93af -F test/tt3_index.c 14f4b0cc3f4c05353e25778cf531c9d6e3b0d27f +F test/tt3_index.c 652630e6b6fc7a48adf2ead60de1ef48e1a34569 F test/tt3_lookaside1.c 0b5b79ba37f21a1eb849cd4a54eed367f4d4aaaf +F test/tt3_stress.c e003a8486ba990c43dc9b8298f4b266cd10eb1c1 +F test/tt3_vacuum.c 6a66e52e2b39fc0cccb71db5a302411f34d09736 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a @@ -1229,7 +1231,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 03c443eaf24413d6faaa91a33575d9dfd3528b5c -R 7d057661252570fbf3877a000bcb607e -U drh -Z 24d2ddd2be9b77d0c401fa6e4cbdad23 +P d8d3e6d04cbb9e3033ad8613e3dbd4ad0b01765a +R 2ae2d517339c22d48400a669b2098fec +U dan +Z 951a78f63dc9b91d226749536e3eb614 diff --git a/manifest.uuid b/manifest.uuid index 82e8f8a854..816a546aae 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d8d3e6d04cbb9e3033ad8613e3dbd4ad0b01765a \ No newline at end of file +f6bf86f907cbff31bed3cbfc922c10c973575498 \ No newline at end of file diff --git a/test/threadtest3.c b/test/threadtest3.c index afa4197ee2..9b12616e3a 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -1399,6 +1399,8 @@ static void dynamic_triggers(int nMs){ #include "tt3_checkpoint.c" #include "tt3_index.c" #include "tt3_lookaside1.c" +#include "tt3_vacuum.c" +#include "tt3_stress.c" int main(int argc, char **argv){ struct ThreadTest { @@ -1420,7 +1422,9 @@ int main(int argc, char **argv){ { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 }, { create_drop_index_1, "create_drop_index_1", 10000 }, - { lookaside1, "lookaside1", 10000 }, + { lookaside1, "lookaside1", 10000 }, + { vacuum1, "vacuum1", 10000 }, + { stress1, "stress1", 10000 }, }; int i; diff --git a/test/tt3_index.c b/test/tt3_index.c index 4bad64aabc..2445f398cd 100644 --- a/test/tt3_index.c +++ b/test/tt3_index.c @@ -22,22 +22,22 @@ static char *create_drop_index_thread(int iTid, int iArg){ opendb(&err, &db, "test.db", 0); sql_script(&err, &db, - "DROP INDEX IF EXISTS i1;" "DROP INDEX IF EXISTS i2;" "DROP INDEX IF EXISTS i3;" "DROP INDEX IF EXISTS i4;" - "CREATE INDEX IF NOT EXISTS i1 ON t1(a);" - "CREATE INDEX IF NOT EXISTS i2 ON t1(b);" - "CREATE INDEX IF NOT EXISTS i3 ON t1(c);" - "CREATE INDEX IF NOT EXISTS i4 ON t1(d);" + "CREATE INDEX IF NOT EXISTS i1 ON t11(a);" + "CREATE INDEX IF NOT EXISTS i2 ON t11(b);" + "CREATE INDEX IF NOT EXISTS i3 ON t11(c);" + "CREATE INDEX IF NOT EXISTS i4 ON t11(d);" - "SELECT * FROM t1 ORDER BY a;" - "SELECT * FROM t1 ORDER BY b;" - "SELECT * FROM t1 ORDER BY c;" - "SELECT * FROM t1 ORDER BY d;" + "SELECT * FROM t11 ORDER BY a;" + "SELECT * FROM t11 ORDER BY b;" + "SELECT * FROM t11 ORDER BY c;" + "SELECT * FROM t11 ORDER BY d;" ); + clear_error(&err, SQLITE_LOCKED); closedb(&err, &db); } @@ -53,9 +53,9 @@ static void create_drop_index_1(int nMs){ opendb(&err, &db, "test.db", 1); sql_script(&err, &db, - "CREATE TABLE t1(a, b, c, d);" + "CREATE TABLE t11(a, b, c, d);" "WITH data(x) AS (SELECT 1 UNION ALL SELECT x+1 FROM data WHERE x<100) " - "INSERT INTO t1 SELECT x,x,x,x FROM data;" + "INSERT INTO t11 SELECT x,x,x,x FROM data;" ); closedb(&err, &db); diff --git a/test/tt3_stress.c b/test/tt3_stress.c new file mode 100644 index 0000000000..93a0d0aa1c --- /dev/null +++ b/test/tt3_stress.c @@ -0,0 +1,161 @@ +/* +** 2014 December 9 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** +*/ + + +/* +** Thread 1. CREATE and DROP a table. +*/ +static char *stress_thread_1(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + + opendb(&err, &db, "test.db", 0); + while( !timetostop(&err) ){ + sql_script(&err, &db, "CREATE TABLE IF NOT EXISTS t1(a PRIMARY KEY, b)"); + clear_error(&err, SQLITE_LOCKED); + sql_script(&err, &db, "DROP TABLE IF EXISTS t1"); + clear_error(&err, SQLITE_LOCKED); + } + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + +/* +** Thread 2. Open and close database connections. +*/ +static char *stress_thread_2(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + while( !timetostop(&err) ){ + opendb(&err, &db, "test.db", 0); + sql_script(&err, &db, "SELECT * FROM sqlite_master;"); + clear_error(&err, SQLITE_LOCKED); + closedb(&err, &db); + } + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + +/* +** Thread 3. Attempt many small SELECT statements. +*/ +static char *stress_thread_3(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + + int i1 = 0; + int i2 = 0; + + opendb(&err, &db, "test.db", 0); + while( !timetostop(&err) ){ + sql_script(&err, &db, "SELECT * FROM t1 ORDER BY a;"); + i1++; + if( err.rc ) i2++; + clear_error(&err, SQLITE_LOCKED); + clear_error(&err, SQLITE_ERROR); + } + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("read t1 %d/%d attempts", i2, i1); +} + +/* +** Thread 5. Attempt INSERT statements. +*/ +static char *stress_thread_4(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + int i1 = 0; + int i2 = 0; + opendb(&err, &db, "test.db", 0); + while( !timetostop(&err) ){ + if( iArg ){ + closedb(&err, &db); + opendb(&err, &db, "test.db", 0); + } + sql_script(&err, &db, + "WITH loop(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM loop LIMIT 200) " + "INSERT INTO t1 VALUES(randomblob(60), randomblob(60));" + ); + i1++; + if( err.rc ) i2++; + clear_error(&err, SQLITE_LOCKED); + clear_error(&err, SQLITE_ERROR); + } + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("wrote t1 %d/%d attempts", i2, i1); +} + +/* +** Thread 6. Attempt DELETE operations. +*/ +static char *stress_thread_5(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + + int i1 = 0; + int i2 = 0; + + opendb(&err, &db, "test.db", 0); + while( !timetostop(&err) ){ + i64 i = (i1 % 4); + if( iArg ){ + closedb(&err, &db); + opendb(&err, &db, "test.db", 0); + } + execsql(&err, &db, "DELETE FROM t1 WHERE (rowid % 4)==:i", &i); + i1++; + if( err.rc ) i2++; + clear_error(&err, SQLITE_LOCKED); + clear_error(&err, SQLITE_ERROR); + } + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("deleted from t1 %d/%d attempts", i2, i1); +} + + +static void stress1(int nMs){ + Error err = {0}; + Sqlite db = {0}; + Threadset threads = {0}; + + + setstoptime(&err, nMs); + + sqlite3_enable_shared_cache(1); + + launch_thread(&err, &threads, stress_thread_1, 0); + launch_thread(&err, &threads, stress_thread_1, 0); + + launch_thread(&err, &threads, stress_thread_2, 0); + launch_thread(&err, &threads, stress_thread_2, 0); + + launch_thread(&err, &threads, stress_thread_3, 0); + launch_thread(&err, &threads, stress_thread_3, 0); + + launch_thread(&err, &threads, stress_thread_4, 0); + launch_thread(&err, &threads, stress_thread_4, 0); + + launch_thread(&err, &threads, stress_thread_5, 0); + launch_thread(&err, &threads, stress_thread_5, 1); + + join_all_threads(&err, &threads); + sqlite3_enable_shared_cache(0); + + print_and_free_err(&err); +} diff --git a/test/tt3_vacuum.c b/test/tt3_vacuum.c new file mode 100644 index 0000000000..126bbfc26e --- /dev/null +++ b/test/tt3_vacuum.c @@ -0,0 +1,90 @@ +/* +** 2014 December 9 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains multi-threaded tests that use shared-cache and +** the VACUUM command. +** +** Tests: +** +** vacuum1 +** +*/ + + +static char *vacuum1_thread_writer(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + opendb(&err, &db, "test.db", 0); + i64 i = 0; + + while( !timetostop(&err) ){ + i++; + + /* Insert lots of rows. Then delete some. */ + execsql(&err, &db, + "WITH loop(i) AS (SELECT 1 UNION ALL SELECT i+1 FROM loop WHERE i<100) " + "INSERT INTO t1 SELECT randomblob(50), randomblob(2500) FROM loop" + ); + + /* Delete lots of rows */ + execsql(&err, &db, "DELETE FROM t1 WHERE rowid = :i", &i); + clear_error(&err, SQLITE_LOCKED); + + /* Select the rows */ + execsql(&err, &db, "SELECT * FROM t1 ORDER BY x"); + clear_error(&err, SQLITE_LOCKED); + } + + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + +static char *vacuum1_thread_vacuumer(int iTid, int iArg){ + Error err = {0}; /* Error code and message */ + Sqlite db = {0}; /* SQLite database connection */ + opendb(&err, &db, "test.db", 0); + + do{ + sql_script(&err, &db, "VACUUM"); + clear_error(&err, SQLITE_LOCKED); + }while( !timetostop(&err) ); + + closedb(&err, &db); + print_and_free_err(&err); + return sqlite3_mprintf("ok"); +} + +static void vacuum1(int nMs){ + Error err = {0}; + Sqlite db = {0}; + Threadset threads = {0}; + + opendb(&err, &db, "test.db", 1); + sql_script(&err, &db, + "CREATE TABLE t1(x PRIMARY KEY, y BLOB);" + "CREATE INDEX i1 ON t1(y);" + ); + closedb(&err, &db); + + setstoptime(&err, nMs); + + sqlite3_enable_shared_cache(1); + launch_thread(&err, &threads, vacuum1_thread_writer, 0); + launch_thread(&err, &threads, vacuum1_thread_writer, 0); + launch_thread(&err, &threads, vacuum1_thread_writer, 0); + launch_thread(&err, &threads, vacuum1_thread_vacuumer, 0); + join_all_threads(&err, &threads); + sqlite3_enable_shared_cache(0); + + print_and_free_err(&err); +} From 9bd3cc46814595f1107d95f372ce912e947e5bf5 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 12 Dec 2014 23:17:54 +0000 Subject: [PATCH 653/710] Add extra tests to threadtest4.c. Fix a benign data race accessing the text encoding using ENC(db). FossilOrigin-Name: d7bb7ea4ab97ad26f4c84c9b8dc2827010093803 --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/main.c | 3 ++- src/pragma.c | 3 ++- src/prepare.c | 2 ++ src/sqliteInt.h | 4 +++- test/threadtest4.c | 31 ++++++++++++++++++++++++++++++- 7 files changed, 51 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 599bf4fb39..ff005f7ef4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests\sto\sthreadtest3. -D 2014-12-12T16:39:38.824 +C Add\sextra\stests\sto\sthreadtest4.c.\s\sFix\sa\sbenign\sdata\srace\saccessing\sthe\ntext\sencoding\susing\sENC(db). +D 2014-12-12T23:17:54.003 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660 -F src/main.c 1f40f66165a6609203a5ff7ecb0292b90b302130 +F src/main.c 48e0410a661c629471ca9061d4153245cc9f853b F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -222,8 +222,8 @@ F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 F src/pcache1.c facbdd3ecc09c8f750089d941305694301328e98 -F src/pragma.c d54cdd40b63d608f2d95b7482c710690e3593a73 -F src/prepare.c b7b7bf020bd4c962f7c8aed5a3c542c7dfe9f9c7 +F src/pragma.c c93be505649183b2d80082c2eef1a56879dabfe6 +F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada @@ -233,7 +233,7 @@ F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf F src/sqlite.h.in 116dc731361549ee3fc79dcebace11b57d24dcfd F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d -F src/sqliteInt.h 073d54f7a631b978b66d50d255c84549fb9e5429 +F src/sqliteInt.h d36da9a07130cae13cbfee0986bf20028cb01465 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc @@ -913,7 +913,7 @@ F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 F test/threadtest3.c bef2bde18b4e638b6cf4b119aa2076123ffdc425 -F test/threadtest4.c 1678c340387c19ae28b18e4d8f71d4a989297e46 +F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 F test/tkt-2a5629202f.test 0521bd25658428baa26665aa53ffed9367d33af2 @@ -1231,7 +1231,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d8d3e6d04cbb9e3033ad8613e3dbd4ad0b01765a -R 2ae2d517339c22d48400a669b2098fec -U dan -Z 951a78f63dc9b91d226749536e3eb614 +P f6bf86f907cbff31bed3cbfc922c10c973575498 +R 97f096b77c750d9e694c3063635e4c8a +U drh +Z 4a2c4212438d559f2811b75a6a368836 diff --git a/manifest.uuid b/manifest.uuid index 816a546aae..2694c2c363 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f6bf86f907cbff31bed3cbfc922c10c973575498 \ No newline at end of file +d7bb7ea4ab97ad26f4c84c9b8dc2827010093803 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 5dfa29279f..c89ccbd55d 100644 --- a/src/main.c +++ b/src/main.c @@ -2758,6 +2758,7 @@ static int openDatabase( } sqlite3BtreeEnter(db->aDb[0].pBt); db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); + if( !db->mallocFailed ) ENC(db) = SCHEMA_ENC(db); sqlite3BtreeLeave(db->aDb[0].pBt); db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); @@ -2916,7 +2917,7 @@ int sqlite3_open16( SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0); assert( *ppDb || rc==SQLITE_NOMEM ); if( rc==SQLITE_OK && !DbHasProperty(*ppDb, 0, DB_SchemaLoaded) ){ - ENC(*ppDb) = SQLITE_UTF16NATIVE; + SCHEMA_ENC(*ppDb) = ENC(*ppDb) = SQLITE_UTF16NATIVE; } }else{ rc = SQLITE_NOMEM; diff --git a/src/pragma.c b/src/pragma.c index ab9a283629..837a15102d 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -2080,7 +2080,8 @@ void sqlite3Pragma( ){ for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ if( 0==sqlite3StrICmp(zRight, pEnc->zName) ){ - ENC(pParse->db) = pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; + SCHEMA_ENC(db) = ENC(db) = + pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; break; } } diff --git a/src/prepare.c b/src/prepare.c index ca9c64b441..97be900d68 100644 --- a/src/prepare.c +++ b/src/prepare.c @@ -394,9 +394,11 @@ int sqlite3Init(sqlite3 *db, char **pzErrMsg){ int commit_internal = !(db->flags&SQLITE_InternChanges); assert( sqlite3_mutex_held(db->mutex) ); + assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) ); assert( db->init.busy==0 ); rc = SQLITE_OK; db->init.busy = 1; + ENC(db) = SCHEMA_ENC(db); for(i=0; rc==SQLITE_OK && inDb; i++){ if( DbHasProperty(db, i, DB_SchemaLoaded) || i==1 ) continue; rc = sqlite3InitOne(db, i, pzErrMsg); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 80f9597557..d90089ba0e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1059,6 +1059,7 @@ struct sqlite3 { int errCode; /* Most recent error code (SQLITE_*) */ int errMask; /* & result codes with this before returning */ u16 dbOptFlags; /* Flags to enable/disable optimizations */ + u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ u8 mallocFailed; /* True if we have seen a malloc failure */ @@ -1160,7 +1161,8 @@ struct sqlite3 { /* ** A macro to discover the encoding of a database. */ -#define ENC(db) ((db)->aDb[0].pSchema->enc) +#define SCHEMA_ENC(db) ((db)->aDb[0].pSchema->enc) +#define ENC(db) ((db)->enc) /* ** Possible values for the sqlite3.flags. diff --git a/test/threadtest4.c b/test/threadtest4.c index 859841c738..da5ce75f05 100644 --- a/test/threadtest4.c +++ b/test/threadtest4.c @@ -305,20 +305,49 @@ static void *worker_thread(void *pArg){ }else if( sqlite3_column_int(pStmt, 0)!=400 ){ worker_error(p, "Wrong result: %d", sqlite3_column_int(pStmt,0)); } - if( p->nErr ) break; sqlite3_finalize(pStmt); + if( p->nErr ) break; if( ((iOuter+p->tid)%3)==0 ){ sqlite3_db_release_memory(p->db); p->nTest++; } + pthread_mutex_lock(p->pWrMutex); + run_sql(p, "BEGIN;"); + run_sql(p, "UPDATE t1 SET c=NULL WHERE a=55"); + run_sql(p, "UPDATE t2 SET f=NULL WHERE d=42"); + run_sql(p, "UPDATE t3 SET z=NULL WHERE x=31"); + run_sql(p, "ROLLBACK;"); + p->nTest++; + pthread_mutex_unlock(p->pWrMutex); + + if( iOuter==p->tid ){ pthread_mutex_lock(p->pWrMutex); run_sql(p, "VACUUM"); pthread_mutex_unlock(p->pWrMutex); } + pStmt = prep_sql(p->db, + "SELECT t1.rowid, t2.rowid, t3.rowid" + " FROM t1, t2, t3" + " WHERE t1.tid=%d AND t2.tid=%d AND t3.tid=%d" + " AND t1.a<>t2.d AND t2.d<>t3.x" + " ORDER BY 1, 2, 3" + ,p->tid, p->tid, p->tid); + worker_trace(p, "query [%s]", sqlite3_sql(pStmt)); + for(i=0; inWorker; i++){ + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + worker_error(p, "Failed to step: %s", sqlite3_sql(pStmt)); + break; + } + sched_yield(); + } + sqlite3_finalize(pStmt); + if( p->nErr ) break; + worker_delete_all_content(p, (p->tid+iOuter)%2); worker_close_connection(p); p->db = 0; From 053542d72a174f16644bba4d580d41da75f038f7 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 13 Dec 2014 17:41:48 +0000 Subject: [PATCH 654/710] Further enhancements to threadtest3 stress tests. FossilOrigin-Name: ba772cff602ca7c3c0c91451e701f52a872e7a14 --- manifest | 24 ++--- manifest.uuid | 2 +- test/threadtest3.c | 83 +++++++++------ test/tt3_checkpoint.c | 2 +- test/tt3_index.c | 2 +- test/tt3_lookaside1.c | 4 +- test/tt3_stress.c | 238 ++++++++++++++++++++++++++++++++++++++++-- test/tt3_vacuum.c | 4 +- 8 files changed, 299 insertions(+), 60 deletions(-) diff --git a/manifest b/manifest index ff005f7ef4..d0026036ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests\sto\sthreadtest4.c.\s\sFix\sa\sbenign\sdata\srace\saccessing\sthe\ntext\sencoding\susing\sENC(db). -D 2014-12-12T23:17:54.003 +C Further\senhancements\sto\sthreadtest3\sstress\stests. +D 2014-12-13T17:41:48.275 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -912,7 +912,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c bef2bde18b4e638b6cf4b119aa2076123ffdc425 +F test/threadtest3.c 05d67ab8fd4ad82978a96200701a22af35faa097 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 @@ -1076,11 +1076,11 @@ F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test a68980c5955d62ee24be6f97129d824f199f9a4c F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 355e9c5cbaed5cd039a60baad1fb2197caeb8e52 -F test/tt3_checkpoint.c 415eccce672d681b297485fc20f44cdf0eac93af -F test/tt3_index.c 652630e6b6fc7a48adf2ead60de1ef48e1a34569 -F test/tt3_lookaside1.c 0b5b79ba37f21a1eb849cd4a54eed367f4d4aaaf -F test/tt3_stress.c e003a8486ba990c43dc9b8298f4b266cd10eb1c1 -F test/tt3_vacuum.c 6a66e52e2b39fc0cccb71db5a302411f34d09736 +F test/tt3_checkpoint.c 5e63ee65ed5f87176e25a996480cb02c6caec8b4 +F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a +F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9 +F test/tt3_stress.c edbb00ed1516535691040315e97cf32c62df22d0 +F test/tt3_vacuum.c 1753f45917699c9c1f66b64c717a717c9379f776 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a @@ -1231,7 +1231,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f6bf86f907cbff31bed3cbfc922c10c973575498 -R 97f096b77c750d9e694c3063635e4c8a -U drh -Z 4a2c4212438d559f2811b75a6a368836 +P d7bb7ea4ab97ad26f4c84c9b8dc2827010093803 +R 5d62798e848b5017221fa2f718c6fa3d +U dan +Z 748bf95122bf9ac21f60dff7348c8009 diff --git a/manifest.uuid b/manifest.uuid index 2694c2c363..4d5b9316c6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d7bb7ea4ab97ad26f4c84c9b8dc2827010093803 \ No newline at end of file +ba772cff602ca7c3c0c91451e701f52a872e7a14 \ No newline at end of file diff --git a/test/threadtest3.c b/test/threadtest3.c index 9b12616e3a..4cdb0f8e9b 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -47,10 +47,13 @@ #define execsql_i64(x,y,...) (SEL(x), execsql_i64_x(x,y,__VA_ARGS__)) #define execsql_text(x,y,z,...) (SEL(x), execsql_text_x(x,y,z,__VA_ARGS__)) #define execsql(x,y,...) (SEL(x), (void)execsql_i64_x(x,y,__VA_ARGS__)) +#define sql_script_printf(x,y,z,...) ( \ + SEL(x), sql_script_printf_x(x,y,z,__VA_ARGS__) \ +) /* Thread functions */ -#define launch_thread(w,x,y,z) (SEL(w), launch_thread_x(w,x,y,z)) -#define join_all_threads(y,z) (SEL(y), join_all_threads_x(y,z)) +#define launch_thread(w,x,y,z) (SEL(w), launch_thread_x(w,x,y,z)) +#define join_all_threads(y,z) (SEL(y), join_all_threads_x(y,z)) /* Timer functions */ #define setstoptime(y,z) (SEL(y), setstoptime_x(y,z)) @@ -64,6 +67,9 @@ #define filesize(y,z) (SEL(y), filesize_x(y,z)) #define filecopy(x,y,z) (SEL(x), filecopy_x(x,y,z)) +#define PTR2INT(x) ((int)((intptr_t)x)) +#define INT2PTR(x) ((void*)((intptr_t)x)) + /* ** End of test code/infrastructure interface macros. *************************************************************************/ @@ -334,7 +340,7 @@ static void MD5Final(unsigned char digest[16], MD5Context *ctx){ MD5Transform(ctx->buf, (uint32 *)ctx->in); byteReverse((unsigned char *)ctx->buf, 4); memcpy(digest, ctx->buf, 16); - memset(ctx, 0, sizeof(ctx)); /* In case it is sensitive */ + memset(ctx, 0, sizeof(*ctx)); /* In case it is sensitive */ } /* @@ -398,9 +404,6 @@ typedef struct Thread Thread; /* Total number of errors in this process so far. */ static int nGlobalErr = 0; -/* Set to true to run in "process" instead of "thread" mode. */ -static int bProcessMode = 0; - struct Error { int rc; int iLine; @@ -421,10 +424,10 @@ struct Statement { struct Thread { int iTid; /* Thread number within test */ - int iArg; /* Integer argument passed by caller */ + void* pArg; /* Pointer argument passed by caller */ pthread_t tid; /* Thread id */ - char *(*xProc)(int, int); /* Thread main proc */ + char *(*xProc)(int, void*); /* Thread main proc */ Thread *pNext; /* Next in this list of threads */ }; @@ -506,8 +509,9 @@ static void opendb_x( ){ if( pErr->rc==SQLITE_OK ){ int rc; + int flags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI; if( bDelete ) unlink(zFile); - rc = sqlite3_open(zFile, &pDb->db); + rc = sqlite3_open_v2(zFile, &pDb->db, flags, 0); if( rc ){ sqlite_error(pErr, pDb, "open"); sqlite3_close(pDb->db); @@ -556,6 +560,22 @@ static void sql_script_x( } } +static void sql_script_printf_x( + Error *pErr, /* IN/OUT: Error code */ + Sqlite *pDb, /* Database handle */ + const char *zFormat, /* SQL printf format string */ + ... /* Printf args */ +){ + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + if( pErr->rc==SQLITE_OK ){ + char *zSql = sqlite3_vmprintf(zFormat, ap); + pErr->rc = sqlite3_exec(pDb->db, zSql, 0, 0, &pErr->zErr); + sqlite3_free(zSql); + } + va_end(ap); +} + static Statement *getSqlStatement( Error *pErr, /* IN/OUT: Error code */ Sqlite *pDb, /* Database handle */ @@ -624,11 +644,9 @@ static i64 execsql_i64_x( if( pErr->rc==SQLITE_OK ){ sqlite3_stmt *pStmt; /* SQL statement to execute */ va_list ap; /* ... arguments */ - int i; /* Used to iterate through parameters */ va_start(ap, pDb); pStmt = getAndBindSqlStatement(pErr, pDb, ap); if( pStmt ){ - int rc; int first = 1; while( SQLITE_ROW==sqlite3_step(pStmt) ){ if( first && sqlite3_column_count(pStmt)>0 ){ @@ -663,11 +681,9 @@ static char * execsql_text_x( if( pErr->rc==SQLITE_OK ){ sqlite3_stmt *pStmt; /* SQL statement to execute */ va_list ap; /* ... arguments */ - int i; /* Used to iterate through parameters */ va_start(ap, iSlot); pStmt = getAndBindSqlStatement(pErr, pDb, ap); if( pStmt ){ - int rc; int first = 1; while( SQLITE_ROW==sqlite3_step(pStmt) ){ if( first && sqlite3_column_count(pStmt)>0 ){ @@ -693,14 +709,13 @@ static void integrity_check_x( ){ if( pErr->rc==SQLITE_OK ){ Statement *pStatement; /* Statement to execute */ - int rc; /* Return code */ char *zErr = 0; /* Integrity check error */ pStatement = getSqlStatement(pErr, pDb, "PRAGMA integrity_check"); if( pStatement ){ sqlite3_stmt *pStmt = pStatement->pStmt; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - const char *z = sqlite3_column_text(pStmt, 0); + const char *z = (const char*)sqlite3_column_text(pStmt, 0); if( strcmp(z, "ok") ){ if( zErr==0 ){ zErr = sqlite3_mprintf("%s", z); @@ -721,14 +736,14 @@ static void integrity_check_x( static void *launch_thread_main(void *pArg){ Thread *p = (Thread *)pArg; - return (void *)p->xProc(p->iTid, p->iArg); + return (void *)p->xProc(p->iTid, p->pArg); } static void launch_thread_x( Error *pErr, /* IN/OUT: Error code */ Threadset *pThreads, /* Thread set */ - char *(*xProc)(int, int), /* Proc to run */ - int iArg /* Argument passed to thread proc */ + char *(*xProc)(int, void*), /* Proc to run */ + void *pArg /* Argument passed to thread proc */ ){ if( pErr->rc==SQLITE_OK ){ int iTid = ++pThreads->iMaxTid; @@ -738,7 +753,7 @@ static void launch_thread_x( p = (Thread *)sqlite3_malloc(sizeof(Thread)); memset(p, 0, sizeof(Thread)); p->iTid = iTid; - p->iArg = iArg; + p->pArg = pArg; p->xProc = xProc; rc = pthread_create(&p->tid, NULL, launch_thread_main, (void *)p); @@ -895,7 +910,7 @@ static int timetostop_x( #define WALTHREAD1_NTHREAD 10 #define WALTHREAD3_NTHREAD 6 -static char *walthread1_thread(int iTid, int iArg){ +static char *walthread1_thread(int iTid, void *pArg){ Error err = {0}; /* Error code and message */ Sqlite db = {0}; /* SQLite database connection */ int nIter = 0; /* Iterations so far */ @@ -934,7 +949,7 @@ static char *walthread1_thread(int iTid, int iArg){ return sqlite3_mprintf("%d iterations", nIter); } -static char *walthread1_ckpt_thread(int iTid, int iArg){ +static char *walthread1_ckpt_thread(int iTid, void *pArg){ Error err = {0}; /* Error code and message */ Sqlite db = {0}; /* SQLite database connection */ int nCkpt = 0; /* Checkpoints so far */ @@ -977,10 +992,11 @@ static void walthread1(int nMs){ print_and_free_err(&err); } -static char *walthread2_thread(int iTid, int iArg){ +static char *walthread2_thread(int iTid, void *pArg){ Error err = {0}; /* Error code and message */ Sqlite db = {0}; /* SQLite database connection */ int anTrans[2] = {0, 0}; /* Number of WAL and Rollback transactions */ + int iArg = PTR2INT(pArg); const char *zJournal = "PRAGMA journal_mode = WAL"; if( iArg ){ zJournal = "PRAGMA journal_mode = DELETE"; } @@ -1026,17 +1042,18 @@ static void walthread2(int nMs){ setstoptime(&err, nMs); launch_thread(&err, &threads, walthread2_thread, 0); launch_thread(&err, &threads, walthread2_thread, 0); - launch_thread(&err, &threads, walthread2_thread, 1); - launch_thread(&err, &threads, walthread2_thread, 1); + launch_thread(&err, &threads, walthread2_thread, (void*)1); + launch_thread(&err, &threads, walthread2_thread, (void*)1); join_all_threads(&err, &threads); print_and_free_err(&err); } -static char *walthread3_thread(int iTid, int iArg){ +static char *walthread3_thread(int iTid, void *pArg){ Error err = {0}; /* Error code and message */ Sqlite db = {0}; /* SQLite database connection */ i64 iNextWrite; /* Next value this thread will write */ + int iArg = PTR2INT(pArg); opendb(&err, &db, "test.db", 0); sql_script(&err, &db, "PRAGMA wal_autocheckpoint = 10"); @@ -1087,14 +1104,14 @@ static void walthread3(int nMs){ setstoptime(&err, nMs); for(i=0; izDb, 0); + while( !timetostop(&err) ){ + pCtx->xProc(&err, &db, i1); + i2 += (err.rc==SQLITE_OK); + clear_error(&err, SQLITE_LOCKED); + clear_error(&err, SQLITE_ERROR); + i1++; + } + + print_and_free_err(&err); + return sqlite3_mprintf("ok %d/%d", i2, i1); +} + +static void stress2_launch_thread_loop( + Error *pErr, /* IN/OUT: Error code */ + Threadset *pThreads, /* Thread set */ + const char *zDb, /* Database name */ + void (*x)(Error*,Sqlite*,int) /* Run this until error or timeout */ +){ + Stress2Ctx *pCtx = sqlite3_malloc(sizeof(Stress2Ctx)); + pCtx->zDb = zDb; + pCtx->xProc = x; + launch_thread(pErr, pThreads, stress2_thread_wrapper, (void*)pCtx); +} + +static void stress2(int nMs){ + struct Stress2Task { + void (*x)(Error*,Sqlite*,int); + } aTask[] = { + { stress2_workload1 }, + { stress2_workload2 }, + { stress2_workload3 }, + { stress2_workload4 }, + { stress2_workload5 }, + { stress2_workload6 }, + { stress2_workload7 }, + { stress2_workload8 }, + { stress2_workload9 }, + { stress2_workload10 }, + { stress2_workload11 }, + { stress2_workload14 }, + { stress2_workload17 }, + }; + const char *azDb[] = { + "test.db", + "file::memory:?cache=shared" + }; + int j; + + for(j=0; j Date: Mon, 15 Dec 2014 08:46:17 +0000 Subject: [PATCH 655/710] Fix errors in threadtest3 tests caused by earlier tests neglecting to close database handles. FossilOrigin-Name: 1d44f1b1a9fefeb2449892775c59765c46784eb1 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/threadtest3.c | 33 ++++++++++++++------------------- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index d0026036ad..56aa9e8cb0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\senhancements\sto\sthreadtest3\sstress\stests. -D 2014-12-13T17:41:48.275 +C Fix\serrors\sin\sthreadtest3\stests\scaused\sby\searlier\stests\sneglecting\sto\sclose\sdatabase\shandles. +D 2014-12-15T08:46:17.172 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -912,7 +912,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c 05d67ab8fd4ad82978a96200701a22af35faa097 +F test/threadtest3.c c8629f836331dd4e5d6ef514b6696605cb593135 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 @@ -1231,7 +1231,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d7bb7ea4ab97ad26f4c84c9b8dc2827010093803 -R 5d62798e848b5017221fa2f718c6fa3d +P ba772cff602ca7c3c0c91451e701f52a872e7a14 +R ed0c57369d0ec456d80773711d0601f8 U dan -Z 748bf95122bf9ac21f60dff7348c8009 +Z bfcc45ad20cf310717047c8b8c88d22b diff --git a/manifest.uuid b/manifest.uuid index 4d5b9316c6..53e85dbd36 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ba772cff602ca7c3c0c91451e701f52a872e7a14 \ No newline at end of file +1d44f1b1a9fefeb2449892775c59765c46784eb1 \ No newline at end of file diff --git a/test/threadtest3.c b/test/threadtest3.c index 4cdb0f8e9b..c8830bb55e 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -1343,6 +1343,7 @@ static char *dynamic_triggers_1(int iTid, void *pArg){ nDrop++; } } + closedb(&err, &db); print_and_free_err(&err); return sqlite3_mprintf("%d created, %d dropped", nCreate, nDrop); @@ -1369,6 +1370,7 @@ static char *dynamic_triggers_2(int iTid, void *pArg){ nDelete++; } while( iVal ); } + closedb(&err, &db); print_and_free_err(&err); return sqlite3_mprintf("%d inserts, %d deletes", nInsert, nDelete); @@ -1393,6 +1395,7 @@ static void dynamic_triggers(int nMs){ "CREATE TABLE t8(x, y);" "CREATE TABLE t9(x, y);" ); + closedb(&err, &db); setstoptime(&err, nMs); @@ -1446,32 +1449,24 @@ int main(int argc, char **argv){ }; int i; - char *zTest = 0; - int nTest = 0; int bTestfound = 0; - int bPrefix = 0; - - if( argc>2 ) goto usage; - if( argc==2 ){ - zTest = argv[1]; - nTest = strlen(zTest); - if( zTest[nTest-1]=='*' ){ - nTest--; - bPrefix = 1; - } - } sqlite3_config(SQLITE_CONFIG_MULTITHREAD); sqlite3_config(SQLITE_CONFIG_MULTITHREAD); for(i=0; i1 ){ + int iArg; + for(iArg=1; iArg0 ? 255 : 0); usage: - printf("Usage: %s [testname|testprefix*]\n", argv[0]); + printf("Usage: %s [testname|testprefix*]...\n", argv[0]); printf("Available tests are:\n"); for(i=0; i Date: Mon, 15 Dec 2014 16:27:12 +0000 Subject: [PATCH 656/710] Add new test file e_walauto.test. FossilOrigin-Name: 62ef45140cdbff5eeb8bef506db8b78ced3ace94 --- manifest | 13 +-- manifest.uuid | 2 +- src/test1.c | 38 +++++++++ test/e_walauto.test | 201 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 247 insertions(+), 7 deletions(-) create mode 100644 test/e_walauto.test diff --git a/manifest b/manifest index 56aa9e8cb0..eeea121a6a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\serrors\sin\sthreadtest3\stests\scaused\sby\searlier\stests\sneglecting\sto\sclose\sdatabase\shandles. -D 2014-12-15T08:46:17.172 +C Add\snew\stest\sfile\se_walauto.test. +D 2014-12-15T16:27:12.622 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -238,7 +238,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc F src/tclsqlite.c c6a21c64da1490e14d53cdc2062d1e2e57942622 -F src/test1.c 7e806af12d7915445e46407d32b275d8ca9db4e7 +F src/test1.c 56e33bf6b1827c6ca7520c189131ddd778fb2267 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -478,6 +478,7 @@ F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a +F test/e_walauto.test a1fa9d36c160cc4001a934d1e009aae597b440b7 F test/e_walckpt.test 3116a98fa0dd9b2c9e493de7c59730adfe436746 F test/e_walhook.test da3ea8b3483d1af72190337bda50155a91a4b664 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea @@ -1231,7 +1232,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ba772cff602ca7c3c0c91451e701f52a872e7a14 -R ed0c57369d0ec456d80773711d0601f8 +P 1d44f1b1a9fefeb2449892775c59765c46784eb1 +R 10b3e0524cfe98a487f450c2b23f40b1 U dan -Z bfcc45ad20cf310717047c8b8c88d22b +Z 762d85cad7a4c7a6623b46eb7c11e324 diff --git a/manifest.uuid b/manifest.uuid index 53e85dbd36..2d88a004d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1d44f1b1a9fefeb2449892775c59765c46784eb1 \ No newline at end of file +62ef45140cdbff5eeb8bef506db8b78ced3ace94 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 851249461c..40acac8360 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5726,6 +5726,43 @@ static int test_wal_checkpoint_v2( return TCL_OK; } +/* +** tclcmd: sqlite3_wal_autocheckpoint db VALUE +*/ +static int test_wal_autocheckpoint( + ClientData clientData, /* Unused */ + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int objc, /* Number of arguments */ + Tcl_Obj *CONST objv[] /* Command arguments */ +){ + sqlite3 *db; + int rc; + int iVal; + + + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DB VALUE"); + return TCL_ERROR; + } + + if( getDbPointer(interp, Tcl_GetString(objv[1]), &db) + || Tcl_GetIntFromObj(0, objv[2], &iVal) + ){ + return TCL_ERROR; + } + + rc = sqlite3_wal_autocheckpoint(db, iVal); + Tcl_ResetResult(interp); + if( rc!=SQLITE_OK ){ + const char *zErrCode = sqlite3ErrName(rc); + Tcl_SetObjResult(interp, Tcl_NewStringObj(zErrCode, -1)); + return TCL_ERROR; + } + + return TCL_OK; +} + + /* ** tclcmd: test_sqlite3_log ?SCRIPT? */ @@ -6787,6 +6824,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ #endif { "sqlite3_wal_checkpoint", test_wal_checkpoint, 0 }, { "sqlite3_wal_checkpoint_v2",test_wal_checkpoint_v2, 0 }, + { "sqlite3_wal_autocheckpoint",test_wal_autocheckpoint, 0 }, { "test_sqlite3_log", test_sqlite3_log, 0 }, #ifndef SQLITE_OMIT_EXPLAIN { "print_explain_query_plan", test_print_eqp, 0 }, diff --git a/test/e_walauto.test b/test/e_walauto.test new file mode 100644 index 0000000000..fc394e9053 --- /dev/null +++ b/test/e_walauto.test @@ -0,0 +1,201 @@ +# 2014 December 04 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/wal_common.tcl +set testprefix e_walauto + + +proc read_nbackfill {} { + seek $::shmfd 96 + binary scan [read $::shmfd 4] i nBackfill + set nBackfill +} +proc read_mxframe {} { + seek $::shmfd 16 + binary scan [read $::shmfd 4] i mxFrame + set mxFrame +} + +# Assuming that the main db for database handle +# +proc do_autocommit_threshold_test {tn value} { + + set nBackfillSaved [read_nbackfill] + while {1} { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + if {[read_mxframe] >= $value} break + } + + set nBackfillNew [read_nbackfill] + uplevel [list do_test $tn "expr $nBackfillNew > $nBackfillSaved" 1] +} + +# EVIDENCE-OF: R-30135-06439 The wal_autocheckpoint pragma can be used +# to invoke this interface from SQL. +# +# All tests in this file are run twice - once using the +# sqlite3_wal_autocheckpoint() API, and once using "PRAGMA +# wal_autocheckpoint". +# +foreach {tn code} { + 1 { + proc autocheckpoint {db value} { + uplevel [list $db eval "PRAGMA wal_autocheckpoint = $value"] + } + } + + 2 { + proc autocheckpoint {db value} { + uplevel [list sqlite3_wal_autocheckpoint $db $value] + return $value + } + } +} { + + eval $code + + reset_db + do_execsql_test 1.$tn.0 { PRAGMA journal_mode = WAL } {wal} + do_execsql_test 1.$tn.1 { CREATE TABLE t1(a, b) } + set shmfd [open "test.db-shm"] + + # EVIDENCE-OF: R-41531-51083 Every new database connection defaults to + # having the auto-checkpoint enabled with a threshold of 1000 or + # SQLITE_DEFAULT_WAL_AUTOCHECKPOINT pages. + # + do_autocommit_threshold_test 1.$tn.2 1000 + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + do_autocommit_threshold_test 1.$tn.3 1000 + + # EVIDENCE-OF: R-38128-34102 The sqlite3_wal_autocheckpoint(D,N) is a + # wrapper around sqlite3_wal_hook() that causes any database on database + # connection D to automatically checkpoint after committing a + # transaction if there are N or more frames in the write-ahead log file. + # + do_test 1.$tn.4 { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + autocheckpoint db 100 + } {100} + do_autocommit_threshold_test 1.$tn.5 100 + + do_test 1.$tn.6 { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + autocheckpoint db 500 + } {500} + do_autocommit_threshold_test 1.$tn.7 500 + + # EVIDENCE-OF: R-26993-43540 Passing zero or a negative value as the + # nFrame parameter disables automatic checkpoints entirely. + # + do_test 1.$tn.7 { + autocheckpoint db 0 ;# Set to zero + for {set i 0} {$i < 10000} {incr i} { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + } + expr {[file size test.db-wal] > (5 * 1024 * 1024)} + } 1 + do_test 1.$tn.8 { + sqlite3_wal_checkpoint_v2 db truncate + file size test.db-wal + } 0 + do_test 1.$tn.9 { + autocheckpoint db -4 ;# Set to a negative value + for {set i 0} {$i < 10000} {incr i} { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + } + expr {[file size test.db-wal] > (5 * 1024 * 1024)} + } 1 + + # EVIDENCE-OF: R-10203-42688 The callback registered by this function + # replaces any existing callback registered using sqlite3_wal_hook(). + # + set ::wal_hook_callback 0 + proc wal_hook_callback {args} { incr ::wal_hook_callback ; return 0 } + do_test 1.$tn.10.1 { + db wal_hook wal_hook_callback + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + set ::wal_hook_callback + } 2 + do_test 1.$tn.10.2 { + autocheckpoint db 100 + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + set ::wal_hook_callback + } 2 + + # EVIDENCE-OF: R-17497-43474 Likewise, registering a callback using + # sqlite3_wal_hook() disables the automatic checkpoint mechanism + # configured by this function. + do_test 1.$tn.11.1 { + sqlite3_wal_checkpoint_v2 db truncate + file size test.db-wal + } 0 + do_test 1.$tn.11.2 { + autocheckpoint db 100 + for {set i 0} {$i < 1000} {incr i} { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + } + expr {[file size test.db-wal] < (1 * 1024 * 1024)} + } 1 + do_test 1.$tn.11.3 { + db wal_hook wal_hook_callback + for {set i 0} {$i < 1000} {incr i} { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + } + expr {[file size test.db-wal] < (1 * 1024 * 1024)} + } 0 + + # EVIDENCE-OF: R-33080-59193 Checkpoints initiated by this mechanism + # are PASSIVE. + # + set ::busy_callback_count 0 + proc busy_callback {args} { + puts Hello + incr ::busy_callback_count + return 0 + } + do_test 1.$tn.12.1 { + sqlite3_wal_checkpoint_v2 db truncate + autocheckpoint db 100 + db busy busy_callback + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + } {} + do_test 1.$tn.12.2 { + sqlite3 db2 test.db + db2 eval { BEGIN; SELECT * FROM t1 LIMIT 10; } + read_nbackfill + } {0} + do_test 1.$tn.12.3 { + for {set i 0} {$i < 1000} {incr i} { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + } + read_nbackfill + } {2} + do_test 1.$tn.12.4 { + set ::busy_callback_count + } {0} + db2 close + + do_test 1.$tn.12.5 { + db eval { INSERT INTO t1 VALUES(randomblob(100), randomblob(100)) } + read_nbackfill + } {1559} + + db close + close $shmfd +} + +finish_test From 1ee46c01b4822c73169bc093ab51ec79f668daf5 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 15 Dec 2014 20:49:26 +0000 Subject: [PATCH 657/710] Changes to threadtest3 so that "stress2" is more similar to the SDS stress test. FossilOrigin-Name: 5648af96d8e2521c5b0cca19f1358374d032394d --- manifest | 17 ++++--- manifest.uuid | 2 +- test/threadtest3.c | 37 ++++++++------- test/tt3_stress.c | 115 ++++++++++++++++++++------------------------- 4 files changed, 82 insertions(+), 89 deletions(-) diff --git a/manifest b/manifest index eeea121a6a..96dffc16a7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\se_walauto.test. -D 2014-12-15T16:27:12.622 +C Changes\sto\sthreadtest3\sso\sthat\s"stress2"\sis\smore\ssimilar\sto\sthe\sSDS\sstress\stest. +D 2014-12-15T20:49:26.942 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -913,7 +913,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c c8629f836331dd4e5d6ef514b6696605cb593135 +F test/threadtest3.c f8c6595664a4c5ef5f28d97a612386fe14dd1940 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 @@ -1080,7 +1080,7 @@ F test/triggerE.test 355e9c5cbaed5cd039a60baad1fb2197caeb8e52 F test/tt3_checkpoint.c 5e63ee65ed5f87176e25a996480cb02c6caec8b4 F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9 -F test/tt3_stress.c edbb00ed1516535691040315e97cf32c62df22d0 +F test/tt3_stress.c c57d804716165811d979d4a719e05baccd79277f F test/tt3_vacuum.c 1753f45917699c9c1f66b64c717a717c9379f776 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84 @@ -1232,7 +1232,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1d44f1b1a9fefeb2449892775c59765c46784eb1 -R 10b3e0524cfe98a487f450c2b23f40b1 +P 62ef45140cdbff5eeb8bef506db8b78ced3ace94 +R 9a6502f6d41e6a843f5a509613ed553f +T *branch * threadtest3 +T *sym-threadtest3 * +T -sym-trunk * U dan -Z 762d85cad7a4c7a6623b46eb7c11e324 +Z 02118ab9cf865ac1e1c9e40290c79fe8 diff --git a/manifest.uuid b/manifest.uuid index 2d88a004d6..038c947b8e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62ef45140cdbff5eeb8bef506db8b78ced3ace94 \ No newline at end of file +5648af96d8e2521c5b0cca19f1358374d032394d \ No newline at end of file diff --git a/test/threadtest3.c b/test/threadtest3.c index c8830bb55e..ff8add5bf6 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -121,7 +121,10 @@ struct MD5Context { int isInit; uint32 buf[4]; uint32 bits[2]; - unsigned char in[64]; + union { + unsigned char in[64]; + uint32 in32[16]; + } u; }; typedef struct MD5Context MD5Context; @@ -270,7 +273,7 @@ void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ /* Handle any leading odd-sized chunks */ if ( t ) { - unsigned char *p = (unsigned char *)ctx->in + t; + unsigned char *p = (unsigned char *)ctx->u.in + t; t = 64-t; if (len < t) { @@ -278,8 +281,8 @@ void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ return; } memcpy(p, buf, t); - byteReverse(ctx->in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->in); + byteReverse(ctx->u.in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); buf += t; len -= t; } @@ -287,16 +290,16 @@ void MD5Update(MD5Context *ctx, const unsigned char *buf, unsigned int len){ /* Process data in 64-byte chunks */ while (len >= 64) { - memcpy(ctx->in, buf, 64); - byteReverse(ctx->in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->in); + memcpy(ctx->u.in, buf, 64); + byteReverse(ctx->u.in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); buf += 64; len -= 64; } /* Handle any remaining bytes of data. */ - memcpy(ctx->in, buf, len); + memcpy(ctx->u.in, buf, len); } /* @@ -312,7 +315,7 @@ static void MD5Final(unsigned char digest[16], MD5Context *ctx){ /* Set the first char of padding to 0x80. This is safe since there is always at least one byte free */ - p = ctx->in + count; + p = ctx->u.in + count; *p++ = 0x80; /* Bytes of padding needed to make 64 bytes */ @@ -322,22 +325,22 @@ static void MD5Final(unsigned char digest[16], MD5Context *ctx){ if (count < 8) { /* Two lots of padding: Pad the first block to 64 bytes */ memset(p, 0, count); - byteReverse(ctx->in, 16); - MD5Transform(ctx->buf, (uint32 *)ctx->in); + byteReverse(ctx->u.in, 16); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); /* Now fill the next block with 56 bytes */ - memset(ctx->in, 0, 56); + memset(ctx->u.in, 0, 56); } else { /* Pad block to 56 bytes */ memset(p, 0, count-8); } - byteReverse(ctx->in, 14); + byteReverse(ctx->u.in, 14); /* Append length in bits and transform */ - ((uint32 *)ctx->in)[ 14 ] = ctx->bits[0]; - ((uint32 *)ctx->in)[ 15 ] = ctx->bits[1]; + ctx->u.in32[14] = ctx->bits[0]; + ctx->u.in32[15] = ctx->bits[1]; - MD5Transform(ctx->buf, (uint32 *)ctx->in); + MD5Transform(ctx->buf, (uint32 *)ctx->u.in); byteReverse((unsigned char *)ctx->buf, 4); memcpy(digest, ctx->buf, 16); memset(ctx, 0, sizeof(*ctx)); /* In case it is sensitive */ @@ -1445,7 +1448,7 @@ int main(int argc, char **argv){ { lookaside1, "lookaside1", 10000 }, { vacuum1, "vacuum1", 10000 }, { stress1, "stress1", 10000 }, - { stress2, "stress2", 10000 }, + { stress2, "stress2", 60000 }, }; int i; diff --git a/test/tt3_stress.c b/test/tt3_stress.c index a1151dd79c..cdfab9c09c 100644 --- a/test/tt3_stress.c +++ b/test/tt3_stress.c @@ -124,7 +124,6 @@ static char *stress_thread_5(int iTid, void *pArg){ i1++; if( err.rc ) i2++; clear_error(&err, SQLITE_LOCKED); - clear_error(&err, SQLITE_ERROR); } closedb(&err, &db); print_and_free_err(&err); @@ -184,73 +183,66 @@ static void stress1(int nMs){ ** 19. Open and close database connections rapidly. */ -#define STRESS2_TABCNT 5 +#define STRESS2_TABCNT 5 /* count1 in SDS test */ + +#define STRESS2_COUNT2 200 /* count2 in SDS test */ +#define STRESS2_COUNT3 57 /* count2 in SDS test */ static void stress2_workload1(Error *pErr, Sqlite *pDb, int i){ - int iTab = (i % STRESS2_TABCNT); + int iTab = (i % (STRESS2_TABCNT-1)) + 1; sql_script_printf(pErr, pDb, "CREATE TABLE IF NOT EXISTS t%d(x PRIMARY KEY, y, z);", iTab ); } static void stress2_workload2(Error *pErr, Sqlite *pDb, int i){ - int iTab = (i % STRESS2_TABCNT); + int iTab = (i % (STRESS2_TABCNT-1)) + 1; sql_script_printf(pErr, pDb, "DROP TABLE IF EXISTS t%d;", iTab); } static void stress2_workload3(Error *pErr, Sqlite *pDb, int i){ - int iTab = (i % STRESS2_TABCNT); - sql_script_printf(pErr, pDb, "SELECT * FROM t%d WHERE z = 'small'", iTab); + sql_script(pErr, pDb, "SELECT * FROM t0 WHERE z = 'small'"); } static void stress2_workload4(Error *pErr, Sqlite *pDb, int i){ - int iTab = (i % STRESS2_TABCNT); - sql_script_printf(pErr, pDb, "SELECT * FROM t%d WHERE z = 'big'", iTab); + sql_script(pErr, pDb, "SELECT * FROM t0 WHERE z = 'big'"); } static void stress2_workload5(Error *pErr, Sqlite *pDb, int i){ - int iTab = (i % STRESS2_TABCNT); - sql_script_printf(pErr, pDb, - "INSERT INTO t%d VALUES(random(), hex(randomblob(57)), 'small');", iTab + sql_script(pErr, pDb, + "INSERT INTO t0 VALUES(hex(random()), hex(randomblob(200)), 'small');" ); } static void stress2_workload6(Error *pErr, Sqlite *pDb, int i){ - int iTab = (i % STRESS2_TABCNT); - sql_script_printf(pErr, pDb, - "INSERT INTO t%d VALUES(random(), hex(randomblob(2000)), 'big');", iTab + sql_script(pErr, pDb, + "INSERT INTO t0 VALUES(hex(random()), hex(randomblob(57)), 'big');" ); } static void stress2_workload7(Error *pErr, Sqlite *pDb, int i){ - int iTab = (i % STRESS2_TABCNT); sql_script_printf(pErr, pDb, - "UPDATE t%d SET y = hex(randomblob(57)) " - "WHERE (x %% 5)==(%d %% 5) AND z='small';" - ,iTab, i + "UPDATE t0 SET y = hex(randomblob(200)) " + "WHERE x LIKE hex((%d %% 5)) AND z='small';" + ,i ); } static void stress2_workload8(Error *pErr, Sqlite *pDb, int i){ - int iTab = (i % STRESS2_TABCNT); sql_script_printf(pErr, pDb, - "UPDATE t%d SET y = hex(randomblob(2000)) " - "WHERE (x %% 5)==(%d %% 5) AND z='big';" - ,iTab, i + "UPDATE t0 SET y = hex(randomblob(57)) " + "WHERE x LIKE hex(%d %% 5) AND z='big';" + ,i ); } static void stress2_workload9(Error *pErr, Sqlite *pDb, int i){ - int iTab = (i % STRESS2_TABCNT); sql_script_printf(pErr, pDb, - "DELETE FROM t%d WHERE (x %% 5)==(%d %% 5) AND z='small';" - ,iTab, i + "DELETE FROM t0 WHERE x LIKE hex(%d %% 5) AND z='small';", i ); } static void stress2_workload10(Error *pErr, Sqlite *pDb, int i){ - int iTab = (i % STRESS2_TABCNT); sql_script_printf(pErr, pDb, - "DELETE FROM t%d WHERE (x %% 5)==(%d %% 5) AND z='big';" - ,iTab, i + "DELETE FROM t0 WHERE x LIKE hex(%d %% 5) AND z='big';", i ); } @@ -296,13 +288,16 @@ static char *stress2_thread_wrapper(int iTid, void *pArg){ int i1 = 0; int i2 = 0; - opendb(&err, &db, pCtx->zDb, 0); while( !timetostop(&err) ){ - pCtx->xProc(&err, &db, i1); - i2 += (err.rc==SQLITE_OK); - clear_error(&err, SQLITE_LOCKED); - clear_error(&err, SQLITE_ERROR); - i1++; + int cnt; + opendb(&err, &db, pCtx->zDb, 0); + for(cnt=0; err.rc==SQLITE_OK && cntxProc(&err, &db, i1); + i2 += (err.rc==SQLITE_OK); + clear_error(&err, SQLITE_LOCKED); + i1++; + } + closedb(&err, &db); } print_and_free_err(&err); @@ -339,41 +334,33 @@ static void stress2(int nMs){ { stress2_workload14 }, { stress2_workload17 }, }; - const char *azDb[] = { - "test.db", - "file::memory:?cache=shared" - }; - int j; + const char *zDb = "test.db"; - for(j=0; j Date: Tue, 16 Dec 2014 00:08:31 +0000 Subject: [PATCH 658/710] Make sure the sqlite3BtreeCount() routine does not leave index cursors in an inconsistent state, as doing so might result in an assertion fault inside of sqlite3BtreeKey() called from saveAllCursors() if content is deleted out from under the statement that issued the sqlite3BtreeCount() call. FossilOrigin-Name: 5b1b697040116048e464b3ebab8395fe088e389a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 2 +- test/pragma.test | 26 +++++++++++++++++++++++++- 4 files changed, 35 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index eeea121a6a..8542d0d893 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\stest\sfile\se_walauto.test. -D 2014-12-15T16:27:12.622 +C Make\ssure\sthe\ssqlite3BtreeCount()\sroutine\sdoes\snot\sleave\sindex\scursors\sin\san\ninconsistent\sstate,\sas\sdoing\sso\smight\sresult\sin\san\sassertion\sfault\sinside\nof\ssqlite3BtreeKey()\scalled\sfrom\ssaveAllCursors()\sif\scontent\sis\sdeleted\sout\nfrom\sunder\sthe\sstatement\sthat\sissued\sthe\ssqlite3BtreeCount()\scall. +D 2014-12-16T00:08:31.918 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c ab4b60fcf9920d862ff4d96efb1d605e4e7701a0 +F src/btree.c 92f745ccd18099973beb28e25fce80148545429e F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 162d84e4833b03f9d07192ef06057b0226f6e543 @@ -782,7 +782,7 @@ F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 F test/permutations.test 4e12d43f4639ea8a0e366d9c64e0009afe2eb544 -F test/pragma.test 49ac8a73c0daa574824538fed28727d1259fe735 +F test/pragma.test aa16dedfe01c02c8895169012f7dfde9c163f0d5 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a @@ -1232,7 +1232,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1d44f1b1a9fefeb2449892775c59765c46784eb1 -R 10b3e0524cfe98a487f450c2b23f40b1 -U dan -Z 762d85cad7a4c7a6623b46eb7c11e324 +P 62ef45140cdbff5eeb8bef506db8b78ced3ace94 +R f292d3e817736f788b9a711a02a9af3e +U drh +Z 46e826bbedddd4a807c643712561f79a diff --git a/manifest.uuid b/manifest.uuid index 2d88a004d6..af4336842d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62ef45140cdbff5eeb8bef506db8b78ced3ace94 \ No newline at end of file +5b1b697040116048e464b3ebab8395fe088e389a \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index a73c831219..d24f6a876a 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8278,7 +8278,7 @@ int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ if( pCur->iPage==0 ){ /* All pages of the b-tree have been visited. Return successfully. */ *pnEntry = nEntry; - return SQLITE_OK; + return moveToRoot(pCur); } moveToParent(pCur); }while ( pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell ); diff --git a/test/pragma.test b/test/pragma.test index e660ab0fe7..09b9b6c14f 100644 --- a/test/pragma.test +++ b/test/pragma.test @@ -454,10 +454,34 @@ do_execsql_test pragma-3.21 { do_execsql_test pragma-3.22 { PRAGMA integrity_check(2); } {{non-unique entry in index t1a} {NULL value in t1x.a}} -do_execsql_test pragma-3.21 { +do_execsql_test pragma-3.23 { PRAGMA integrity_check(1); } {{non-unique entry in index t1a}} +# PRAGMA integrity check (or more specifically the sqlite3BtreeCount() +# interface) used to leave index cursors in an inconsistent state +# which could result in an assertion fault in sqlite3BtreeKey() +# called from saveCursorPosition() if content is removed from the +# index while the integrity_check is still running. This test verifies +# that problem has been fixed. +# +do_test pragma-3.30 { + db close + delete_file test.db + sqlite3 db test.db + db eval { + CREATE TABLE t1(a,b,c); + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<100) + INSERT INTO t1(a,b,c) SELECT i, printf('xyz%08x',i), 2000-i FROM c; + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1bc ON t1(b,c); + } + db eval {PRAGMA integrity_check} { + db eval {DELETE FROM t1} + } +} {} + # Test modifying the cache_size of an attached database. ifcapable pager_pragmas&&attach { do_test pragma-4.1 { From e8d1777a2de65611519adffa20997b2b19ece6b7 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 16 Dec 2014 12:46:38 +0000 Subject: [PATCH 659/710] Fix the e_walauto.test script so that it works on windows. FossilOrigin-Name: 7d092ebb6724c3c0fdc05dc94ca767d158933fb5 --- manifest | 13 ++++++------- manifest.uuid | 2 +- test/e_walauto.test | 2 +- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 012153429f..269b2e6b45 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhanced\s"stress2"\stesting\sin\sthe\sthreadtest3.c\stest\sprogram. -D 2014-12-16T00:20:07.236 +C Fix\sthe\se_walauto.test\sscript\sso\sthat\sit\sworks\son\swindows. +D 2014-12-16T12:46:38.635 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -478,7 +478,7 @@ F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a -F test/e_walauto.test a1fa9d36c160cc4001a934d1e009aae597b440b7 +F test/e_walauto.test eab3bedddbc3fd19795d51e618da41a48e19a3e3 F test/e_walckpt.test 3116a98fa0dd9b2c9e493de7c59730adfe436746 F test/e_walhook.test da3ea8b3483d1af72190337bda50155a91a4b664 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea @@ -1232,8 +1232,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5b1b697040116048e464b3ebab8395fe088e389a 5648af96d8e2521c5b0cca19f1358374d032394d -R 64bec50dd05bb00af986c569d5edb044 -T +closed 5648af96d8e2521c5b0cca19f1358374d032394d +P ae43539e62e76676a3daf561b629a1b9b4e2d2c9 +R 6cb5309eecfa4835aff26790e68b0f80 U drh -Z 1075019e3631a6f364d3a72648c7dd6d +Z 6fc407d3934d070a5bb755e43a0ba0ad diff --git a/manifest.uuid b/manifest.uuid index 209ef7a57a..fd507f2f23 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae43539e62e76676a3daf561b629a1b9b4e2d2c9 \ No newline at end of file +7d092ebb6724c3c0fdc05dc94ca767d158933fb5 \ No newline at end of file diff --git a/test/e_walauto.test b/test/e_walauto.test index fc394e9053..79a2702d04 100644 --- a/test/e_walauto.test +++ b/test/e_walauto.test @@ -68,7 +68,7 @@ foreach {tn code} { reset_db do_execsql_test 1.$tn.0 { PRAGMA journal_mode = WAL } {wal} do_execsql_test 1.$tn.1 { CREATE TABLE t1(a, b) } - set shmfd [open "test.db-shm"] + set shmfd [open "test.db-shm" rb] # EVIDENCE-OF: R-41531-51083 Every new database connection defaults to # having the auto-checkpoint enabled with a threshold of 1000 or From 0466883300819e9b95cd07212136e1ee2bb8aee9 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 16 Dec 2014 20:13:30 +0000 Subject: [PATCH 660/710] Experimental opimizations to speed up FK constraint CASCADE and SET NULL action processing. FossilOrigin-Name: 35a20a5f22245c70faa51965951e8cc011defa93 --- manifest | 21 ++++++------ manifest.uuid | 2 +- src/fkey.c | 63 +++++++++++++++++++++++++----------- src/vdbeaux.c | 11 ++++--- test/fkey8.test | 85 +++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+), 32 deletions(-) create mode 100644 test/fkey8.test diff --git a/manifest b/manifest index 012153429f..d7fc48ce45 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhanced\s"stress2"\stesting\sin\sthe\sthreadtest3.c\stest\sprogram. -D 2014-12-16T00:20:07.236 +C Experimental\sopimizations\sto\sspeed\sup\sFK\sconstraint\sCASCADE\sand\sSET\sNULL\saction\sprocessing. +D 2014-12-16T20:13:30.440 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -184,7 +184,7 @@ F src/date.c 93594514aae68de117ca4a2a0d6cc63eddf26744 F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 F src/expr.c 00da3072f362b06f39ce4052baa1d4ce2bb36d1c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c da985ae673efef2c712caef825a5d2edb087ead7 +F src/fkey.c eab56799b0339b04c258233b0f462b6e9317f90f F src/func.c 6d3c4ebd72aa7923ce9b110a7dc15f9b8c548430 F src/global.c 6ded36dda9466fc1c9a3c5492ded81d79bf3977d F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 @@ -295,7 +295,7 @@ F src/vdbe.c 1a9e671c9cfc259e4d2affc71f7df4a4c00a842c F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78 F src/vdbeapi.c 4bc511a46b9839392ae0e90844a71dc96d9dbd71 -F src/vdbeaux.c 6f7f39c3fcf0f5923758df8561bb5d843908a553 +F src/vdbeaux.c 07ef87c6d4b5abdf13ff33babb10205702fdab0a F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c c150803a3e98fbc68bd07772cbbd4328a0a7212d @@ -504,6 +504,7 @@ F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d F test/fkey5.test 8a1fde4e7721ae00b05b3178888833726ca2df8d F test/fkey6.test abb59f866c1b44926fd02d1fdd217d831fe04f48 F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13 +F test/fkey8.test 2d58cfb990cdd56b2fbac9f4ae54ee53968b3e06 F test/fkey_malloc.test 594a7ea1fbab553c036c70813cd8bd9407d63749 F test/format4.test 1f0cac8ff3895e9359ed87e41aaabee982a812eb F test/fts-9fd058691.test 78b887e30ae6816df0e1fed6259de4b5a64ad33c @@ -1232,8 +1233,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5b1b697040116048e464b3ebab8395fe088e389a 5648af96d8e2521c5b0cca19f1358374d032394d -R 64bec50dd05bb00af986c569d5edb044 -T +closed 5648af96d8e2521c5b0cca19f1358374d032394d -U drh -Z 1075019e3631a6f364d3a72648c7dd6d +P ae43539e62e76676a3daf561b629a1b9b4e2d2c9 +R ea6275c5fe67bcf602365c1ede77fced +T *branch * experimental-fk-actions +T *sym-experimental-fk-actions * +T -sym-trunk * +U dan +Z b6352d484674a0d7b3ae6d8d076cd106 diff --git a/manifest.uuid b/manifest.uuid index 209ef7a57a..859f607ae0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ae43539e62e76676a3daf561b629a1b9b4e2d2c9 \ No newline at end of file +35a20a5f22245c70faa51965951e8cc011defa93 \ No newline at end of file diff --git a/src/fkey.c b/src/fkey.c index e816bd95da..7ea48f5058 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -437,7 +437,7 @@ static void fkLookupParent( OE_Abort, 0, P4_STATIC, P5_ConstraintFK); }else{ if( nIncr>0 && pFKey->isDeferred==0 ){ - sqlite3ParseToplevel(pParse)->mayAbort = 1; + sqlite3MayAbort(pParse); } sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); } @@ -509,6 +509,10 @@ static Expr *exprTableColumn( ** code for an SQL UPDATE operation, this function may be called twice - ** once to "delete" the old row and once to "insert" the new row. ** +** Parameter nIncr is passed -1 when inserting a row (as this may decrease +** the number of FK violations in the db) or +1 when deleting one (as this +** may increase the number of FK constraint problems). +** ** The code generated by this function scans through the rows in the child ** table that correspond to the parent table row being deleted or inserted. ** For each child row found, one of the following actions is taken: @@ -629,9 +633,6 @@ static void fkScanChildren( ** each row found. Otherwise, for deferred constraints, increment the ** deferred constraint counter by nIncr for each row selected. */ pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); - if( nIncr>0 && pFKey->isDeferred==0 ){ - sqlite3ParseToplevel(pParse)->mayAbort = 1; - } sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); if( pWInfo ){ sqlite3WhereEnd(pWInfo); @@ -810,6 +811,24 @@ static int fkParentIsModified( return 0; } +/* +** Return true if the parser passed as the first argument is being +** used to code a trigger that is really a "SET NULL" action belonging +** to trigger pFKey. +*/ +static int isSetNullAction(Parse *pParse, FKey *pFKey){ + Parse *pTop = sqlite3ParseToplevel(pParse); + if( pTop->pTriggerPrg ){ + Trigger *p = pTop->pTriggerPrg->pTrigger; + if( (p==pFKey->apTrigger[0] && pFKey->aAction[0]==OE_SetNull) + || (p==pFKey->apTrigger[1] && pFKey->aAction[1]==OE_SetNull) + ){ + return 1; + } + } + return 0; +} + /* ** This function is called when inserting, deleting or updating a row of ** table pTab to generate VDBE code to perform foreign key constraint @@ -862,7 +881,7 @@ void sqlite3FkCheck( int *aiCol; int iCol; int i; - int isIgnore = 0; + int bIgnore = 0; if( aChange && sqlite3_stricmp(pTab->zName, pFKey->zTo)!=0 @@ -921,7 +940,7 @@ void sqlite3FkCheck( int rcauth; char *zCol = pTo->aCol[pIdx ? pIdx->aiColumn[i] : pTo->iPKey].zName; rcauth = sqlite3AuthReadCol(pParse, pTo->zName, zCol, iDb); - isIgnore = (rcauth==SQLITE_IGNORE); + bIgnore = (rcauth==SQLITE_IGNORE); } #endif } @@ -936,12 +955,18 @@ void sqlite3FkCheck( /* A row is being removed from the child table. Search for the parent. ** If the parent does not exist, removing the child row resolves an ** outstanding foreign key constraint violation. */ - fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1,isIgnore); + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1, bIgnore); } - if( regNew!=0 ){ + if( regNew!=0 && !isSetNullAction(pParse, pFKey) ){ /* A row is being added to the child table. If a parent row cannot - ** be found, adding the child row has violated the FK constraint. */ - fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1,isIgnore); + ** be found, adding the child row has violated the FK constraint. + ** + ** If this operation is being performed as part of a trigger program + ** that is actually a "SET NULL" action belonging to this very + ** foreign key, then omit this scan altogether. As the child keys + ** values are guaranteed to be NULL, it is not possible for adding + ** this row to cause an FK violation. */ + fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1, bIgnore); } sqlite3DbFree(db, aiFree); @@ -962,8 +987,8 @@ void sqlite3FkCheck( && !pParse->pToplevel && !pParse->isMultiWrite ){ assert( regOld==0 && regNew!=0 ); - /* Inserting a single row into a parent table cannot cause an immediate - ** foreign key violation. So do nothing in this case. */ + /* Inserting a single row into a parent table cannot cause (or fix) + ** an immediate foreign key violation. So do nothing in this case. */ continue; } @@ -987,13 +1012,15 @@ void sqlite3FkCheck( fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regNew, -1); } if( regOld!=0 ){ - /* If there is a RESTRICT action configured for the current operation - ** on the parent table of this FK, then throw an exception - ** immediately if the FK constraint is violated, even if this is a - ** deferred trigger. That's what RESTRICT means. To defer checking - ** the constraint, the FK should specify NO ACTION (represented - ** using OE_None). NO ACTION is the default. */ + int eAction = pFKey->aAction[aChange!=0]; fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); + /* If this is a deferred FK constraint, or a CASCADE or SET NULL + ** action applies, do not set the may-abort flag on this statement. + ** The flag may be set on this statement for some other reason, but + ** not as a result of this FK constraint. */ + if( !pFKey->isDeferred && eAction!=OE_Cascade && eAction!=OE_SetNull ){ + sqlite3MayAbort(pParse); + } } pItem->zName = 0; sqlite3SrcListDelete(db, pSrc); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 7900bd52ac..03c229c994 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -396,6 +396,7 @@ static Op *opIterNext(VdbeOpIter *p){ */ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ int hasAbort = 0; + int hasFkCounter = 0; Op *pOp; VdbeOpIter sIter; memset(&sIter, 0, sizeof(sIter)); @@ -404,15 +405,17 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ while( (pOp = opIterNext(&sIter))!=0 ){ int opcode = pOp->opcode; if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename -#ifndef SQLITE_OMIT_FOREIGN_KEY - || (opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1) -#endif || ((opcode==OP_Halt || opcode==OP_HaltIfNull) && ((pOp->p1&0xff)==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) ){ hasAbort = 1; break; } +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){ + hasFkCounter = 1; + } +#endif } sqlite3DbFree(v->db, sIter.apSub); @@ -421,7 +424,7 @@ int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ ** through all opcodes and hasAbort may be set incorrectly. Return ** true for this case to prevent the assert() in the callers frame ** from failing. */ - return ( v->db->mallocFailed || hasAbort==mayAbort ); + return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter ); } #endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ diff --git a/test/fkey8.test b/test/fkey8.test new file mode 100644 index 0000000000..e9f3093930 --- /dev/null +++ b/test/fkey8.test @@ -0,0 +1,85 @@ +# 2001 September 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests for foreign keys. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix fkey8 + +ifcapable {!foreignkey} { + finish_test + return +} +do_execsql_test 1.0 { PRAGMA foreign_keys = 1; } + + +foreach {tn use_stmt sql schema} { + 1 1 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1); + } + + 2.1 0 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE); + } + 2.2 0 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE SET NULL); + } + 2.3 1 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE SET DEFAULT); + } + + 3 1 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE); + CREATE TRIGGER ct1 AFTER DELETE ON c1 BEGIN + INSERT INTO p1 VALUES('x'); + END; + } + + 4 1 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE, c PRIMARY KEY); + CREATE TABLE cc1(d REFERENCES c1); + } + + 5.1 0 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE, c PRIMARY KEY); + CREATE TABLE cc1(d REFERENCES c1 ON DELETE CASCADE); + } + 5.2 0 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE, c PRIMARY KEY); + CREATE TABLE cc1(d REFERENCES c1 ON DELETE SET NULL); + } + +} { + drop_all_tables + do_test 1.$tn { + execsql $schema + set stmt [sqlite3_prepare_v2 db $sql -1 dummy] + set ret [uses_stmt_journal $stmt] + sqlite3_finalize $stmt + set ret + } $use_stmt +} + + + + +finish_test From d457271130985a26c10b2b2d2c853fb27cfcdcb7 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 17 Dec 2014 14:38:45 +0000 Subject: [PATCH 661/710] Fix some comments in fkey.c. Add tests to fkey8.test. FossilOrigin-Name: 210cb2a6aaf780365064a26c0c99926bd6346e19 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/fkey.c | 28 ++++++++++++++++++++-------- test/fkey8.test | 25 +++++++++++++++++++++++-- 4 files changed, 51 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index d7fc48ce45..ec31cc9e8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\sopimizations\sto\sspeed\sup\sFK\sconstraint\sCASCADE\sand\sSET\sNULL\saction\sprocessing. -D 2014-12-16T20:13:30.440 +C Fix\ssome\scomments\sin\sfkey.c.\sAdd\stests\sto\sfkey8.test. +D 2014-12-17T14:38:45.643 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -184,7 +184,7 @@ F src/date.c 93594514aae68de117ca4a2a0d6cc63eddf26744 F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 F src/expr.c 00da3072f362b06f39ce4052baa1d4ce2bb36d1c F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb -F src/fkey.c eab56799b0339b04c258233b0f462b6e9317f90f +F src/fkey.c e0444b61bed271a76840cbe6182df93a9baa3f12 F src/func.c 6d3c4ebd72aa7923ce9b110a7dc15f9b8c548430 F src/global.c 6ded36dda9466fc1c9a3c5492ded81d79bf3977d F src/hash.c 4263fbc955f26c2e8cdc0cf214bc42435aa4e4f5 @@ -504,7 +504,7 @@ F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d F test/fkey5.test 8a1fde4e7721ae00b05b3178888833726ca2df8d F test/fkey6.test abb59f866c1b44926fd02d1fdd217d831fe04f48 F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13 -F test/fkey8.test 2d58cfb990cdd56b2fbac9f4ae54ee53968b3e06 +F test/fkey8.test 8f08203458321e6c19a263829de4cfc936274ab0 F test/fkey_malloc.test 594a7ea1fbab553c036c70813cd8bd9407d63749 F test/format4.test 1f0cac8ff3895e9359ed87e41aaabee982a812eb F test/fts-9fd058691.test 78b887e30ae6816df0e1fed6259de4b5a64ad33c @@ -1233,10 +1233,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ae43539e62e76676a3daf561b629a1b9b4e2d2c9 -R ea6275c5fe67bcf602365c1ede77fced -T *branch * experimental-fk-actions -T *sym-experimental-fk-actions * -T -sym-trunk * +P 35a20a5f22245c70faa51965951e8cc011defa93 +R c39a2128ba39a16276f2dc7a158fbe28 U dan -Z b6352d484674a0d7b3ae6d8d076cd106 +Z 96b8a9e537739194255aed7e579f65ad diff --git a/manifest.uuid b/manifest.uuid index 859f607ae0..367ac4e6aa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -35a20a5f22245c70faa51965951e8cc011defa93 \ No newline at end of file +210cb2a6aaf780365064a26c0c99926bd6346e19 \ No newline at end of file diff --git a/src/fkey.c b/src/fkey.c index 7ea48f5058..fa148ba6a3 100644 --- a/src/fkey.c +++ b/src/fkey.c @@ -629,9 +629,8 @@ static void fkScanChildren( sqlite3ResolveExprNames(&sNameContext, pWhere); /* Create VDBE to loop through the entries in pSrc that match the WHERE - ** clause. If the constraint is not deferred, throw an exception for - ** each row found. Otherwise, for deferred constraints, increment the - ** deferred constraint counter by nIncr for each row selected. */ + ** clause. For each row found, increment either the deferred or immediate + ** foreign key constraint counter. */ pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); if( pWInfo ){ @@ -963,9 +962,9 @@ void sqlite3FkCheck( ** ** If this operation is being performed as part of a trigger program ** that is actually a "SET NULL" action belonging to this very - ** foreign key, then omit this scan altogether. As the child keys + ** foreign key, then omit this scan altogether. As all child key ** values are guaranteed to be NULL, it is not possible for adding - ** this row to cause an FK violation. */ + ** this row to cause an FK violation. */ fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regNew, +1, bIgnore); } @@ -1015,9 +1014,22 @@ void sqlite3FkCheck( int eAction = pFKey->aAction[aChange!=0]; fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); /* If this is a deferred FK constraint, or a CASCADE or SET NULL - ** action applies, do not set the may-abort flag on this statement. - ** The flag may be set on this statement for some other reason, but - ** not as a result of this FK constraint. */ + ** action applies, then any foreign key violations caused by + ** removing the parent key will be rectified by the action trigger. + ** So do not set the "may-abort" flag in this case. + ** + ** Note 1: If the FK is declared "ON UPDATE CASCADE", then the + ** may-abort flag will eventually be set on this statement anyway + ** (when this function is called as part of processing the UPDATE + ** within the action trigger). + ** + ** Note 2: At first glance it may seem like SQLite could simply omit + ** all OP_FkCounter related scans when either CASCADE or SET NULL + ** applies. The trouble starts if the CASCADE or SET NULL action + ** trigger causes other triggers or action rules attached to the + ** child table to fire. In these cases the fk constraint counters + ** might be set incorrectly if any OP_FkCounter related scans are + ** omitted. */ if( !pFKey->isDeferred && eAction!=OE_Cascade && eAction!=OE_SetNull ){ sqlite3MayAbort(pParse); } diff --git a/test/fkey8.test b/test/fkey8.test index e9f3093930..60bb8e4640 100644 --- a/test/fkey8.test +++ b/test/fkey8.test @@ -67,6 +67,28 @@ foreach {tn use_stmt sql schema} { CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE, c PRIMARY KEY); CREATE TABLE cc1(d REFERENCES c1 ON DELETE SET NULL); } + 5.3 1 "DELETE FROM p1" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON DELETE CASCADE, c PRIMARY KEY); + CREATE TABLE cc1(d REFERENCES c1 ON DELETE SET DEFAULT); + } + + 6.1 1 "UPDATE p1 SET a = ?" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON UPDATE SET NULL, c); + } + 6.2 0 "UPDATE OR IGNORE p1 SET a = ?" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON UPDATE SET NULL, c); + } + 6.3 1 "UPDATE OR IGNORE p1 SET a = ?" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b REFERENCES p1 ON UPDATE CASCADE, c); + } + 6.4 1 "UPDATE OR IGNORE p1 SET a = ?" { + CREATE TABLE p1(a PRIMARY KEY); + CREATE TABLE c1(b NOT NULL REFERENCES p1 ON UPDATE SET NULL, c); + } } { drop_all_tables @@ -80,6 +102,5 @@ foreach {tn use_stmt sql schema} { } - - finish_test + From 8e755e7e19613a71feb59775cfd456cad3b5a2e4 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Dec 2014 18:49:55 +0000 Subject: [PATCH 662/710] Simplify the implementation of the "header-value" pragmas (schema_version, user_version, freelist_count, and application_id) by making them more table-driven. FossilOrigin-Name: da27a09d1d991583b59997f6cc67efa28ffd9d6a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pragma.c | 30 ++++++++---------------------- tool/mkpragmatab.tcl | 5 +++++ 4 files changed, 22 insertions(+), 31 deletions(-) diff --git a/manifest b/manifest index 1752ddd915..940c131970 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\sopimizations\sto\sspeed\sup\sFK\sconstraint\sCASCADE\sand\sSET\sNULL\saction\sprocessing. -D 2014-12-17T15:03:50.611 +C Simplify\sthe\simplementation\sof\sthe\s"header-value"\spragmas\s(schema_version,\nuser_version,\sfreelist_count,\sand\sapplication_id)\sby\smaking\sthem\smore\ntable-driven. +D 2014-12-19T18:49:55.326 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 F src/pcache1.c facbdd3ecc09c8f750089d941305694301328e98 -F src/pragma.c c93be505649183b2d80082c2eef1a56879dabfe6 +F src/pragma.c 7337e5d9f14664e987be3e6ed4d73fad16e48ad1 F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 @@ -1197,7 +1197,7 @@ F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh 5dc5010e2e748a9e1bba67baca5956a2c2deda7b F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl cce51d8f60c7f145d8fccabe6b5dfdedf31c5f5c +F tool/mkpragmatab.tcl f7facbe8988ce61946f8069d34827ffc6af92521 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 88a1e3b0c769773fb7a9ebb363ffc603a4ac21d8 F tool/mksqlite3c.tcl e72c0c97fe1a105fa9616483e652949be2199fe6 @@ -1233,7 +1233,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7d092ebb6724c3c0fdc05dc94ca767d158933fb5 210cb2a6aaf780365064a26c0c99926bd6346e19 -R 2edc2fe620223f33e078e9395972a943 -U dan -Z c2959ecd9d3a1afd4baaa55723226ea7 +P 8c5dd6cc259e0cdaaddaa52ccfa96fee6b166906 +R 8ba8633e296b4b7fe64ae8b30f249024 +U drh +Z d788c0d876adfa7410350841952e3917 diff --git a/manifest.uuid b/manifest.uuid index ee9030a0e4..e0e3164dca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8c5dd6cc259e0cdaaddaa52ccfa96fee6b166906 \ No newline at end of file +da27a09d1d991583b59997f6cc67efa28ffd9d6a \ No newline at end of file diff --git a/src/pragma.c b/src/pragma.c index 837a15102d..cbeb81eee6 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -71,6 +71,7 @@ #define PragTyp_LOCK_STATUS 40 #define PragTyp_PARSER_TRACE 41 #define PragFlag_NeedSchema 0x01 +#define PragFlag_ReadOnly 0x02 static const struct sPragmaNames { const char *const zName; /* Name of pragma */ u8 ePragTyp; /* PragTyp_XXX value */ @@ -87,7 +88,7 @@ static const struct sPragmaNames { { /* zName: */ "application_id", /* ePragTyp: */ PragTyp_HEADER_VALUE, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ BTREE_APPLICATION_ID }, #endif #if !defined(SQLITE_OMIT_AUTOVACUUM) { /* zName: */ "auto_vacuum", @@ -208,8 +209,8 @@ static const struct sPragmaNames { #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) { /* zName: */ "freelist_count", /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* ePragFlag: */ PragFlag_ReadOnly, + /* iArg: */ BTREE_FREE_PAGE_COUNT }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) { /* zName: */ "full_column_names", @@ -361,7 +362,7 @@ static const struct sPragmaNames { { /* zName: */ "schema_version", /* ePragTyp: */ PragTyp_HEADER_VALUE, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ BTREE_SCHEMA_VERSION }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) { /* zName: */ "secure_delete", @@ -427,7 +428,7 @@ static const struct sPragmaNames { { /* zName: */ "user_version", /* ePragTyp: */ PragTyp_HEADER_VALUE, /* ePragFlag: */ 0, - /* iArg: */ 0 }, + /* iArg: */ BTREE_USER_VERSION }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if defined(SQLITE_DEBUG) @@ -2126,24 +2127,9 @@ void sqlite3Pragma( ** applications for any purpose. */ case PragTyp_HEADER_VALUE: { - int iCookie; /* Cookie index. 1 for schema-cookie, 6 for user-cookie. */ + int iCookie = aPragmaNames[mid].iArg; /* Which cookie to read or write */ sqlite3VdbeUsesBtree(v, iDb); - switch( zLeft[0] ){ - case 'a': case 'A': - iCookie = BTREE_APPLICATION_ID; - break; - case 'f': case 'F': - iCookie = BTREE_FREE_PAGE_COUNT; - break; - case 's': case 'S': - iCookie = BTREE_SCHEMA_VERSION; - break; - default: - iCookie = BTREE_USER_VERSION; - break; - } - - if( zRight && iCookie!=BTREE_FREE_PAGE_COUNT ){ + if( zRight && (aPragmaNames[mid].mPragFlag & PragFlag_ReadOnly)==0 ){ /* Write the specified cookie value */ static const VdbeOpList setCookie[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index aa7c8078c5..2c9b84b735 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -246,18 +246,23 @@ set pragma_def { NAME: schema_version TYPE: HEADER_VALUE + ARG: BTREE_SCHEMA_VERSION IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) NAME: user_version TYPE: HEADER_VALUE + ARG: BTREE_USER_VERSION IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) NAME: freelist_count TYPE: HEADER_VALUE + ARG: BTREE_FREE_PAGE_COUNT + FLAG: ReadOnly IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) NAME: application_id TYPE: HEADER_VALUE + ARG: BTREE_APPLICATION_ID IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) NAME: compile_options From 91618564955d2da0e70c795076f214b5cb974b40 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Dec 2014 19:28:02 +0000 Subject: [PATCH 663/710] Experimental "PRAGMA data_version" command for detecting when another process has changed the database file. FossilOrigin-Name: 43db1f44bce5a0ee50197b95ab0d844540b69d86 --- manifest | 25 ++++++++++++++----------- manifest.uuid | 2 +- src/btree.c | 13 ++++++++++++- src/btree.h | 8 +++++++- src/pager.c | 12 +++++++++++- src/pager.h | 1 + src/pragma.c | 8 +++++++- tool/mkpragmatab.tcl | 6 ++++++ 8 files changed, 59 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 940c131970..f22a0f1b43 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Simplify\sthe\simplementation\sof\sthe\s"header-value"\spragmas\s(schema_version,\nuser_version,\sfreelist_count,\sand\sapplication_id)\sby\smaking\sthem\smore\ntable-driven. -D 2014-12-19T18:49:55.326 +C Experimental\s"PRAGMA\sdata_version"\scommand\sfor\sdetecting\swhen\sanother\sprocess\nhas\schanged\sthe\sdatabase\sfile. +D 2014-12-19T19:28:02.465 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,8 +173,8 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 92f745ccd18099973beb28e25fce80148545429e -F src/btree.h e31a3a3ebdedb1caf9bda3ad5dbab3db9b780f6e +F src/btree.c b2b2bd0aa02430fe86bc891295db919fcafb0d64 +F src/btree.h 94277c1d30c0b75705974bcc8b0c05e79c03d474 F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 162d84e4833b03f9d07192ef06057b0226f6e543 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 @@ -216,13 +216,13 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c ecb04a0dad2fa6fa659931a9d8f0f3aca33f908a F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 7a5c5bc0e29b9b16834f5558a9d5d22bbae59a08 -F src/pager.h d1eee3c3f741be247ce6d82752a178515fc8578b +F src/pager.c abe022cd72e112a05db5fd1d6d46f5ed93284247 +F src/pager.h c3476e7c89cdf1c6914e50a11f3714e30b4e0a77 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 F src/pcache1.c facbdd3ecc09c8f750089d941305694301328e98 -F src/pragma.c 7337e5d9f14664e987be3e6ed4d73fad16e48ad1 +F src/pragma.c bd33aa24456f043bb6f6d32a918bbeed41d8c591 F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696 @@ -1197,7 +1197,7 @@ F tool/logest.c eef612f8adf4d0993dafed0416064cf50d5d33c6 F tool/mkautoconfamal.sh 5dc5010e2e748a9e1bba67baca5956a2c2deda7b F tool/mkkeywordhash.c dfff09dbbfaf950e89af294f48f902181b144670 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e -F tool/mkpragmatab.tcl f7facbe8988ce61946f8069d34827ffc6af92521 +F tool/mkpragmatab.tcl 07a5124cf2dbafa1b375eefcf8ac4227028b0f8b F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 88a1e3b0c769773fb7a9ebb363ffc603a4ac21d8 F tool/mksqlite3c.tcl e72c0c97fe1a105fa9616483e652949be2199fe6 @@ -1233,7 +1233,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 8c5dd6cc259e0cdaaddaa52ccfa96fee6b166906 -R 8ba8633e296b4b7fe64ae8b30f249024 +P da27a09d1d991583b59997f6cc67efa28ffd9d6a +R df4854633d0919cb180185852b6a45b5 +T *branch * data_version_pragma +T *sym-data_version_pragma * +T -sym-trunk * U drh -Z d788c0d876adfa7410350841952e3917 +Z 81defb06f28b03737e8cb7607139b4d9 diff --git a/manifest.uuid b/manifest.uuid index e0e3164dca..92b0ead781 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da27a09d1d991583b59997f6cc67efa28ffd9d6a \ No newline at end of file +43db1f44bce5a0ee50197b95ab0d844540b69d86 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index d24f6a876a..1223f78e48 100644 --- a/src/btree.c +++ b/src/btree.c @@ -8177,6 +8177,13 @@ int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ ** The schema layer numbers meta values differently. At the schema ** layer (and the SetCookie and ReadCookie opcodes) the number of ** free pages is not visible. So Cookie[0] is the same as Meta[1]. +** +** This routine treats Meta[BTREE_DATA_VERSION] as a special case. Instead +** of reading the value out of the header, it instead loads the "DataVersion" +** from the pager. The BTREE_DATA_VERSION value is not actually stored in the +** database file. It is a number computed by the pager. But its access +** pattern is the same as header meta values, and so it is convenient to +** read it from this routine. */ void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ BtShared *pBt = p->pBt; @@ -8187,7 +8194,11 @@ void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ assert( pBt->pPage1 ); assert( idx>=0 && idx<=15 ); - *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); + if( idx==BTREE_DATA_VERSION ){ + *pMeta = sqlite3PagerDataVersion(pBt->pPager); + }else{ + *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); + } /* If auto-vacuum is disabled in this build and this is an auto-vacuum ** database, mark the database as read-only. */ diff --git a/src/btree.h b/src/btree.h index 281e57dafd..b57d500c77 100644 --- a/src/btree.h +++ b/src/btree.h @@ -19,7 +19,7 @@ /* TODO: This definition is just included so other modules compile. It ** needs to be revisited. */ -#define SQLITE_N_BTREE_META 10 +#define SQLITE_N_BTREE_META 16 /* ** If defined as non-zero, auto-vacuum is enabled by default. Otherwise @@ -134,6 +134,11 @@ int sqlite3BtreeNewDb(Btree *p); ** For example, the free-page-count field is located at byte offset 36 of ** the database file header. The incr-vacuum-flag field is located at ** byte offset 64 (== 36+4*7). +** +** The BTREE_DATA_VERSION value is not really a value stored in the header. +** It is a read-only number computed by the pager. But we merge it with +** the header value access routines since its access pattern is the same. +** Call it a "virtual meta value". */ #define BTREE_FREE_PAGE_COUNT 0 #define BTREE_SCHEMA_VERSION 1 @@ -144,6 +149,7 @@ int sqlite3BtreeNewDb(Btree *p); #define BTREE_USER_VERSION 6 #define BTREE_INCR_VACUUM 7 #define BTREE_APPLICATION_ID 8 +#define BTREE_DATA_VERSION 15 /* A virtual meta-value */ /* ** Values that may be OR'd together to form the second argument of an diff --git a/src/pager.c b/src/pager.c index a2ae9cc410..97dd78e35b 100644 --- a/src/pager.c +++ b/src/pager.c @@ -646,6 +646,7 @@ struct Pager { u8 setMaster; /* True if a m-j name has been written to jrnl */ u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 subjInMemory; /* True to use in-memory sub-journals */ + u8 bUseFetch; /* True to use xFetch() */ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ @@ -663,9 +664,9 @@ struct Pager { sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ + u32 nReset; /* Number of calls to pager_reset() */ char dbFileVers[16]; /* Changes whenever database file changes */ - u8 bUseFetch; /* True to use xFetch() */ int nMmapOut; /* Number of mmap pages currently outstanding */ sqlite3_int64 szMmap; /* Desired maximum mmap size */ PgHdr *pMmapFreelist; /* List of free mmap page headers (pDirty) */ @@ -1681,10 +1682,19 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ ** Discard the entire contents of the in-memory page-cache. */ static void pager_reset(Pager *pPager){ + pPager->nReset++; sqlite3BackupRestart(pPager->pBackup); sqlite3PcacheClear(pPager->pPCache); } +/* +** Return the pPager->nReset value +*/ +u32 sqlite3PagerDataVersion(Pager *pPager){ + assert( pPager->eState>PAGER_OPEN ); + return pPager->nReset; +} + /* ** Free all structures in the Pager.aSavepoint[] array and set both ** Pager.aSavepoint and Pager.nSavepoint to zero. Close the sub-journal diff --git a/src/pager.h b/src/pager.h index 764c7bd197..e07fa88fc5 100644 --- a/src/pager.h +++ b/src/pager.h @@ -172,6 +172,7 @@ int sqlite3PagerSharedLock(Pager *pPager); /* Functions used to query pager state and configuration. */ u8 sqlite3PagerIsreadonly(Pager*); +u32 sqlite3PagerDataVersion(Pager*); int sqlite3PagerRefcount(Pager*); int sqlite3PagerMemUsed(Pager*); const char *sqlite3PagerFilename(Pager*, int); diff --git a/src/pragma.c b/src/pragma.c index cbeb81eee6..1312beef04 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -154,6 +154,12 @@ static const struct sPragmaNames { /* ePragFlag: */ 0, /* iArg: */ 0 }, #endif +#if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + { /* zName: */ "data_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlag: */ PragFlag_ReadOnly, + /* iArg: */ BTREE_DATA_VERSION }, +#endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) { /* zName: */ "database_list", /* ePragTyp: */ PragTyp_DATABASE_LIST, @@ -471,7 +477,7 @@ static const struct sPragmaNames { /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, #endif }; -/* Number of pragmas: 57 on by default, 70 total. */ +/* Number of pragmas: 58 on by default, 71 total. */ /* End of the automatically generated pragma table. ***************************************************************************/ diff --git a/tool/mkpragmatab.tcl b/tool/mkpragmatab.tcl index 2c9b84b735..21503d16ee 100644 --- a/tool/mkpragmatab.tcl +++ b/tool/mkpragmatab.tcl @@ -254,6 +254,12 @@ set pragma_def { ARG: BTREE_USER_VERSION IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + NAME: data_version + TYPE: HEADER_VALUE + ARG: BTREE_DATA_VERSION + FLAG: ReadOnly + IF: !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) + NAME: freelist_count TYPE: HEADER_VALUE ARG: BTREE_FREE_PAGE_COUNT From 0d339e44a0fa01a8f99ca2bda5971aecbd860922 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 19 Dec 2014 20:27:02 +0000 Subject: [PATCH 664/710] Adding test cases for the "PRAGMA data_version" command. FossilOrigin-Name: c5fb7d6a106d46f10e71abe3a6d4243b21ed02a5 --- manifest | 14 ++++------ manifest.uuid | 2 +- test/pragma3.test | 70 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 9 deletions(-) create mode 100644 test/pragma3.test diff --git a/manifest b/manifest index f22a0f1b43..221ed91de4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experimental\s"PRAGMA\sdata_version"\scommand\sfor\sdetecting\swhen\sanother\sprocess\nhas\schanged\sthe\sdatabase\sfile. -D 2014-12-19T19:28:02.465 +C Adding\stest\scases\sfor\sthe\s"PRAGMA\sdata_version"\scommand. +D 2014-12-19T20:27:02.112 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -785,6 +785,7 @@ F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 F test/permutations.test 4e12d43f4639ea8a0e366d9c64e0009afe2eb544 F test/pragma.test aa16dedfe01c02c8895169012f7dfde9c163f0d5 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 +F test/pragma3.test 117ef9768f6c8d11823de7bbe3231b35c426b013 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d @@ -1233,10 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P da27a09d1d991583b59997f6cc67efa28ffd9d6a -R df4854633d0919cb180185852b6a45b5 -T *branch * data_version_pragma -T *sym-data_version_pragma * -T -sym-trunk * +P 43db1f44bce5a0ee50197b95ab0d844540b69d86 +R 2feb9c2cda6af5a5f57185acec7e4041 U drh -Z 81defb06f28b03737e8cb7607139b4d9 +Z 8c3ff0ee7ae569dc5dbcf90ed4307270 diff --git a/manifest.uuid b/manifest.uuid index 92b0ead781..bcdaad2219 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -43db1f44bce5a0ee50197b95ab0d844540b69d86 \ No newline at end of file +c5fb7d6a106d46f10e71abe3a6d4243b21ed02a5 \ No newline at end of file diff --git a/test/pragma3.test b/test/pragma3.test new file mode 100644 index 0000000000..f3229ee528 --- /dev/null +++ b/test/pragma3.test @@ -0,0 +1,70 @@ +# 2014-12-19 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# +# This file implements tests for PRAGMA data_version command. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test pragma3-100 { + PRAGMA data_version; +} {1} +do_execsql_test pragma3-101 { + PRAGMA temp.data_version; +} {1} + +# Writing is a no-op +do_execsql_test pragma3-102 { + PRAGMA main.data_version=1234; + PRAGMA main.data_version; +} {1 1} + +do_execsql_test pragma3-110 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(100),(200),(300); + SELECT * FROM t1; + PRAGMA data_version; +} {100 200 300 1} + +sqlite3 db2 test.db +do_test pragma3-120 { + db2 eval { + SELECT * FROM t1; + PRAGMA data_version; + } +} {100 200 300 1} + +do_execsql_test pragma3-130 { + INSERT INTO t1 VALUES(400),(500); + SELECT * FROM t1; + PRAGMA data_version; +} {100 200 300 400 500 1} + +do_test pragma3-140 { + db2 eval { + SELECT * FROM t1; + PRAGMA data_version; + UPDATE t1 SET a=a+1; + SELECT * FROM t1; + PRAGMA data_version; + } +} {100 200 300 400 500 2 101 201 301 401 501 2} + +do_execsql_test pragma3-150 { + SELECT * FROM t1; + PRAGMA data_version; +} {101 201 301 401 501 2} + + +db2 close +finish_test From d7107b3852793d77c7e5e4ac679ec99300a42507 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 20 Dec 2014 14:34:02 +0000 Subject: [PATCH 665/710] Update the PRAGMA data_version command so that it reponse to changes made by a shared-cache database connection, and also to changes made by the same database connection. Add test cases to verify the new behavior. FossilOrigin-Name: 44ee538374940c50198949f2cbb9213ba2375b6a --- manifest | 14 +++---- manifest.uuid | 2 +- src/pager.c | 9 ++-- test/pragma3.test | 102 ++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 108 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 221ed91de4..affc6751c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adding\stest\scases\sfor\sthe\s"PRAGMA\sdata_version"\scommand. -D 2014-12-19T20:27:02.112 +C Update\sthe\sPRAGMA\sdata_version\scommand\sso\sthat\sit\sreponse\sto\schanges\smade\nby\sa\sshared-cache\sdatabase\sconnection,\sand\salso\sto\schanges\smade\sby\sthe\ssame\ndatabase\sconnection.\s\sAdd\stest\scases\sto\sverify\sthe\snew\sbehavior. +D 2014-12-20T14:34:02.508 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -216,7 +216,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 F src/os_win.c ecb04a0dad2fa6fa659931a9d8f0f3aca33f908a F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c abe022cd72e112a05db5fd1d6d46f5ed93284247 +F src/pager.c 2cbaf886a6157c53a8061ea7e677f81620ff46eb F src/pager.h c3476e7c89cdf1c6914e50a11f3714e30b4e0a77 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 @@ -785,7 +785,7 @@ F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 F test/permutations.test 4e12d43f4639ea8a0e366d9c64e0009afe2eb544 F test/pragma.test aa16dedfe01c02c8895169012f7dfde9c163f0d5 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 -F test/pragma3.test 117ef9768f6c8d11823de7bbe3231b35c426b013 +F test/pragma3.test af097524ab7fdac11d3a37a5bab4f947f2d5d16c F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 43db1f44bce5a0ee50197b95ab0d844540b69d86 -R 2feb9c2cda6af5a5f57185acec7e4041 +P c5fb7d6a106d46f10e71abe3a6d4243b21ed02a5 +R 4d00079d15bfd5a1a2ef4485d7808f56 U drh -Z 8c3ff0ee7ae569dc5dbcf90ed4307270 +Z 7090cd431f3e847d9d9c987c1e3b054c diff --git a/manifest.uuid b/manifest.uuid index bcdaad2219..5bbac1f6ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c5fb7d6a106d46f10e71abe3a6d4243b21ed02a5 \ No newline at end of file +44ee538374940c50198949f2cbb9213ba2375b6a \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 97dd78e35b..67b95f7186 100644 --- a/src/pager.c +++ b/src/pager.c @@ -664,7 +664,7 @@ struct Pager { sqlite3_backup *pBackup; /* Pointer to list of ongoing backup processes */ PagerSavepoint *aSavepoint; /* Array of active savepoints */ int nSavepoint; /* Number of elements in aSavepoint[] */ - u32 nReset; /* Number of calls to pager_reset() */ + u32 iDataVersion; /* Changes whenever database content changes */ char dbFileVers[16]; /* Changes whenever database file changes */ int nMmapOut; /* Number of mmap pages currently outstanding */ @@ -1682,17 +1682,17 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ ** Discard the entire contents of the in-memory page-cache. */ static void pager_reset(Pager *pPager){ - pPager->nReset++; + pPager->iDataVersion++; sqlite3BackupRestart(pPager->pBackup); sqlite3PcacheClear(pPager->pPCache); } /* -** Return the pPager->nReset value +** Return the pPager->iDataVersion value */ u32 sqlite3PagerDataVersion(Pager *pPager){ assert( pPager->eState>PAGER_OPEN ); - return pPager->nReset; + return pPager->iDataVersion; } /* @@ -6317,6 +6317,7 @@ int sqlite3PagerCommitPhaseTwo(Pager *pPager){ } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); + pPager->iDataVersion++; rc = pager_end_transaction(pPager, pPager->setMaster, 1); return pager_error(pPager, rc); } diff --git a/test/pragma3.test b/test/pragma3.test index f3229ee528..a070c9b42c 100644 --- a/test/pragma3.test +++ b/test/pragma3.test @@ -23,18 +23,34 @@ do_execsql_test pragma3-101 { PRAGMA temp.data_version; } {1} -# Writing is a no-op +# Writing to the pragma is a no-op do_execsql_test pragma3-102 { PRAGMA main.data_version=1234; PRAGMA main.data_version; } {1 1} +# EVIDENCE-OF: R-27726-60934 The "PRAGMA data_version" command provides +# an indication that the database file has been modified. +# +# EVIDENCE-OF: R-30058-27547 The integer values returned by two +# invocations of "PRAGMA data_version" will be different if changes +# where committed to that database in between the two invocations. +# +# EVIDENCE-OF: R-10201-09349 The "PRAGMA data_version" command responses +# to changes committed by the same database connection, by database +# connections sharing a cache in shared cache mode, and by completely +# independent database connections including connections in separate +# threads and processes. +# +# In this test, it response to two separate changes on the same database +# connection. +# do_execsql_test pragma3-110 { CREATE TABLE t1(a); INSERT INTO t1 VALUES(100),(200),(300); SELECT * FROM t1; PRAGMA data_version; -} {100 200 300 1} +} {100 200 300 3} sqlite3 db2 test.db do_test pragma3-120 { @@ -48,8 +64,17 @@ do_execsql_test pragma3-130 { INSERT INTO t1 VALUES(400),(500); SELECT * FROM t1; PRAGMA data_version; -} {100 200 300 400 500 1} +} {100 200 300 400 500 4} +# EVIDENCE-OF: R-10201-09349 The "PRAGMA data_version" command responses +# to changes committed by the same database connection, by database +# connections sharing a cache in shared cache mode, and by completely +# independent database connections including connections in separate +# threads and processes. +# +# In these test, it response to changes in a different database connection +# part of the same process. +# do_test pragma3-140 { db2 eval { SELECT * FROM t1; @@ -58,13 +83,76 @@ do_test pragma3-140 { SELECT * FROM t1; PRAGMA data_version; } -} {100 200 300 400 500 2 101 201 301 401 501 2} - +} {100 200 300 400 500 2 101 201 301 401 501 3} do_execsql_test pragma3-150 { SELECT * FROM t1; PRAGMA data_version; -} {101 201 301 401 501 2} - +} {101 201 301 401 501 5} +# EVIDENCE-OF: R-10201-09349 The "PRAGMA data_version" command responses +# to changes committed by the same database connection, by database +# connections sharing a cache in shared cache mode, and by completely +# independent database connections including connections in separate +# threads and processes. +# +# This test verifies behavior when a separate process changes the database +# file. +# +do_test pragma3-200 { + set fd [open pragma3.txt wb] + puts $fd { + sqlite3 db test.db; + db eval {DELETE FROM t1 WHERE a>300}; + db close; + exit; + } + close $fd + exec [info nameofexec] pragma3.txt + forcedelete pragma3.txt + db eval { + PRAGMA data_version; + SELECT * FROM t1; + } +} {6 101 201} db2 close +db close + +# EVIDENCE-OF: R-10201-09349 The "PRAGMA data_version" command responses +# to changes committed by the same database connection, by database +# connections sharing a cache in shared cache mode, and by completely +# independent database connections including connections in separate +# threads and processes. +# +# The next series of tests verifies the behavior for shared-cache +# database connections. +# +ifcapable shared_cache { + set ::enable_shared_cache [sqlite3_enable_shared_cache 1] + sqlite3 db test.db + sqlite3 db2 test.db + do_test pragma3-300 { + db eval { + PRAGMA data_version; + CREATE TABLE t3(a,b,c); + PRAGMA data_version; + } + } {1 2} + do_test pragma3-310 { + db2 eval { + PRAGMA data_version; + INSERT INTO t3(a,b,c) VALUES('abc','def','ghi'); + SELECT * FROM t3; + PRAGMA data_version; + } + } {2 abc def ghi 3} + do_test pragma3-320 { + db eval { + PRAGMA data_version; + SELECT * FROM t3; + } + } {3 abc def ghi} + db2 close + sqlite3_enable_shared_cache $::enable_shared_cache +} + finish_test From cd54bab6fe5946c641979abfe7e16c42cf9436f1 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 20 Dec 2014 21:14:14 +0000 Subject: [PATCH 666/710] Minor fixes and enhancements to the SQLITE_ENABLE_API_ARMOR functionality. FossilOrigin-Name: cb3e4219ac9560d2773b85453aafda54b7c9346f --- Makefile.msc | 38 ++++++++++++++++++++++---------------- manifest | 25 ++++++++++++------------- manifest.uuid | 2 +- src/complete.c | 14 +++++++------- src/main.c | 6 ++++-- src/mutex_w32.c | 6 ++++++ src/table.c | 2 +- src/tokenize.c | 3 +++ 8 files changed, 56 insertions(+), 40 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 64175a0679..3fc675ee4f 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -108,11 +108,12 @@ WIN32HEAP = 0 # levels. Currently, the recognized values for DEBUG are: # # 0 == NDEBUG: Disables assert() and other runtime diagnostics. -# 1 == Disables NDEBUG and all optimizations and then enables PDBs. -# 2 == SQLITE_DEBUG: Enables various diagnostics messages and code. -# 3 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call. -# 4 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros. -# 5 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros. +# 1 == SQLITE_ENABLE_API_ARMOR: extra attempts to detect misuse of the API. +# 2 == Disables NDEBUG and all optimizations and then enables PDBs. +# 3 == SQLITE_DEBUG: Enables various diagnostics messages and code. +# 4 == SQLITE_WIN32_MALLOC_VALIDATE: Validate the Win32 native heap per call. +# 5 == SQLITE_DEBUG_OS_TRACE: Enables output from the OSTRACE() macros. +# 6 == SQLITE_ENABLE_IOTRACE: Enables output from the IOTRACE() macros. # !IFNDEF DEBUG DEBUG = 0 @@ -280,7 +281,7 @@ RCC = $(RCC) -DWINAPI_FAMILY=WINAPI_FAMILY_APP # MSVC runtime library. # !IF $(FOR_WINRT)!=0 || $(USE_CRT_DLL)!=0 -!IF $(DEBUG)>0 +!IF $(DEBUG)>1 TCC = $(TCC) -MDd BCC = $(BCC) -MDd !ELSE @@ -288,7 +289,7 @@ TCC = $(TCC) -MD BCC = $(BCC) -MD !ENDIF !ELSE -!IF $(DEBUG)>0 +!IF $(DEBUG)>1 TCC = $(TCC) -MTd BCC = $(BCC) -MTd !ELSE @@ -313,7 +314,7 @@ RCC = $(RCC) -I$(TOP)\ext\rtree # options are necessary in order to allow debugging symbols to # work correctly with Visual Studio when using the amalgamation. # -!IF $(DEBUG)>0 +!IF $(DEBUG)>1 MKSQLITE3C_ARGS = --linemacros !ELSE MKSQLITE3C_ARGS = @@ -329,17 +330,22 @@ BCC = $(BCC) -DNDEBUG RCC = $(RCC) -DNDEBUG !ENDIF -!IF $(DEBUG)>1 +!IF $(DEBUG)>0 +TCC = $(TCC) -DSQLITE_ENABLE_API_ARMOR +RCC = $(RCC) -DSQLITE_ENABLE_API_ARMOR +!ENDIF + +!IF $(DEBUG)>2 TCC = $(TCC) -DSQLITE_DEBUG RCC = $(RCC) -DSQLITE_DEBUG !ENDIF -!IF $(DEBUG)>3 +!IF $(DEBUG)>4 TCC = $(TCC) -DSQLITE_DEBUG_OS_TRACE=1 RCC = $(RCC) -DSQLITE_DEBUG_OS_TRACE=1 !ENDIF -!IF $(DEBUG)>4 +!IF $(DEBUG)>5 TCC = $(TCC) -DSQLITE_ENABLE_IOTRACE RCC = $(RCC) -DSQLITE_ENABLE_IOTRACE !ENDIF @@ -371,7 +377,7 @@ RCC = $(RCC) -DSQLITE_WIN32_MALLOC=1 # Validate the heap on every call into the native Win32 heap subsystem? # -!IF $(DEBUG)>2 +!IF $(DEBUG)>3 TCC = $(TCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 RCC = $(RCC) -DSQLITE_WIN32_MALLOC_VALIDATE=1 !ENDIF @@ -482,7 +488,7 @@ RCC = $(RCC) $(OPTS) # If compiling for debugging, add some defines. # -!IF $(DEBUG)>0 +!IF $(DEBUG)>1 TCC = $(TCC) -D_DEBUG BCC = $(BCC) -D_DEBUG RCC = $(RCC) -D_DEBUG @@ -491,7 +497,7 @@ RCC = $(RCC) -D_DEBUG # If optimizations are enabled or disabled (either implicitly or # explicitly), add the necessary flags. # -!IF $(DEBUG)>0 || $(OPTIMIZATIONS)==0 +!IF $(DEBUG)>1 || $(OPTIMIZATIONS)==0 TCC = $(TCC) -Od BCC = $(BCC) -Od !ELSEIF $(OPTIMIZATIONS)>=3 @@ -507,7 +513,7 @@ BCC = $(BCC) -O1 # If symbols are enabled (or compiling for debugging), enable PDBs. # -!IF $(DEBUG)>0 || $(SYMBOLS)!=0 +!IF $(DEBUG)>1 || $(SYMBOLS)!=0 TCC = $(TCC) -Zi BCC = $(BCC) -Zi !ENDIF @@ -592,7 +598,7 @@ LTLINKOPTS = $(LTLINKOPTS) /NODEFAULTLIB:kernel32.lib /NODEFAULTLIB:ole32.lib # If either debugging or symbols are enabled, enable PDBs. # -!IF $(DEBUG)>0 || $(SYMBOLS)!=0 +!IF $(DEBUG)>1 || $(SYMBOLS)!=0 LDFLAGS = /DEBUG !ENDIF diff --git a/manifest b/manifest index b4b2c48ca5..6353c101f7 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Add\sthe\s"PRAGMA\sdata_version"\scommand\sfor\schecking\sto\ssee\sif\sa\sdatabase\shas\nbeen\smodified. -D 2014-12-20T14:50:28.363 +C Minor\sfixes\sand\senhancements\sto\sthe\sSQLITE_ENABLE_API_ARMOR\sfunctionality. +D 2014-12-20T21:14:14.374 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc 10720782f88648bf2b5dcedf4c1524b067d43e47 +F Makefile.msc b363b90fe1bfc3b87d190f2f728a126c00d9ce09 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 F VERSION d846487aff892625eb8e75960234e7285f0462fe @@ -178,7 +178,7 @@ F src/btree.h 94277c1d30c0b75705974bcc8b0c05e79c03d474 F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 F src/build.c 162d84e4833b03f9d07192ef06057b0226f6e543 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 -F src/complete.c c4ba6e0626bb94bc77a0861735f3382fcf7cc818 +F src/complete.c 198a0066ba60ab06fc00fba1998d870a4d575463 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a F src/date.c 93594514aae68de117ca4a2a0d6cc63eddf26744 F src/delete.c 0750b1eb4d96cd3fb2c798599a3a7c85e92f1417 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660 -F src/main.c 48e0410a661c629471ca9061d4153245cc9f853b +F src/main.c ee498c89735ba7b581fbd58d2ad2e7679a0475b2 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -207,7 +207,7 @@ F src/mutex.c 19bf9acba69ca2f367c3761080f8a9f0cf4670a8 F src/mutex.h 779d588e3b7756ec3ecf7d78cde1d84aba414f85 F src/mutex_noop.c f3f09fd7a2eb4287cfc799753ffc30380e7b71a1 F src/mutex_unix.c 551e2f25f0fa0ee8fd7a43f50fc3d8be00e95dde -F src/mutex_w32.c 06bfff9a3a83b53389a51a967643db3967032e1e +F src/mutex_w32.c df48fe07562a45c5c927c45b8d5873a27f98bbb0 F src/notify.c 9711a7575036f0d3040ba61bc6e217f13a9888e7 F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf @@ -236,7 +236,7 @@ F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h d36da9a07130cae13cbfee0986bf20028cb01465 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f -F src/table.c f142bba7903e93ca8d113a5b8877a108ad1a27dc +F src/table.c e7a09215315a978057fb42c640f890160dbcc45e F src/tclsqlite.c c6a21c64da1490e14d53cdc2062d1e2e57942622 F src/test1.c 56e33bf6b1827c6ca7520c189131ddd778fb2267 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 @@ -285,7 +285,7 @@ F src/test_vfs.c f84075a388527892ff184988f43b69ce69b8083c F src/test_vfstrace.c bab9594adc976cbe696ff3970728830b4c5ed698 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/threads.c 6de09362b657f19ba83e5fa521ee715787ce9fee -F src/tokenize.c cc9016e5007fc5e76789079616d2f26741bcc689 +F src/tokenize.c e00458c9938072b0ea711c850b8dcf4ddcb5fe18 F src/trigger.c 25571661fdeae8c7f975ff40ffec205520a3f92f F src/update.c 3c4ecc282accf12d39edb8d524cf089645e55a13 F src/utf.c fc6b889ba0779b7722634cdeaa25f1930d93820c @@ -1234,8 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P da27a09d1d991583b59997f6cc67efa28ffd9d6a 44ee538374940c50198949f2cbb9213ba2375b6a -R 6a9e76520ab206bdf596f781a2c88e65 -T +closed 44ee538374940c50198949f2cbb9213ba2375b6a -U drh -Z 9a9f87b94c56980445c2924a0392e880 +P de50f25ce3226fa4929b8236c72c88b739859d5f +R c123b89e18af5f5ce6fabce3b17d012e +U mistachkin +Z 086c85a234ecd6ff163dc839ee55f963 diff --git a/manifest.uuid b/manifest.uuid index 59533b96a3..31b35da2d7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -de50f25ce3226fa4929b8236c72c88b739859d5f \ No newline at end of file +cb3e4219ac9560d2773b85453aafda54b7c9346f \ No newline at end of file diff --git a/src/complete.c b/src/complete.c index c439cfe181..f7a35cc6f3 100644 --- a/src/complete.c +++ b/src/complete.c @@ -105,13 +105,6 @@ int sqlite3_complete(const char *zSql){ u8 state = 0; /* Current state, using numbers defined in header comment */ u8 token; /* Value of the next token */ -#ifdef SQLITE_ENABLE_API_ARMOR - if( zSql==0 ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - #ifndef SQLITE_OMIT_TRIGGER /* A complex statement machine used to detect the end of a CREATE TRIGGER ** statement. This is the normal case. @@ -141,6 +134,13 @@ int sqlite3_complete(const char *zSql){ }; #endif /* SQLITE_OMIT_TRIGGER */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + while( *zSql ){ switch( *zSql ){ case ';': { /* A semicolon */ diff --git a/src/main.c b/src/main.c index c89ccbd55d..76af9a2e95 100644 --- a/src/main.c +++ b/src/main.c @@ -3659,13 +3659,14 @@ Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ ** connection. */ const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ + Btree *pBt; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ (void)SQLITE_MISUSE_BKPT; return 0; } #endif - Btree *pBt = sqlite3DbNameToBtree(db, zDbName); + pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeGetFilename(pBt) : 0; } @@ -3674,12 +3675,13 @@ const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ ** no such database exists. */ int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ + Btree *pBt; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ (void)SQLITE_MISUSE_BKPT; return -1; } #endif - Btree *pBt = sqlite3DbNameToBtree(db, zDbName); + pBt = sqlite3DbNameToBtree(db, zDbName); return pBt ? sqlite3BtreeIsReadonly(pBt) : -1; } diff --git a/src/mutex_w32.c b/src/mutex_w32.c index da7d73f7c5..a799c86159 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -209,6 +209,12 @@ static sqlite3_mutex *winMutexAlloc(int iType){ break; } default: { +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2<0 || iType-2>=ArraySize(winMutex_staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif assert( iType-2 >= 0 ); assert( iType-2 < ArraySize(winMutex_staticMutexes) ); assert( winMutex_isInit==1 ); diff --git a/src/table.c b/src/table.c index 6e1df30643..235d8dd3df 100644 --- a/src/table.c +++ b/src/table.c @@ -127,7 +127,7 @@ int sqlite3_get_table( TabResult res; #ifdef SQLITE_ENABLE_API_ARMOR - if( pazResult==0 ) return SQLITE_MISUSE_BKPT; + if( !sqlite3SafetyCheckOk(db) || pazResult==0 ) return SQLITE_MISUSE_BKPT; #endif *pazResult = 0; if( pnColumn ) *pnColumn = 0; diff --git a/src/tokenize.c b/src/tokenize.c index 5bb9155460..f0360eef61 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -391,6 +391,9 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ int mxSqlLen; /* Max length of an SQL string */ +#ifdef SQLITE_ENABLE_API_ARMOR + if( zSql==0 || pzErrMsg==0 ) return SQLITE_MISUSE_BKPT; +#endif mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; if( db->nVdbeActive==0 ){ db->u1.isInterrupted = 0; From d9d812fc3878495b7435d6c25b64209b04330f2c Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 20 Dec 2014 22:21:49 +0000 Subject: [PATCH 667/710] Support manually disabling overlapped file I/O for Windows sub-platforms other than Windows CE. FossilOrigin-Name: b9330b887cc8bed2b6b3e6c1b269788e08ccf50d --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 14 +++++++------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 6353c101f7..24ba3c7b0a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\sfixes\sand\senhancements\sto\sthe\sSQLITE_ENABLE_API_ARMOR\sfunctionality. -D 2014-12-20T21:14:14.374 +C Support\smanually\sdisabling\soverlapped\sfile\sI/O\sfor\sWindows\ssub-platforms\sother\sthan\sWindows\sCE. +D 2014-12-20T22:21:49.905 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -214,7 +214,7 @@ F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 -F src/os_win.c ecb04a0dad2fa6fa659931a9d8f0f3aca33f908a +F src/os_win.c 91d3d08e33ec0258d180d4c8255492f47d15e007 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 2cbaf886a6157c53a8061ea7e677f81620ff46eb F src/pager.h c3476e7c89cdf1c6914e50a11f3714e30b4e0a77 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P de50f25ce3226fa4929b8236c72c88b739859d5f -R c123b89e18af5f5ce6fabce3b17d012e +P cb3e4219ac9560d2773b85453aafda54b7c9346f +R ac9484154ad6b155e9718a0c54a2f6d5 U mistachkin -Z 086c85a234ecd6ff163dc839ee55f963 +Z f0b985b941d4fc82caba937840400408 diff --git a/manifest.uuid b/manifest.uuid index 31b35da2d7..b62cc74d7e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb3e4219ac9560d2773b85453aafda54b7c9346f \ No newline at end of file +b9330b887cc8bed2b6b3e6c1b269788e08ccf50d \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 80ce97a6cd..e2f9aecac6 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2479,7 +2479,7 @@ static int winRead( int amt, /* Number of bytes to read */ sqlite3_int64 offset /* Begin reading at this offset */ ){ -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) OVERLAPPED overlapped; /* The offset for ReadFile. */ #endif winFile *pFile = (winFile*)id; /* file handle */ @@ -2511,7 +2511,7 @@ static int winRead( } #endif -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) if( winSeekFile(pFile, offset) ){ OSTRACE(("READ file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; @@ -2583,13 +2583,13 @@ static int winWrite( } #endif -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) rc = winSeekFile(pFile, offset); if( rc==0 ){ #else { #endif -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) OVERLAPPED overlapped; /* The offset for WriteFile. */ #endif u8 *aRem = (u8 *)pBuf; /* Data yet to be written */ @@ -2597,14 +2597,14 @@ static int winWrite( DWORD nWrite; /* Bytes written by each WriteFile() call */ DWORD lastErrno = NO_ERROR; /* Value returned by GetLastError() */ -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) memset(&overlapped, 0, sizeof(OVERLAPPED)); overlapped.Offset = (LONG)(offset & 0xffffffff); overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); #endif while( nRem>0 ){ -#if SQLITE_OS_WINCE +#if SQLITE_OS_WINCE || defined(SQLITE_WIN32_NO_OVERLAPPED) if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, 0) ){ #else if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ @@ -2617,7 +2617,7 @@ static int winWrite( lastErrno = osGetLastError(); break; } -#if !SQLITE_OS_WINCE +#if !SQLITE_OS_WINCE && !defined(SQLITE_WIN32_NO_OVERLAPPED) offset += nWrite; overlapped.Offset = (LONG)(offset & 0xffffffff); overlapped.OffsetHigh = (LONG)((offset>>32) & 0x7fffffff); From dd31b4e709b65916083061a60f6bc055c5122cac Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 21 Dec 2014 11:56:02 +0000 Subject: [PATCH 668/710] Fixes to the README.md file. No changes to code. FossilOrigin-Name: ef4b734d1ff3dbb9c802c60dc1384033fdfd87e5 --- README.md | 24 ++++++++++++------------ manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 5e52dea507..ff8dc34aef 100644 --- a/README.md +++ b/README.md @@ -53,10 +53,10 @@ to the "sqlite3.dll" command line above. When debugging into the SQLite code, adding the "DEBUG=1" argument to one of the above command lines is recommended. -SQLite does not require Tcl to run, but a Tcl installation is required -by the makefiles (including those for MSVC). SQLite contains a lot of -generated code and Tcl is used to do much of that code generation. The -makefiles also require AWK. +SQLite does not require [Tcl](http://www.tcl.tk/) to run, but a Tcl installation +is required by the makefiles (including those for MSVC). SQLite contains +a lot of generated code and Tcl is used to do much of that code generation. +The makefiles also require AWK. ## Source Code Tour @@ -95,14 +95,14 @@ manually-edited files and automatically-generated files. The SQLite interface is defined by the **sqlite3.h** header file, which is generated from src/sqlite.h.in, ./manifest.uuid, and ./VERSION. The -Tcl script at tool/mksqlite3h.tcl does the conversion. The manifest.uuid -file contains the SHA1 hash of the particular check-in and is used to generate -the SQLITE_SOURCE_ID macro. The VERSION file contains the current SQLite -version number. The sqlite3.h header is really just a copy of src/sqlite.h.in -with the source-id and version number inserted at just the right spots. -Note that comment text in the sqlite3.h file is used to generate much of -the SQLite API documentation. The Tcl scripts used to generate that -documentation are in a separate source repository. +[Tcl script](http://www.tcl.tk) at tool/mksqlite3h.tcl does the conversion. +The manifest.uuid file contains the SHA1 hash of the particular check-in +and is used to generate the SQLITE\_SOURCE\_ID macro. The VERSION file +contains the current SQLite version number. The sqlite3.h header is really +just a copy of src/sqlite.h.in with the source-id and version number inserted +at just the right spots. Note that comment text in the sqlite3.h file is +used to generate much of the SQLite API documentation. The Tcl scripts +used to generate that documentation are in a separate source repository. The SQL language parser is **parse.c** which is generate from a grammar in the src/parse.y file. The conversion of "parse.y" into "parse.c" is done diff --git a/manifest b/manifest index 24ba3c7b0a..d7e15132e7 100644 --- a/manifest +++ b/manifest @@ -1,11 +1,11 @@ -C Support\smanually\sdisabling\soverlapped\sfile\sI/O\sfor\sWindows\ssub-platforms\sother\sthan\sWindows\sCE. -D 2014-12-20T22:21:49.905 +C Fixes\sto\sthe\sREADME.md\sfile.\s\sNo\schanges\sto\scode. +D 2014-12-21T11:56:02.079 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc b363b90fe1bfc3b87d190f2f728a126c00d9ce09 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 -F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 +F README.md d58e3bebc0a4145e0f2a87994015fdb575a8e866 F VERSION d846487aff892625eb8e75960234e7285f0462fe F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cb3e4219ac9560d2773b85453aafda54b7c9346f -R ac9484154ad6b155e9718a0c54a2f6d5 -U mistachkin -Z f0b985b941d4fc82caba937840400408 +P b9330b887cc8bed2b6b3e6c1b269788e08ccf50d +R 92e5107af7d9564747f56fe6b5bf8f20 +U drh +Z 2bbd439edb2e1185c83f3c807f675b2f diff --git a/manifest.uuid b/manifest.uuid index b62cc74d7e..08ca74492b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b9330b887cc8bed2b6b3e6c1b269788e08ccf50d \ No newline at end of file +ef4b734d1ff3dbb9c802c60dc1384033fdfd87e5 \ No newline at end of file From 3da9c04712ed96e74427c8168fcfac5527413e1c Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Dec 2014 18:41:21 +0000 Subject: [PATCH 669/710] Redefine the way PRAGMA data_version works: It continues to change when any other connection commits, including shared-cache connections, but does not change if the local connection commits. FossilOrigin-Name: 7a97826f33460f3b4f3890c9cf97116c3355eeda --- manifest | 16 ++--- manifest.uuid | 2 +- src/btree.c | 3 +- src/btreeInt.h | 1 + test/pragma3.test | 146 ++++++++++++++++++++++++++++++++-------------- 5 files changed, 114 insertions(+), 54 deletions(-) diff --git a/manifest b/manifest index d7e15132e7..8ae7e71d4a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sto\sthe\sREADME.md\sfile.\s\sNo\schanges\sto\scode. -D 2014-12-21T11:56:02.079 +C Redefine\sthe\sway\sPRAGMA\sdata_version\sworks:\s\sIt\scontinues\sto\schange\swhen\nany\sother\sconnection\scommits,\sincluding\sshared-cache\sconnections,\sbut\sdoes\snot\nchange\sif\sthe\slocal\sconnection\scommits. +D 2014-12-22T18:41:21.243 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,9 +173,9 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c b2b2bd0aa02430fe86bc891295db919fcafb0d64 +F src/btree.c 1de0560426ecde85ff3ea95d7c94261d7652e284 F src/btree.h 94277c1d30c0b75705974bcc8b0c05e79c03d474 -F src/btreeInt.h 3363e18fd76f69a27a870b25221b2345b3fd4d21 +F src/btreeInt.h a3d0ae1d511365e1a2b76ad10960dbe55c286f34 F src/build.c 162d84e4833b03f9d07192ef06057b0226f6e543 F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 198a0066ba60ab06fc00fba1998d870a4d575463 @@ -785,7 +785,7 @@ F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 F test/permutations.test 4e12d43f4639ea8a0e366d9c64e0009afe2eb544 F test/pragma.test aa16dedfe01c02c8895169012f7dfde9c163f0d5 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 -F test/pragma3.test 1935dfdd0082250df4cf4caed52bdfef527c34ff +F test/pragma3.test 3da08d907ba027c50ede8d6e95418f32898971a5 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b9330b887cc8bed2b6b3e6c1b269788e08ccf50d -R 92e5107af7d9564747f56fe6b5bf8f20 +P ef4b734d1ff3dbb9c802c60dc1384033fdfd87e5 +R 717423e0d3aaa97f5eb5c0e7357ce13c U drh -Z 2bbd439edb2e1185c83f3c807f675b2f +Z 78337e4052167193689952903ddb287b diff --git a/manifest.uuid b/manifest.uuid index 08ca74492b..327aa0952b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef4b734d1ff3dbb9c802c60dc1384033fdfd87e5 \ No newline at end of file +7a97826f33460f3b4f3890c9cf97116c3355eeda \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 1223f78e48..e6a7e2c2b9 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3550,6 +3550,7 @@ int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ sqlite3BtreeLeave(p); return rc; } + p->iDataVersion--; /* Compensate for pPager->iDataVersion++; */ pBt->inTransaction = TRANS_READ; btreeClearHasContent(pBt); } @@ -8195,7 +8196,7 @@ void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ assert( idx>=0 && idx<=15 ); if( idx==BTREE_DATA_VERSION ){ - *pMeta = sqlite3PagerDataVersion(pBt->pPager); + *pMeta = sqlite3PagerDataVersion(pBt->pPager) + p->iDataVersion; }else{ *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); } diff --git a/src/btreeInt.h b/src/btreeInt.h index a28a6a297e..ed4d75ee9f 100644 --- a/src/btreeInt.h +++ b/src/btreeInt.h @@ -351,6 +351,7 @@ struct Btree { u8 locked; /* True if db currently has pBt locked */ int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ int nBackup; /* Number of backup operations reading this btree */ + u32 iDataVersion; /* Combines with pBt->pPager->iDataVersion */ Btree *pNext; /* List of other sharable Btrees from the same db */ Btree *pPrev; /* Back pointer of the same list */ #ifndef SQLITE_OMIT_SHARED_CACHE diff --git a/test/pragma3.test b/test/pragma3.test index 5918d0f0ed..e6069f6e53 100644 --- a/test/pragma3.test +++ b/test/pragma3.test @@ -32,25 +32,20 @@ do_execsql_test pragma3-102 { # EVIDENCE-OF: R-27726-60934 The "PRAGMA data_version" command provides # an indication that the database file has been modified. # -# EVIDENCE-OF: R-30058-27547 The integer values returned by two -# invocations of "PRAGMA data_version" will be different if changes -# where committed to that database in between the two invocations. -# -# EVIDENCE-OF: R-10201-09349 The "PRAGMA data_version" command responses -# to changes committed by the same database connection, by database -# connections sharing a cache in shared cache mode, and by completely -# independent database connections including connections in separate -# threads and processes. -# -# In this test, it response to two separate changes on the same database -# connection. +# EVIDENCE-OF: R-25838-33704 The "PRAGMA data_version" value is +# unchanced for commits made on the same database connection. # do_execsql_test pragma3-110 { + PRAGMA data_version; + BEGIN IMMEDIATE; + PRAGMA data_version; CREATE TABLE t1(a); INSERT INTO t1 VALUES(100),(200),(300); + PRAGMA data_version; + COMMIT; SELECT * FROM t1; PRAGMA data_version; -} {100 200 300 3} +} {1 1 1 100 200 300 1} sqlite3 db2 test.db do_test pragma3-120 { @@ -61,44 +56,88 @@ do_test pragma3-120 { } {100 200 300 1} do_execsql_test pragma3-130 { + PRAGMA data_version; + BEGIN IMMEDIATE; + PRAGMA data_version; INSERT INTO t1 VALUES(400),(500); + PRAGMA data_version; + COMMIT; SELECT * FROM t1; PRAGMA data_version; -} {100 200 300 400 500 4} +} {1 1 1 100 200 300 400 500 1} -# EVIDENCE-OF: R-10201-09349 The "PRAGMA data_version" command responses -# to changes committed by the same database connection, by database -# connections sharing a cache in shared cache mode, and by completely -# independent database connections including connections in separate -# threads and processes. +# EVIDENCE-OF: R-63005-41812 The integer values returned by two +# invocations of "PRAGMA data_version" from the same connection will be +# different if changes were committed to the database by any other +# connection in the interim. # -# In these test, it response to changes in a different database connection -# part of the same process. +# Value went from 1 in pragma3-120 to 2 here. # do_test pragma3-140 { db2 eval { SELECT * FROM t1; PRAGMA data_version; + BEGIN IMMEDIATE; + PRAGMA data_version; UPDATE t1 SET a=a+1; + COMMIT; SELECT * FROM t1; PRAGMA data_version; } -} {100 200 300 400 500 2 101 201 301 401 501 3} +} {100 200 300 400 500 2 2 101 201 301 401 501 2} do_execsql_test pragma3-150 { SELECT * FROM t1; PRAGMA data_version; -} {101 201 301 401 501 5} +} {101 201 301 401 501 2} -# EVIDENCE-OF: R-10201-09349 The "PRAGMA data_version" command responses -# to changes committed by the same database connection, by database -# connections sharing a cache in shared cache mode, and by completely -# independent database connections including connections in separate -# threads and processes. # -# This test verifies behavior when a separate process changes the database -# file. +do_test pragma3-160 { + db eval { + BEGIN; + PRAGMA data_version; + UPDATE t1 SET a=555 WHERE a=501; + PRAGMA data_version; + SELECT * FROM t1 ORDER BY a; + PRAGMA data_version; + } +} {2 2 101 201 301 401 555 2} +do_test pragma3-170 { + db2 eval { + PRAGMA data_version; + } +} {2} +do_test pragma3-180 { + db eval { + COMMIT; + PRAGMA data_version; + } +} {2} +do_test pragma3-190 { + db2 eval { + PRAGMA data_version; + } +} {3} + +# EVIDENCE-OF: R-19326-44825 The "PRAGMA data_version" value is a local +# property of each database connection and so values returned by two +# concurrent invocations of "PRAGMA data_version" on separate database +# connections are often different even though the underlying database is +# identical. +# +do_test pragma3-195 { + expr {[db eval {PRAGMA data_version}]!=[db2 eval {PRAGMA data_version}]} +} {1} + +# EVIDENCE-OF: R-54562-06892 The behavior of "PRAGMA data_version" is +# the same for all database connections, including database connections +# in separate processes and shared cache database connections. +# +# The next block checks the behavior for separate processes. # do_test pragma3-200 { + db eval {PRAGMA data_version; SELECT * FROM t1;} +} {2 101 201 301 401 555} +do_test pragma3-201 { set fd [open pragma3.txt wb] puts $fd { sqlite3 db test.db; @@ -113,18 +152,15 @@ do_test pragma3-200 { PRAGMA data_version; SELECT * FROM t1; } -} {6 101 201} +} {3 101 201} db2 close db close -# EVIDENCE-OF: R-10201-09349 The "PRAGMA data_version" command responses -# to changes committed by the same database connection, by database -# connections sharing a cache in shared cache mode, and by completely -# independent database connections including connections in separate -# threads and processes. +# EVIDENCE-OF: R-54562-06892 The behavior of "PRAGMA data_version" is +# the same for all database connections, including database connections +# in separate processes and shared cache database connections. # -# The next series of tests verifies the behavior for shared-cache -# database connections. +# The next block checks that behavior is the same for shared-cache. # ifcapable shared_cache { set ::enable_shared_cache [sqlite3_enable_shared_cache 1] @@ -133,24 +169,46 @@ ifcapable shared_cache { do_test pragma3-300 { db eval { PRAGMA data_version; + BEGIN; CREATE TABLE t3(a,b,c); + CREATE TABLE t4(x,y,z); + INSERT INTO t4 VALUES(123,456,789); + PRAGMA data_version; + COMMIT; PRAGMA data_version; } - } {1 2} + } {1 1 1} do_test pragma3-310 { db2 eval { PRAGMA data_version; + BEGIN; INSERT INTO t3(a,b,c) VALUES('abc','def','ghi'); SELECT * FROM t3; PRAGMA data_version; } - } {2 abc def ghi 3} + } {2 abc def ghi 2} + # The transaction in db2 has not yet committed, so the data_version in + # db is unchanged. do_test pragma3-320 { db eval { PRAGMA data_version; - SELECT * FROM t3; + SELECT * FROM t4; } - } {3 abc def ghi} + } {1 123 456 789} + do_test pragma3-330 { + db2 eval { + COMMIT; + PRAGMA data_version; + SELECT * FROM t4; + } + } {2 123 456 789} + do_test pragma3-340 { + db eval { + PRAGMA data_version; + SELECT * FROM t3; + SELECT * FROM t4; + } + } {2 abc def ghi 123 456 789} db2 close db close sqlite3_enable_shared_cache $::enable_shared_cache @@ -168,7 +226,7 @@ ifcapable wal { PRAGMA journal_mode; SELECT * FROM t1; } - } {3 wal 101 201} + } {2 wal 101 201} do_test pragma3-410 { db2 eval { PRAGMA data_version; @@ -178,7 +236,7 @@ ifcapable wal { } {2 wal 101 201} do_test pragma3-420 { db eval {UPDATE t1 SET a=111*(a/100); PRAGMA data_version; SELECT * FROM t1} - } {4 111 222} + } {2 111 222} do_test pragma3-430 { db2 eval {PRAGMA data_version; SELECT * FROM t1;} } {3 111 222} From 4a86d0016225d78abd3bb170c29d6e9ffbd0985e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 22 Dec 2014 22:02:20 +0000 Subject: [PATCH 670/710] Fix a typo in an evidence mark on a test script. No changes to code. FossilOrigin-Name: a08b0c7512287ce5ae3fffe02c092d0eb25b3a25 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/pragma3.test | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8ae7e71d4a..545fed12be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Redefine\sthe\sway\sPRAGMA\sdata_version\sworks:\s\sIt\scontinues\sto\schange\swhen\nany\sother\sconnection\scommits,\sincluding\sshared-cache\sconnections,\sbut\sdoes\snot\nchange\sif\sthe\slocal\sconnection\scommits. -D 2014-12-22T18:41:21.243 +C Fix\sa\stypo\sin\san\sevidence\smark\son\sa\stest\sscript.\s\sNo\schanges\sto\scode. +D 2014-12-22T22:02:20.146 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -785,7 +785,7 @@ F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 F test/permutations.test 4e12d43f4639ea8a0e366d9c64e0009afe2eb544 F test/pragma.test aa16dedfe01c02c8895169012f7dfde9c163f0d5 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 -F test/pragma3.test 3da08d907ba027c50ede8d6e95418f32898971a5 +F test/pragma3.test 4f141da233358783ba443eb685e6739ce0eb1d90 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef4b734d1ff3dbb9c802c60dc1384033fdfd87e5 -R 717423e0d3aaa97f5eb5c0e7357ce13c +P 7a97826f33460f3b4f3890c9cf97116c3355eeda +R 75921305dcdb8922f71566b00cf9fa0c U drh -Z 78337e4052167193689952903ddb287b +Z 6c9fa3fa92317b5bccccc5c856a1593a diff --git a/manifest.uuid b/manifest.uuid index 327aa0952b..9504503c19 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7a97826f33460f3b4f3890c9cf97116c3355eeda \ No newline at end of file +a08b0c7512287ce5ae3fffe02c092d0eb25b3a25 \ No newline at end of file diff --git a/test/pragma3.test b/test/pragma3.test index e6069f6e53..f3d531164b 100644 --- a/test/pragma3.test +++ b/test/pragma3.test @@ -32,8 +32,8 @@ do_execsql_test pragma3-102 { # EVIDENCE-OF: R-27726-60934 The "PRAGMA data_version" command provides # an indication that the database file has been modified. # -# EVIDENCE-OF: R-25838-33704 The "PRAGMA data_version" value is -# unchanced for commits made on the same database connection. +# EVIDENCE-OF: R-47505-58569 The "PRAGMA data_version" value is +# unchanged for commits made on the same database connection. # do_execsql_test pragma3-110 { PRAGMA data_version; From d477eee312abf8de89426949976fab10fbcb6e1e Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Dec 2014 19:40:51 +0000 Subject: [PATCH 671/710] Rework the test/releasetest.tcl script so that it uses the autoconf makefile instead of the test/releasetest.mk makefile. Also add options like --dryrun and --buildonly. Omit the --makefile option and replace it with --srcdir with the default computed relative to the releasetest.tcl script itself. FossilOrigin-Name: 7c85e831153f9aef2afaf981d53db38a28091be5 --- manifest | 16 ++-- manifest.uuid | 2 +- test/releasetest.mk | 14 ---- test/releasetest.tcl | 185 +++++++++++++++++++++++++------------------ 4 files changed, 118 insertions(+), 99 deletions(-) delete mode 100644 test/releasetest.mk diff --git a/manifest b/manifest index 545fed12be..63ed4c71f6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\san\sevidence\smark\son\sa\stest\sscript.\s\sNo\schanges\sto\scode. -D 2014-12-22T22:02:20.146 +C Rework\sthe\stest/releasetest.tcl\sscript\sso\sthat\sit\suses\sthe\sautoconf\smakefile\ninstead\sof\sthe\stest/releasetest.mk\smakefile.\s\sAlso\sadd\soptions\slike\s--dryrun\nand\s--buildonly.\s\sOmit\sthe\s--makefile\soption\sand\sreplace\sit\swith\s--srcdir\nwith\sthe\sdefault\scomputed\srelative\sto\sthe\sreleasetest.tcl\sscript\sitself. +D 2014-12-23T19:40:51.746 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -801,8 +801,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.mk 2eced2f9ae701fd0a29e714a241760503ccba25a -F test/releasetest.tcl a4279c890698584feb2ffc86735857a4e4474180 +F test/releasetest.tcl b57a20ccc7aa64be8b52906d211b1ed4c0998cc2 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -1234,7 +1233,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7a97826f33460f3b4f3890c9cf97116c3355eeda -R 75921305dcdb8922f71566b00cf9fa0c +P a08b0c7512287ce5ae3fffe02c092d0eb25b3a25 +R d1cf8fd2536e09346a06ad4dd6ebb5ea +T *branch * releasetest-refactor +T *sym-releasetest-refactor * +T -sym-trunk * U drh -Z 6c9fa3fa92317b5bccccc5c856a1593a +Z 1d662dddfc70aa251729f355b7103297 diff --git a/manifest.uuid b/manifest.uuid index 9504503c19..f2873c82b4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a08b0c7512287ce5ae3fffe02c092d0eb25b3a25 \ No newline at end of file +7c85e831153f9aef2afaf981d53db38a28091be5 \ No newline at end of file diff --git a/test/releasetest.mk b/test/releasetest.mk deleted file mode 100644 index 5d217c645b..0000000000 --- a/test/releasetest.mk +++ /dev/null @@ -1,14 +0,0 @@ -######################################################## -TOP=/home/drh/sqlite/sqlite - -TCL_FLAGS=-I/home/drh/tcltk/86linux -LIBTCL=/home/drh/tcltk/86linux/libtcl8.6.a -lm -ldl -lpthread - -BCC = gcc -TCC = gcc -ansi -g $(CFLAGS) -NAWK = awk -AR = ar cr -RANLIB = ranlib -THREADLIB = -lpthread -ldl -lz -include $(TOP)/main.mk -######################################################## diff --git a/test/releasetest.tcl b/test/releasetest.tcl index d2a1bd2bb0..fae7d3c8f6 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -1,6 +1,5 @@ - -set rcsid {$Id: $} - +#!/usr/bin/tclsh +# # Documentation for this script. This may be output to stderr # if the script is invoked incorrectly. See the [process_options] # proc below. @@ -10,51 +9,23 @@ This Tcl script is used to test the various configurations required before releasing a new version. Supported command line options (all optional) are: - -makefile PATH-TO-MAKEFILE (default "releasetest.mk") - -platform PLATFORM (see below) - -quick BOOLEAN (default "0") - -config CONFIGNAME (Run only CONFIGNAME) + --srcdir TOP-OF-SQLITE-TREE (see below) + --platform PLATFORM (see below) + --config CONFIGNAME (Run only CONFIGNAME) + --quick (Run "veryquick.test" only) + --buildonly (Just build testfixture - do not run) + --dryrun (Print what would have happened) + --info (Show diagnostic info) -The default value for -makefile is "./releasetest.mk". +The default value for --srcdir is the parent of the directory holding +this script. -The script determines the default value for -platform using the +The script determines the default value for --platform using the $tcl_platform(os) and $tcl_platform(machine) variables. Supported platforms are "Linux-x86", "Linux-x86_64" and "Darwin-i386". -If the -quick option is set to true, then the "veryquick.test" script -is run for all compilation configurations. Otherwise, sometimes "all.test" -is run, sometimes "veryquick.test". - -Almost any SQLite makefile (except those generated by configure - see below) -should work. The following properties are required: - - * The makefile should support the "fulltest" target. - * The makefile should support the variable "OPTS" as a way to pass - options from the make command line to lemon and the C compiler. - -More precisely, the following invocation must be supported: - - make -f $::MAKEFILE fulltest OPTS="-DSQLITE_SECURE_DELETE=1 -DSQLITE_DEBUG=1" - -Makefiles generated by the sqlite configure program cannot be used as -they do not respect the OPTS variable. - -Example Makefile contents: - - ######################################################## - TOP=/home/dan/work/sqlite/sqlite - - TCL_FLAGS=-I/home/dan/tcl/include - LIBTCL=-L/home/dan/tcl/lib -ltcl - - BCC = gcc - TCC = gcc -ansi -g $(CFLAGS) - NAWK = awk - AR = ar cr - RANLIB = ranlib - THREADLIB = -lpthread -ldl - include $(TOP)/main.mk - ######################################################## +Every test begins with a fresh run of the configure script at the top +of the SQLite source tree. } array set ::Configs { @@ -228,13 +199,12 @@ foreach {key value} [array get ::Platforms] { } proc run_test_suite {name testtarget config} { - # Tcl variable $opts is used to build up the value used to set the # OPTS Makefile variable. Variable $cflags holds the value for # CFLAGS. The makefile will pass OPTS to both gcc and lemon, but # CFLAGS is only passed to gcc. # - set cflags "" + set cflags "-g" set opts "" foreach arg $config { if {[string match -D* $arg]} { @@ -258,30 +228,46 @@ proc run_test_suite {name testtarget config} { append opts " -DSQLITE_OS_UNIX=1" } - # Run the test. - # - set makefile [file normalize $::MAKEFILE] - file mkdir $dir - puts -nonewline "Testing configuration \"$name\" (logfile=$dir/test.log)..." - flush stdout + dryrun file mkdir $dir + if {!$::DRYRUN} { + set n [string length $name] + puts -nonewline "${name}[string repeat . [expr {40-$n}]]" + flush stdout + } - set makecmd [concat \ - [list exec make -C $dir -f $makefile clean] \ - $testtarget \ - [list CFLAGS=$cflags OPTS=$opts >& $dir/test.log] \ - ] - - set tm1 [clock seconds] - set rc [catch $makecmd] + set tm1 [clock seconds] + set origdir [pwd] + dryrun cd $dir + set rc [catch [list dryrun exec $::SRCDIR/configure >& test.log]] + if {!$rc} { + set rc [catch [list dryrun exec make clean $testtarget \ + CFLAGS=$cflags OPTS=$opts >>& test.log]] + } set tm2 [clock seconds] + dryrun cd $origdir - set minutes [expr {($tm2-$tm1)/60}] - set seconds [expr {($tm2-$tm1)%60}] - puts -nonewline [format " (%d:%.2d) " $minutes $seconds] - if {$rc} { - puts "FAILED." + if {!$::DRYRUN} { + set minutes [expr {($tm2-$tm1)/60}] + set seconds [expr {($tm2-$tm1)%60}] + set tm [format (%02d:%02d) $minutes $seconds] + if {$rc} { + puts " FAIL $tm" + incr ::NERR + } else { + puts " Ok $tm" + } + } +} + +# The following procedure either prints its arguments (if ::DRYRUN is true) +# or executes the command of its arguments in the calling context +# (if ::DRYRUN is false). +# +proc dryrun {args} { + if {$::DRYRUN} { + puts $args } else { - puts "Ok." + uplevel 1 $args } } @@ -292,16 +278,21 @@ proc run_test_suite {name testtarget config} { # option. # proc process_options {argv} { - set ::MAKEFILE releasetest.mk ;# Default value - set ::QUICK 0 ;# Default value + set ::SRCDIR [file normalize [file dirname [file dirname $::argv0]]] + set ::QUICK 0 + set ::BUILDONLY 0 + set ::DRYRUN 0 + set ::EXEC exec set config {} set platform $::tcl_platform(os)-$::tcl_platform(machine) for {set i 0} {$i < [llength $argv]} {incr i} { - switch -- [lindex $argv $i] { - -makefile { + set x [lindex $argv $i] + if {[regexp {^--[a-z]} $x]} {set x [string range $x 1 end]} + switch -- $x { + -srcdir { incr i - set ::MAKEFILE [lindex $argv $i] + set ::SRCDIR [file normalize [lindex $argv $i]] } -platform { @@ -310,14 +301,40 @@ proc process_options {argv} { } -quick { - incr i - set ::QUICK [lindex $argv $i] + set ::QUICK 1 } -config { incr i set config [lindex $argv $i] } + + -buildonly { + set ::BUILDONLY 1 + } + + -dryrun { + set ::DRYRUN 1 + } + + -info { + puts "Command-line Options:" + puts " --srcdir $::SRCDIR" + puts " --platform [list $platform]" + puts " --config [list $config]" + if {$::QUICK} {puts " --quick"} + if {$::BUILDONLY} {puts " --buildonly"} + if {$::DRYRUN} {puts " --dryrun"} + puts "\nAvailable --platform options:" + foreach y [lsort [array names ::Platforms]] { + puts " [list $y]" + } + puts "\nAvailable --config options:" + foreach y [lsort [array names ::Configs]] { + puts " [list $y]" + } + exit + } default { puts stderr "" @@ -327,8 +344,6 @@ proc process_options {argv} { } } - set ::MAKEFILE [file normalize $::MAKEFILE] - if {0==[info exists ::Platforms($platform)]} { puts "Unknown platform: $platform" puts -nonewline "Set the -platform option to " @@ -347,8 +362,13 @@ proc process_options {argv} { } else { set ::CONFIGLIST $::Platforms($platform) } - puts "Running the following configurations for $platform:" + puts "Running the following test configurations for $platform:" puts " [string trim $::CONFIGLIST]" + puts -nonewline "Flags:" + if {$::DRYRUN} {puts -nonewline " --dryrun"} + if {$::BUILDONLY} {puts -nonewline " --buildonly"} + if {$::QUICK} {puts -nonewline " --quick"} + puts "" } # Main routine. @@ -357,30 +377,41 @@ proc main {argv} { # Process any command line options. process_options $argv + puts [string repeat * 70] + set NERR 0 + set NTEST 0 + set STARTTIME [clock seconds] foreach {zConfig target} $::CONFIGLIST { if {$::QUICK} {set target test} + if {$::BUILDONLY} {set target testfixture} set config_options $::Configs($zConfig) + incr NTEST run_test_suite $zConfig $target $config_options # If the configuration included the SQLITE_DEBUG option, then remove # it and run veryquick.test. If it did not include the SQLITE_DEBUG option # add it and run veryquick.test. - if {$target!="checksymbols"} { + if {$target!="checksymbols" && !$::BUILDONLY} { set debug_idx [lsearch -glob $config_options -DSQLITE_DEBUG*] if {$debug_idx < 0} { + incr NTEST run_test_suite "${zConfig}_debug" test [ concat $config_options -DSQLITE_DEBUG=1 ] } else { + incr NTEST run_test_suite "${zConfig}_ndebug" test [ lreplace $config_options $debug_idx $debug_idx ] } } - } + + set elapsetime [expr {[clock seconds]-$STARTTIME}] + puts [string repeat * 70] + puts "$NERR failures of $NTEST test suites run in $elapsetime seconds" } main $argv From 00fa334a0b698bb6fb197961d9312e05c67ae5f1 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Dec 2014 19:52:26 +0000 Subject: [PATCH 672/710] Add the "checksymbols" target to Makefile.in FossilOrigin-Name: 5bd73dba5e278db81f7d728c75e3142c02d37ff7 --- Makefile.in | 7 +++++++ manifest | 15 ++++++--------- manifest.uuid | 2 +- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Makefile.in b/Makefile.in index 4929ce313c..db4c23b5bd 100644 --- a/Makefile.in +++ b/Makefile.in @@ -969,6 +969,13 @@ wordcount$(TEXE): $(TOP)/test/wordcount.c sqlite3.c speedtest1$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo $(LTLINK) -o $@ $(TOP)/test/speedtest1.c sqlite3.lo $(TLIBS) +# This target will fail if the SQLite amalgamation contains any exported +# symbols that do not begin with "sqlite3_". It is run as part of the +# releasetest.tcl script. +# +checksymbols: sqlite3.o + nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 + # Standard install and cleanup targets # lib_install: libsqlite3.la diff --git a/manifest b/manifest index 63ed4c71f6..d608fa8851 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Rework\sthe\stest/releasetest.tcl\sscript\sso\sthat\sit\suses\sthe\sautoconf\smakefile\ninstead\sof\sthe\stest/releasetest.mk\smakefile.\s\sAlso\sadd\soptions\slike\s--dryrun\nand\s--buildonly.\s\sOmit\sthe\s--makefile\soption\sand\sreplace\sit\swith\s--srcdir\nwith\sthe\sdefault\scomputed\srelative\sto\sthe\sreleasetest.tcl\sscript\sitself. -D 2014-12-23T19:40:51.746 +C Add\sthe\s"checksymbols"\starget\sto\sMakefile.in +D 2014-12-23T19:52:26.814 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 +F Makefile.in 37aec21b4fbb469ba42985a4454adf424c84f8e4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc b363b90fe1bfc3b87d190f2f728a126c00d9ce09 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 @@ -1233,10 +1233,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a08b0c7512287ce5ae3fffe02c092d0eb25b3a25 -R d1cf8fd2536e09346a06ad4dd6ebb5ea -T *branch * releasetest-refactor -T *sym-releasetest-refactor * -T -sym-trunk * +P 7c85e831153f9aef2afaf981d53db38a28091be5 +R c05ea3aeb104c4afb51e4fdc443cf2a6 U drh -Z 1d662dddfc70aa251729f355b7103297 +Z 02c9f8fd295b123fa2397cb2b2094ed6 diff --git a/manifest.uuid b/manifest.uuid index f2873c82b4..47558f25fb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7c85e831153f9aef2afaf981d53db38a28091be5 \ No newline at end of file +5bd73dba5e278db81f7d728c75e3142c02d37ff7 \ No newline at end of file From e5d7bf1e4c0f20277b5d98227dcc8461919f7853 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Dec 2014 20:05:19 +0000 Subject: [PATCH 673/710] Fix a potential segfault following OOM error in the test harness. The SQLite core itself is not at fault. FossilOrigin-Name: 1bb26695ff28a96d740752e327c2e0a0da1d067e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test1.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 545fed12be..eb52499b91 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\san\sevidence\smark\son\sa\stest\sscript.\s\sNo\schanges\sto\scode. -D 2014-12-22T22:02:20.146 +C Fix\sa\spotential\ssegfault\sfollowing\sOOM\serror\sin\sthe\stest\sharness.\s\sThe\nSQLite\score\sitself\sis\snot\sat\sfault. +D 2014-12-23T20:05:19.241 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -238,7 +238,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 81712116e826b0089bb221b018929536b2b5406f F src/table.c e7a09215315a978057fb42c640f890160dbcc45e F src/tclsqlite.c c6a21c64da1490e14d53cdc2062d1e2e57942622 -F src/test1.c 56e33bf6b1827c6ca7520c189131ddd778fb2267 +F src/test1.c 041c4edf2f9c49a329add297e26ee86a83852f51 F src/test2.c 98049e51a17dc62606a99a9eb95ee477f9996712 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7a97826f33460f3b4f3890c9cf97116c3355eeda -R 75921305dcdb8922f71566b00cf9fa0c +P a08b0c7512287ce5ae3fffe02c092d0eb25b3a25 +R 6f17c6ca29a22c9a696402dcec6c7766 U drh -Z 6c9fa3fa92317b5bccccc5c856a1593a +Z 3a6a677c416cfd077a0efd463f488c83 diff --git a/manifest.uuid b/manifest.uuid index 9504503c19..83787366b0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a08b0c7512287ce5ae3fffe02c092d0eb25b3a25 \ No newline at end of file +1bb26695ff28a96d740752e327c2e0a0da1d067e \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 40acac8360..032e3f338c 100644 --- a/src/test1.c +++ b/src/test1.c @@ -3670,7 +3670,7 @@ static int test_prepare_v2( assert(rc==SQLITE_OK || pStmt==0); Tcl_ResetResult(interp); if( sqlite3TestErrCode(interp, db, rc) ) return TCL_ERROR; - if( zTail && objc>=5 ){ + if( rc==SQLITE_OK && zTail && objc>=5 ){ if( bytes>=0 ){ bytes = bytes - (int)(zTail-zSql); } From 7284056ad665013f991eaa74e6c23bcd69bab68f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 23 Dec 2014 20:22:57 +0000 Subject: [PATCH 674/710] Draft of changes necessary to make releasetest work on Windows via MinGW. FossilOrigin-Name: af166c5c64216d845269410d1ac2493310694b86 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/releasetest.tcl | 44 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 42 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index d608fa8851..ac6be8ffcb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"checksymbols"\starget\sto\sMakefile.in -D 2014-12-23T19:52:26.814 +C Draft\sof\schanges\snecessary\sto\smake\sreleasetest\swork\son\sWindows\svia\sMinGW. +D 2014-12-23T20:22:57.094 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 37aec21b4fbb469ba42985a4454adf424c84f8e4 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -801,7 +801,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl b57a20ccc7aa64be8b52906d211b1ed4c0998cc2 +F test/releasetest.tcl ed2f3293c318e0c80f6dfd099dde529f60e6d048 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -1233,7 +1233,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 7c85e831153f9aef2afaf981d53db38a28091be5 -R c05ea3aeb104c4afb51e4fdc443cf2a6 -U drh -Z 02c9f8fd295b123fa2397cb2b2094ed6 +P 5bd73dba5e278db81f7d728c75e3142c02d37ff7 +R 9f40447b09389fdd147e8f95f68f235c +U mistachkin +Z 2d56af309587de973049953da48ad3d0 diff --git a/manifest.uuid b/manifest.uuid index 47558f25fb..5a8726daed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5bd73dba5e278db81f7d728c75e3142c02d37ff7 \ No newline at end of file +af166c5c64216d845269410d1ac2493310694b86 \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index fae7d3c8f6..e0738d9c4f 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -6,7 +6,7 @@ # set ::USAGE_MESSAGE { This Tcl script is used to test the various configurations required -before releasing a new version. Supported command line options (all +before releasing a new version. Supported command line options (all optional) are: --srcdir TOP-OF-SQLITE-TREE (see below) @@ -14,14 +14,14 @@ optional) are: --config CONFIGNAME (Run only CONFIGNAME) --quick (Run "veryquick.test" only) --buildonly (Just build testfixture - do not run) - --dryrun (Print what would have happened) + --dryrun (Print what would have happened) --info (Show diagnostic info) The default value for --srcdir is the parent of the directory holding this script. The script determines the default value for --platform using the -$tcl_platform(os) and $tcl_platform(machine) variables. Supported +$tcl_platform(os) and $tcl_platform(machine) variables. Supported platforms are "Linux-x86", "Linux-x86_64" and "Darwin-i386". Every test begins with a fresh run of the configure script at the top @@ -132,7 +132,7 @@ array set ::Configs { -DSQLITE_DEFAULT_CACHE_SIZE=1000 -DSQLITE_MAX_LENGTH=2147483645 -DSQLITE_MAX_VARIABLE_NUMBER=500000 - -DSQLITE_DEBUG=1 + -DSQLITE_DEBUG=1 -DSQLITE_PREFER_PROXY_LOCKING=1 } "Extra-Robustness" { @@ -182,6 +182,9 @@ array set ::Platforms { "Locking-Style" test "OS-X" "threadtest fulltest" } + "Windows NT-intel" { + "Default" "threadtest fulltest" + } } @@ -199,7 +202,7 @@ foreach {key value} [array get ::Platforms] { } proc run_test_suite {name testtarget config} { - # Tcl variable $opts is used to build up the value used to set the + # Tcl variable $opts is used to build up the value used to set the # OPTS Makefile variable. Variable $cflags holds the value for # CFLAGS. The makefile will pass OPTS to both gcc and lemon, but # CFLAGS is only passed to gcc. @@ -238,10 +241,9 @@ proc run_test_suite {name testtarget config} { set tm1 [clock seconds] set origdir [pwd] dryrun cd $dir - set rc [catch [list dryrun exec $::SRCDIR/configure >& test.log]] + set rc [catch [configureCommand]] if {!$rc} { - set rc [catch [list dryrun exec make clean $testtarget \ - CFLAGS=$cflags OPTS=$opts >>& test.log]] + set rc [catch [makeCommand $testtarget $cflags $opts]] } set tm2 [clock seconds] dryrun cd $origdir @@ -259,8 +261,30 @@ proc run_test_suite {name testtarget config} { } } +# The following procedure returns the "configure" command to be exectued for +# the current platform, which may be Windows (via MinGW, etc). +# +proc configureCommand {} { + set result [list dryrun exec] + if {$::tcl_platform(platform)=="windows"} { + lappend result sh + } + lappend result $::SRCDIR/configure >& test.log +} + +# The following procedure returns the "make" command to be executed for the +# specified targets, compiler flags, and options. +# +proc makeCommand { targets cflags opts } { + set result [list dryrun exec make clean] + foreach target $targets { + lappend result $target + } + lappend result CFLAGS=$cflags OPTS=$opts >>& test.log +} + # The following procedure either prints its arguments (if ::DRYRUN is true) -# or executes the command of its arguments in the calling context +# or executes the command of its arguments in the calling context # (if ::DRYRUN is false). # proc dryrun {args} { @@ -335,7 +359,7 @@ proc process_options {argv} { } exit } - + default { puts stderr "" puts stderr [string trim $::USAGE_MESSAGE] From 069b8f2cb3623b9b0905673433b1ada1325fbcb5 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Dec 2014 20:31:43 +0000 Subject: [PATCH 675/710] Add the threadtest target to Makefile.in. Add --enable-load-extension to the configure issued by releasetest.tcl. FossilOrigin-Name: cb128067faabf0503dff1298ed29934f484f71bb --- Makefile.in | 17 +++++++++++++++++ manifest | 16 ++++++++-------- manifest.uuid | 2 +- test/releasetest.tcl | 2 +- 4 files changed, 27 insertions(+), 10 deletions(-) diff --git a/Makefile.in b/Makefile.in index db4c23b5bd..04305350be 100644 --- a/Makefile.in +++ b/Makefile.in @@ -976,6 +976,23 @@ speedtest1$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo checksymbols: sqlite3.o nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 +# The next two rules are used to support the "threadtest" target. Building +# threadtest runs a few thread-safety tests that are implemented in C. This +# target is invoked by the releasetest.tcl script. +# +THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ + $(TOP)/test/tt3_checkpoint.c \ + $(TOP)/test/tt3_index.c \ + $(TOP)/test/tt3_vacuum.c \ + $(TOP)/test/tt3_stress.c \ + $(TOP)/test/tt3_lookaside1.c + +threadtest3$(TEXE): sqlite3.o $(THREADTEST3_SRC) + $(LTLINK) $(TOP)/test/threadtest3.c sqlite3.o -o $@ $(TLIBS) + +threadtest: threadtest3$(TEXE) + ./threadtest3$(TEXE) + # Standard install and cleanup targets # lib_install: libsqlite3.la diff --git a/manifest b/manifest index ac6be8ffcb..36c6416518 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Draft\sof\schanges\snecessary\sto\smake\sreleasetest\swork\son\sWindows\svia\sMinGW. -D 2014-12-23T20:22:57.094 +C Add\sthe\sthreadtest\starget\sto\sMakefile.in.\s\sAdd\s--enable-load-extension\sto\nthe\sconfigure\sissued\sby\sreleasetest.tcl. +D 2014-12-23T20:31:43.428 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 37aec21b4fbb469ba42985a4454adf424c84f8e4 +F Makefile.in c5bad35715abe0ffcb6a11fbafb157de7405d51a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc b363b90fe1bfc3b87d190f2f728a126c00d9ce09 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 @@ -801,7 +801,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl ed2f3293c318e0c80f6dfd099dde529f60e6d048 +F test/releasetest.tcl 8563ba7df4b8cb9a9ee9ccd042f14f0553e62f63 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -1233,7 +1233,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5bd73dba5e278db81f7d728c75e3142c02d37ff7 -R 9f40447b09389fdd147e8f95f68f235c -U mistachkin -Z 2d56af309587de973049953da48ad3d0 +P af166c5c64216d845269410d1ac2493310694b86 +R 84b62c16990cff93a3eab955a0ae7d8d +U drh +Z 40c138478ca76e8e420aceec7c5661be diff --git a/manifest.uuid b/manifest.uuid index 5a8726daed..5424c8d2ee 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -af166c5c64216d845269410d1ac2493310694b86 \ No newline at end of file +cb128067faabf0503dff1298ed29934f484f71bb \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index e0738d9c4f..f7aef83c1c 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -269,7 +269,7 @@ proc configureCommand {} { if {$::tcl_platform(platform)=="windows"} { lappend result sh } - lappend result $::SRCDIR/configure >& test.log + lappend result $::SRCDIR/configure -enable-load-extension >& test.log } # The following procedure returns the "make" command to be executed for the From e43ff920ceeac2fb2c2127e639cbde892d7033bf Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 23 Dec 2014 20:41:13 +0000 Subject: [PATCH 676/710] In the releasetest.tcl script, show the test target for each configuration that is run. And show the time in HH:MM:SS. FossilOrigin-Name: 2295e9e0a22220ccb1b9cc2b031c1d8e702a1888 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/releasetest.tcl | 10 ++++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 36c6416518..e882294d9d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sthreadtest\starget\sto\sMakefile.in.\s\sAdd\s--enable-load-extension\sto\nthe\sconfigure\sissued\sby\sreleasetest.tcl. -D 2014-12-23T20:31:43.428 +C In\sthe\sreleasetest.tcl\sscript,\sshow\sthe\stest\starget\sfor\seach\sconfiguration\nthat\sis\srun.\s\sAnd\sshow\sthe\stime\sin\sHH:MM:SS. +D 2014-12-23T20:41:13.506 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c5bad35715abe0ffcb6a11fbafb157de7405d51a F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -801,7 +801,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl 8563ba7df4b8cb9a9ee9ccd042f14f0553e62f63 +F test/releasetest.tcl 84b224e46e8d31c26591518ff91daf88c602decb F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -1233,7 +1233,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P af166c5c64216d845269410d1ac2493310694b86 -R 84b62c16990cff93a3eab955a0ae7d8d +P cb128067faabf0503dff1298ed29934f484f71bb +R e47d011ee548c05856b81e40855ed9d3 U drh -Z 40c138478ca76e8e420aceec7c5661be +Z 930cf73b25702ad0596e5aa07bf5b1dd diff --git a/manifest.uuid b/manifest.uuid index 5424c8d2ee..8208554d56 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb128067faabf0503dff1298ed29934f484f71bb \ No newline at end of file +2295e9e0a22220ccb1b9cc2b031c1d8e702a1888 \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index f7aef83c1c..b7090143cc 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -233,8 +233,9 @@ proc run_test_suite {name testtarget config} { dryrun file mkdir $dir if {!$::DRYRUN} { - set n [string length $name] - puts -nonewline "${name}[string repeat . [expr {40-$n}]]" + set title ${name}($testtarget) + set n [string length $title] + puts -nonewline "${title}[string repeat . [expr {54-$n}]]" flush stdout } @@ -249,9 +250,10 @@ proc run_test_suite {name testtarget config} { dryrun cd $origdir if {!$::DRYRUN} { - set minutes [expr {($tm2-$tm1)/60}] + set hours [expr {($tm2-$tm2)/3600}] + set minutes [expr {(($tm2-$tm1)/60)%60}] set seconds [expr {($tm2-$tm1)%60}] - set tm [format (%02d:%02d) $minutes $seconds] + set tm [format (%02d:%02d:%02d) $hours $minutes $seconds] if {$rc} { puts " FAIL $tm" incr ::NERR From 054450f0bdc3bdae1da45281d96659a60aaebac8 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 23 Dec 2014 20:42:48 +0000 Subject: [PATCH 677/710] Permit sqlite3_shutdown() to be called with OMIT_WSD enabled and without having called sqlite3_initialize() first. FossilOrigin-Name: 3f7dbdb5df38bd4b8cd49d22a23b8412b8d506e5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/main.c | 7 +++++++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index eb52499b91..46ba432235 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\ssegfault\sfollowing\sOOM\serror\sin\sthe\stest\sharness.\s\sThe\nSQLite\score\sitself\sis\snot\sat\sfault. -D 2014-12-23T20:05:19.241 +C Permit\ssqlite3_shutdown()\sto\sbe\scalled\swith\sOMIT_WSD\senabled\sand\swithout\shaving\scalled\ssqlite3_initialize()\sfirst. +D 2014-12-23T20:42:48.779 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 6c4f961fa91d0b4fa121946a19f9e5eac2f2f809 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660 -F src/main.c ee498c89735ba7b581fbd58d2ad2e7679a0475b2 +F src/main.c 784f3d613f08d8bfba238a5baa7398770a641de2 F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a08b0c7512287ce5ae3fffe02c092d0eb25b3a25 -R 6f17c6ca29a22c9a696402dcec6c7766 -U drh -Z 3a6a677c416cfd077a0efd463f488c83 +P 1bb26695ff28a96d740752e327c2e0a0da1d067e +R 69846a8eed33bdc4ac13d4f2cdf5a199 +U mistachkin +Z c9331cbf8b5ecd4eca34d95e3015ec1f diff --git a/manifest.uuid b/manifest.uuid index 83787366b0..5e20eb47ce 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1bb26695ff28a96d740752e327c2e0a0da1d067e \ No newline at end of file +3f7dbdb5df38bd4b8cd49d22a23b8412b8d506e5 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 76af9a2e95..16e9a1babb 100644 --- a/src/main.c +++ b/src/main.c @@ -271,6 +271,13 @@ int sqlite3_initialize(void){ ** when this routine is invoked, then this routine is a harmless no-op. */ int sqlite3_shutdown(void){ +#ifdef SQLITE_OMIT_WSD + int rc = sqlite3_wsd_init(4096, 24); + if( rc!=SQLITE_OK ){ + return rc; + } +#endif + if( sqlite3GlobalConfig.isInit ){ #ifdef SQLITE_EXTRA_SHUTDOWN void SQLITE_EXTRA_SHUTDOWN(void); From 4d03a381f82dd43250686939d88501fbafefedf9 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 23 Dec 2014 21:03:09 +0000 Subject: [PATCH 678/710] Fix typo in library object file name. FossilOrigin-Name: f49566a79d05e630a86ea7b2acc04672d76d6337 --- Makefile.in | 8 ++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Makefile.in b/Makefile.in index 04305350be..6a944fdded 100644 --- a/Makefile.in +++ b/Makefile.in @@ -973,8 +973,8 @@ speedtest1$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo # symbols that do not begin with "sqlite3_". It is run as part of the # releasetest.tcl script. # -checksymbols: sqlite3.o - nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 +checksymbols: sqlite3.lo + nm -g --defined-only sqlite3.lo | grep -v " sqlite3_" ; test $$? -ne 0 # The next two rules are used to support the "threadtest" target. Building # threadtest runs a few thread-safety tests that are implemented in C. This @@ -987,8 +987,8 @@ THREADTEST3_SRC = $(TOP)/test/threadtest3.c \ $(TOP)/test/tt3_stress.c \ $(TOP)/test/tt3_lookaside1.c -threadtest3$(TEXE): sqlite3.o $(THREADTEST3_SRC) - $(LTLINK) $(TOP)/test/threadtest3.c sqlite3.o -o $@ $(TLIBS) +threadtest3$(TEXE): sqlite3.lo $(THREADTEST3_SRC) + $(LTLINK) $(TOP)/test/threadtest3.c sqlite3.lo -o $@ $(TLIBS) threadtest: threadtest3$(TEXE) ./threadtest3$(TEXE) diff --git a/manifest b/manifest index e882294d9d..90187d60b9 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C In\sthe\sreleasetest.tcl\sscript,\sshow\sthe\stest\starget\sfor\seach\sconfiguration\nthat\sis\srun.\s\sAnd\sshow\sthe\stime\sin\sHH:MM:SS. -D 2014-12-23T20:41:13.506 +C Fix\stypo\sin\slibrary\sobject\sfile\sname. +D 2014-12-23T21:03:09.146 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in c5bad35715abe0ffcb6a11fbafb157de7405d51a +F Makefile.in 9aaaaf6b5b0e4b42aa909db5165dc042db9bfcd5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc b363b90fe1bfc3b87d190f2f728a126c00d9ce09 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 @@ -1233,7 +1233,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cb128067faabf0503dff1298ed29934f484f71bb -R e47d011ee548c05856b81e40855ed9d3 -U drh -Z 930cf73b25702ad0596e5aa07bf5b1dd +P 2295e9e0a22220ccb1b9cc2b031c1d8e702a1888 +R a6e41f52bbcb02764e8ea99c7fc0d456 +U mistachkin +Z 6ab7423cea251d6a6862aa20cc19aa93 diff --git a/manifest.uuid b/manifest.uuid index 8208554d56..60a6c38f0f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2295e9e0a22220ccb1b9cc2b031c1d8e702a1888 \ No newline at end of file +f49566a79d05e630a86ea7b2acc04672d76d6337 \ No newline at end of file From 29f98f307764438cef900bfe76b74c2aedab102a Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 23 Dec 2014 21:10:38 +0000 Subject: [PATCH 679/710] Remove 'threadtest' as a target on Windows, due to lack of pthreads. Also, change 'fulltest' to 'fulltestonly'. FossilOrigin-Name: a010c404b5140104b68087dcbb0698b7a85eef65 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/releasetest.tcl | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 90187d60b9..89f8b08461 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\stypo\sin\slibrary\sobject\sfile\sname. -D 2014-12-23T21:03:09.146 +C Remove\s'threadtest'\sas\sa\starget\son\sWindows,\sdue\sto\slack\sof\spthreads.\s\sAlso,\schange\s'fulltest'\sto\s'fulltestonly'. +D 2014-12-23T21:10:38.485 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 9aaaaf6b5b0e4b42aa909db5165dc042db9bfcd5 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -801,7 +801,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl 84b224e46e8d31c26591518ff91daf88c602decb +F test/releasetest.tcl ba6e760aed00b829bd0718d61d9e605d56b0bd8d F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -1233,7 +1233,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 2295e9e0a22220ccb1b9cc2b031c1d8e702a1888 -R a6e41f52bbcb02764e8ea99c7fc0d456 +P f49566a79d05e630a86ea7b2acc04672d76d6337 +R 333eb3cbf92231ab2930505109652a4c U mistachkin -Z 6ab7423cea251d6a6862aa20cc19aa93 +Z 00b07e8b2b9d92ea23f97a4876d4552f diff --git a/manifest.uuid b/manifest.uuid index 60a6c38f0f..89f5792bcc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f49566a79d05e630a86ea7b2acc04672d76d6337 \ No newline at end of file +a010c404b5140104b68087dcbb0698b7a85eef65 \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index b7090143cc..2459422c5e 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -183,7 +183,7 @@ array set ::Platforms { "OS-X" "threadtest fulltest" } "Windows NT-intel" { - "Default" "threadtest fulltest" + "Default" "fulltestonly" } } From 8038953c15707754a842ce4f1e9b8bc9d69ce58d Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 24 Dec 2014 17:17:30 +0000 Subject: [PATCH 680/710] When building the amalgamation with SQLITE_ENABLE_IOTRACE defined, do not mark symbol sqlite3IoTrace as static. FossilOrigin-Name: 5b7ca013b7171a6807b15b128e140ce160f526d3 --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- src/main.c | 2 +- tool/mksqlite3c.tcl | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 7175c91a50..cb39ca6035 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\snew\sand\simproved\sreleasetest.tcl\sscript\sinto\strunk.\s\sAdd\sa\n"make\sreleasetest"\starget\sto\sthe\sautoconf\smakefile. -D 2014-12-23T21:17:58.595 +C When\sbuilding\sthe\samalgamation\swith\sSQLITE_ENABLE_IOTRACE\sdefined,\sdo\snot\smark\ssymbol\ssqlite3IoTrace\sas\sstatic. +D 2014-12-24T17:17:30.707 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c20e37499a3d664a3732257ed042352eba777a4d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -195,7 +195,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c ba1863ea58c4c840335a84ec276fc2b25e22bc4e F src/lempar.c 7274c97d24bb46631e504332ccd3bd1b37841770 F src/loadext.c 86bd4e2fccd520b748cba52492ab60c4a770f660 -F src/main.c 784f3d613f08d8bfba238a5baa7398770a641de2 +F src/main.c fa2128ef7d6a3dcd6770b2b1f3c284b696f11a2a F src/malloc.c 740db54387204c9a2eb67c6d98e68b08e9ef4eab F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c faf615aafd8be74a71494dfa027c113ea5c6615f @@ -1200,7 +1200,7 @@ F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 07a5124cf2dbafa1b375eefcf8ac4227028b0f8b F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 F tool/mksqlite3c-noext.tcl 88a1e3b0c769773fb7a9ebb363ffc603a4ac21d8 -F tool/mksqlite3c.tcl e72c0c97fe1a105fa9616483e652949be2199fe6 +F tool/mksqlite3c.tcl e94bdc37b531bba50d421e82efbe3738d0c1e950 F tool/mksqlite3h.tcl ba24038056f51fde07c0079c41885ab85e2cff12 F tool/mksqlite3internalh.tcl b6514145a7d5321b47e64e19b8116cc44f973eb1 F tool/mkvsix.tcl 52a4c613707ac34ae9c226e5ccc69cb948556105 @@ -1233,8 +1233,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3f7dbdb5df38bd4b8cd49d22a23b8412b8d506e5 a010c404b5140104b68087dcbb0698b7a85eef65 -R c03b15cfdf1585f3588eb5fb232886aa -T +closed a010c404b5140104b68087dcbb0698b7a85eef65 -U drh -Z 23d8a858cc383cd4bc2fa2df9f7a8bc3 +P 1deb00ec758c6d213da71ef64294cc816e204338 +R 94b7be58e117bbe257a3aeec77eff05b +U dan +Z 725f19b25ac60ae68e0c4f9942138bfa diff --git a/manifest.uuid b/manifest.uuid index 9aa2bbe3dc..0b171fb5d1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1deb00ec758c6d213da71ef64294cc816e204338 \ No newline at end of file +5b7ca013b7171a6807b15b128e140ce160f526d3 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 16e9a1babb..3220cfe8f6 100644 --- a/src/main.c +++ b/src/main.c @@ -62,7 +62,7 @@ int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } ** I/O active are written using this function. These messages ** are intended for debugging activity only. */ -void (*sqlite3IoTrace)(const char*, ...) = 0; +/* not-private */ void (*sqlite3IoTrace)(const char*, ...) = 0; #endif /* diff --git a/tool/mksqlite3c.tcl b/tool/mksqlite3c.tcl index 0e979234f3..72098c7357 100644 --- a/tool/mksqlite3c.tcl +++ b/tool/mksqlite3c.tcl @@ -212,7 +212,7 @@ proc copy_file {filename} { } } elseif {[regexp {^(SQLITE_EXTERN )?void \(\*sqlite3IoTrace\)} $line]} { regsub {^SQLITE_EXTERN } $line {} line - puts $out "SQLITE_PRIVATE $line" + puts $out $line } elseif {[regexp {^void \(\*sqlite3Os} $line]} { puts $out "SQLITE_PRIVATE $line" } else { From ee7172f12ad51ee528da22fa5ea72073c23dd5f4 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 24 Dec 2014 18:11:50 +0000 Subject: [PATCH 681/710] Fix a failing assert() in balance_nonroot(). FossilOrigin-Name: e2e323145f66cca31babe1a979db6ef97038b879 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/btree.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index cb39ca6035..d97e03238b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sbuilding\sthe\samalgamation\swith\sSQLITE_ENABLE_IOTRACE\sdefined,\sdo\snot\smark\ssymbol\ssqlite3IoTrace\sas\sstatic. -D 2014-12-24T17:17:30.707 +C Fix\sa\sfailing\sassert()\sin\sbalance_nonroot(). +D 2014-12-24T18:11:50.466 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c20e37499a3d664a3732257ed042352eba777a4d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 1de0560426ecde85ff3ea95d7c94261d7652e284 +F src/btree.c 904d30478685fe0723ad9092fc800a655544c69a F src/btree.h 94277c1d30c0b75705974bcc8b0c05e79c03d474 F src/btreeInt.h a3d0ae1d511365e1a2b76ad10960dbe55c286f34 F src/build.c 162d84e4833b03f9d07192ef06057b0226f6e543 @@ -1233,7 +1233,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 1deb00ec758c6d213da71ef64294cc816e204338 -R 94b7be58e117bbe257a3aeec77eff05b +P 5b7ca013b7171a6807b15b128e140ce160f526d3 +R b11f421779b35c3a19d382d5b3ba189c U dan -Z 725f19b25ac60ae68e0c4f9942138bfa +Z 0b37a721b79ad0d2f596c72e79efffa8 diff --git a/manifest.uuid b/manifest.uuid index 0b171fb5d1..8989984104 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5b7ca013b7171a6807b15b128e140ce160f526d3 \ No newline at end of file +e2e323145f66cca31babe1a979db6ef97038b879 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e6a7e2c2b9..0554f3fa3c 100644 --- a/src/btree.c +++ b/src/btree.c @@ -6865,8 +6865,8 @@ static int balance_nonroot( /* Do not allow any cells smaller than 4 bytes. If a smaller cell ** does exist, pad it with 0x00 bytes. */ assert( szCell[nCell]==3 ); - assert( apCell[nCell]==&pTemp[iSpace1-3] ); - pTemp[iSpace1++] = 0x00; + assert( apCell[nCell]==&aSpace1[iSpace1-3] ); + aSpace1[iSpace1++] = 0x00; szCell[nCell] = 4; } } From 97876ee666274615c4034cc3658a52b976aa2493 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 24 Dec 2014 23:35:36 +0000 Subject: [PATCH 682/710] Fix the error counter in releasetest.tcl. And report the total time in HH:MM:SS instead of just seconds. FossilOrigin-Name: 6396f8046242286298fecd1748a6e8e786e6794e --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/releasetest.tcl | 10 +++++++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d97e03238b..f5b01362a2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfailing\sassert()\sin\sbalance_nonroot(). -D 2014-12-24T18:11:50.466 +C Fix\sthe\serror\scounter\sin\sreleasetest.tcl.\s\sAnd\sreport\sthe\stotal\stime\sin\nHH:MM:SS\sinstead\sof\sjust\sseconds. +D 2014-12-24T23:35:36.745 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c20e37499a3d664a3732257ed042352eba777a4d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -801,7 +801,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl ba6e760aed00b829bd0718d61d9e605d56b0bd8d +F test/releasetest.tcl 80d10f058667e6d0c8ab379d6e5bb0a60ecb40e2 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -1233,7 +1233,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 5b7ca013b7171a6807b15b128e140ce160f526d3 -R b11f421779b35c3a19d382d5b3ba189c -U dan -Z 0b37a721b79ad0d2f596c72e79efffa8 +P e2e323145f66cca31babe1a979db6ef97038b879 +R 5d68ae4c7a1de514625f0040c5883c44 +U drh +Z 5948198e5a17f619d6c7388ef4763f03 diff --git a/manifest.uuid b/manifest.uuid index 8989984104..70b92936b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e2e323145f66cca31babe1a979db6ef97038b879 \ No newline at end of file +6396f8046242286298fecd1748a6e8e786e6794e \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index 2459422c5e..879b7f8fb9 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -405,8 +405,8 @@ proc main {argv} { process_options $argv puts [string repeat * 70] - set NERR 0 - set NTEST 0 + set ::NERR 0 + set ::NTEST 0 set STARTTIME [clock seconds] foreach {zConfig target} $::CONFIGLIST { if {$::QUICK} {set target test} @@ -436,8 +436,12 @@ proc main {argv} { } set elapsetime [expr {[clock seconds]-$STARTTIME}] + set hr [expr {$elapsetime/3600}] + set min [expr {($elapsetime/60)%60}] + set sec [expr {$elapsetime%60}] + set etime [format (%02d:%02d:%02d) $hr $min $sec] puts [string repeat * 70] - puts "$NERR failures of $NTEST test suites run in $elapsetime seconds" + puts "$::NERR failures of $::NTEST test suites run in $etime" } main $argv From 0ab0e05c6b3b5eee7f16bd76f8a59f397449c4d8 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 25 Dec 2014 12:19:56 +0000 Subject: [PATCH 683/710] Update the SQLITE_CONFIG_PAGECACHE documentation so that the maximum page size is correctly stated to be 65536. FossilOrigin-Name: 3286424b4d30035de69b88ef0b2897365ff848f9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/sqlite.h.in | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f5b01362a2..50e33c046d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\serror\scounter\sin\sreleasetest.tcl.\s\sAnd\sreport\sthe\stotal\stime\sin\nHH:MM:SS\sinstead\sof\sjust\sseconds. -D 2014-12-24T23:35:36.745 +C Update\sthe\sSQLITE_CONFIG_PAGECACHE\sdocumentation\sso\sthat\sthe\smaximum\spage\nsize\sis\scorrectly\sstated\sto\sbe\s65536. +D 2014-12-25T12:19:56.744 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c20e37499a3d664a3732257ed042352eba777a4d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -230,7 +230,7 @@ F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e F src/select.c f377fb8a5c73c10678ea74f3400f7913943e3d75 F src/shell.c 45d9c9bd7cde07845af957f2d849933b990773cf -F src/sqlite.h.in 116dc731361549ee3fc79dcebace11b57d24dcfd +F src/sqlite.h.in 47cb601ed2b2ea7f01119e2763185c809d8e82fa F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d F src/sqliteInt.h d36da9a07130cae13cbfee0986bf20028cb01465 @@ -1233,7 +1233,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P e2e323145f66cca31babe1a979db6ef97038b879 -R 5d68ae4c7a1de514625f0040c5883c44 +P 6396f8046242286298fecd1748a6e8e786e6794e +R 6fef442856f5d39a46d82b3795b528e7 U drh -Z 5948198e5a17f619d6c7388ef4763f03 +Z 07073e8eb87986ae3ece689929973238 diff --git a/manifest.uuid b/manifest.uuid index 70b92936b1..e53bf043df 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6396f8046242286298fecd1748a6e8e786e6794e \ No newline at end of file +3286424b4d30035de69b88ef0b2897365ff848f9 \ No newline at end of file diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 701e6de2e3..b525b396f4 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -1565,7 +1565,7 @@ struct sqlite3_mem_methods { ** 8-byte aligned ** memory, the size of each page buffer (sz), and the number of pages (N). ** The sz argument should be the size of the largest database page -** (a power of two between 512 and 32768) plus some extra bytes for each +** (a power of two between 512 and 65536) plus some extra bytes for each ** page header. ^The number of extra bytes needed by the page header ** can be determined using the [SQLITE_CONFIG_PCACHE_HDRSZ] option ** to [sqlite3_config()]. From e385d8876e86b8b8e615f2437112f10560ac427c Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 28 Dec 2014 22:10:51 +0000 Subject: [PATCH 684/710] Fix WITHOUT ROWID tables so that they correctly deal with PRIMARY KEYs that contain redundant columns. FossilOrigin-Name: 0dfef6757056ef0bdea8f049f7469ccf6960e2cb --- manifest | 13 +++++++------ manifest.uuid | 2 +- src/build.c | 13 +++++++++++++ test/without_rowid6.test | 41 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 7 deletions(-) create mode 100644 test/without_rowid6.test diff --git a/manifest b/manifest index 50e33c046d..ab14a67a57 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sSQLITE_CONFIG_PAGECACHE\sdocumentation\sso\sthat\sthe\smaximum\spage\nsize\sis\scorrectly\sstated\sto\sbe\s65536. -D 2014-12-25T12:19:56.744 +C Fix\sWITHOUT\sROWID\stables\sso\sthat\sthey\scorrectly\sdeal\swith\sPRIMARY\sKEYs\sthat\ncontain\sredundant\scolumns. +D 2014-12-28T22:10:51.114 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in c20e37499a3d664a3732257ed042352eba777a4d F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -176,7 +176,7 @@ F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 F src/btree.c 904d30478685fe0723ad9092fc800a655544c69a F src/btree.h 94277c1d30c0b75705974bcc8b0c05e79c03d474 F src/btreeInt.h a3d0ae1d511365e1a2b76ad10960dbe55c286f34 -F src/build.c 162d84e4833b03f9d07192ef06057b0226f6e543 +F src/build.c f5cfd7b32216f695b995bbc7c1a395f6d451d11f F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0 F src/complete.c 198a0066ba60ab06fc00fba1998d870a4d575463 F src/ctime.c df19848891c8a553c80e6f5a035e768280952d1a @@ -1178,6 +1178,7 @@ F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 F test/without_rowid3.test 1081aabf60a1e1123b7f9a8f6ae19954351843b0 F test/without_rowid4.test 4e08bcbaee0399f35d58b5581881e7a6243d458a F test/without_rowid5.test 61256715b686359df48ca1742db50cc7e3e7b862 +F test/without_rowid6.test deddb78ef539c355bddec00cdfaea6c56efd8b3f F test/wordcount.c 9915e06cb33d8ca8109b8700791afe80d305afda F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test cf6748bad89553cc1632be51a6f54e487e4039ac @@ -1233,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 6396f8046242286298fecd1748a6e8e786e6794e -R 6fef442856f5d39a46d82b3795b528e7 +P 3286424b4d30035de69b88ef0b2897365ff848f9 +R fdb1da8fdb04c896d0536c98941bdfa0 U drh -Z 07073e8eb87986ae3ece689929973238 +Z ccbcde8f60a3b3f04760cafd27073216 diff --git a/manifest.uuid b/manifest.uuid index e53bf043df..e8065acc31 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3286424b4d30035de69b88ef0b2897365ff848f9 \ No newline at end of file +0dfef6757056ef0bdea8f049f7469ccf6960e2cb \ No newline at end of file diff --git a/src/build.c b/src/build.c index db954647cc..f02989bffe 100644 --- a/src/build.c +++ b/src/build.c @@ -1713,6 +1713,19 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ pTab->iPKey = -1; }else{ pPk = sqlite3PrimaryKeyIndex(pTab); + /* + ** Remove all redundant columns from the PRIMARY KEY. For example, change + ** "PRIMARY KEY(a,b,a,b,c,b,c,d)" into just "PRIMARY KEY(a,b,c,d)". Later + ** code assumes the PRIMARY KEY contains no repeated columns. + */ + for(i=j=1; inKeyCol; i++){ + if( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ){ + pPk->nColumn--; + }else{ + pPk->aiColumn[j++] = pPk->aiColumn[i]; + } + } + pPk->nKeyCol = j; } pPk->isCovering = 1; assert( pPk!=0 ); diff --git a/test/without_rowid6.test b/test/without_rowid6.test new file mode 100644 index 0000000000..e827ccab90 --- /dev/null +++ b/test/without_rowid6.test @@ -0,0 +1,41 @@ +# 2014-12-28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Verify that WITHOUT ROWID tables work correctly when the PRIMARY KEY +# has redundant columns. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl + +do_execsql_test without_rowid6-100 { + CREATE TABLE t1(a,b,c,d,e, PRIMARY KEY(a,b,c,a,b,c,d,a,b,c)) WITHOUT ROWID; + CREATE INDEX t1a ON t1(b, b); + WITH RECURSIVE + c(i) AS (VALUES(1) UNION ALL SELECT i+1 FROM c WHERE i<1000) + INSERT INTO t1(a,b,c,d,e) SELECT i, i+1000, printf('x%dy',i), 0, 0 FROM c; + ANALYZE; +} {} +do_execsql_test without_rowid6-110 { + SELECT c FROM t1 WHERE a=123; +} {x123y} +do_execsql_test without_rowid6-120 { + SELECT c FROM t1 WHERE b=1123; +} {x123y} +do_execsql_test without_rowid6-130 { + SELECT c FROM t1 ORDER BY a DESC LIMIT 5; +} {x1000y x999y x998y x997y x996y} +do_execsql_test without_rowid6-140 { + SELECT c FROM t1 ORDER BY b LIMIT 5; +} {x1y x2y x3y x4y x5y} + + +finish_test From 277b4e446cb4400b107df38fb91553d8585e5ff1 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 29 Dec 2014 02:55:58 +0000 Subject: [PATCH 685/710] Fix the "checksymbols" target in Makefile.in so that it actually works. Enhance the releasetest.tcl script to count the total number of tests run over all configurations. FossilOrigin-Name: 4eda1c746043acbdb7ef3e1f95bf8b01ee976479 --- Makefile.in | 3 ++- manifest | 14 +++++++------- manifest.uuid | 2 +- test/releasetest.tcl | 39 +++++++++++++++++++++++++++++++++++++-- 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/Makefile.in b/Makefile.in index 887f99e0ee..6867afcf52 100644 --- a/Makefile.in +++ b/Makefile.in @@ -974,7 +974,8 @@ speedtest1$(TEXE): $(TOP)/test/wordcount.c sqlite3.lo # releasetest.tcl script. # checksymbols: sqlite3.lo - nm -g --defined-only sqlite3.lo | grep -v " sqlite3_" ; test $$? -ne 0 + nm -g --defined-only sqlite3.o | grep -v " sqlite3_" ; test $$? -ne 0 + echo '0 errors out of 1 tests' # The next two rules are used to support the "threadtest" target. Building # threadtest runs a few thread-safety tests that are implemented in C. This diff --git a/manifest b/manifest index ab14a67a57..8891c37497 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Fix\sWITHOUT\sROWID\stables\sso\sthat\sthey\scorrectly\sdeal\swith\sPRIMARY\sKEYs\sthat\ncontain\sredundant\scolumns. -D 2014-12-28T22:10:51.114 +C Fix\sthe\s"checksymbols"\starget\sin\sMakefile.in\sso\sthat\sit\sactually\sworks.\nEnhance\sthe\sreleasetest.tcl\sscript\sto\scount\sthe\stotal\snumber\sof\stests\nrun\sover\sall\sconfigurations. +D 2014-12-29T02:55:58.026 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in c20e37499a3d664a3732257ed042352eba777a4d +F Makefile.in 40326b6d788007dd5e00587c54adcd2621832bb3 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc b363b90fe1bfc3b87d190f2f728a126c00d9ce09 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 @@ -801,7 +801,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl 80d10f058667e6d0c8ab379d6e5bb0a60ecb40e2 +F test/releasetest.tcl 14552a8741165a0489cd9ec3e9a651ba1f1b3567 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 3286424b4d30035de69b88ef0b2897365ff848f9 -R fdb1da8fdb04c896d0536c98941bdfa0 +P 0dfef6757056ef0bdea8f049f7469ccf6960e2cb +R 74cab324b9af5a87ce922fe9e97d1a6b U drh -Z ccbcde8f60a3b3f04760cafd27073216 +Z 3ab79c0378fea7c029a9124c0c9f4912 diff --git a/manifest.uuid b/manifest.uuid index e8065acc31..5ec8724985 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0dfef6757056ef0bdea8f049f7469ccf6960e2cb \ No newline at end of file +4eda1c746043acbdb7ef3e1f95bf8b01ee976479 \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index 879b7f8fb9..6a91ac585b 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -65,7 +65,6 @@ array set ::Configs { -DSQLITE_SECURE_DELETE=1 -DSQLITE_SOUNDEX=1 -DSQLITE_ENABLE_ATOMIC_WRITE=1 - -DSQLITE_ENABLE_IOTRACE=1 -DSQLITE_ENABLE_MEMORY_MANAGEMENT=1 -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1 } @@ -201,6 +200,37 @@ foreach {key value} [array get ::Platforms] { } } +# Open the file $logfile and look for a report on the number of errors +# and the number of test cases run. Add these values to the global +# $::NERRCASE and $::NTESTCASE variables. +# +# If any errors occur, then write into $errmsgVar the text of an appropriate +# one-line error message to show on the output. +# +proc count_tests_and_errors {logfile rcVar errmsgVar} { + upvar 1 $rcVar rc $errmsgVar errmsg + set fd [open $logfile rb] + set seen 0 + while {![eof $fd]} { + set line [gets $fd] + if {[regexp {^(\d+) errors out of (\d+) tests} $line all nerr ntest]} { + incr ::NERRCASE $nerr + incr ::NTESTCASE $ntest + set seen 1 + if {$nerr>0} { + set rc 1 + set errmsg $line + } + break; + } + } + close $fd + if {!$seen} { + set rc 1 + set errmsg "Test did not complete" + } +} + proc run_test_suite {name testtarget config} { # Tcl variable $opts is used to build up the value used to set the # OPTS Makefile variable. Variable $cflags holds the value for @@ -242,9 +272,11 @@ proc run_test_suite {name testtarget config} { set tm1 [clock seconds] set origdir [pwd] dryrun cd $dir + set errmsg {} set rc [catch [configureCommand]] if {!$rc} { set rc [catch [makeCommand $testtarget $cflags $opts]] + count_tests_and_errors test.log rc errmsg } set tm2 [clock seconds] dryrun cd $origdir @@ -257,6 +289,7 @@ proc run_test_suite {name testtarget config} { if {$rc} { puts " FAIL $tm" incr ::NERR + if {$errmsg!=""} {puts " $errmsg"} } else { puts " Ok $tm" } @@ -407,6 +440,8 @@ proc main {argv} { set ::NERR 0 set ::NTEST 0 + set ::NTESTCASE 0 + set ::NERRCASE 0 set STARTTIME [clock seconds] foreach {zConfig target} $::CONFIGLIST { if {$::QUICK} {set target test} @@ -441,7 +476,7 @@ proc main {argv} { set sec [expr {$elapsetime%60}] set etime [format (%02d:%02d:%02d) $hr $min $sec] puts [string repeat * 70] - puts "$::NERR failures of $::NTEST test suites run in $etime" + puts "$::NERRCASE failures of $::NTESTCASE tests run in $etime" } main $argv From 622a53d54aa4f585ab8e465b9b9e7a208146ad9d Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 29 Dec 2014 11:50:39 +0000 Subject: [PATCH 686/710] Reinstate an assert() by adding an "|| CORRUPT_DB" term. FossilOrigin-Name: 95ce20348d9b868a0407adccdb222a0e4c762945 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/wal.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8891c37497..c6dc73b170 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s"checksymbols"\starget\sin\sMakefile.in\sso\sthat\sit\sactually\sworks.\nEnhance\sthe\sreleasetest.tcl\sscript\sto\scount\sthe\stotal\snumber\sof\stests\nrun\sover\sall\sconfigurations. -D 2014-12-29T02:55:58.026 +C Reinstate\san\sassert()\sby\sadding\san\s"||\sCORRUPT_DB"\sterm. +D 2014-12-29T11:50:39.465 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 40326b6d788007dd5e00587c54adcd2621832bb3 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -301,7 +301,7 @@ F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f F src/vdbesort.c c150803a3e98fbc68bd07772cbbd4328a0a7212d F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 -F src/wal.c 847692349eb6e1fb8543dbc97e69ddbfa4cc7ea7 +F src/wal.c 85353539f2d9d0c91ebd057c32525b1e1aa3335e F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c c253b95b4ee44b21c406e2a1052636c31ea27804 F src/where.c d46de821bc604a4fd36fa3928c086950e91aafb1 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0dfef6757056ef0bdea8f049f7469ccf6960e2cb -R 74cab324b9af5a87ce922fe9e97d1a6b +P 4eda1c746043acbdb7ef3e1f95bf8b01ee976479 +R 110d8f73632ac0551fa4d638f24a9811 U drh -Z 3ab79c0378fea7c029a9124c0c9f4912 +Z b99a612326485f12a8c134680408ea8b diff --git a/manifest.uuid b/manifest.uuid index 5ec8724985..bf6224e944 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4eda1c746043acbdb7ef3e1f95bf8b01ee976479 \ No newline at end of file +95ce20348d9b868a0407adccdb222a0e4c762945 \ No newline at end of file diff --git a/src/wal.c b/src/wal.c index 2b80c7a95e..f2738a6727 100644 --- a/src/wal.c +++ b/src/wal.c @@ -2412,7 +2412,7 @@ int sqlite3WalFindFrame( for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ u32 iFrame = aHash[iKey] + iZero; if( iFrame<=iLast && aPgno[aHash[iKey]]==pgno ){ - /* assert( iFrame>iRead ); -- not true if there is corruption */ + assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } if( (nCollide--)==0 ){ From 620315840c0f9797b85a09cebe94672575602caa Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 29 Dec 2014 12:02:31 +0000 Subject: [PATCH 687/710] Fix some recently added tests so that they work with SQLITE_DEFAULT_AUTOVACUUM=1. FossilOrigin-Name: ef0626ab20f753f01090ca8e8a94e8b516eea55e --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- test/e_walauto.test | 1 + test/e_walckpt.test | 2 ++ test/wal5.test | 11 ++++++----- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index c6dc73b170..ff08366760 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reinstate\san\sassert()\sby\sadding\san\s"||\sCORRUPT_DB"\sterm. -D 2014-12-29T11:50:39.465 +C Fix\ssome\srecently\sadded\stests\sso\sthat\sthey\swork\swith\sSQLITE_DEFAULT_AUTOVACUUM=1. +D 2014-12-29T12:02:31.823 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 40326b6d788007dd5e00587c54adcd2621832bb3 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -478,8 +478,8 @@ F test/e_update.test 312cb8f5ccfe41515a6bb092f8ea562a9bd54d52 F test/e_uri.test 5ae33760fb2039c61aa2d90886f1664664173585 F test/e_vacuum.test 5bfbdc21b65c0abf24398d0ba31dc88d93ca77a9 F test/e_wal.test 0967f0b8f1dfda871dc7b9b5574198f1f4f7d69a -F test/e_walauto.test eab3bedddbc3fd19795d51e618da41a48e19a3e3 -F test/e_walckpt.test 3116a98fa0dd9b2c9e493de7c59730adfe436746 +F test/e_walauto.test d2dfc6681aade1f1306a8a336f5a258d8b62becd +F test/e_walckpt.test 65e29b6631e51f210f83e4ff11571e647ba93608 F test/e_walhook.test da3ea8b3483d1af72190337bda50155a91a4b664 F test/enc.test e54531cd6bf941ee6760be041dff19a104c7acea F test/enc2.test 83437a79ba1545a55fb549309175c683fb334473 @@ -1123,7 +1123,7 @@ F test/wal.test 885f32b2b390b30b4aa3dbb0e568f8f78d40f5cc F test/wal2.test 1f841d2048080d32f552942e333fd99ce541dada F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c -F test/wal5.test 174cc1512e304a7dfa28ac30527e28ea02fc37df +F test/wal5.test 11b8658dd4d5448f4604124bebd9b68be5bc3e66 F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 4eda1c746043acbdb7ef3e1f95bf8b01ee976479 -R 110d8f73632ac0551fa4d638f24a9811 -U drh -Z b99a612326485f12a8c134680408ea8b +P 95ce20348d9b868a0407adccdb222a0e4c762945 +R 0db2cbd8617dad79686d47ebce6896f7 +U dan +Z 03a26819b4fc4a14b459978f03e3e164 diff --git a/manifest.uuid b/manifest.uuid index bf6224e944..57a4b2b13b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -95ce20348d9b868a0407adccdb222a0e4c762945 \ No newline at end of file +ef0626ab20f753f01090ca8e8a94e8b516eea55e \ No newline at end of file diff --git a/test/e_walauto.test b/test/e_walauto.test index 79a2702d04..239adc3b95 100644 --- a/test/e_walauto.test +++ b/test/e_walauto.test @@ -66,6 +66,7 @@ foreach {tn code} { eval $code reset_db + execsql { PRAGMA auto_vacuum = 0 } do_execsql_test 1.$tn.0 { PRAGMA journal_mode = WAL } {wal} do_execsql_test 1.$tn.1 { CREATE TABLE t1(a, b) } set shmfd [open "test.db-shm" rb] diff --git a/test/e_walckpt.test b/test/e_walckpt.test index e6b6566bed..e022f840cf 100644 --- a/test/e_walckpt.test +++ b/test/e_walckpt.test @@ -247,6 +247,7 @@ foreach {tn script} { do_test $tn.3.2.1 { db2 eval { + PRAGMA auto_vacuum = 0; PRAGMA journal_mode = WAL; CREATE TABLE t1(x, y); INSERT INTO t1 VALUES(1,2); @@ -706,6 +707,7 @@ sqlite3 db2 test.db do_test 6.1 { execsql { + PRAGMA auto_vacuum = 0; PRAGMA journal_mode = WAL; CREATE TABLE t1(a, b); INSERT INTO t1 VALUES(1, 2); diff --git a/test/wal5.test b/test/wal5.test index 09c7d76040..8c1ec8bcc7 100644 --- a/test/wal5.test +++ b/test/wal5.test @@ -363,9 +363,10 @@ foreach {testprefix do_wal_checkpoint} { code2 $do_wal_checkpoint code3 $do_wal_checkpoint - do_test 3.$tn.1 { + do_test 4.$tn.1 { sql1 { PRAGMA page_size = 1024; + PRAGMA auto_vacuum = 0; PRAGMA journal_mode = WAL; PRAGMA synchronous = normal; CREATE TABLE t1(x, y); @@ -376,14 +377,14 @@ foreach {testprefix do_wal_checkpoint} { file size test.db-wal } [wal_file_size 8 1024] - do_test 3.$tn.2 { do_wal_checkpoint db -mode truncate } {0 0 0} - do_test 3.$tn.3 { file size test.db-wal } 0 + do_test 4.$tn.2 { do_wal_checkpoint db -mode truncate } {0 0 0} + do_test 4.$tn.3 { file size test.db-wal } 0 - do_test 3.$tn.4 { + do_test 4.$tn.4 { sql2 { SELECT * FROM t1 } } {1 2 3 4} - do_test 3.$tn.5 { + do_test 4.$tn.5 { sql2 { INSERT INTO t1 VALUES('a', 'b') } file size test.db-wal } [wal_file_size 2 1024] From cb281a9a2c3887f2e80aed8a59bcdf8c241c141b Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 29 Dec 2014 19:54:10 +0000 Subject: [PATCH 688/710] Fix the --dryrun option in releasetest.tcl. FossilOrigin-Name: 0f9e549643ab94b0465e6891384dd20506708a8f --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/releasetest.tcl | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ff08366760..101d767d16 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\srecently\sadded\stests\sso\sthat\sthey\swork\swith\sSQLITE_DEFAULT_AUTOVACUUM=1. -D 2014-12-29T12:02:31.823 +C Fix\sthe\s--dryrun\soption\sin\sreleasetest.tcl. +D 2014-12-29T19:54:10.833 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 40326b6d788007dd5e00587c54adcd2621832bb3 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -801,7 +801,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl 14552a8741165a0489cd9ec3e9a651ba1f1b3567 +F test/releasetest.tcl 3ffd8d99da7a087917f057147053ed35ce1bf90b F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 95ce20348d9b868a0407adccdb222a0e4c762945 -R 0db2cbd8617dad79686d47ebce6896f7 -U dan -Z 03a26819b4fc4a14b459978f03e3e164 +P ef0626ab20f753f01090ca8e8a94e8b516eea55e +R fa81eb180b12cb206ba390f700c75ee2 +U drh +Z 819655494d4a64bf8e32c2a6559858d5 diff --git a/manifest.uuid b/manifest.uuid index 57a4b2b13b..8a65bd3f3d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ef0626ab20f753f01090ca8e8a94e8b516eea55e \ No newline at end of file +0f9e549643ab94b0465e6891384dd20506708a8f \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index 6a91ac585b..bcd7d3d060 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -208,6 +208,7 @@ foreach {key value} [array get ::Platforms] { # one-line error message to show on the output. # proc count_tests_and_errors {logfile rcVar errmsgVar} { + if {$::DRYRUN} return upvar 1 $rcVar rc $errmsgVar errmsg set fd [open $logfile rb] set seen 0 From 37c057b8ff93ad750623b0a4f032722849954551 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Dec 2014 00:57:29 +0000 Subject: [PATCH 689/710] Round all object sizes that go into computing SQLITE_CONFIG_PCACHE_HDRSZ up to a multiple of 8 bytes. FossilOrigin-Name: b28ce75f2d3a6343dc20d581dc55afae89ab5efa --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 2 +- src/pcache.c | 2 +- src/pcache1.c | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 101d767d16..750bbbaad6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s--dryrun\soption\sin\sreleasetest.tcl. -D 2014-12-29T19:54:10.833 +C Round\sall\sobject\ssizes\sthat\sgo\sinto\scomputing\sSQLITE_CONFIG_PCACHE_HDRSZ\sup\nto\sa\smultiple\sof\s8\sbytes. +D 2014-12-30T00:57:29.979 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 40326b6d788007dd5e00587c54adcd2621832bb3 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/auth.c b56c78ebe40a2110fd361379f7e8162d23f92240 F src/backup.c 7ddee9c7d505e07e959a575b18498f17c71e53ea F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 49ca66250c7dfa844a4d4cb8272b87420d27d3a5 -F src/btree.c 904d30478685fe0723ad9092fc800a655544c69a +F src/btree.c 4c098bb6e8678e4596983862abf78f7a0fcb807e F src/btree.h 94277c1d30c0b75705974bcc8b0c05e79c03d474 F src/btreeInt.h a3d0ae1d511365e1a2b76ad10960dbe55c286f34 F src/build.c f5cfd7b32216f695b995bbc7c1a395f6d451d11f @@ -219,9 +219,9 @@ F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 2cbaf886a6157c53a8061ea7e677f81620ff46eb F src/pager.h c3476e7c89cdf1c6914e50a11f3714e30b4e0a77 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 -F src/pcache.c ace1b67632deeaa84859b4c16c27711dfb7db3d4 +F src/pcache.c 8ee2ce15c4f49b44e27e8ffb8ce683c5e506e736 F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 -F src/pcache1.c facbdd3ecc09c8f750089d941305694301328e98 +F src/pcache1.c ff599891da0e4993c814eeae3e7e2801f47364f3 F src/pragma.c bd33aa24456f043bb6f6d32a918bbeed41d8c591 F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ef0626ab20f753f01090ca8e8a94e8b516eea55e -R fa81eb180b12cb206ba390f700c75ee2 +P 0f9e549643ab94b0465e6891384dd20506708a8f +R e296d2763d3effbc9c21b29aae15693b U drh -Z 819655494d4a64bf8e32c2a6559858d5 +Z 2369afc171a95fab3bda365a2e52134d diff --git a/manifest.uuid b/manifest.uuid index 8a65bd3f3d..51d7bf7cd2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0f9e549643ab94b0465e6891384dd20506708a8f \ No newline at end of file +b28ce75f2d3a6343dc20d581dc55afae89ab5efa \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 0554f3fa3c..f9f76c2ebb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -9146,4 +9146,4 @@ int sqlite3BtreeIsReadonly(Btree *p){ /* ** Return the size of the header added to each page by this module. */ -int sqlite3HeaderSizeBtree(void){ return sizeof(MemPage); } +int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } diff --git a/src/pcache.c b/src/pcache.c index 13551872d1..3139429699 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -655,7 +655,7 @@ void sqlite3PcacheShrink(PCache *pCache){ ** Return the size of the header added by this middleware layer ** in the page-cache hierarchy. */ -int sqlite3HeaderSizePcache(void){ return sizeof(PgHdr); } +int sqlite3HeaderSizePcache(void){ return ROUND8(sizeof(PgHdr)); } #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) diff --git a/src/pcache1.c b/src/pcache1.c index cad41b1b73..990f51591f 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -984,7 +984,7 @@ void sqlite3PCacheSetDefault(void){ /* ** Return the size of the header on each page of this PCACHE implementation. */ -int sqlite3HeaderSizePcache1(void){ return sizeof(PgHdr1); } +int sqlite3HeaderSizePcache1(void){ return ROUND8(sizeof(PgHdr1)); } #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* From 8ab4b9e96459f756749edad0cfb2eb6faef771bc Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 30 Dec 2014 12:03:35 +0000 Subject: [PATCH 690/710] Change notify2.test to check that sqlite3_blocking_step() uses CPU more efficiently than sqlite3_step(), not that it results in greater overall throughput for any specific number of threads. FossilOrigin-Name: d904d29354a5ed85d33bafe4a7143f3c5ecee790 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/notify2.test | 41 +++++++++++++++++++++++++++++++++-------- 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 750bbbaad6..68e968d12e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Round\sall\sobject\ssizes\sthat\sgo\sinto\scomputing\sSQLITE_CONFIG_PCACHE_HDRSZ\sup\nto\sa\smultiple\sof\s8\sbytes. -D 2014-12-30T00:57:29.979 +C Change\snotify2.test\sto\scheck\sthat\ssqlite3_blocking_step()\suses\sCPU\smore\sefficiently\sthan\ssqlite3_step(),\snot\sthat\sit\sresults\sin\sgreater\soverall\sthroughput\sfor\sany\sspecific\snumber\sof\sthreads. +D 2014-12-30T12:03:35.117 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 40326b6d788007dd5e00587c54adcd2621832bb3 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -755,7 +755,7 @@ F test/mutex2.test bfeaeac2e73095b2ac32285d2756e3a65e681660 F test/nan.test e9648b9d007c7045242af35e11a984d4b169443a F test/nolock.test 0540dd96f39b8876e3ffdd8814fad0ea425efeee F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf -F test/notify2.test ce23eb522c9e1fff6443f96376fe67872202061c +F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161 F test/notify3.test 10ff25cde502e72a92053a2f215d64bece4ef934 F test/notnull.test f8fcf58669ddba79274daa2770d61dfad8274f62 F test/null.test a8b09b8ed87852742343b33441a9240022108993 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 0f9e549643ab94b0465e6891384dd20506708a8f -R e296d2763d3effbc9c21b29aae15693b -U drh -Z 2369afc171a95fab3bda365a2e52134d +P b28ce75f2d3a6343dc20d581dc55afae89ab5efa +R 05d3fab766dc5317a84b8e8fc66a7683 +U dan +Z 9e6a15ccef1a8c8d1ccc67c36a60ea41 diff --git a/manifest.uuid b/manifest.uuid index 51d7bf7cd2..f0f3fc30da 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b28ce75f2d3a6343dc20d581dc55afae89ab5efa \ No newline at end of file +d904d29354a5ed85d33bafe4a7143f3c5ecee790 \ No newline at end of file diff --git a/test/notify2.test b/test/notify2.test index 9e40ed695b..12c6a537ef 100644 --- a/test/notify2.test +++ b/test/notify2.test @@ -101,6 +101,8 @@ set sql $zSql # This loop runs for ~20 seconds. # set iStart [clock_seconds] + set nOp 0 + set nAttempt 0 while { ([clock_seconds]-$iStart) < $nSecond } { # Each transaction does 3 operations. Each operation is either a read @@ -128,6 +130,7 @@ set sql $zSql # Execute the SQL transaction. # + incr nAttempt set rc [catch { execsql_blocking $::DB " BEGIN; $SQL(1); @@ -154,13 +157,14 @@ set sql $zSql # returned "1". Otherwise, the invariant was false, indicating that # some malfunction has occurred. foreach r $msg { if {$r != 1} { puts "Invariant check failed: $msg" } } + incr nOp } } # Close the database connection and return 0. # sqlite3_close $::DB - expr 0 + list $nOp $nAttempt } foreach {iTest xStep xPrepare} { @@ -204,7 +208,9 @@ foreach {iTest xStep xPrepare} { for {set ii 0} {$ii < $nThread} {incr ii} { do_test notify2-$iTest.2.$ii { if {![info exists finished($ii)]} { vwait finished($ii) } - set finished($ii) + incr anSuccess($xStep) [lindex $finished($ii) 0] + incr anAttempt($xStep) [lindex $finished($ii) 1] + expr 0 } {0} } @@ -225,17 +231,36 @@ foreach {iTest xStep xPrepare} { } # The following tests checks to make sure sqlite3_blocking_step() is -# faster than sqlite3_step(). blocking_step() is always faster on -# multi-core and is usually faster on single-core. But sometimes, by -# chance, step() will be faster on a single core, in which case the +# faster than sqlite3_step(). "Faster" in this case means uses fewer +# CPU cycles. This is not always the same as faster in wall-clock time +# for this type of test. The number of CPU cycles per transaction is +# roughly proportional to the number of attempts made (i.e. one plus the +# number of SQLITE_BUSY or SQLITE_LOCKED errors that require the transaction +# to be retried). So this test just measures that a greater percentage of +# transactions attempted using blocking_step() succeed. +# +# The blocking_step() function is almost always faster on multi-core and is +# usually faster on single-core. But sometimes, by chance, step() will be +# faster on a single core, in which case the # following test will fail. # puts "The following test seeks to demonstrate that the sqlite3_unlock_notify()" -puts "interface helps multi-core systems to run faster. This test sometimes" -puts "fails on single-core machines." +puts "interface helps multi-core systems to run more efficiently. This test" +puts "sometimes fails on single-core machines." puts [array get anWrite] do_test notify2-3 { - expr {$anWrite(sqlite3_blocking_step) > $anWrite(sqlite3_step)} + set blocking [expr { + double($anSuccess(sqlite3_blocking_step)) / + double($anAttempt(sqlite3_blocking_step)) + }] + set non [expr { + double($anSuccess(sqlite3_step)) / + double($anAttempt(sqlite3_step)) + }] + puts -nonewline [format " blocking: %.1f%% non-blocking %.1f%% ..." \ + [expr $blocking*100.0] [expr $non*100.0]] + + expr {$blocking > $non} } {1} sqlite3_enable_shared_cache $::enable_shared_cache From 51dc84eb7053963a6236e0a10dd3936bf4bd1378 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Dec 2014 13:04:25 +0000 Subject: [PATCH 691/710] Make SQLITE_CONFIG_PCACHE_HDRSZ accurate (not an over-estimate) on 32-bit systems. FossilOrigin-Name: 340b347758e570db3e739b56af0dcf3fc34525be --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pcache.c | 3 ++- src/pcache1.c | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 68e968d12e..e2c3330813 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\snotify2.test\sto\scheck\sthat\ssqlite3_blocking_step()\suses\sCPU\smore\sefficiently\sthan\ssqlite3_step(),\snot\sthat\sit\sresults\sin\sgreater\soverall\sthroughput\sfor\sany\sspecific\snumber\sof\sthreads. -D 2014-12-30T12:03:35.117 +C Make\sSQLITE_CONFIG_PCACHE_HDRSZ\saccurate\s(not\san\sover-estimate)\son\s32-bit\nsystems. +D 2014-12-30T13:04:25.944 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 40326b6d788007dd5e00587c54adcd2621832bb3 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,9 +219,9 @@ F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 2cbaf886a6157c53a8061ea7e677f81620ff46eb F src/pager.h c3476e7c89cdf1c6914e50a11f3714e30b4e0a77 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 -F src/pcache.c 8ee2ce15c4f49b44e27e8ffb8ce683c5e506e736 +F src/pcache.c b83d160ce81ca101f98f0d27498e6d6bd49f1599 F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8 -F src/pcache1.c ff599891da0e4993c814eeae3e7e2801f47364f3 +F src/pcache1.c 1e77432b40b7d3288327d9cdf399dcdfd2b6d3bf F src/pragma.c bd33aa24456f043bb6f6d32a918bbeed41d8c591 F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9 F src/printf.c 9e75a6a0b55bf61cfff7d7e19d89834a1b938236 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b28ce75f2d3a6343dc20d581dc55afae89ab5efa -R 05d3fab766dc5317a84b8e8fc66a7683 -U dan -Z 9e6a15ccef1a8c8d1ccc67c36a60ea41 +P d904d29354a5ed85d33bafe4a7143f3c5ecee790 +R 10e3253f54695b7f7d1030889f62fe40 +U drh +Z 96098511eaa9438036b4e314651b0800 diff --git a/manifest.uuid b/manifest.uuid index f0f3fc30da..d790a34fba 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d904d29354a5ed85d33bafe4a7143f3c5ecee790 \ No newline at end of file +340b347758e570db3e739b56af0dcf3fc34525be \ No newline at end of file diff --git a/src/pcache.c b/src/pcache.c index 3139429699..0194f63bb9 100644 --- a/src/pcache.c +++ b/src/pcache.c @@ -196,7 +196,8 @@ int sqlite3PcacheSetPageSize(PCache *pCache, int szPage){ if( pCache->szPage ){ sqlite3_pcache *pNew; pNew = sqlite3GlobalConfig.pcache2.xCreate( - szPage, pCache->szExtra + sizeof(PgHdr), pCache->bPurgeable + szPage, pCache->szExtra + ROUND8(sizeof(PgHdr)), + pCache->bPurgeable ); if( pNew==0 ) return SQLITE_NOMEM; sqlite3GlobalConfig.pcache2.xCachesize(pNew, numberOfCachePages(pCache)); diff --git a/src/pcache1.c b/src/pcache1.c index 990f51591f..f5f7893714 100644 --- a/src/pcache1.c +++ b/src/pcache1.c @@ -296,7 +296,7 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache){ pPg = 0; } #else - pPg = pcache1Alloc(sizeof(PgHdr1) + pCache->szPage + pCache->szExtra); + pPg = pcache1Alloc(ROUND8(sizeof(PgHdr1)) + pCache->szPage + pCache->szExtra); p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; #endif pcache1EnterMutex(pCache->pGroup); From d348c66e2971fe9de3ec832e54c24348bfcec8ec Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 30 Dec 2014 14:40:53 +0000 Subject: [PATCH 692/710] If the sorter uses mmap'd temp files, ensure all pages of the temp file have been allocated before it is accessed. Otherwise, a disk-full condition might result in a SIGBUS exception. FossilOrigin-Name: 776648412c30dce206f1024ff849c2cb025bb006 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 8 ++++---- src/vdbesort.c | 12 ++++++------ 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index e2c3330813..553eab2472 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sSQLITE_CONFIG_PCACHE_HDRSZ\saccurate\s(not\san\sover-estimate)\son\s32-bit\nsystems. -D 2014-12-30T13:04:25.944 +C If\sthe\ssorter\suses\smmap'd\stemp\sfiles,\sensure\sall\spages\sof\sthe\stemp\sfile\shave\sbeen\sallocated\sbefore\sit\sis\saccessed.\sOtherwise,\sa\sdisk-full\scondition\smight\sresult\sin\sa\sSIGBUS\sexception. +D 2014-12-30T14:40:53.460 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 40326b6d788007dd5e00587c54adcd2621832bb3 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c fb587121840f690101336879adfa6d0b2cd0e8c7 +F src/os_unix.c 7f9ed5f05e4a9eb7275d1216e46d245d0cebfebb F src/os_win.c 91d3d08e33ec0258d180d4c8255492f47d15e007 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 2cbaf886a6157c53a8061ea7e677f81620ff46eb @@ -298,7 +298,7 @@ F src/vdbeapi.c 4bc511a46b9839392ae0e90844a71dc96d9dbd71 F src/vdbeaux.c 07ef87c6d4b5abdf13ff33babb10205702fdab0a F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f -F src/vdbesort.c c150803a3e98fbc68bd07772cbbd4328a0a7212d +F src/vdbesort.c 80e40d889ebb536cb7a5ac4c12fa2a4662bc9181 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010 F src/vtab.c c08ec66f45919eaa726bf88aa53eb08379d607f9 F src/wal.c 85353539f2d9d0c91ebd057c32525b1e1aa3335e @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P d904d29354a5ed85d33bafe4a7143f3c5ecee790 -R 10e3253f54695b7f7d1030889f62fe40 -U drh -Z 96098511eaa9438036b4e314651b0800 +P 340b347758e570db3e739b56af0dcf3fc34525be +R e81d03c815814a7da9c21e19ec0ae799 +U dan +Z 9955da99fb103f07b93ca8aef676d75e diff --git a/manifest.uuid b/manifest.uuid index d790a34fba..40b0399bb8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -340b347758e570db3e739b56af0dcf3fc34525be \ No newline at end of file +776648412c30dce206f1024ff849c2cb025bb006 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index a9344ee830..8314e4f678 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3718,16 +3718,16 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ int nBlk = buf.st_blksize; /* File-system block size */ i64 iWrite; /* Next offset to write to */ - if( robust_ftruncate(pFile->h, nSize) ){ - pFile->lastErrno = errno; - return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); - } iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; while( iWriteh, nSize) ){ + pFile->lastErrno = errno; + return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + } #endif } } diff --git a/src/vdbesort.c b/src/vdbesort.c index 78ecc1efb9..d022af9af6 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -1132,12 +1132,12 @@ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ */ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ if( nByte<=(i64)(db->nMaxSorterMmap) && pFd->pMethods->iVersion>=3 ){ - int rc = sqlite3OsTruncate(pFd, nByte); - if( rc==SQLITE_OK ){ - void *p = 0; - sqlite3OsFetch(pFd, 0, (int)nByte, &p); - sqlite3OsUnfetch(pFd, 0, p); - } + void *p = 0; + int chunksize = 4*1024; + sqlite3OsFileControlHint(pFd, SQLITE_FCNTL_CHUNK_SIZE, &chunksize); + sqlite3OsFileControlHint(pFd, SQLITE_FCNTL_SIZE_HINT, &nByte); + sqlite3OsFetch(pFd, 0, (int)nByte, &p); + sqlite3OsUnfetch(pFd, 0, p); } } #else From 55e115f060fae26e3f3ea3fee78cb24012035dfc Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 30 Dec 2014 18:07:34 +0000 Subject: [PATCH 693/710] Fix problems with the "inmemory_journal" permutation. FossilOrigin-Name: 79693f0412ffb0486b974ee6c63b4231cfff5a77 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/permutations.test | 2 +- test/pragma3.test | 6 ++++++ 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 553eab2472..c8eb6467d4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthe\ssorter\suses\smmap'd\stemp\sfiles,\sensure\sall\spages\sof\sthe\stemp\sfile\shave\sbeen\sallocated\sbefore\sit\sis\saccessed.\sOtherwise,\sa\sdisk-full\scondition\smight\sresult\sin\sa\sSIGBUS\sexception. -D 2014-12-30T14:40:53.460 +C Fix\sproblems\swith\sthe\s"inmemory_journal"\spermutation. +D 2014-12-30T18:07:34.789 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 40326b6d788007dd5e00587c54adcd2621832bb3 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -782,10 +782,10 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 4e12d43f4639ea8a0e366d9c64e0009afe2eb544 +F test/permutations.test d408cd2d48e7a61c9c51ae528d94c3c48ce97477 F test/pragma.test aa16dedfe01c02c8895169012f7dfde9c163f0d5 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 -F test/pragma3.test 4f141da233358783ba443eb685e6739ce0eb1d90 +F test/pragma3.test 0ca2aea1499a7c2dcee235419e520d825dac958d F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 340b347758e570db3e739b56af0dcf3fc34525be -R e81d03c815814a7da9c21e19ec0ae799 +P 776648412c30dce206f1024ff849c2cb025bb006 +R d8ec23380207a0894ebd8ce1599deb6c U dan -Z 9955da99fb103f07b93ca8aef676d75e +Z 8cab0a0cf2d75b3466c28901d182f0a0 diff --git a/manifest.uuid b/manifest.uuid index 40b0399bb8..e8c8d8da23 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -776648412c30dce206f1024ff849c2cb025bb006 \ No newline at end of file +79693f0412ffb0486b974ee6c63b4231cfff5a77 \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index ff61bf644b..63f8dc589e 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -690,7 +690,7 @@ test_suite "inmemory_journal" -description { zerodamage.test # WAL mode is different. - wal* tkt-2d1a5c67d.test backcompat.test + wal* tkt-2d1a5c67d.test backcompat.test e_wal* }] ifcapable mem3 { diff --git a/test/pragma3.test b/test/pragma3.test index f3d531164b..4654578df2 100644 --- a/test/pragma3.test +++ b/test/pragma3.test @@ -216,7 +216,12 @@ ifcapable shared_cache { # Make sure this also works in WAL mode # +# This will not work with the in-memory journal permutation, as opening +# [db2] switches the journal mode back to "memory" +# ifcapable wal { +if {[permutation]!="inmemory_journal"} { + sqlite3 db test.db db eval {PRAGMA journal_mode=WAL} sqlite3 db2 test.db @@ -242,5 +247,6 @@ ifcapable wal { } {3 111 222} db2 close } +} finish_test From 9486c1b020767a6cc5962470605cc72305717b25 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Dec 2014 19:26:07 +0000 Subject: [PATCH 694/710] Update the threadtest3 test program so that its output summary is compatible with releasetest.tcl. In threadtest3, do not record errors that contain the string "no such table" as being fatal errors, since they happen sometimes in a race condition in stress1. FossilOrigin-Name: 98cb56e2401ae7e113b071df8997ba62265821d3 --- manifest | 16 ++++----- manifest.uuid | 2 +- test/threadtest3.c | 80 ++++++++++++++++++++----------------------- test/tt3_checkpoint.c | 4 +-- 4 files changed, 48 insertions(+), 54 deletions(-) diff --git a/manifest b/manifest index c8eb6467d4..6c2e6952e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblems\swith\sthe\s"inmemory_journal"\spermutation. -D 2014-12-30T18:07:34.789 +C Update\sthe\sthreadtest3\stest\sprogram\sso\sthat\sits\soutput\ssummary\sis\s\ncompatible\swith\sreleasetest.tcl.\s\sIn\sthreadtest3,\sdo\snot\srecord\serrors\nthat\scontain\sthe\sstring\s"no\ssuch\stable"\sas\sbeing\sfatal\serrors,\ssince\sthey\nhappen\ssometimes\sin\sa\srace\scondition\sin\sstress1. +D 2014-12-30T19:26:07.267 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 40326b6d788007dd5e00587c54adcd2621832bb3 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -914,7 +914,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c f8c6595664a4c5ef5f28d97a612386fe14dd1940 +F test/threadtest3.c 66c2693d888f9ed256d54d70dd60f569d92c12cf F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 @@ -1078,7 +1078,7 @@ F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe F test/triggerC.test a68980c5955d62ee24be6f97129d824f199f9a4c F test/triggerD.test 8e7f3921a92a5797d472732108109e44575fa650 F test/triggerE.test 355e9c5cbaed5cd039a60baad1fb2197caeb8e52 -F test/tt3_checkpoint.c 5e63ee65ed5f87176e25a996480cb02c6caec8b4 +F test/tt3_checkpoint.c 9e75cf7c1c364f52e1c47fd0f14c4340a9db0fe1 F test/tt3_index.c 39eec10a35f57672225be4d182862152896dee4a F test/tt3_lookaside1.c 0377e202c3c2a50d688cb65ba203afeda6fafeb9 F test/tt3_stress.c c57d804716165811d979d4a719e05baccd79277f @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 776648412c30dce206f1024ff849c2cb025bb006 -R d8ec23380207a0894ebd8ce1599deb6c -U dan -Z 8cab0a0cf2d75b3466c28901d182f0a0 +P 79693f0412ffb0486b974ee6c63b4231cfff5a77 +R 5cf8bac225464c2ed58e93713858c6ef +U drh +Z 61949e8b38696d2e4ed6889c396c567f diff --git a/manifest.uuid b/manifest.uuid index e8c8d8da23..cee491506b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -79693f0412ffb0486b974ee6c63b4231cfff5a77 \ No newline at end of file +98cb56e2401ae7e113b071df8997ba62265821d3 \ No newline at end of file diff --git a/test/threadtest3.c b/test/threadtest3.c index ff8add5bf6..d700d7d47a 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -1,41 +1,41 @@ - /* +** 2010-07-22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** ** The code in this file runs a few multi-threaded test cases using the ** SQLite library. It can be compiled to an executable on unix using the ** following command: ** ** gcc -O2 threadtest3.c sqlite3.c -ldl -lpthread -lm ** -** Then run the compiled program. The exit status is non-zero if any tests -** failed (hopefully there is also some output to stdout to clarify what went -** wrong). +** Even though threadtest3.c is the only C source code file mentioned on +** the compiler command-line, #include macros are used to pull in additional +** C code files named "tt3_*.c". ** -** There are three parts to the code in this file, in the following order: +** After compiling, run this program with an optional argument telling +** which test to run. All tests are run if no argument is given. The +** argument can be a glob pattern to match multiple tests. Examples: ** -** 1. Code for the SQL aggregate function md5sum() copied from -** tclsqlite.c in the SQLite distribution. The names of all the -** types and functions in this section begin with "MD5" or "md5". +** ./a.out -- Run all tests +** ./a.out walthread3 -- Run the "walthread3" test +** ./a.out 'wal*' -- Run all of the wal* tests +** ./a.out --help -- List all available tests ** -** 2. A set of utility functions that may be used to implement -** multi-threaded test cases. These are all called by test code -** via macros that help with error reporting. The macros are defined -** immediately below this comment. -** -** 3. The test code itself. And a main() routine to drive the test -** code. +** The exit status is non-zero if any test fails. */ -/************************************************************************* -** Start of test code/infrastructure interface macros. -** -** The following macros constitute the interface between the test -** programs and the test infrastructure. Test infrastructure code -** does not itself use any of these macros. Test code should not -** call any of the macroname_x() functions directly. -** -** See the header comments above the corresponding macroname_x() -** function for a description of each interface. +/* +** The "Set Error Line" macro. */ +#define SEL(e) ((e)->iLine = ((e)->rc ? (e)->iLine : __LINE__)) /* Database functions */ #define opendb(w,x,y,z) (SEL(w), opendb_x(w,x,y,z)) @@ -391,9 +391,9 @@ static void md5finalize(sqlite3_context *context){ sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); } -/************************************************************************* +/* ** End of copied md5sum() code. -*/ +**************************************************************************/ typedef sqlite3_int64 i64; @@ -448,7 +448,8 @@ static void free_err(Error *p){ static void print_err(Error *p){ if( p->rc!=SQLITE_OK ){ printf("Error: (%d) \"%s\" at line %d\n", p->rc, p->zErr, p->iLine); - nGlobalErr++; + if( sqlite3_strglob("* - no such table: *",p->zErr)!=0 ) nGlobalErr++; + fflush(stdout); } } @@ -785,6 +786,7 @@ static void join_all_threads_x( if( pErr->rc==SQLITE_OK ) system_error(pErr, rc); }else{ printf("Thread %d says: %s\n", p->iTid, (ret==0 ? "..." : (char *)ret)); + fflush(stdout); } sqlite3_free(p); } @@ -898,11 +900,6 @@ static int timetostop_x( return ret; } -/* -** The "Set Error Line" macro. -*/ -#define SEL(e) ((e)->iLine = ((e)->rc ? (e)->iLine : __LINE__)) - /************************************************************************* ************************************************************************** @@ -1427,9 +1424,9 @@ static void dynamic_triggers(int nMs){ int main(int argc, char **argv){ struct ThreadTest { - void (*xTest)(int); - const char *zTest; - int nMs; + void (*xTest)(int); /* Routine for running this test */ + const char *zTest; /* Name of this test */ + int nMs; /* How long to run this test, in milliseconds */ } aTest[] = { { walthread1, "walthread1", 20000 }, { walthread2, "walthread2", 20000 }, @@ -1452,7 +1449,7 @@ int main(int argc, char **argv){ }; int i; - int bTestfound = 0; + int nTestfound = 0; sqlite3_config(SQLITE_CONFIG_MULTITHREAD); sqlite3_config(SQLITE_CONFIG_MULTITHREAD); @@ -1468,12 +1465,13 @@ int main(int argc, char **argv){ } printf("Running %s for %d seconds...\n", z, aTest[i].nMs/1000); + fflush(stdout); aTest[i].xTest(aTest[i].nMs); - bTestfound++; + nTestfound++; } - if( bTestfound==0 ) goto usage; + if( nTestfound==0 ) goto usage; - printf("Total of %d errors across all tests\n", nGlobalErr); + printf("%d errors out of %d tests\n", nGlobalErr, nTestfound); return (nGlobalErr>0 ? 255 : 0); usage: @@ -1485,5 +1483,3 @@ int main(int argc, char **argv){ return 254; } - - diff --git a/test/tt3_checkpoint.c b/test/tt3_checkpoint.c index db237d6d9f..060a698211 100644 --- a/test/tt3_checkpoint.c +++ b/test/tt3_checkpoint.c @@ -1,5 +1,5 @@ /* -** 2001 September 15 +** 2011-02-02 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -146,5 +146,3 @@ static void checkpoint_starvation_2(int nMs){ } print_and_free_err(&err); } - - From 592bf7faf5a855a77b04abfeab3c66d5a5f3fa7b Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 30 Dec 2014 19:58:31 +0000 Subject: [PATCH 695/710] Ensure that when a file is extended using FCNTL_SIZE_HINT the last page is allocated on disk, even if the file will only use part of it. FossilOrigin-Name: c7f84717d61197afa9e0ac607c4b349361e6e2b7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 23 +++++++++++++---------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 6c2e6952e9..b8c410c164 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sthreadtest3\stest\sprogram\sso\sthat\sits\soutput\ssummary\sis\s\ncompatible\swith\sreleasetest.tcl.\s\sIn\sthreadtest3,\sdo\snot\srecord\serrors\nthat\scontain\sthe\sstring\s"no\ssuch\stable"\sas\sbeing\sfatal\serrors,\ssince\sthey\nhappen\ssometimes\sin\sa\srace\scondition\sin\sstress1. -D 2014-12-30T19:26:07.267 +C Ensure\sthat\swhen\sa\sfile\sis\sextended\susing\sFCNTL_SIZE_HINT\sthe\slast\spage\sis\sallocated\son\sdisk,\seven\sif\sthe\sfile\swill\sonly\suse\spart\sof\sit. +D 2014-12-30T19:58:31.340 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 40326b6d788007dd5e00587c54adcd2621832bb3 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/os.c 8fd25588eeba74068d41102d26810e216999b6c8 F src/os.h 3e57a24e2794a94d3cf2342c6d9a884888cd96bf F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa -F src/os_unix.c 7f9ed5f05e4a9eb7275d1216e46d245d0cebfebb +F src/os_unix.c 08c0346d2ea5e5ffd5b1a796f9becf1976d648d7 F src/os_win.c 91d3d08e33ec0258d180d4c8255492f47d15e007 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 F src/pager.c 2cbaf886a6157c53a8061ea7e677f81620ff46eb @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 79693f0412ffb0486b974ee6c63b4231cfff5a77 -R 5cf8bac225464c2ed58e93713858c6ef -U drh -Z 61949e8b38696d2e4ed6889c396c567f +P 98cb56e2401ae7e113b071df8997ba62265821d3 +R 42e94e61b93d775e47b58099deb363fd +U dan +Z 6763ecde934c72a96b9e8144b7d09293 diff --git a/manifest.uuid b/manifest.uuid index cee491506b..1e1eaa02a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -98cb56e2401ae7e113b071df8997ba62265821d3 \ No newline at end of file +c7f84717d61197afa9e0ac607c4b349361e6e2b7 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 8314e4f678..f802d9cd19 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -3709,24 +3709,27 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ }while( err==EINTR ); if( err ) return SQLITE_IOERR_WRITE; #else - /* If the OS does not have posix_fallocate(), fake it. First use - ** ftruncate() to set the file size, then write a single byte to - ** the last byte in each block within the extended region. This - ** is the same technique used by glibc to implement posix_fallocate() - ** on systems that do not have a real fallocate() system call. + /* If the OS does not have posix_fallocate(), fake it. Write a + ** single byte to the last byte in each block that falls entirely + ** within the extended region. Then, if required, a single byte + ** at offset (nSize-1), to set the size of the file correctly. + ** This is a similar technique to that used by glibc on systems + ** that do not have a real fallocate() call. */ int nBlk = buf.st_blksize; /* File-system block size */ i64 iWrite; /* Next offset to write to */ iWrite = ((buf.st_size + 2*nBlk - 1)/nBlk)*nBlk-1; - while( iWrite=buf.st_size ); + assert( (iWrite/nBlk)==((buf.st_size+nBlk-1)/nBlk) ); + assert( ((iWrite+1)%nBlk)==0 ); + for(/*no-op*/; iWriteh, nSize) ){ - pFile->lastErrno = errno; - return unixLogError(SQLITE_IOERR_TRUNCATE, "ftruncate", pFile->zPath); + if( nSize%nBlk ){ + int nWrite = seekAndWrite(pFile, nSize-1, "", 1); + if( nWrite!=1 ) return SQLITE_IOERR_WRITE; } #endif } From bd41d566292dec6f65994b7f56a6d4f508222988 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 30 Dec 2014 20:40:32 +0000 Subject: [PATCH 696/710] Add the "mptester" tests to releasetest.tcl. FossilOrigin-Name: 93094a68d3a6178779878cbbe0e5e09ef31a323f --- Makefile.in | 5 +++++ manifest | 18 +++++++++--------- manifest.uuid | 2 +- mptest/mptest.c | 2 +- test/releasetest.tcl | 22 +++++++++++----------- 5 files changed, 27 insertions(+), 22 deletions(-) diff --git a/Makefile.in b/Makefile.in index 6867afcf52..cd024e950a 100644 --- a/Makefile.in +++ b/Makefile.in @@ -530,6 +530,11 @@ mptester$(EXE): sqlite3.c $(TOP)/mptest/mptest.c $(LTLINK) -o $@ -I. $(TOP)/mptest/mptest.c sqlite3.c \ $(TLIBS) -rpath "$(libdir)" +mptest: mptester$(EXE) + rm -f mptest1.db + ./mptester$(EXE) mptest1.db $(TOP)/mptest/crash01.test + rm -f mptest2.db + ./mptester$(EXE) mptest2.db $(TOP)/mptest/multiwrite01.test # This target creates a directory named "tsrc" and fills it with # copies of all of the C source code and header files needed to diff --git a/manifest b/manifest index b8c410c164..f0f77e4215 100644 --- a/manifest +++ b/manifest @@ -1,7 +1,7 @@ -C Ensure\sthat\swhen\sa\sfile\sis\sextended\susing\sFCNTL_SIZE_HINT\sthe\slast\spage\sis\sallocated\son\sdisk,\seven\sif\sthe\sfile\swill\sonly\suse\spart\sof\sit. -D 2014-12-30T19:58:31.340 +C Add\sthe\s"mptester"\stests\sto\sreleasetest.tcl. +D 2014-12-30T20:40:32.817 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f -F Makefile.in 40326b6d788007dd5e00587c54adcd2621832bb3 +F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc b363b90fe1bfc3b87d190f2f728a126c00d9ce09 F Makefile.vxworks 034289efa9d591b04b1a73598623119c306cbba0 @@ -160,7 +160,7 @@ F mptest/config01.test 3c6adcbc50b991866855f1977ff172eb6d901271 F mptest/config02.test 4415dfe36c48785f751e16e32c20b077c28ae504 F mptest/crash01.test cce8e306d8596d5a2e497e27112dae1f6e5e3538 F mptest/crash02.subtest f4ef05adcd15d60e5d2bd654204f2c008b519df8 -F mptest/mptest.c 499a74af4be293b7c1c7c3d40f332b67227dd739 +F mptest/mptest.c 24c5f72415df2eab7088ef8c9f99f163aed590c8 F mptest/multiwrite01.test 499ad0310da8dff8e8f98d2e272fc2a8aa741b2e F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b @@ -801,7 +801,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl 3ffd8d99da7a087917f057147053ed35ce1bf90b +F test/releasetest.tcl 8f35e5073901e48a634a649462fa2e7e522e9dc0 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 98cb56e2401ae7e113b071df8997ba62265821d3 -R 42e94e61b93d775e47b58099deb363fd -U dan -Z 6763ecde934c72a96b9e8144b7d09293 +P c7f84717d61197afa9e0ac607c4b349361e6e2b7 +R 0933bc5647d1041ce6d28bf44cb1ff01 +U drh +Z 8ac9cef8267dc35aa4f2371bf41435dc diff --git a/manifest.uuid b/manifest.uuid index 1e1eaa02a9..daccc74158 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c7f84717d61197afa9e0ac607c4b349361e6e2b7 \ No newline at end of file +93094a68d3a6178779878cbbe0e5e09ef31a323f \ No newline at end of file diff --git a/mptest/mptest.c b/mptest/mptest.c index 059ae102fa..7b56b61902 100644 --- a/mptest/mptest.c +++ b/mptest/mptest.c @@ -1395,7 +1395,7 @@ int main(int argc, char **argv){ maybeClose(g.pLog); maybeClose(g.pErrLog); if( iClient==0 ){ - printf("Summary: %d errors in %d tests\n", g.nError, g.nTest); + printf("Summary: %d errors out of %d tests\n", g.nError, g.nTest); } return g.nError>0; } diff --git a/test/releasetest.tcl b/test/releasetest.tcl index bcd7d3d060..e2311d8f9c 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -158,7 +158,7 @@ array set ::Configs { array set ::Platforms { Linux-x86_64 { "Check-Symbols" checksymbols - "Debug-One" test + "Debug-One" "mptest test" "Secure-Delete" test "Unlock-Notify" "QUICKTEST_INCLUDE=notify2.test test" "Update-Delete-Limit" test @@ -178,11 +178,11 @@ array set ::Platforms { "Default" "threadtest fulltest" } Darwin-i386 { - "Locking-Style" test + "Locking-Style" "mptest test" "OS-X" "threadtest fulltest" } "Windows NT-intel" { - "Default" "fulltestonly" + "Default" "mptest fulltestonly" } } @@ -214,7 +214,7 @@ proc count_tests_and_errors {logfile rcVar errmsgVar} { set seen 0 while {![eof $fd]} { set line [gets $fd] - if {[regexp {^(\d+) errors out of (\d+) tests} $line all nerr ntest]} { + if {[regexp {(\d+) errors out of (\d+) tests} $line all nerr ntest]} { incr ::NERRCASE $nerr incr ::NTESTCASE $ntest set seen 1 @@ -222,7 +222,6 @@ proc count_tests_and_errors {logfile rcVar errmsgVar} { set rc 1 set errmsg $line } - break; } } close $fd @@ -457,16 +456,17 @@ proc main {argv} { # add it and run veryquick.test. if {$target!="checksymbols" && !$::BUILDONLY} { set debug_idx [lsearch -glob $config_options -DSQLITE_DEBUG*] + set xtarget $target + regsub -all {fulltest[a-z]+} $xtarget test xtarget if {$debug_idx < 0} { incr NTEST - run_test_suite "${zConfig}_debug" test [ - concat $config_options -DSQLITE_DEBUG=1 - ] + append config_options " -DSQLITE_DEBUG=1" + run_test_suite "${zConfig}_debug" $xtarget $config_options } else { incr NTEST - run_test_suite "${zConfig}_ndebug" test [ - lreplace $config_options $debug_idx $debug_idx - ] + regsub { *-DSQLITE_MEMDEBUG[^ ]* *} $config_options { } config_options + regsub { *-DSQLITE_DEBUG[^ ]* *} $config_options { } config_options + run_test_suite "${zConfig}_ndebug" $xtarget $config_options } } } From e895b873894149268b661c14cd93adcafb5eee4e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 31 Dec 2014 09:52:15 +0000 Subject: [PATCH 697/710] Do not run pragma3.test as part of the mmap permutation. FossilOrigin-Name: 94101011966243d599519a69c99c202ea31b928d --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/permutations.test | 8 ++++++-- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f0f77e4215..9942d4b320 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"mptester"\stests\sto\sreleasetest.tcl. -D 2014-12-30T20:40:32.817 +C Do\snot\srun\spragma3.test\sas\spart\sof\sthe\smmap\spermutation. +D 2014-12-31T09:52:15.410 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -782,7 +782,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test d408cd2d48e7a61c9c51ae528d94c3c48ce97477 +F test/permutations.test e1c603ec095e29de3d1f1566d704ea270f9c3f89 F test/pragma.test aa16dedfe01c02c8895169012f7dfde9c163f0d5 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/pragma3.test 0ca2aea1499a7c2dcee235419e520d825dac958d @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P c7f84717d61197afa9e0ac607c4b349361e6e2b7 -R 0933bc5647d1041ce6d28bf44cb1ff01 -U drh -Z 8ac9cef8267dc35aa4f2371bf41435dc +P 93094a68d3a6178779878cbbe0e5e09ef31a323f +R 568e484006de3cb2669a32ac139fa8fe +U dan +Z 9ed2b5eb1c7f4568141d7b75c646d319 diff --git a/manifest.uuid b/manifest.uuid index daccc74158..9a0f1e309b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -93094a68d3a6178779878cbbe0e5e09ef31a323f \ No newline at end of file +94101011966243d599519a69c99c202ea31b928d \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index 63f8dc589e..28572ca0a7 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -141,11 +141,15 @@ test_suite "veryquick" -prefix "" -description { ] test_suite "mmap" -prefix "mm-" -description { - Similar to veryquick. Except with memory mapping disabled. + Similar to veryquick. Except with memory mapping enabled. } -presql { pragma mmap_size = 268435456; } -files [ - test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test + # Do not run pragma3.test, as it depends on the values returned by + # "PRAGMA data_version". And if mmap is being used these are often, + # but not always, one greater than if it were not. + test_set $allquicktests -exclude *malloc* *ioerr* *fault* pragma3.test \ + -include malloc.test ] test_suite "valgrind" -prefix "" -description { From 542d55865cb1ec11421783060212b4b13e9f96e4 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Dec 2014 14:18:48 +0000 Subject: [PATCH 698/710] Make sure PRAGMA data_version is updated even if the cache is empty when another connection changes the database. FossilOrigin-Name: cf48eb608af9102a8def2a5b7f5f7b348548116f --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/pager.c | 21 ++++++++++----------- test/pragma3.test | 1 + 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 9942d4b320..18d0f09e4c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\srun\spragma3.test\sas\spart\sof\sthe\smmap\spermutation. -D 2014-12-31T09:52:15.410 +C Make\ssure\sPRAGMA\sdata_version\sis\supdated\seven\sif\sthe\scache\sis\sempty\swhen\nanother\sconnection\schanges\sthe\sdatabase. +D 2014-12-31T14:18:48.679 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -216,7 +216,7 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa F src/os_unix.c 08c0346d2ea5e5ffd5b1a796f9becf1976d648d7 F src/os_win.c 91d3d08e33ec0258d180d4c8255492f47d15e007 F src/os_win.h 09e751b20bbc107ffbd46e13555dc73576d88e21 -F src/pager.c 2cbaf886a6157c53a8061ea7e677f81620ff46eb +F src/pager.c 4120a49ecd37697e28f5ed807f470b9c0b88410c F src/pager.h c3476e7c89cdf1c6914e50a11f3714e30b4e0a77 F src/parse.y 5dfead8aed90cb0c7c1115898ee2266804daff45 F src/pcache.c b83d160ce81ca101f98f0d27498e6d6bd49f1599 @@ -785,7 +785,7 @@ F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 F test/permutations.test e1c603ec095e29de3d1f1566d704ea270f9c3f89 F test/pragma.test aa16dedfe01c02c8895169012f7dfde9c163f0d5 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 -F test/pragma3.test 0ca2aea1499a7c2dcee235419e520d825dac958d +F test/pragma3.test 6f849ccffeee7e496d2f2b5e74152306c0b8757c F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 F test/printf2.test b4acd4bf8734243257f01ddefa17c4fb090acc8a F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 93094a68d3a6178779878cbbe0e5e09ef31a323f -R 568e484006de3cb2669a32ac139fa8fe -U dan -Z 9ed2b5eb1c7f4568141d7b75c646d319 +P 94101011966243d599519a69c99c202ea31b928d +R ce88be0f3eb17006a4da57569dcf8731 +U drh +Z eb99526801ba8c2c5e7e84b0513387c7 diff --git a/manifest.uuid b/manifest.uuid index 9a0f1e309b..a48eb47585 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -94101011966243d599519a69c99c202ea31b928d \ No newline at end of file +cf48eb608af9102a8def2a5b7f5f7b348548116f \ No newline at end of file diff --git a/src/pager.c b/src/pager.c index 67b95f7186..2230174e5c 100644 --- a/src/pager.c +++ b/src/pager.c @@ -647,6 +647,7 @@ struct Pager { u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 subjInMemory; /* True to use in-memory sub-journals */ u8 bUseFetch; /* True to use xFetch() */ + u8 hasBeenUsed; /* True if any content previously read from this pager*/ Pgno dbSize; /* Number of pages in the database */ Pgno dbOrigSize; /* dbSize before the current transaction */ Pgno dbFileSize; /* Number of pages in the database file */ @@ -3897,7 +3898,7 @@ static int pagerAcquireMapPage( PgHdr **ppPage /* OUT: Acquired page object */ ){ PgHdr *p; /* Memory mapped page to return */ - + if( pPager->pMmapFreelist ){ *ppPage = p = pPager->pMmapFreelist; pPager->pMmapFreelist = p->pDirty; @@ -5128,16 +5129,12 @@ int sqlite3PagerSharedLock(Pager *pPager){ ); } - if( !pPager->tempFile && ( - pPager->pBackup - || sqlite3PcachePagecount(pPager->pPCache)>0 - || USEFETCH(pPager) - )){ - /* The shared-lock has just been acquired on the database file - ** and there are already pages in the cache (from a previous - ** read or write transaction). Check to see if the database - ** has been modified. If the database has changed, flush the - ** cache. + if( !pPager->tempFile && pPager->hasBeenUsed ){ + /* The shared-lock has just been acquired then check to + ** see if the database has been modified. If the database has changed, + ** flush the cache. The pPager->hasBeenUsed flag prevents this from + ** occurring on the very first access to a file, in order to save a + ** single unnecessary sqlite3OsRead() call at the start-up. ** ** Database changes is detected by looking at 15 bytes beginning ** at offset 24 into the file. The first 4 of these 16 bytes are @@ -5302,6 +5299,7 @@ int sqlite3PagerAcquire( if( pgno==0 ){ return SQLITE_CORRUPT_BKPT; } + pPager->hasBeenUsed = 1; /* If the pager is in the error state, return an error immediately. ** Otherwise, request the page from the PCache layer. */ @@ -5451,6 +5449,7 @@ DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ assert( pgno!=0 ); assert( pPager->pPCache!=0 ); pPage = sqlite3PcacheFetch(pPager->pPCache, pgno, 0); + assert( pPage==0 || pPager->hasBeenUsed ); return sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pPage); } diff --git a/test/pragma3.test b/test/pragma3.test index 4654578df2..b7ea4d3fc6 100644 --- a/test/pragma3.test +++ b/test/pragma3.test @@ -64,6 +64,7 @@ do_execsql_test pragma3-130 { COMMIT; SELECT * FROM t1; PRAGMA data_version; + PRAGMA shrink_memory; } {1 1 1 100 200 300 400 500 1} # EVIDENCE-OF: R-63005-41812 The integer values returned by two From c67d650264c37d024074736a451ba48480b0c4d2 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Dec 2014 15:14:29 +0000 Subject: [PATCH 699/710] Change the width of output lines in releasetest.tcl from 70 to 79 characters. FossilOrigin-Name: a468d96700c05d1a01a745930d13ce89c09ce4fa --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/releasetest.tcl | 7 ++++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 18d0f09e4c..847c4e9ac3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sPRAGMA\sdata_version\sis\supdated\seven\sif\sthe\scache\sis\sempty\swhen\nanother\sconnection\schanges\sthe\sdatabase. -D 2014-12-31T14:18:48.679 +C Change\sthe\swidth\sof\soutput\slines\sin\sreleasetest.tcl\sfrom\s70\sto\s79\scharacters. +D 2014-12-31T15:14:29.242 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -801,7 +801,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl 8f35e5073901e48a634a649462fa2e7e522e9dc0 +F test/releasetest.tcl fad4402d19b94021a3a9032b7b4a9855bf1ef498 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 94101011966243d599519a69c99c202ea31b928d -R ce88be0f3eb17006a4da57569dcf8731 +P cf48eb608af9102a8def2a5b7f5f7b348548116f +R 0eaceead134311c4ebac703045ebc24b U drh -Z eb99526801ba8c2c5e7e84b0513387c7 +Z b3362a619542185f9ea20d6df6dfbfd7 diff --git a/manifest.uuid b/manifest.uuid index a48eb47585..f7f3930580 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cf48eb608af9102a8def2a5b7f5f7b348548116f \ No newline at end of file +a468d96700c05d1a01a745930d13ce89c09ce4fa \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index e2311d8f9c..8b97f7f4f3 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -133,6 +133,7 @@ array set ::Configs { -DSQLITE_MAX_VARIABLE_NUMBER=500000 -DSQLITE_DEBUG=1 -DSQLITE_PREFER_PROXY_LOCKING=1 + -DSQLITE_ENABLE_API_ARMOR=1 } "Extra-Robustness" { -DSQLITE_ENABLE_OVERSIZE_CELL_CHECK=1 @@ -265,7 +266,7 @@ proc run_test_suite {name testtarget config} { if {!$::DRYRUN} { set title ${name}($testtarget) set n [string length $title] - puts -nonewline "${title}[string repeat . [expr {54-$n}]]" + puts -nonewline "${title}[string repeat . [expr {63-$n}]]" flush stdout } @@ -436,7 +437,7 @@ proc main {argv} { # Process any command line options. process_options $argv - puts [string repeat * 70] + puts [string repeat * 79] set ::NERR 0 set ::NTEST 0 @@ -476,7 +477,7 @@ proc main {argv} { set min [expr {($elapsetime/60)%60}] set sec [expr {$elapsetime%60}] set etime [format (%02d:%02d:%02d) $hr $min $sec] - puts [string repeat * 70] + puts [string repeat * 79] puts "$::NERRCASE failures of $::NTESTCASE tests run in $etime" } From 23f8a7cd2f7f035fe2abfd19a75d6fc98c742e9f Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 31 Dec 2014 18:08:46 +0000 Subject: [PATCH 700/710] Do run pragma3.test as part of the mmap permutation. As it works as of [cf48eb608a]. FossilOrigin-Name: 11057e2645d92876fd0fb6d702757408077b7354 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/permutations.test | 6 +----- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 847c4e9ac3..c70155dc52 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\swidth\sof\soutput\slines\sin\sreleasetest.tcl\sfrom\s70\sto\s79\scharacters. -D 2014-12-31T15:14:29.242 +C Do\srun\spragma3.test\sas\spart\sof\sthe\smmap\spermutation.\sAs\sit\sworks\sas\sof\s[cf48eb608a]. +D 2014-12-31T18:08:46.501 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -782,7 +782,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test e1c603ec095e29de3d1f1566d704ea270f9c3f89 +F test/permutations.test 59e2d8aba8c4f5842edba1a10a158b798690d3ba F test/pragma.test aa16dedfe01c02c8895169012f7dfde9c163f0d5 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 F test/pragma3.test 6f849ccffeee7e496d2f2b5e74152306c0b8757c @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P cf48eb608af9102a8def2a5b7f5f7b348548116f -R 0eaceead134311c4ebac703045ebc24b -U drh -Z b3362a619542185f9ea20d6df6dfbfd7 +P a468d96700c05d1a01a745930d13ce89c09ce4fa +R 2124f8dd0c13e87cf432312e601e89d3 +U dan +Z 1d23743f5f5d9d87deea6bb5afeffc32 diff --git a/manifest.uuid b/manifest.uuid index f7f3930580..61668af109 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a468d96700c05d1a01a745930d13ce89c09ce4fa \ No newline at end of file +11057e2645d92876fd0fb6d702757408077b7354 \ No newline at end of file diff --git a/test/permutations.test b/test/permutations.test index 28572ca0a7..fea70a60e8 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -145,11 +145,7 @@ test_suite "mmap" -prefix "mm-" -description { } -presql { pragma mmap_size = 268435456; } -files [ - # Do not run pragma3.test, as it depends on the values returned by - # "PRAGMA data_version". And if mmap is being used these are often, - # but not always, one greater than if it were not. - test_set $allquicktests -exclude *malloc* *ioerr* *fault* pragma3.test \ - -include malloc.test + test_set $allquicktests -exclude *malloc* *ioerr* *fault* -include malloc.test ] test_suite "valgrind" -prefix "" -description { From fb212d02c51c3f9beaa7ac60858ee13fd6618a4c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Dec 2014 18:10:07 +0000 Subject: [PATCH 701/710] Only run walthread5 once when running all tests in threadtest3. FossilOrigin-Name: ca2e4a5b22e1dc9ea79f77853e328d401138f747 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/threadtest3.c | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c70155dc52..de184d9294 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\srun\spragma3.test\sas\spart\sof\sthe\smmap\spermutation.\sAs\sit\sworks\sas\sof\s[cf48eb608a]. -D 2014-12-31T18:08:46.501 +C Only\srun\swalthread5\sonce\swhen\srunning\sall\stests\sin\sthreadtest3. +D 2014-12-31T18:10:07.569 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -914,7 +914,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c 66c2693d888f9ed256d54d70dd60f569d92c12cf +F test/threadtest3.c 7bbedc9df2b8fb0a3f75c5f2f66941c7f18ce1db F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a468d96700c05d1a01a745930d13ce89c09ce4fa -R 2124f8dd0c13e87cf432312e601e89d3 -U dan -Z 1d23743f5f5d9d87deea6bb5afeffc32 +P 11057e2645d92876fd0fb6d702757408077b7354 +R 977b3277256e94a381a08f38ec63eac7 +U drh +Z 206269a2c6d9b5759d40a6ff53cbafad diff --git a/manifest.uuid b/manifest.uuid index 61668af109..de99444ace 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -11057e2645d92876fd0fb6d702757408077b7354 \ No newline at end of file +ca2e4a5b22e1dc9ea79f77853e328d401138f747 \ No newline at end of file diff --git a/test/threadtest3.c b/test/threadtest3.c index d700d7d47a..9d7edd18cd 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -1433,7 +1433,6 @@ int main(int argc, char **argv){ { walthread3, "walthread3", 20000 }, { walthread4, "walthread4", 20000 }, { walthread5, "walthread5", 1000 }, - { walthread5, "walthread5", 1000 }, { cgt_pager_1, "cgt_pager_1", 0 }, { dynamic_triggers, "dynamic_triggers", 20000 }, From b8a9d8db872b048e30440de359176f87a32906f0 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 31 Dec 2014 18:25:21 +0000 Subject: [PATCH 702/710] Add a missing sqlite3_close() call to threadtest3.c. FossilOrigin-Name: a65a44f3ff851b71c9a79e96b9575c6a7006d2dd --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/threadtest3.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index de184d9294..5e3ede2339 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Only\srun\swalthread5\sonce\swhen\srunning\sall\stests\sin\sthreadtest3. -D 2014-12-31T18:10:07.569 +C Add\sa\smissing\ssqlite3_close()\scall\sto\sthreadtest3.c. +D 2014-12-31T18:25:21.460 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -914,7 +914,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c 7bbedc9df2b8fb0a3f75c5f2f66941c7f18ce1db +F test/threadtest3.c 14aa134981cffb4deefe8acf1154b9fa6b6cdf78 F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 11057e2645d92876fd0fb6d702757408077b7354 -R 977b3277256e94a381a08f38ec63eac7 -U drh -Z 206269a2c6d9b5759d40a6ff53cbafad +P ca2e4a5b22e1dc9ea79f77853e328d401138f747 +R b1064b21ce1900d12ee259ac9e26c073 +U dan +Z aa685526ef0fa03163cab11b3ca440f6 diff --git a/manifest.uuid b/manifest.uuid index de99444ace..1b893c5708 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca2e4a5b22e1dc9ea79f77853e328d401138f747 \ No newline at end of file +a65a44f3ff851b71c9a79e96b9575c6a7006d2dd \ No newline at end of file diff --git a/test/threadtest3.c b/test/threadtest3.c index 9d7edd18cd..cc51447804 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -981,6 +981,7 @@ static void walthread1(int nMs){ "INSERT INTO t1 VALUES(randomblob(100));" "INSERT INTO t1 SELECT md5sum(x) FROM t1;" ); + closedb(&err, &db); setstoptime(&err, nMs); for(i=0; i Date: Wed, 31 Dec 2014 18:28:59 +0000 Subject: [PATCH 703/710] Update the command-line parsing for threadtest3 so that tests are run in the order they are specified on the command-line. FossilOrigin-Name: f489bc31165f043dc10570e6c1250a292673660e --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/threadtest3.c | 38 +++++++++++++++++++++----------------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/manifest b/manifest index 5e3ede2339..c3a71d4329 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\smissing\ssqlite3_close()\scall\sto\sthreadtest3.c. -D 2014-12-31T18:25:21.460 +C Update\sthe\scommand-line\sparsing\sfor\sthreadtest3\sso\sthat\stests\sare\srun\sin\sthe\norder\sthey\sare\sspecified\son\sthe\scommand-line. +D 2014-12-31T18:28:59.230 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -914,7 +914,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c 14aa134981cffb4deefe8acf1154b9fa6b6cdf78 +F test/threadtest3.c 7331b9f7f79e4b22341dc87a4ee7074a8a62ad7f F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ca2e4a5b22e1dc9ea79f77853e328d401138f747 -R b1064b21ce1900d12ee259ac9e26c073 -U dan -Z aa685526ef0fa03163cab11b3ca440f6 +P a65a44f3ff851b71c9a79e96b9575c6a7006d2dd +R f32e62f7b458009e8ff83ca7f7bb9ebe +U drh +Z 7baa195e642be2449fd61a3f6b6a2b7e diff --git a/manifest.uuid b/manifest.uuid index 1b893c5708..906c60ab4e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a65a44f3ff851b71c9a79e96b9575c6a7006d2dd \ No newline at end of file +f489bc31165f043dc10570e6c1250a292673660e \ No newline at end of file diff --git a/test/threadtest3.c b/test/threadtest3.c index cc51447804..edc56527d2 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -1447,27 +1447,31 @@ int main(int argc, char **argv){ { stress1, "stress1", 10000 }, { stress2, "stress2", 60000 }, }; - - int i; + static char *substArgv[] = { 0, "*", 0 }; + int i, iArg; int nTestfound = 0; sqlite3_config(SQLITE_CONFIG_MULTITHREAD); - sqlite3_config(SQLITE_CONFIG_MULTITHREAD); - - for(i=0; i1 ){ - int iArg; - for(iArg=1; iArg=sizeof(aTest)/sizeof(aTest[0]) ) goto usage; + } + for(iArg=1; iArg Date: Wed, 31 Dec 2014 18:55:09 +0000 Subject: [PATCH 704/710] Change threadtest3.c so that SQLITE_SCHEMA returns result in warnings not hard errors. Add the Darwin-x86_64 platform to releasetest.tcl. FossilOrigin-Name: b09a139c9e2e1a45a3d53395ac1376e952d459e5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/releasetest.tcl | 4 ++++ test/threadtest3.c | 8 ++++++-- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index c3a71d4329..f3e7704fed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\scommand-line\sparsing\sfor\sthreadtest3\sso\sthat\stests\sare\srun\sin\sthe\norder\sthey\sare\sspecified\son\sthe\scommand-line. -D 2014-12-31T18:28:59.230 +C Change\sthreadtest3.c\sso\sthat\sSQLITE_SCHEMA\sreturns\sresult\sin\swarnings\nnot\shard\serrors.\s\sAdd\sthe\sDarwin-x86_64\splatform\sto\sreleasetest.tcl. +D 2014-12-31T18:55:09.661 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -801,7 +801,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl fad4402d19b94021a3a9032b7b4a9855bf1ef498 +F test/releasetest.tcl f8845aae7b19f0c5418b27bf7723da84a336d0d5 F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -914,7 +914,7 @@ F test/thread2.test f35d2106452b77523b3a2b7d1dcde2e5ee8f9e46 F test/thread_common.tcl 334639cadcb9f912bf82aa73f49efd5282e6cadd F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9 -F test/threadtest3.c 7331b9f7f79e4b22341dc87a4ee7074a8a62ad7f +F test/threadtest3.c 9ab4b168681c3a6f70f6c833ba08e0d48dd4af9b F test/threadtest4.c c1e67136ceb6c7ec8184e56ac61db28f96bd2925 F test/tkt-02a8e81d44.test 6c80d9c7514e2a42d4918bf87bf6bc54f379110c F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P a65a44f3ff851b71c9a79e96b9575c6a7006d2dd -R f32e62f7b458009e8ff83ca7f7bb9ebe +P f489bc31165f043dc10570e6c1250a292673660e +R fb381e088e2268612fdb185243c3a797 U drh -Z 7baa195e642be2449fd61a3f6b6a2b7e +Z 09baa910ae3dee28dd8ef1c911c03495 diff --git a/manifest.uuid b/manifest.uuid index 906c60ab4e..5518ac9017 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f489bc31165f043dc10570e6c1250a292673660e \ No newline at end of file +b09a139c9e2e1a45a3d53395ac1376e952d459e5 \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index 8b97f7f4f3..cfd0ba900f 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -182,6 +182,10 @@ array set ::Platforms { "Locking-Style" "mptest test" "OS-X" "threadtest fulltest" } + Darwin-x86_64 { + "Locking-Style" "mptest test" + "OS-X" "threadtest fulltest" + } "Windows NT-intel" { "Default" "mptest fulltestonly" } diff --git a/test/threadtest3.c b/test/threadtest3.c index edc56527d2..25caeb89f9 100644 --- a/test/threadtest3.c +++ b/test/threadtest3.c @@ -447,8 +447,12 @@ static void free_err(Error *p){ static void print_err(Error *p){ if( p->rc!=SQLITE_OK ){ - printf("Error: (%d) \"%s\" at line %d\n", p->rc, p->zErr, p->iLine); - if( sqlite3_strglob("* - no such table: *",p->zErr)!=0 ) nGlobalErr++; + int isWarn = 0; + if( p->rc==SQLITE_SCHEMA ) isWarn = 1; + if( sqlite3_strglob("* - no such table: *",p->zErr)==0 ) isWarn = 1; + printf("%s: (%d) \"%s\" at line %d\n", isWarn ? "Warning" : "Error", + p->rc, p->zErr, p->iLine); + if( !isWarn ) nGlobalErr++; fflush(stdout); } } From 2e5021d5eaab7008fcde02520b4248b0ded63f7d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Dec 2014 19:58:32 +0000 Subject: [PATCH 705/710] Disable the bigsort.test module on machine with less than 8GB of available RAM or machine, to avoid thrashing. FossilOrigin-Name: 9d4fe11641043af4e663085e979f637676599da0 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/bigsort.test | 11 +++++++++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index f3e7704fed..6474745a56 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthreadtest3.c\sso\sthat\sSQLITE_SCHEMA\sreturns\sresult\sin\swarnings\nnot\shard\serrors.\s\sAdd\sthe\sDarwin-x86_64\splatform\sto\sreleasetest.tcl. -D 2014-12-31T18:55:09.661 +C Disable\sthe\sbigsort.test\smodule\son\smachine\swith\sless\sthan\s8GB\sof\savailable\nRAM\sor\smachine,\sto\savoid\sthrashing. +D 2014-12-31T19:58:32.224 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -366,7 +366,7 @@ F test/between.test 34d375fb5ce1ae283ffe82b6b233e9f38e84fc6c F test/bigfile.test aa74f4e5db51c8e54a1d9de9fa65d01d1eb20b59 F test/bigfile2.test 1b489a3a39ae90c7f027b79110d6b4e1dbc71bfc F test/bigrow.test f0aeb7573dcb8caaafea76454be3ade29b7fc747 -F test/bigsort.test 835478d0ce83bd1e5b05c90571dedd9871a09196 +F test/bigsort.test 8299fa9298f4f1e02fc7d2712e8b77d6cd60e5a2 F test/bind.test 3c7b320969000c441a70952b0b15938fbb66237c F test/bindxfer.test efecd12c580c14df5f4ad3b3e83c667744a4f7e0 F test/bitvec.test 75894a880520164d73b1305c1c3f96882615e142 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P f489bc31165f043dc10570e6c1250a292673660e -R fb381e088e2268612fdb185243c3a797 +P b09a139c9e2e1a45a3d53395ac1376e952d459e5 +R 958bbb31d1168aa8cd0ac8a1eba90de6 U drh -Z 09baa910ae3dee28dd8ef1c911c03495 +Z 1f37f8136326091af864e56108cf76a8 diff --git a/manifest.uuid b/manifest.uuid index 5518ac9017..d365d2d171 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b09a139c9e2e1a45a3d53395ac1376e952d459e5 \ No newline at end of file +9d4fe11641043af4e663085e979f637676599da0 \ No newline at end of file diff --git a/test/bigsort.test b/test/bigsort.test index 259adc3747..c711515973 100644 --- a/test/bigsort.test +++ b/test/bigsort.test @@ -20,6 +20,15 @@ set testprefix bigsort # loop if the product was also an integer multiple of 2^32, or # inefficiency otherwise. # +# This test causes thrashing on machines with smaller amounts of +# memory. Make sure the host has at least 8GB available before running +# this test. +# +if {[catch {exec free | grep Mem:} out] || [lindex $out 1]<8000000} { + finish_test + return +} + do_execsql_test 1.0 { PRAGMA page_size = 1024; CREATE TABLE t1(a, b); @@ -39,5 +48,3 @@ do_execsql_test 1.1 { finish_test - - From 9854248609bb4cefe1114eb63b9c21e4e6949057 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Dec 2014 20:19:20 +0000 Subject: [PATCH 706/710] Fix a floating-point round-off error problem in the percentile.test module. FossilOrigin-Name: 456948ea64c6980dab79dac30a538b5a6ab8773d --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/percentile.test | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 6474745a56..2b97b2d733 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\sbigsort.test\smodule\son\smachine\swith\sless\sthan\s8GB\sof\savailable\nRAM\sor\smachine,\sto\savoid\sthrashing. -D 2014-12-31T19:58:32.224 +C Fix\sa\sfloating-point\sround-off\serror\sproblem\sin\sthe\spercentile.test\smodule. +D 2014-12-31T20:19:20.731 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -781,7 +781,7 @@ F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6 F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 -F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 +F test/percentile.test 4243af26b8f3f4555abe166f723715a1f74c77ff F test/permutations.test 59e2d8aba8c4f5842edba1a10a158b798690d3ba F test/pragma.test aa16dedfe01c02c8895169012f7dfde9c163f0d5 F test/pragma2.test aea7b3d82c76034a2df2b38a13745172ddc0bc13 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P b09a139c9e2e1a45a3d53395ac1376e952d459e5 -R 958bbb31d1168aa8cd0ac8a1eba90de6 +P 9d4fe11641043af4e663085e979f637676599da0 +R efe4238b2b9db9ced84ff6eed3896298 U drh -Z 1f37f8136326091af864e56108cf76a8 +Z 00ece672f6a64adef12b5a8614c66e28 diff --git a/manifest.uuid b/manifest.uuid index d365d2d171..aacc249381 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d4fe11641043af4e663085e979f637676599da0 \ No newline at end of file +456948ea64c6980dab79dac30a538b5a6ab8773d \ No newline at end of file diff --git a/test/percentile.test b/test/percentile.test index 9d471dfcde..b2bd061e86 100644 --- a/test/percentile.test +++ b/test/percentile.test @@ -200,7 +200,7 @@ ifcapable vtab { } { do_test percentile-2.1.$in { execsql { - SELECT percentile(x, $in) from t3; + SELECT round(percentile(x, $in),1) from t3; } } $out } From 0265eb666be73fd3bd6d23f6c3bb8ad8e2a76d2a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Dec 2014 20:25:33 +0000 Subject: [PATCH 707/710] Adjust the memsubsys1-5.5 test case to avoid occasional false positives. FossilOrigin-Name: 432413187f41061a08ecff2697ce85c935fa6fa8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/memsubsys1.test | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 2b97b2d733..287183c9b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sfloating-point\sround-off\serror\sproblem\sin\sthe\spercentile.test\smodule. -D 2014-12-31T20:19:20.731 +C Adjust\sthe\smemsubsys1-5.5\stest\scase\sto\savoid\soccasional\sfalse\spositives. +D 2014-12-31T20:25:33.588 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -727,7 +727,7 @@ F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test fcb5297b321b562084fc79d64d5a12a1cd2b639b F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 -F test/memsubsys1.test bf270964ab83bc2da5927960f78304a866fb9a9d +F test/memsubsys1.test e33072ea2ebbca54104f00bc0b6452295ff6268e F test/memsubsys2.test 3a1c1a9de48e5726faa85108b02459fae8cb9ee9 F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 9d4fe11641043af4e663085e979f637676599da0 -R efe4238b2b9db9ced84ff6eed3896298 +P 456948ea64c6980dab79dac30a538b5a6ab8773d +R faffc322ba435b37e085a2179210ef63 U drh -Z 00ece672f6a64adef12b5a8614c66e28 +Z 4cc37900b3dc79a667b61580e5a01937 diff --git a/manifest.uuid b/manifest.uuid index aacc249381..ac386cfe2a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -456948ea64c6980dab79dac30a538b5a6ab8773d \ No newline at end of file +432413187f41061a08ecff2697ce85c935fa6fa8 \ No newline at end of file diff --git a/test/memsubsys1.test b/test/memsubsys1.test index 8cc7c2afc1..7402d515be 100644 --- a/test/memsubsys1.test +++ b/test/memsubsys1.test @@ -176,13 +176,13 @@ do_test memsubsys1-4.6 { set s_used [lindex [sqlite3_status SQLITE_STATUS_SCRATCH_USED 0] 2] } 1 -# Test 5: Activate both PAGECACHE and SCRATCH. But make the page size +# Test 5: Activate both PAGECACHE and SCRATCH. But make the page size is # such that the SCRATCH allocations are too small. # db close sqlite3_shutdown sqlite3_config_pagecache [expr 4096+$xtra_size] 24 -sqlite3_config_scratch 6000 2 +sqlite3_config_scratch 4000 2 sqlite3_initialize reset_highwater_marks build_test_db memsubsys1-5 {PRAGMA page_size=4096} From af700b3971a968d4539e43bad0e14e555c796e4f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 31 Dec 2014 20:35:11 +0000 Subject: [PATCH 708/710] Fix to releasetest.tcl: When doing the secondary _debug runs, convert "fulltest" to just "test". FossilOrigin-Name: ec264bdee5ab8047fda9a36af27c8a2c8d964112 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/releasetest.tcl | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 287183c9b9..23cde5e727 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sthe\smemsubsys1-5.5\stest\scase\sto\savoid\soccasional\sfalse\spositives. -D 2014-12-31T20:25:33.588 +C Fix\sto\sreleasetest.tcl:\s\sWhen\sdoing\sthe\ssecondary\s_debug\sruns,\sconvert\n"fulltest"\sto\sjust\s"test". +D 2014-12-31T20:35:11.375 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -801,7 +801,7 @@ F test/randexpr1.test eda062a97e60f9c38ae8d806b03b0ddf23d796df F test/rdonly.test dd30a4858d8e0fbad2304c2bd74a33d4df36412a F test/regexp1.test 497ea812f264d12b6198d6e50a76be4a1973a9d8 F test/reindex.test 44edd3966b474468b823d481eafef0c305022254 -F test/releasetest.tcl f8845aae7b19f0c5418b27bf7723da84a336d0d5 +F test/releasetest.tcl 9440b21870bc25c9303e7a2adc0708257249295a F test/resolver01.test 33abf37ff8335e6bf98f2b45a0af3e06996ccd9a F test/rollback.test 458fe73eb3ffdfdf9f6ba3e9b7350a6220414dea F test/rollback2.test fc14cf6d1a2b250d2735ef16124b971bce152f14 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 456948ea64c6980dab79dac30a538b5a6ab8773d -R faffc322ba435b37e085a2179210ef63 +P 432413187f41061a08ecff2697ce85c935fa6fa8 +R eab92cc42f1c6f3a82dbafd90384e659 U drh -Z 4cc37900b3dc79a667b61580e5a01937 +Z efe20f1bf1699435938e7d67d2cf6478 diff --git a/manifest.uuid b/manifest.uuid index ac386cfe2a..4765ac41a9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -432413187f41061a08ecff2697ce85c935fa6fa8 \ No newline at end of file +ec264bdee5ab8047fda9a36af27c8a2c8d964112 \ No newline at end of file diff --git a/test/releasetest.tcl b/test/releasetest.tcl index cfd0ba900f..96e7ffb4b2 100644 --- a/test/releasetest.tcl +++ b/test/releasetest.tcl @@ -462,7 +462,7 @@ proc main {argv} { if {$target!="checksymbols" && !$::BUILDONLY} { set debug_idx [lsearch -glob $config_options -DSQLITE_DEBUG*] set xtarget $target - regsub -all {fulltest[a-z]+} $xtarget test xtarget + regsub -all {fulltest[a-z]*} $xtarget test xtarget if {$debug_idx < 0} { incr NTEST append config_options " -DSQLITE_DEBUG=1" From fddfacc371c75c20603e0b1122c1a5afef4ea72a Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 Jan 2015 14:06:24 +0000 Subject: [PATCH 709/710] Fix the fkey-7.1 test so that it sorts its answer and hence always gives the same answer. FossilOrigin-Name: 5830c557f7ed048056e2faeb82a8705ee7ecfdd7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/fkey5.test | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 23cde5e727..8890c6573d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sto\sreleasetest.tcl:\s\sWhen\sdoing\sthe\ssecondary\s_debug\sruns,\sconvert\n"fulltest"\sto\sjust\s"test". -D 2014-12-31T20:35:11.375 +C Fix\sthe\sfkey-7.1\stest\sso\sthat\sit\ssorts\sits\sanswer\sand\shence\salways\sgives\sthe\nsame\sanswer. +D 2015-01-01T14:06:24.026 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -501,7 +501,7 @@ F test/fkey1.test e1d1fa84cde579185ea01358436839703e415a5b F test/fkey2.test 1db212cda86b0d3ce72714001f7b6381c321341c F test/fkey3.test 76d475c80b84ee7a5d062e56ccb6ea68882e2b49 F test/fkey4.test 86446017011273aad8f9a99c1a65019e7bd9ca9d -F test/fkey5.test 8a1fde4e7721ae00b05b3178888833726ca2df8d +F test/fkey5.test 488601fbda8350619b3029487e56830447056fd2 F test/fkey6.test abb59f866c1b44926fd02d1fdd217d831fe04f48 F test/fkey7.test 72e915890ee4a005daaf3002cb208e8fe973ac13 F test/fkey8.test 8f08203458321e6c19a263829de4cfc936274ab0 @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P 432413187f41061a08ecff2697ce85c935fa6fa8 -R eab92cc42f1c6f3a82dbafd90384e659 +P ec264bdee5ab8047fda9a36af27c8a2c8d964112 +R 8acff839204fa9ef6e40b09dc0daac28 U drh -Z efe20f1bf1699435938e7d67d2cf6478 +Z e6b9572729fe8b314ffd25cb175a4396 diff --git a/manifest.uuid b/manifest.uuid index 4765ac41a9..d65542baa3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec264bdee5ab8047fda9a36af27c8a2c8d964112 \ No newline at end of file +5830c557f7ed048056e2faeb82a8705ee7ecfdd7 \ No newline at end of file diff --git a/test/fkey5.test b/test/fkey5.test index 5aa8b1d4b7..dc866cbdb8 100644 --- a/test/fkey5.test +++ b/test/fkey5.test @@ -252,13 +252,17 @@ do_test fkey5-6.5 { } {c12 1 p4 0 c12 3 p4 0 c12 6 p4 0} do_test fkey5-7.1 { + set res {} db eval { INSERT OR IGNORE INTO c13 SELECT * FROM c12; INSERT OR IGNORE INTO C14 SELECT * FROM c12; DELETE FROM c12; PRAGMA foreign_key_check; + } { + lappend res [list $table $rowid $fkid $parent] } -} {c14 1 p4 0 c14 3 p4 0 c14 6 p4 0 c13 1 p3 0 c13 2 p3 0 c13 3 p3 0 c13 4 p3 0 c13 5 p3 0 c13 6 p3 0} + lsort $res +} {{c13 1 0 p3} {c13 2 0 p3} {c13 3 0 p3} {c13 4 0 p3} {c13 5 0 p3} {c13 6 0 p3} {c14 1 0 p4} {c14 3 0 p4} {c14 6 0 p4}} do_test fkey5-7.2 { db eval { PRAGMA foreign_key_check(c14); From a0de826c9ff84bd19de76ebbc0d4bdafa9686d3a Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 1 Jan 2015 14:13:45 +0000 Subject: [PATCH 710/710] Fix two test cases in memsubsys1 so that they work with the mmap permutation. FossilOrigin-Name: 66269d0d8e49eb3dc7f508714753584f648bb022 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/memsubsys1.test | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 8890c6573d..4094f7cc8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sfkey-7.1\stest\sso\sthat\sit\ssorts\sits\sanswer\sand\shence\salways\sgives\sthe\nsame\sanswer. -D 2015-01-01T14:06:24.026 +C Fix\stwo\stest\scases\sin\smemsubsys1\sso\sthat\sthey\swork\swith\sthe\smmap\spermutation. +D 2015-01-01T14:13:45.725 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 7cd23e4fc91004a6bd081623e1bc6932e44828c0 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -727,7 +727,7 @@ F test/manydb.test 28385ae2087967aa05c38624cec7d96ec74feb3e F test/mem5.test c6460fba403c5703141348cd90de1c294188c68f F test/memdb.test fcb5297b321b562084fc79d64d5a12a1cd2b639b F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2 -F test/memsubsys1.test e33072ea2ebbca54104f00bc0b6452295ff6268e +F test/memsubsys1.test 690d142525a7d31efb638b0d232e25fac3afeb1a F test/memsubsys2.test 3a1c1a9de48e5726faa85108b02459fae8cb9ee9 F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc @@ -1234,7 +1234,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f -P ec264bdee5ab8047fda9a36af27c8a2c8d964112 -R 8acff839204fa9ef6e40b09dc0daac28 +P 5830c557f7ed048056e2faeb82a8705ee7ecfdd7 +R 9ee216f4d87447d224436bd4fa1652bd U drh -Z e6b9572729fe8b314ffd25cb175a4396 +Z f94d536f9021f3614a2a0370bbcf66ff diff --git a/manifest.uuid b/manifest.uuid index d65542baa3..6e46921ab5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5830c557f7ed048056e2faeb82a8705ee7ecfdd7 \ No newline at end of file +66269d0d8e49eb3dc7f508714753584f648bb022 \ No newline at end of file diff --git a/test/memsubsys1.test b/test/memsubsys1.test index 7402d515be..891558fd52 100644 --- a/test/memsubsys1.test +++ b/test/memsubsys1.test @@ -189,7 +189,7 @@ build_test_db memsubsys1-5 {PRAGMA page_size=4096} #show_memstats do_test memsubsys1-5.3 { set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] -} 24 +} {/^2[34]$/} do_test memsubsys1-5.4 { set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] expr {$maxreq>4096} @@ -215,7 +215,7 @@ build_test_db memsubsys1-6 {PRAGMA page_size=4096} #show_memstats do_test memsubsys1-6.3 { set pg_used [lindex [sqlite3_status SQLITE_STATUS_PAGECACHE_USED 0] 2] -} 24 +} {/^2[34]$/} #do_test memsubsys1-6.4 { # set maxreq [lindex [sqlite3_status SQLITE_STATUS_MALLOC_SIZE 0] 2] # expr {$maxreq>4096 && $maxreq<=(4096+$xtra_size)}