From fe98081e18e03e4f3a3389f94b44eeb201276721 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Jan 2014 14:00:13 +0000 Subject: [PATCH 01/74] Enhance sqlite3_randomness(N,P) such that it resets the internal PRNG if N is less than 1. Subsequent calls to sqlite3_randomness() will reinitialize the internal PRNG by calling the xRandomness() method of the default VFS. FossilOrigin-Name: a221aa82bb5496885fd0bf76e4601443799511de --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/main.c | 2 +- src/random.c | 14 +++++++++----- src/sqlite.h.in | 10 ++++++---- src/sqliteInt.h | 1 - 6 files changed, 26 insertions(+), 21 deletions(-) diff --git a/manifest b/manifest index 56fb6db832..df2e8029ab 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\s-I.\sto\sthe\smain.mk\smakefile\sfor\sbuilding\ssqlite3.o\sand\sspeedtest1. -D 2013-12-24T12:04:24.811 +C Enhance\ssqlite3_randomness(N,P)\ssuch\sthat\sit\sresets\sthe\sinternal\sPRNG\nif\sN\sis\sless\sthan\s1.\s\sSubsequent\scalls\sto\ssqlite3_randomness()\swill\sreinitialize\nthe\sinternal\sPRNG\sby\scalling\sthe\sxRandomness()\smethod\sof\sthe\sdefault\sVFS. +D 2014-01-01T14:00:13.905 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -188,7 +188,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 5307601c58b6e08153a15846098399a59c166d8d +F src/main.c 37f55de2000f6c882414ad3e1bcdb751c531fddd F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -216,15 +216,15 @@ F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b F src/pragma.c 5ab7279d132143feb77f773688a24ab05da75fd7 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 -F src/random.c 0b2dbc37fdfbfa6bd455b091dfcef5bdb32dba68 +F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 819bb090c9a348d17f69f136cad2bfa9ee9cbb41 F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 -F src/sqlite.h.in 4ef56464aeaa3785a2c5ca37fb3a0fb229d68b2e +F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h fdb8c1e4bc5424ad82a9394a845781abf0d7d849 +F src/sqliteInt.h 62664868458b53a815380733f059ff9d92cb08be F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1147,7 +1147,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c289a253c0c053ac8fc344efe138262c327d8096 -R 2bb3d3295c5bbe58e528a8069b6e67b9 +P cc72c5aec7fe93d4a1368517aab949dc98d33003 +R 4dbb82094f79fe324b8594c5661b9532 U drh -Z 2be15cf525011fea6a70c12bcaf2ac37 +Z 7f33812905741a2d8942dba59b43dd66 diff --git a/manifest.uuid b/manifest.uuid index 5e0ef5b21c..691c3bd9a8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cc72c5aec7fe93d4a1368517aab949dc98d33003 \ No newline at end of file +a221aa82bb5496885fd0bf76e4601443799511de \ No newline at end of file diff --git a/src/main.c b/src/main.c index fff1bc42e6..13ad2b9081 100644 --- a/src/main.c +++ b/src/main.c @@ -3101,7 +3101,7 @@ int sqlite3_test_control(int op, ...){ ** to the xRandomness method of the default VFS. */ case SQLITE_TESTCTRL_PRNG_RESET: { - sqlite3PrngResetState(); + sqlite3_randomness(0,0); break; } diff --git a/src/random.c b/src/random.c index 7afff50885..b82566524c 100644 --- a/src/random.c +++ b/src/random.c @@ -52,6 +52,12 @@ void sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); #endif + if( N<=0 ){ + wsdPrng.isInit = 0; + sqlite3_mutex_leave(mutex); + return; + } + /* Initialize the state of the random number generator once, ** the first time this routine is called. The seed value does ** not need to contain a lot of randomness since we are not @@ -79,7 +85,8 @@ void sqlite3_randomness(int N, void *pBuf){ wsdPrng.isInit = 1; } - while( N-- ){ + assert( N>0 ); + do{ wsdPrng.i++; t = wsdPrng.s[wsdPrng.i]; wsdPrng.j += t; @@ -87,7 +94,7 @@ void sqlite3_randomness(int N, void *pBuf){ wsdPrng.s[wsdPrng.j] = t; t += wsdPrng.s[wsdPrng.i]; *(zBuf++) = wsdPrng.s[t]; - } + }while( --N ); sqlite3_mutex_leave(mutex); } @@ -116,7 +123,4 @@ void sqlite3PrngRestoreState(void){ sizeof(sqlite3Prng) ); } -void sqlite3PrngResetState(void){ - GLOBAL(struct sqlite3PrngType, sqlite3Prng).isInit = 0; -} #endif /* SQLITE_OMIT_BUILTIN_TEST */ diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 7b0e648bfc..5012f864a5 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2395,11 +2395,13 @@ sqlite3_int64 sqlite3_memory_highwater(int resetFlag); ** applications to access the same PRNG for other purposes. ** ** ^A call to this routine stores N bytes of randomness into buffer P. +** ^If N is less than one, then P can be a NULL pointer. ** -** ^The first time this routine is invoked (either internally or by -** the application) the PRNG is seeded using randomness obtained -** from the xRandomness method of the default [sqlite3_vfs] object. -** ^On all subsequent invocations, the pseudo-randomness is generated +** ^If this routine has not been previously called or if the previous +** call had N less than one, then the PRNG is seeded using randomness +** obtained from the xRandomness method of the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more then +** the pseudo-randomness is generated ** internally and without recourse to the [sqlite3_vfs] xRandomness ** method. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e7dc69cc73..77ed38374f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2972,7 +2972,6 @@ int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); Vdbe *sqlite3GetVdbe(Parse*); void sqlite3PrngSaveState(void); void sqlite3PrngRestoreState(void); -void sqlite3PrngResetState(void); void sqlite3RollbackAll(sqlite3*,int); void sqlite3CodeVerifySchema(Parse*, int); void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); From b00d8621392397de9104e5d191ad1d6f018dcd7a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 1 Jan 2014 15:18:36 +0000 Subject: [PATCH 02/74] Try to detect process ID changes due to fork() calls in os_unix.c and reset the PRNG when a process ID change is detected. FossilOrigin-Name: e1eba1fb09d7db49d77928bd115b27b8002ae640 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 26 +++++++++++++++++++++----- 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index df2e8029ab..e254e7cba0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\ssqlite3_randomness(N,P)\ssuch\sthat\sit\sresets\sthe\sinternal\sPRNG\nif\sN\sis\sless\sthan\s1.\s\sSubsequent\scalls\sto\ssqlite3_randomness()\swill\sreinitialize\nthe\sinternal\sPRNG\sby\scalling\sthe\sxRandomness()\smethod\sof\sthe\sdefault\sVFS. -D 2014-01-01T14:00:13.905 +C Try\sto\sdetect\sprocess\sID\schanges\sdue\sto\sfork()\scalls\sin\sos_unix.c\sand\nreset\sthe\sPRNG\swhen\sa\sprocess\sID\schange\sis\sdetected. +D 2014-01-01T15:18:36.453 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -205,7 +205,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 60a7b3b23e6fcf83a50d1e320b280b551724e11f +F src/os_unix.c abeb9d54036aaea6f4395050ce823f51217ae4d4 F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1147,7 +1147,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P cc72c5aec7fe93d4a1368517aab949dc98d33003 -R 4dbb82094f79fe324b8594c5661b9532 +P a221aa82bb5496885fd0bf76e4601443799511de +R 0b5cbfb8643918ccfe030368af8a9f9c U drh -Z 7f33812905741a2d8942dba59b43dd66 +Z 8d0b25dd26eff92b0e8709464c57dd81 diff --git a/manifest.uuid b/manifest.uuid index 691c3bd9a8..002b2fbf68 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a221aa82bb5496885fd0bf76e4601443799511de \ No newline at end of file +e1eba1fb09d7db49d77928bd115b27b8002ae640 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 485b32fd90..4b76d4fec4 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -260,6 +260,12 @@ struct unixFile { #endif }; +/* This variable holds the process id (pid) from when the xRandomness() +** method was called. If xOpen() is called from a different process id, +** indicating that a fork() has occurred, the PRNG will be reset. +*/ +static int randomnessPid = 0; + /* ** Allowed values for the unixFile.ctrlFlags bitmask: */ @@ -5651,6 +5657,16 @@ static int unixOpen( || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); + /* Detect a pid change and reset the PRNG. There is a race condition + ** here such that two or more threads all trying to open databases at + ** the same instant might all reset the PRNG. But multiple resets + ** are harmless. + */ + if( randomnessPid!=getpid() ){ + randomnessPid = getpid(); + sqlite3_randomness(0,0); + } + memset(p, 0, sizeof(unixFile)); if( eType==SQLITE_OPEN_MAIN_DB ){ @@ -6038,18 +6054,18 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ ** tests repeatable. */ memset(zBuf, 0, nBuf); + randomnessPid = getpid(); #if !defined(SQLITE_TEST) { - int pid, fd, got; + int fd, got; fd = robust_open("/dev/urandom", O_RDONLY, 0); if( fd<0 ){ time_t t; time(&t); memcpy(zBuf, &t, sizeof(t)); - pid = getpid(); - memcpy(&zBuf[sizeof(t)], &pid, sizeof(pid)); - assert( sizeof(t)+sizeof(pid)<=(size_t)nBuf ); - nBuf = sizeof(t) + sizeof(pid); + memcpy(&zBuf[sizeof(t)], &randomnessPid, sizeof(randomnessPid)); + assert( sizeof(t)+sizeof(randomnessPid)<=(size_t)nBuf ); + nBuf = sizeof(t) + sizeof(randomnessPid); }else{ do{ got = osRead(fd, zBuf, nBuf); }while( got<0 && errno==EINTR ); robust_close(0, fd, __LINE__); From 991a19851112ffb03946a003350e9e148d631348 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Jan 2014 17:57:16 +0000 Subject: [PATCH 03/74] Be more aggressive in optimizing constant conditional expressions. FossilOrigin-Name: b7e39851a75b87ebca747b26a39989560fd6362b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/expr.c | 47 ++++++++++++++++++++++++++++++++++------------- 3 files changed, 41 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index e254e7cba0..142a875f5d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Try\sto\sdetect\sprocess\sID\schanges\sdue\sto\sfork()\scalls\sin\sos_unix.c\sand\nreset\sthe\sPRNG\swhen\sa\sprocess\sID\schange\sis\sdetected. -D 2014-01-01T15:18:36.453 +C Be\smore\saggressive\sin\soptimizing\sconstant\sconditional\sexpressions. +D 2014-01-02T17:57:16.874 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 821da82527c24496bef0677ed4f61a53b44c27ee -F src/expr.c ffe4bc79c66f711f450a6113fbd1943b9b2380f7 +F src/expr.c ca0959386f01d6e5d4dc67b362a4e4c912ebd054 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -1147,7 +1147,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a221aa82bb5496885fd0bf76e4601443799511de -R 0b5cbfb8643918ccfe030368af8a9f9c +P e1eba1fb09d7db49d77928bd115b27b8002ae640 +R 8c33ee4ecd82143d9dd9bc5270372c1d U drh -Z 8d0b25dd26eff92b0e8709464c57dd81 +Z 948115126ed1b38f36b2ba1121c24109 diff --git a/manifest.uuid b/manifest.uuid index 002b2fbf68..38faece72d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e1eba1fb09d7db49d77928bd115b27b8002ae640 \ No newline at end of file +b7e39851a75b87ebca747b26a39989560fd6362b \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 8ee73acb78..0db51fe43f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -523,16 +523,25 @@ Expr *sqlite3PExpr( } /* -** Return 1 if an expression must be FALSE in all cases and 0 if the -** expression might be true. This is an optimization. If is OK to -** return 0 here even if the expression really is always false (a -** false negative). But it is a bug to return 1 if the expression -** might be true in some rare circumstances (a false positive.) +** If the expression is always either TRUE or FALSE (respectively), +** then return 1. If one cannot determine the truth value of the +** expression at compile-time return 0. +** +** This is an optimization. If is OK to return 0 here even if +** the expression really is always false or false (a false negative). +** But it is a bug to return 1 if the expression might have different +** boolean values in different circumstances (a false positive.) ** ** Note that if the expression is part of conditional for a ** LEFT JOIN, then we cannot determine at compile-time whether or not ** is it true or false, so always return 0. */ +static int exprAlwaysTrue(Expr *p){ + int v = 0; + if( ExprHasProperty(p, EP_FromJoin) ) return 0; + if( !sqlite3ExprIsInteger(p, &v) ) return 0; + return v!=0; +} static int exprAlwaysFalse(Expr *p){ int v = 0; if( ExprHasProperty(p, EP_FromJoin) ) return 0; @@ -3614,10 +3623,16 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } #endif default: { - r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); - sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); - testcase( regFree1==0 ); - testcase( jumpIfNull==0 ); + if( exprAlwaysTrue(pExpr) ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + }else if( exprAlwaysFalse(pExpr) ){ + /* No-op */ + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_If, r1, dest, jumpIfNull!=0); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + } break; } } @@ -3759,10 +3774,16 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ } #endif default: { - r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); - sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); - testcase( regFree1==0 ); - testcase( jumpIfNull==0 ); + if( exprAlwaysFalse(pExpr) ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, dest); + }else if( exprAlwaysTrue(pExpr) ){ + /* no-op */ + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); + sqlite3VdbeAddOp3(v, OP_IfNot, r1, dest, jumpIfNull!=0); + testcase( regFree1==0 ); + testcase( jumpIfNull==0 ); + } break; } } From 762c1c4071bbc4c73ec1aa70d017ee0e9c03d729 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Jan 2014 19:35:30 +0000 Subject: [PATCH 04/74] Avoid unnecessary affinity transformations when building indices using data from a table. FossilOrigin-Name: 10d851353c2abeadbd2852c210a7ff9f7f513e5d --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/delete.c | 18 +++++++++--------- src/sqliteInt.h | 2 +- src/test1.c | 1 - src/vdbe.h | 1 + src/vdbeaux.c | 7 +++++++ 7 files changed, 29 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 142a875f5d..b647b373c8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Be\smore\saggressive\sin\soptimizing\sconstant\sconditional\sexpressions. -D 2014-01-02T17:57:16.874 +C Avoid\sunnecessary\saffinity\stransformations\swhen\sbuilding\sindices\susing\ndata\sfrom\sa\stable. +D 2014-01-02T19:35:30.181 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -174,7 +174,7 @@ F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 -F src/delete.c 821da82527c24496bef0677ed4f61a53b44c27ee +F src/delete.c 22f3dc3fd6b82ebce3dabf951a4b157026943ac1 F src/expr.c ca0959386f01d6e5d4dc67b362a4e4c912ebd054 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 @@ -224,12 +224,12 @@ F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 62664868458b53a815380733f059ff9d92cb08be +F src/sqliteInt.h a77cb6c7404d1a20e536211be4a2e4be01cde213 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c c43379f77f90399802b0e215faa71c0adc3a4d2e -F src/test1.c 633e5e6a116acf4473b9289240bcceb5320a9d93 +F src/test1.c db16ba651453b15001c7f2838c446284dde4ecaf F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -281,10 +281,10 @@ F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c e71f19b272f05c8695cf747b4bac1732685f9e5c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c b110887e415b5d2af58c2374c4dfdcf774c5d46c -F src/vdbe.h c3278ab2b410f17acf61faf91be7bce3fd466e8b +F src/vdbe.h 298e0492827ef9db626934d060df62ed27ba56bc F src/vdbeInt.h 42db251e9f863401ff847b90d5fe1614c89a6a56 F src/vdbeapi.c ce4e68ea4842cc6081046f533d088dcf01d247ad -F src/vdbeaux.c bc6799db8795cd1480f2cab6cb22acf47e3e640f +F src/vdbeaux.c 915fda7ca876aa2df2a13b0621ad0326342c0638 F src/vdbeblob.c bc40f98f256f0b34116d6a44b114da4a81a15d33 F src/vdbemem.c 0e69351b2c6ff7d8b638688c0ae336a26befa6b2 F src/vdbesort.c 9d83601f9d6243fe70dd0169a2820c5ddfd48147 @@ -1147,7 +1147,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e1eba1fb09d7db49d77928bd115b27b8002ae640 -R 8c33ee4ecd82143d9dd9bc5270372c1d +P b7e39851a75b87ebca747b26a39989560fd6362b +R 8c4d4f58f19d587102e868c570c261da U drh -Z 948115126ed1b38f36b2ba1121c24109 +Z 387b8179fd8db58dae028273b1bc8ad7 diff --git a/manifest.uuid b/manifest.uuid index 38faece72d..b39edb4e71 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b7e39851a75b87ebca747b26a39989560fd6362b \ No newline at end of file +10d851353c2abeadbd2852c210a7ff9f7f513e5d \ No newline at end of file diff --git a/src/delete.c b/src/delete.c index 8746cd324a..0180ec48c8 100644 --- a/src/delete.c +++ b/src/delete.c @@ -784,18 +784,18 @@ int sqlite3GenerateIndexKey( for(j=0; jaiColumn[j], regBase+j); + /* If the column affinity is REAL but the number is an integer, then it + ** might be stored in the table as an integer (using a compact + ** representation) then converted to REAL by an OP_RealAffinity opcode. + ** But we are getting ready to store this value back into an index, where + ** it should be converted by to INTEGER again. So omit the OP_RealAffinity + ** opcode if it is present */ + if( sqlite3VdbeGetOp(v, -1)->opcode==OP_RealAffinity ){ + sqlite3VdbeDeleteLastOpcode(v); + } } if( regOut ){ - const char *zAff; - if( pTab->pSelect - || OptimizationDisabled(pParse->db, SQLITE_IdxRealAsInt) - ){ - zAff = 0; - }else{ - zAff = sqlite3IndexAffinityStr(v, pIdx); - } sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regOut); - sqlite3VdbeChangeP4(v, -1, zAff, P4_TRANSIENT); } sqlite3ReleaseTempRange(pParse, regBase, nCol); return regBase; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 77ed38374f..5943ced2a0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1056,7 +1056,7 @@ struct sqlite3 { #define SQLITE_ColumnCache 0x0002 /* Column cache */ #define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ #define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ -#define SQLITE_IdxRealAsInt 0x0010 /* Store REAL as INT in indices */ +/* not used 0x0010 // Was: SQLITE_IdxRealAsInt */ #define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ #define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ #define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ diff --git a/src/test1.c b/src/test1.c index a96b298661..109f90143f 100644 --- a/src/test1.c +++ b/src/test1.c @@ -6160,7 +6160,6 @@ static int optimization_control( { "column-cache", SQLITE_ColumnCache }, { "groupby-order", SQLITE_GroupByOrder }, { "factor-constants", SQLITE_FactorOutConst }, - { "real-as-int", SQLITE_IdxRealAsInt }, { "distinct-opt", SQLITE_DistinctOpt }, { "cover-idx-scan", SQLITE_CoverIdxScan }, { "order-by-idx-join", SQLITE_OrderByIdxJoin }, diff --git a/src/vdbe.h b/src/vdbe.h index 51cf60b520..46bc89cee8 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -175,6 +175,7 @@ void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); void sqlite3VdbeChangeP5(Vdbe*, u8 P5); void sqlite3VdbeJumpHere(Vdbe*, int addr); void sqlite3VdbeChangeToNoop(Vdbe*, int addr); +void sqlite3VdbeDeleteLastOpcode(Vdbe*); void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); void sqlite3VdbeUsesBtree(Vdbe*, int); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index d917e8540e..e0b3c5485f 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -726,6 +726,13 @@ void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ } } +/* +** Remove the last opcode inserted +*/ +void sqlite3VdbeDeleteLastOpcode(Vdbe *p){ + p->nOp--; +} + /* ** Change the value of the P4 operand for a specific instruction. ** This routine is useful when a large program is loaded from a From 759e8588117ef04a0ef331b0128e2a684049f00c Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 2 Jan 2014 21:05:10 +0000 Subject: [PATCH 05/74] Try to factor constant subcomponents of the WHERE clause out of the loop. FossilOrigin-Name: 9d05777fe24e1a5ce71762de38db840931ef0bc8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 9 ++++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b647b373c8..07bca20bb5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sunnecessary\saffinity\stransformations\swhen\sbuilding\sindices\susing\ndata\sfrom\sa\stable. -D 2014-01-02T19:35:30.181 +C Try\sto\sfactor\sconstant\ssubcomponents\sof\sthe\sWHERE\sclause\sout\sof\sthe\sloop. +D 2014-01-02T21:05:10.654 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 60bc8c5b00e2292c24a42455d022eeeda33a16f1 +F src/where.c 8e3cb2030eaa242defb804aa30115b5d870e5cd4 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1147,7 +1147,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b7e39851a75b87ebca747b26a39989560fd6362b -R 8c4d4f58f19d587102e868c570c261da +P 10d851353c2abeadbd2852c210a7ff9f7f513e5d +R c2b1a12fe347b39aeab4feaef9309c2b U drh -Z 387b8179fd8db58dae028273b1bc8ad7 +Z c170c664d17a9fc9f22716e0d87d4dc7 diff --git a/manifest.uuid b/manifest.uuid index b39edb4e71..a55e61dfa1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -10d851353c2abeadbd2852c210a7ff9f7f513e5d \ No newline at end of file +9d05777fe24e1a5ce71762de38db840931ef0bc8 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 59028b9921..0acc0db53f 100644 --- a/src/where.c +++ b/src/where.c @@ -5430,9 +5430,12 @@ WhereInfo *sqlite3WhereBegin( /* Special case: a WHERE clause that is constant. Evaluate the ** expression and either jump over all of the code or fall thru. */ - if( pWhere && (nTabList==0 || sqlite3ExprIsConstantNotJoin(pWhere)) ){ - sqlite3ExprIfFalse(pParse, pWhere, pWInfo->iBreak, SQLITE_JUMPIFNULL); - pWhere = 0; + for(ii=0; iinTerm; ii++){ + if( nTabList==0 || sqlite3ExprIsConstantNotJoin(sWLB.pWC->a[ii].pExpr) ){ + sqlite3ExprIfFalse(pParse, sWLB.pWC->a[ii].pExpr, pWInfo->iBreak, + SQLITE_JUMPIFNULL); + sWLB.pWC->a[ii].wtFlags |= TERM_CODED; + } } /* Special case: No FROM clause From 5426d80948c18c9dda289aa92f48328003e36193 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 3 Jan 2014 16:03:43 +0000 Subject: [PATCH 06/74] Avoid some unnecessary OP_SCopy operations when inserting into a table with multiple indices. FossilOrigin-Name: 429018b19cb525a4bb0843d20955457b53900d4b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/insert.c | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 07bca20bb5..3c6dfd1b82 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Try\sto\sfactor\sconstant\ssubcomponents\sof\sthe\sWHERE\sclause\sout\sof\sthe\sloop. -D 2014-01-02T21:05:10.654 +C Avoid\ssome\sunnecessary\sOP_SCopy\soperations\swhen\sinserting\sinto\sa\stable\nwith\smultiple\sindices. +D 2014-01-03T16:03:43.440 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -183,7 +183,7 @@ F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c f165ba414eda735b327c8126def0a2428f9138de +F src/insert.c e6b3d7847aaf06170967ff6ea9076717fbc08d3f F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b @@ -1147,7 +1147,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 10d851353c2abeadbd2852c210a7ff9f7f513e5d -R c2b1a12fe347b39aeab4feaef9309c2b +P 9d05777fe24e1a5ce71762de38db840931ef0bc8 +R 3a4d1de2d6235e13c562ba377a9da47e U drh -Z c170c664d17a9fc9f22716e0d87d4dc7 +Z f5e03e0d190fce062c51c7968e7436da diff --git a/manifest.uuid b/manifest.uuid index a55e61dfa1..ce9867dde2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d05777fe24e1a5ce71762de38db840931ef0bc8 \ No newline at end of file +429018b19cb525a4bb0843d20955457b53900d4b \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index b3b8444f34..abc34c03ca 100644 --- a/src/insert.c +++ b/src/insert.c @@ -1233,6 +1233,7 @@ void sqlite3GenerateConstraintChecks( int ipkTop = 0; /* Top of the rowid change constraint check */ int ipkBottom = 0; /* Bottom of the rowid change constraint check */ u8 isUpdate; /* True if this is an UPDATE operation */ + int regRowid = -1; /* Register holding ROWID value */ isUpdate = regOldData!=0; db = pParse->db; @@ -1463,7 +1464,9 @@ void sqlite3GenerateConstraintChecks( int iField = pIdx->aiColumn[i]; int x; if( iField<0 || iField==pTab->iPKey ){ + if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ x = regNewData; + regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; }else{ x = iField + regNewData + 1; } From 42899543f8423aaebf0f2e4e436330d99a6e4529 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Jan 2014 14:16:06 +0000 Subject: [PATCH 07/74] Have the planner detect cases where a MATCH constraint is applied to a virtual table, and ignore any plans that do not allow the virtual table implementation to implement the MATCH filtering. FossilOrigin-Name: 19f3208b26597625728d1ef716d4e44407cf96ac --- manifest | 18 ++++++++----- manifest.uuid | 2 +- src/where.c | 5 ++++ test/fts3join.test | 66 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 test/fts3join.test diff --git a/manifest b/manifest index 3c6dfd1b82..32c49321f0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\ssome\sunnecessary\sOP_SCopy\soperations\swhen\sinserting\sinto\sa\stable\nwith\smultiple\sindices. -D 2014-01-03T16:03:43.440 +C Have\sthe\splanner\sdetect\scases\swhere\sa\sMATCH\sconstraint\sis\sapplied\sto\sa\svirtual\stable,\sand\signore\sany\splans\sthat\sdo\snot\sallow\sthe\svirtual\stable\simplementation\sto\simplement\sthe\sMATCH\sfiltering. +D 2014-01-04T14:16:06.902 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 8e3cb2030eaa242defb804aa30115b5d870e5cd4 +F src/where.c 14403429f7ad11b13b6ae0e466d089b8875ba195 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -547,6 +547,7 @@ F test/fts3expr3.test 9e91b8edbcb197bf2e92161aa7696446d96dce5f F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660 F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 +F test/fts3join.test bc8e5ca3e6b6577cf98e81b5271291e33e020e87 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6 F test/fts3matchinfo.test ff423e73faab8fc6d7adeefedf74dd8e2b0b14e0 F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905 @@ -1147,7 +1148,10 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9d05777fe24e1a5ce71762de38db840931ef0bc8 -R 3a4d1de2d6235e13c562ba377a9da47e -U drh -Z f5e03e0d190fce062c51c7968e7436da +P 429018b19cb525a4bb0843d20955457b53900d4b +R 6571bb57c7cea9164f2f0721e5f9106f +T *branch * avoid-unusable-match +T *sym-avoid-unusable-match * +T -sym-trunk * +U dan +Z 6e1a68d01b84e17020c8f255059d7b13 diff --git a/manifest.uuid b/manifest.uuid index ce9867dde2..d3155e3c85 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -429018b19cb525a4bb0843d20955457b53900d4b \ No newline at end of file +19f3208b26597625728d1ef716d4e44407cf96ac \ No newline at end of file diff --git a/src/where.c b/src/where.c index 0acc0db53f..83ec748679 100644 --- a/src/where.c +++ b/src/where.c @@ -4338,6 +4338,7 @@ static int whereLoopAddVirtual( int nConstraint; int seenIn = 0; /* True if an IN operator is seen */ int seenVar = 0; /* True if a non-constant constraint is seen */ + int seenVarMatch = 0; /* If a non-constant MATCH constraint is seen */ int iPhase; /* 0: const w/o IN, 1: const, 2: no IN, 2: IN */ WhereLoop *pNew; int rc = SQLITE_OK; @@ -4382,6 +4383,7 @@ static int whereLoopAddVirtual( } if( pTerm->prereqRight!=0 ){ seenVar = 1; + if( pTerm->eOperator & WO_MATCH ) seenVarMatch = 1; }else if( (pTerm->eOperator & WO_IN)==0 ){ pIdxCons->usable = 1; } @@ -4400,6 +4402,9 @@ static int whereLoopAddVirtual( break; } } + /* The following line ensures that, if there exists a MATCH constraint, + ** no plans for which the MATCH constraint is not usable are considered. */ + if( seenVarMatch && iPhase<=1 ) continue; memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); pIdxInfo->idxStr = 0; diff --git a/test/fts3join.test b/test/fts3join.test new file mode 100644 index 0000000000..f02e21513e --- /dev/null +++ b/test/fts3join.test @@ -0,0 +1,66 @@ +# 2014 January 4 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#************************************************************************* +# This file implements regression tests for SQLite library. The +# focus of this script is testing the FTS3 module. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix fts3join + +# If SQLITE_ENABLE_FTS3 is defined, omit this file. +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.0 { + CREATE VIRTUAL TABLE ft1 USING fts4(x); + INSERT INTO ft1 VALUES('aaa aaa'); + INSERT INTO ft1 VALUES('aaa bbb'); + INSERT INTO ft1 VALUES('bbb aaa'); + INSERT INTO ft1 VALUES('bbb bbb'); + + CREATE TABLE t1(id, y); + INSERT INTO t1 VALUES(1, 'aaa'); + INSERT INTO t1 VALUES(2, 'bbb'); +} + +do_execsql_test 1.1 { + SELECT docid FROM ft1, t1 WHERE ft1 MATCH y AND id=1; +} {1 2 3} + +do_execsql_test 1.2 { + SELECT docid FROM ft1, t1 WHERE ft1 MATCH y AND id=1 ORDER BY docid; +} {1 2 3} + +do_execsql_test 2.0 { + CREATE VIRTUAL TABLE ft2 USING fts4(x); + CREATE VIRTUAL TABLE ft3 USING fts4(y); + + INSERT INTO ft2 VALUES('abc'); + INSERT INTO ft2 VALUES('def'); + INSERT INTO ft3 VALUES('ghi'); + INSERT INTO ft3 VALUES('abc'); +} + +do_execsql_test 2.1 { SELECT * FROM ft2, ft3 WHERE x MATCH y; } {abc abc} +do_execsql_test 2.2 { SELECT * FROM ft2, ft3 WHERE y MATCH x; } {abc abc} +do_execsql_test 2.3 { SELECT * FROM ft3, ft2 WHERE x MATCH y; } {abc abc} +do_execsql_test 2.4 { SELECT * FROM ft3, ft2 WHERE y MATCH x; } {abc abc} + +do_catchsql_test 2.5 { + SELECT * FROM ft3, ft2 WHERE y MATCH x AND x MATCH y; +} {1 {no query solution}} + +finish_test + + From 69014393f87e6b89b8c32207ff1aee2202a665e2 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Jan 2014 14:42:14 +0000 Subject: [PATCH 08/74] Instead of having the planner ignore plans with unusable MATCH constraints, have FTS assign extremely high costs to such plans in order to discourage the planner from using them. FossilOrigin-Name: 24f84b38131866f7b435ffe641bb2f6991a70db2 --- ext/fts3/fts3.c | 28 +++++++++++++++++++++++++++- manifest | 19 ++++++++----------- manifest.uuid | 2 +- src/where.c | 5 ----- test/fts3join.test | 2 +- 5 files changed, 37 insertions(+), 19 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 3b9efee54c..62ff1437b9 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1472,6 +1472,19 @@ static int fts3CreateMethod( return fts3InitVtab(1, db, pAux, argc, argv, ppVtab, pzErr); } +/* +** Set the pIdxInfo->estimatedRows variable to nRow. Unless this +** extension is currently being used by a version of SQLite too old to +** support estimatedRows. In that case this function is a no-op. +*/ +static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +#if SQLITE_VERSION_NUMBER>=3008002 + if( sqlite3_libversion_number()>=3008002 ){ + pIdxInfo->estimatedRows = nRow; + } +#endif +} + /* ** Implementation of the xBestIndex method for FTS3 tables. There ** are three possible strategies, in order of preference: @@ -1499,7 +1512,20 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ for(i=0; inConstraint; i++){ int bDocid; /* True if this constraint is on docid */ struct sqlite3_index_constraint *pCons = &pInfo->aConstraint[i]; - if( pCons->usable==0 ) continue; + if( pCons->usable==0 ){ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* There exists an unusable MATCH constraint. This means that if + ** the planner does elect to use the results of this call as part + ** of the overall query plan the user will see an "unable to use + ** function MATCH in the requested context" error. To discourage + ** this, return a very high cost here. */ + pInfo->idxNum = FTS3_FULLSCAN_SEARCH; + pInfo->estimatedCost = 1e50; + setEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + return SQLITE_OK; + } + continue; + } bDocid = (pCons->iColumn<0 || pCons->iColumn==p->nColumn+1); diff --git a/manifest b/manifest index 32c49321f0..8e2e2e7378 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sthe\splanner\sdetect\scases\swhere\sa\sMATCH\sconstraint\sis\sapplied\sto\sa\svirtual\stable,\sand\signore\sany\splans\sthat\sdo\snot\sallow\sthe\svirtual\stable\simplementation\sto\simplement\sthe\sMATCH\sfiltering. -D 2014-01-04T14:16:06.902 +C Instead\sof\shaving\sthe\splanner\signore\splans\swith\sunusable\sMATCH\sconstraints,\shave\sFTS\sassign\sextremely\shigh\scosts\sto\ssuch\splans\sin\sorder\sto\sdiscourage\sthe\splanner\sfrom\susing\sthem. +D 2014-01-04T14:42:14.057 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 1e667eacb3fe4b4ad6f863920da4286f071f6e07 +F ext/fts3/fts3.c 2af2cb2e742461b79710c132c7969fc7d949a59a F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h eb5f8029589f3d8f1dc7fd50c773326a640388b1 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 14403429f7ad11b13b6ae0e466d089b8875ba195 +F src/where.c 8e3cb2030eaa242defb804aa30115b5d870e5cd4 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -547,7 +547,7 @@ F test/fts3expr3.test 9e91b8edbcb197bf2e92161aa7696446d96dce5f F test/fts3fault.test cb72dccb0a3b9f730f16c5240f3fcb9303eb1660 F test/fts3fault2.test 3198eef2804deea7cac8403e771d9cbcb752d887 F test/fts3first.test dbdedd20914c8d539aa3206c9b34a23775644641 -F test/fts3join.test bc8e5ca3e6b6577cf98e81b5271291e33e020e87 +F test/fts3join.test 53e66a0c21eb568580674a43b21c059acb26f499 F test/fts3malloc.test b0e4c133b8d61d4f6d112d8110f8320e9e453ef6 F test/fts3matchinfo.test ff423e73faab8fc6d7adeefedf74dd8e2b0b14e0 F test/fts3near.test 7e3354d46f155a822b59c0e957fd2a70c1d7e905 @@ -1148,10 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 429018b19cb525a4bb0843d20955457b53900d4b -R 6571bb57c7cea9164f2f0721e5f9106f -T *branch * avoid-unusable-match -T *sym-avoid-unusable-match * -T -sym-trunk * +P 19f3208b26597625728d1ef716d4e44407cf96ac +R 2d783783dbaa85fb88df6c2aa4454d7b U dan -Z 6e1a68d01b84e17020c8f255059d7b13 +Z 4059190156e7afb8f8755a31ed7148b9 diff --git a/manifest.uuid b/manifest.uuid index d3155e3c85..4eab7c326e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -19f3208b26597625728d1ef716d4e44407cf96ac \ No newline at end of file +24f84b38131866f7b435ffe641bb2f6991a70db2 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 83ec748679..0acc0db53f 100644 --- a/src/where.c +++ b/src/where.c @@ -4338,7 +4338,6 @@ static int whereLoopAddVirtual( int nConstraint; int seenIn = 0; /* True if an IN operator is seen */ int seenVar = 0; /* True if a non-constant constraint is seen */ - int seenVarMatch = 0; /* If a non-constant MATCH constraint is seen */ int iPhase; /* 0: const w/o IN, 1: const, 2: no IN, 2: IN */ WhereLoop *pNew; int rc = SQLITE_OK; @@ -4383,7 +4382,6 @@ static int whereLoopAddVirtual( } if( pTerm->prereqRight!=0 ){ seenVar = 1; - if( pTerm->eOperator & WO_MATCH ) seenVarMatch = 1; }else if( (pTerm->eOperator & WO_IN)==0 ){ pIdxCons->usable = 1; } @@ -4402,9 +4400,6 @@ static int whereLoopAddVirtual( break; } } - /* The following line ensures that, if there exists a MATCH constraint, - ** no plans for which the MATCH constraint is not usable are considered. */ - if( seenVarMatch && iPhase<=1 ) continue; memset(pUsage, 0, sizeof(pUsage[0])*pIdxInfo->nConstraint); if( pIdxInfo->needToFreeIdxStr ) sqlite3_free(pIdxInfo->idxStr); pIdxInfo->idxStr = 0; diff --git a/test/fts3join.test b/test/fts3join.test index f02e21513e..64363639db 100644 --- a/test/fts3join.test +++ b/test/fts3join.test @@ -59,7 +59,7 @@ do_execsql_test 2.4 { SELECT * FROM ft3, ft2 WHERE y MATCH x; } {abc abc} do_catchsql_test 2.5 { SELECT * FROM ft3, ft2 WHERE y MATCH x AND x MATCH y; -} {1 {no query solution}} +} {1 {unable to use function MATCH in the requested context}} finish_test From 54e2adb5f3b2d9ddd561fec1b36aec5ac85693b4 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 4 Jan 2014 15:17:04 +0000 Subject: [PATCH 09/74] Improvements to the column-cache for nested AND/OR operators. FossilOrigin-Name: 4e725f53131d3584319c710c8710a068989543c6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 10 +++++++--- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 4e1d000e17..6bd605599f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Have\sFTS\sassign\sextremely\shigh\scosts\sto\splans\sthat\sfeature\sunusable\sMATCH\sconstraints.\sThis\sdiscourages\sthe\splanner\sfrom\schoosing\ssuch\splans,\swhich\slead\sto\s"unable\sto\suse\sfunction\sMATCH\sin\sthe\srequested\scontext"\serrors. -D 2014-01-04T14:46:39.188 +C Improvements\sto\sthe\scolumn-cache\sfor\snested\sAND/OR\soperators. +D 2014-01-04T15:17:04.888 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 22f3dc3fd6b82ebce3dabf951a4b157026943ac1 -F src/expr.c ca0959386f01d6e5d4dc67b362a4e4c912ebd054 +F src/expr.c 15a86b7632da09924ccffb53fafa86e7d8727b70 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 429018b19cb525a4bb0843d20955457b53900d4b 24f84b38131866f7b435ffe641bb2f6991a70db2 -R 2d783783dbaa85fb88df6c2aa4454d7b -U dan -Z 183032db794c33dfd03471ac339127f5 +P fa8be488a3ad290722dae7a1a7396c77277b2149 +R 7b52b4a00e9a7616715e494e48bd3657 +U drh +Z 701e96c9216858714def7e53249e15db diff --git a/manifest.uuid b/manifest.uuid index 0232dbf208..b3d3207e4a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa8be488a3ad290722dae7a1a7396c77277b2149 \ No newline at end of file +4e725f53131d3584319c710c8710a068989543c6 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 0db51fe43f..67aae870d9 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3388,7 +3388,7 @@ void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){ sqlite3ExplainPrintf(pOut, "item[%d] = ", i); sqlite3ExplainPush(pOut); sqlite3ExplainExpr(pOut, pList->a[i].pExpr); - sqlite3ExplainPop(pOut); + sqlite3ExplainPop(pOut, 1); if( pList->a[i].zName ){ sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); } @@ -3538,8 +3538,8 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_AND: { int d2 = sqlite3VdbeMakeLabel(v); testcase( jumpIfNull==0 ); - sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2,jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); sqlite3VdbeResolveLabel(v, d2); sqlite3ExprCachePop(pParse, 1); @@ -3548,7 +3548,9 @@ void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_OR: { testcase( jumpIfNull==0 ); sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3ExprCachePop(pParse, 1); break; } case TK_NOT: { @@ -3695,14 +3697,16 @@ void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int jumpIfNull){ case TK_AND: { testcase( jumpIfNull==0 ); sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3ExprCachePop(pParse, 1); break; } case TK_OR: { int d2 = sqlite3VdbeMakeLabel(v); testcase( jumpIfNull==0 ); - sqlite3ExprCachePush(pParse); sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprCachePush(pParse); sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); sqlite3VdbeResolveLabel(v, d2); sqlite3ExprCachePop(pParse, 1); From 61019c788362b46aa9a2a89380f7525359d13a43 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 4 Jan 2014 16:49:02 +0000 Subject: [PATCH 10/74] Omit OP_Close operations that occur immediately prior to OP_Halt and which cannot be jumped over. FossilOrigin-Name: 874b7e9999811c288ad41d07709f88e458d2d497 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/build.c | 1 + src/delete.c | 4 +--- src/sqliteInt.h | 1 + src/vdbe.h | 2 +- src/vdbeaux.c | 13 ++++++++++--- 7 files changed, 25 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 6bd605599f..6426172ac4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improvements\sto\sthe\scolumn-cache\sfor\snested\sAND/OR\soperators. -D 2014-01-04T15:17:04.888 +C Omit\sOP_Close\soperations\sthat\soccur\simmediately\sprior\sto\sOP_Halt\sand\swhich\ncannot\sbe\sjumped\sover. +D 2014-01-04T16:49:02.617 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,12 +169,12 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 11e29ef8cf16a42925fde036bcffbeffd9cc82df F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 -F src/build.c 47ef8209e56d840d2b35b8a243c6ee567ad52bda +F src/build.c 5597ffa6d60759f89da3dd8a326c291a75e83c23 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 -F src/delete.c 22f3dc3fd6b82ebce3dabf951a4b157026943ac1 +F src/delete.c f7c47fbbcbddf83c4af5df3681aa05a741a8d536 F src/expr.c 15a86b7632da09924ccffb53fafa86e7d8727b70 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 @@ -224,7 +224,7 @@ F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h a77cb6c7404d1a20e536211be4a2e4be01cde213 +F src/sqliteInt.h 40fa66cb72bc88bcd0a16967f4ad58cff7e2c4fd F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -281,10 +281,10 @@ F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c e71f19b272f05c8695cf747b4bac1732685f9e5c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c b110887e415b5d2af58c2374c4dfdcf774c5d46c -F src/vdbe.h 298e0492827ef9db626934d060df62ed27ba56bc +F src/vdbe.h e6c4c610fcabad4fa80ebb1efc6822a9367e2b26 F src/vdbeInt.h 42db251e9f863401ff847b90d5fe1614c89a6a56 F src/vdbeapi.c ce4e68ea4842cc6081046f533d088dcf01d247ad -F src/vdbeaux.c 915fda7ca876aa2df2a13b0621ad0326342c0638 +F src/vdbeaux.c 9f4bfc52672acbb0bb4493d6a03603dc5a595ac1 F src/vdbeblob.c bc40f98f256f0b34116d6a44b114da4a81a15d33 F src/vdbemem.c 0e69351b2c6ff7d8b638688c0ae336a26befa6b2 F src/vdbesort.c 9d83601f9d6243fe70dd0169a2820c5ddfd48147 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P fa8be488a3ad290722dae7a1a7396c77277b2149 -R 7b52b4a00e9a7616715e494e48bd3657 +P 4e725f53131d3584319c710c8710a068989543c6 +R 9d927180209ef7717259744d3fe6bde8 U drh -Z 701e96c9216858714def7e53249e15db +Z 5260028c0eafa272f25baa07be63a049 diff --git a/manifest.uuid b/manifest.uuid index b3d3207e4a..eb18276f5b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4e725f53131d3584319c710c8710a068989543c6 \ No newline at end of file +874b7e9999811c288ad41d07709f88e458d2d497 \ No newline at end of file diff --git a/src/build.c b/src/build.c index c83ce57cfa..5c8c9300f6 100644 --- a/src/build.c +++ b/src/build.c @@ -140,6 +140,7 @@ void sqlite3FinishCoding(Parse *pParse){ assert( !pParse->isMultiWrite || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ + while( sqlite3VdbeDeletePriorOpcode(v, OP_Close) ){} sqlite3VdbeAddOp0(v, OP_Halt); /* The cookie mask contains one bit for each database file open. diff --git a/src/delete.c b/src/delete.c index 0180ec48c8..641b4a763f 100644 --- a/src/delete.c +++ b/src/delete.c @@ -790,9 +790,7 @@ int sqlite3GenerateIndexKey( ** But we are getting ready to store this value back into an index, where ** it should be converted by to INTEGER again. So omit the OP_RealAffinity ** opcode if it is present */ - if( sqlite3VdbeGetOp(v, -1)->opcode==OP_RealAffinity ){ - sqlite3VdbeDeleteLastOpcode(v); - } + sqlite3VdbeDeletePriorOpcode(v, OP_RealAffinity); } if( regOut ){ sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regOut); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5943ced2a0..035e590bfe 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2293,6 +2293,7 @@ struct Parse { int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ int nLabel; /* Number of labels used */ int *aLabel; /* Space to hold the labels */ + int iFixedOp; /* Never back out opcodes iFixedOp-1 or earlier */ int ckBase; /* Base register of data during check constraints */ int iPartIdxTab; /* Table corresponding to a partial index */ int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ diff --git a/src/vdbe.h b/src/vdbe.h index 46bc89cee8..620f2eaf72 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -175,7 +175,7 @@ void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); void sqlite3VdbeChangeP5(Vdbe*, u8 P5); void sqlite3VdbeJumpHere(Vdbe*, int addr); void sqlite3VdbeChangeToNoop(Vdbe*, int addr); -void sqlite3VdbeDeleteLastOpcode(Vdbe*); +int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); void sqlite3VdbeUsesBtree(Vdbe*, int); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index e0b3c5485f..faabaf75d7 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -276,6 +276,7 @@ void sqlite3VdbeResolveLabel(Vdbe *v, int x){ if( j>=0 && p->aLabel ){ p->aLabel[j] = v->nOp; } + p->iFixedOp = v->nOp - 1; } /* @@ -624,7 +625,8 @@ void sqlite3VdbeChangeP5(Vdbe *p, u8 val){ ** the address of the next instruction to be coded. */ void sqlite3VdbeJumpHere(Vdbe *p, int addr){ - if( ALWAYS(addr>=0) ) sqlite3VdbeChangeP2(p, addr, p->nOp); + sqlite3VdbeChangeP2(p, addr, p->nOp); + p->pParse->iFixedOp = p->nOp - 1; } @@ -729,8 +731,13 @@ void sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ /* ** Remove the last opcode inserted */ -void sqlite3VdbeDeleteLastOpcode(Vdbe *p){ - p->nOp--; +int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ + if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){ + sqlite3VdbeChangeToNoop(p, p->nOp-1); + return 1; + }else{ + return 0; + } } /* From 1c2c0b776fdd1e12e7d844c8766157b82e4265a0 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 4 Jan 2014 19:27:05 +0000 Subject: [PATCH 11/74] Avoid redundant register loads during index key generation when doing a DELETE or INTEGRITY_CHECK on a table with multiple indices. FossilOrigin-Name: 8f6e6149a165f516be6395fd753e163d52ffd52e --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/build.c | 2 +- src/delete.c | 24 +++++++++++++++++++++--- src/pragma.c | 7 +++++-- src/sqliteInt.h | 2 +- src/where.c | 2 +- 7 files changed, 40 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 6426172ac4..66e96c4c62 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Omit\sOP_Close\soperations\sthat\soccur\simmediately\sprior\sto\sOP_Halt\sand\swhich\ncannot\sbe\sjumped\sover. -D 2014-01-04T16:49:02.617 +C Avoid\sredundant\sregister\sloads\sduring\sindex\skey\sgeneration\swhen\sdoing\sa\nDELETE\sor\sINTEGRITY_CHECK\son\sa\stable\swith\smultiple\sindices. +D 2014-01-04T19:27:05.589 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,12 +169,12 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 11e29ef8cf16a42925fde036bcffbeffd9cc82df F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 -F src/build.c 5597ffa6d60759f89da3dd8a326c291a75e83c23 +F src/build.c 3609c8aa26947d7a035faa23eb1cb2cfc54b4680 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 -F src/delete.c f7c47fbbcbddf83c4af5df3681aa05a741a8d536 +F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff F src/expr.c 15a86b7632da09924ccffb53fafa86e7d8727b70 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 @@ -213,7 +213,7 @@ F src/parse.y acee1a9958539e21263362b194594c5255ad2fca F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b -F src/pragma.c 5ab7279d132143feb77f773688a24ab05da75fd7 +F src/pragma.c ed409ce4104cf4d9de6ead40ace70974f124853b F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece @@ -224,7 +224,7 @@ F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 40fa66cb72bc88bcd0a16967f4ad58cff7e2c4fd +F src/sqliteInt.h 672ef78e9e6f8b03fc9d37884f9784fb02e6ce2d F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 8e3cb2030eaa242defb804aa30115b5d870e5cd4 +F src/where.c 18f07fd0fd116a5880773c689eb7cd8e60175107 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4e725f53131d3584319c710c8710a068989543c6 -R 9d927180209ef7717259744d3fe6bde8 +P 874b7e9999811c288ad41d07709f88e458d2d497 +R fc5dbcd7a08b2e4b11ffb23dc1241ea1 U drh -Z 5260028c0eafa272f25baa07be63a049 +Z 86dc65664e674593ae8c26084106bce8 diff --git a/manifest.uuid b/manifest.uuid index eb18276f5b..cc46b176ca 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -874b7e9999811c288ad41d07709f88e458d2d497 \ No newline at end of file +8f6e6149a165f516be6395fd753e163d52ffd52e \ No newline at end of file diff --git a/src/build.c b/src/build.c index 5c8c9300f6..15430fd28b 100644 --- a/src/build.c +++ b/src/build.c @@ -2678,7 +2678,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); regRecord = sqlite3GetTempReg(pParse); - sqlite3GenerateIndexKey(pParse, pIndex, iTab, regRecord, 0, &iPartIdxLabel); + sqlite3GenerateIndexKey(pParse,pIndex,iTab,regRecord,0,&iPartIdxLabel,0,0); sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); sqlite3VdbeResolveLabel(v, iPartIdxLabel); sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); diff --git a/src/delete.c b/src/delete.c index 641b4a763f..e448e47b02 100644 --- a/src/delete.c +++ b/src/delete.c @@ -716,9 +716,10 @@ void sqlite3GenerateRowIndexDelete( int *aRegIdx /* Only delete if aRegIdx!=0 && aRegIdx[i]>0 */ ){ int i; /* Index loop counter */ - int r1; /* Register holding an index key */ + int r1 = -1; /* Register holding an index key */ int iPartIdxLabel; /* Jump destination for skipping partial index entries */ Index *pIdx; /* Current index */ + Index *pPrior = 0; /* Prior index */ Vdbe *v; /* The prepared statement under construction */ Index *pPk; /* PRIMARY KEY index, or NULL for rowid tables */ @@ -729,10 +730,12 @@ void sqlite3GenerateRowIndexDelete( if( aRegIdx!=0 && aRegIdx[i]==0 ) continue; if( pIdx==pPk ) continue; VdbeModuleComment((v, "GenRowIdxDel for %s", pIdx->zName)); - r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, &iPartIdxLabel); + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 1, + &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); sqlite3VdbeResolveLabel(v, iPartIdxLabel); + pPrior = pIdx; } } @@ -754,6 +757,17 @@ void sqlite3GenerateRowIndexDelete( ** to false or null. If pIdx is not a partial index, *piPartIdxLabel ** will be set to zero which is an empty label that is ignored by ** sqlite3VdbeResolveLabel(). +** +** The pPrior and regPrior parameters are used to implement a cache to +** avoid unnecessary register loads. If pPrior is not NULL, then it is +** a pointer to a different index for which an index key has just been +** computed into register regPrior. If the current pIdx index is generating +** its key into the same sequence of registers and if pPrior and pIdx share +** a column in common, then the register corresponding to that column already +** holds the correct value and the loading of that register is skipped. +** This optimization is helpful when doing a DELETE or an INTEGRITY_CHECK +** on a table with multiple indices, and especially with the ROWID or +** PRIMARY KEY columns of the index. */ int sqlite3GenerateIndexKey( Parse *pParse, /* Parsing context */ @@ -761,7 +775,9 @@ int sqlite3GenerateIndexKey( int iDataCur, /* Cursor number from which to take column data */ int regOut, /* Put the new key into this register if not 0 */ int prefixOnly, /* Compute only a unique prefix of the key */ - int *piPartIdxLabel /* OUT: Jump to this label to skip partial index */ + int *piPartIdxLabel, /* OUT: Jump to this label to skip partial index */ + Index *pPrior, /* Previously generated index key */ + int regPrior /* Register holding previous generated key */ ){ Vdbe *v = pParse->pVdbe; int j; @@ -781,7 +797,9 @@ int sqlite3GenerateIndexKey( } nCol = (prefixOnly && pIdx->uniqNotNull) ? pIdx->nKeyCol : pIdx->nColumn; regBase = sqlite3GetTempRange(pParse, nCol); + if( pPrior && (regBase!=regPrior || pPrior->pPartIdxWhere) ) pPrior = 0; for(j=0; jaiColumn[j]==pIdx->aiColumn[j] ) continue; sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pIdx->aiColumn[j], regBase+j); /* If the column affinity is REAL but the number is an integer, then it diff --git a/src/pragma.c b/src/pragma.c index bbd27b8c18..7383bce96f 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -1881,8 +1881,10 @@ void sqlite3Pragma( for(x=sqliteHashFirst(pTbls); x && !isQuick; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx, *pPk; + Index *pPrior = 0; int loopTop; int iDataCur, iIdxCur; + int r1 = -1; if( pTab->pIndex==0 ) continue; pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); @@ -1901,9 +1903,10 @@ void sqlite3Pragma( loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ int jmp2, jmp3, jmp4; - int r1; if( pPk==pIdx ) continue; - r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3); + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, + pPrior, r1); + pPrior = pIdx; sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1); /* increment entry count */ jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, 0, r1, pIdx->nColumn); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 035e590bfe..d3e993c135 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2992,7 +2992,7 @@ int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); int sqlite3IsRowid(const char*); void sqlite3GenerateRowDelete(Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8); void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*); -int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*); +int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, u8,u8,int,int*); void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); diff --git a/src/where.c b/src/where.c index 0acc0db53f..d5444a6054 100644 --- a/src/where.c +++ b/src/where.c @@ -1710,7 +1710,7 @@ static void constructAutomaticIndex( /* Fill the automatic index with content */ addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); regRecord = sqlite3GetTempReg(pParse); - sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0); + sqlite3GenerateIndexKey(pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); From 2e08486a08a41e251b594566c8ef37e51c9facf8 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 4 Jan 2014 19:58:29 +0000 Subject: [PATCH 12/74] Add the usual "fts3" prefix to new static method setEstimatedRows() in fts3.c. This fixes a problem when compiling the amalgamation, as the r-tree module also contains a static method named setEstimatedRows. FossilOrigin-Name: d6fcfc8890489b942e5b3f1bc271835d77c5ef96 --- ext/fts3/fts3.c | 4 ++-- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 62ff1437b9..44b7f431df 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1477,7 +1477,7 @@ static int fts3CreateMethod( ** extension is currently being used by a version of SQLite too old to ** support estimatedRows. In that case this function is a no-op. */ -static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ +static void fts3SetEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ #if SQLITE_VERSION_NUMBER>=3008002 if( sqlite3_libversion_number()>=3008002 ){ pIdxInfo->estimatedRows = nRow; @@ -1521,7 +1521,7 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ ** this, return a very high cost here. */ pInfo->idxNum = FTS3_FULLSCAN_SEARCH; pInfo->estimatedCost = 1e50; - setEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); + fts3SetEstimatedRows(pInfo, ((sqlite3_int64)1) << 50); return SQLITE_OK; } continue; diff --git a/manifest b/manifest index 66e96c4c62..20a04a96bd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sredundant\sregister\sloads\sduring\sindex\skey\sgeneration\swhen\sdoing\sa\nDELETE\sor\sINTEGRITY_CHECK\son\sa\stable\swith\smultiple\sindices. -D 2014-01-04T19:27:05.589 +C Add\sthe\susual\s"fts3"\sprefix\sto\snew\sstatic\smethod\ssetEstimatedRows()\sin\sfts3.c.\sThis\sfixes\sa\sproblem\swhen\scompiling\sthe\samalgamation,\sas\sthe\sr-tree\smodule\salso\scontains\sa\sstatic\smethod\snamed\ssetEstimatedRows. +D 2014-01-04T19:58:29.359 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,7 +78,7 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 2af2cb2e742461b79710c132c7969fc7d949a59a +F ext/fts3/fts3.c 3fe91e36a0304ad4b35020f0e22ff37e95873166 F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe F ext/fts3/fts3Int.h eb5f8029589f3d8f1dc7fd50c773326a640388b1 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 874b7e9999811c288ad41d07709f88e458d2d497 -R fc5dbcd7a08b2e4b11ffb23dc1241ea1 -U drh -Z 86dc65664e674593ae8c26084106bce8 +P 8f6e6149a165f516be6395fd753e163d52ffd52e +R 2ae7b961fa0019d77bff10a5332fc93d +U dan +Z 855ecbcf8fda10bc6c9a4ba8bd1399e1 diff --git a/manifest.uuid b/manifest.uuid index cc46b176ca..d709dd888d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8f6e6149a165f516be6395fd753e163d52ffd52e \ No newline at end of file +d6fcfc8890489b942e5b3f1bc271835d77c5ef96 \ No newline at end of file From b66e21fda587a78079413d3588da69155b538141 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 4 Jan 2014 20:00:14 +0000 Subject: [PATCH 13/74] Fix an typo that breaks the build when SQLITE_ENABLE_TREE_EXPLAIN is defined. FossilOrigin-Name: f461e2b3973d0fe6a7b8cb7a3aaab8a30b3e16c0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 20a04a96bd..1d0ad0ed3f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\susual\s"fts3"\sprefix\sto\snew\sstatic\smethod\ssetEstimatedRows()\sin\sfts3.c.\sThis\sfixes\sa\sproblem\swhen\scompiling\sthe\samalgamation,\sas\sthe\sr-tree\smodule\salso\scontains\sa\sstatic\smethod\snamed\ssetEstimatedRows. -D 2014-01-04T19:58:29.359 +C Fix\san\stypo\sthat\sbreaks\sthe\sbuild\swhen\sSQLITE_ENABLE_TREE_EXPLAIN\sis\sdefined. +D 2014-01-04T20:00:14.172 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff -F src/expr.c 15a86b7632da09924ccffb53fafa86e7d8727b70 +F src/expr.c 4115ad67088cdd55f4fa0ef3ddd22cb8da8f9c94 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8f6e6149a165f516be6395fd753e163d52ffd52e -R 2ae7b961fa0019d77bff10a5332fc93d -U dan -Z 855ecbcf8fda10bc6c9a4ba8bd1399e1 +P d6fcfc8890489b942e5b3f1bc271835d77c5ef96 +R 92a605458ee112680310df2ab331e715 +U drh +Z 1b6a595171700e1d6836f451072293d3 diff --git a/manifest.uuid b/manifest.uuid index d709dd888d..89e098a77e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d6fcfc8890489b942e5b3f1bc271835d77c5ef96 \ No newline at end of file +f461e2b3973d0fe6a7b8cb7a3aaab8a30b3e16c0 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 67aae870d9..aad6cd1cda 100644 --- a/src/expr.c +++ b/src/expr.c @@ -3388,7 +3388,7 @@ void sqlite3ExplainExprList(Vdbe *pOut, ExprList *pList){ sqlite3ExplainPrintf(pOut, "item[%d] = ", i); sqlite3ExplainPush(pOut); sqlite3ExplainExpr(pOut, pList->a[i].pExpr); - sqlite3ExplainPop(pOut, 1); + sqlite3ExplainPop(pOut); if( pList->a[i].zName ){ sqlite3ExplainPrintf(pOut, " AS %s", pList->a[i].zName); } From 1bcbc621dfbea87d52cba3988dbbf176ea041aa4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 9 Jan 2014 13:39:07 +0000 Subject: [PATCH 14/74] Fix harmless compiler warning in unixUnfetch(). FossilOrigin-Name: 618f248f4ea9fb0b6ff019a4c2cd72857389301f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_unix.c | 7 +++++-- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 1d0ad0ed3f..d4fa6de5fd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\stypo\sthat\sbreaks\sthe\sbuild\swhen\sSQLITE_ENABLE_TREE_EXPLAIN\sis\sdefined. -D 2014-01-04T20:00:14.172 +C Fix\sharmless\scompiler\swarning\sin\sunixUnfetch(). +D 2014-01-09T13:39:07.144 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -205,7 +205,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c abeb9d54036aaea6f4395050ce823f51217ae4d4 +F src/os_unix.c 9270957b8ebab7a6c930cc6891f98cf396771d9d F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d6fcfc8890489b942e5b3f1bc271835d77c5ef96 -R 92a605458ee112680310df2ab331e715 +P f461e2b3973d0fe6a7b8cb7a3aaab8a30b3e16c0 +R d5e72e34ba57bc2641d05089f26a27b4 U drh -Z 1b6a595171700e1d6836f451072293d3 +Z 8832e3e8b28de8e986889f1676a8ae62 diff --git a/manifest.uuid b/manifest.uuid index 89e098a77e..6db219b4e4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f461e2b3973d0fe6a7b8cb7a3aaab8a30b3e16c0 \ No newline at end of file +618f248f4ea9fb0b6ff019a4c2cd72857389301f \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 4b76d4fec4..420275b3f9 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4848,10 +4848,10 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ ** may now be invalid and should be unmapped. */ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ - unixFile *pFd = (unixFile *)fd; /* The underlying database file */ UNUSED_PARAMETER(iOff); - #if SQLITE_MAX_MMAP_SIZE>0 + unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + /* If p==0 (unmap the entire file) then there must be no outstanding ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), ** then there must be at least one outstanding. */ @@ -4867,6 +4867,9 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ } assert( pFd->nFetchOut>=0 ); +#else + UNUSED_PARAMETER(fd); + UNUSED_PARAMETER(p); #endif return SQLITE_OK; } From 9871c59a2d18e0b07ef97118aa13843f666a6bb0 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 10 Jan 2014 16:40:21 +0000 Subject: [PATCH 15/74] Fix another harmless compiler warning in unixUnfetch(). FossilOrigin-Name: 0484549bb82ca5246488330c8d266e429ccd19b9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_unix.c | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index d4fa6de5fd..19878b4a97 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sin\sunixUnfetch(). -D 2014-01-09T13:39:07.144 +C Fix\sanother\sharmless\scompiler\swarning\sin\sunixUnfetch(). +D 2014-01-10T16:40:21.047 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -205,7 +205,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 9270957b8ebab7a6c930cc6891f98cf396771d9d +F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968 F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P f461e2b3973d0fe6a7b8cb7a3aaab8a30b3e16c0 -R d5e72e34ba57bc2641d05089f26a27b4 -U drh -Z 8832e3e8b28de8e986889f1676a8ae62 +P 618f248f4ea9fb0b6ff019a4c2cd72857389301f +R 680b9cb5772eadf3cf119ad0606b2737 +U dan +Z 5d71bf2b25857007a3e37b3d6409191a diff --git a/manifest.uuid b/manifest.uuid index 6db219b4e4..d04b2a2897 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -618f248f4ea9fb0b6ff019a4c2cd72857389301f \ No newline at end of file +0484549bb82ca5246488330c8d266e429ccd19b9 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 420275b3f9..96cd5e6191 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4848,9 +4848,9 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ ** may now be invalid and should be unmapped. */ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ - UNUSED_PARAMETER(iOff); #if SQLITE_MAX_MMAP_SIZE>0 unixFile *pFd = (unixFile *)fd; /* The underlying database file */ + UNUSED_PARAMETER(iOff); /* If p==0 (unmap the entire file) then there must be no outstanding ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), @@ -4870,6 +4870,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ #else UNUSED_PARAMETER(fd); UNUSED_PARAMETER(p); + UNUSED_PARAMETER(iOff); #endif return SQLITE_OK; } From c740752470a0557b03299c5fd5036e5e096b78e3 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Jan 2014 20:38:12 +0000 Subject: [PATCH 16/74] Fix CREATE TABLE ... AS so that it works with column names that are empty strings. FossilOrigin-Name: 632045f21c553e10f59a14c772d50d7824ca0c2c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/build.c | 8 ++++---- test/misc1.test | 13 +++++++++++++ 4 files changed, 26 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index 19878b4a97..ef3aae89ea 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sanother\sharmless\scompiler\swarning\sin\sunixUnfetch(). -D 2014-01-10T16:40:21.047 +C Fix\sCREATE\sTABLE\s...\sAS\sso\sthat\sit\sworks\swith\scolumn\snames\sthat\sare\sempty\nstrings. +D 2014-01-10T20:38:12.815 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,7 +169,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 11e29ef8cf16a42925fde036bcffbeffd9cc82df F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 -F src/build.c 3609c8aa26947d7a035faa23eb1cb2cfc54b4680 +F src/build.c 8c56d91447770a746b16d08a6510109c161dbc1a F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd @@ -693,7 +693,7 @@ F test/minmax.test 42fbad0e81afaa6e0de41c960329f2b2c3526efd F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354 F test/minmax4.test 536a3360470633a177e42fbc19660d146b51daef -F test/misc1.test 9bed1bd334065a57dc841cff969d4fc1eeb6d49b +F test/misc1.test 441a0fafc7087f841db09fbfca54e7aea9f5a84c F test/misc2.test 00d7de54eda90e237fc9a38b9e5ccc769ebf6d4d F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 618f248f4ea9fb0b6ff019a4c2cd72857389301f -R 680b9cb5772eadf3cf119ad0606b2737 -U dan -Z 5d71bf2b25857007a3e37b3d6409191a +P 0484549bb82ca5246488330c8d266e429ccd19b9 +R 00ef541b0e35ee7a66a11f656f6dc3e2 +U drh +Z 1394adca743ceab6563433db8e16d260 diff --git a/manifest.uuid b/manifest.uuid index d04b2a2897..7a50b80671 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0484549bb82ca5246488330c8d266e429ccd19b9 \ No newline at end of file +632045f21c553e10f59a14c772d50d7824ca0c2c \ No newline at end of file diff --git a/src/build.c b/src/build.c index 15430fd28b..fa82d56cc7 100644 --- a/src/build.c +++ b/src/build.c @@ -1452,10 +1452,10 @@ static void identPut(char *z, int *pIdx, char *zSignedIdent){ for(j=0; zIdent[j]; j++){ if( !sqlite3Isalnum(zIdent[j]) && zIdent[j]!='_' ) break; } - needQuote = sqlite3Isdigit(zIdent[0]) || sqlite3KeywordCode(zIdent, j)!=TK_ID; - if( !needQuote ){ - needQuote = zIdent[j]; - } + needQuote = sqlite3Isdigit(zIdent[0]) + || sqlite3KeywordCode(zIdent, j)!=TK_ID + || zIdent[j]!=0 + || j==0; if( needQuote ) z[i++] = '"'; for(j=0; zIdent[j]; j++){ diff --git a/test/misc1.test b/test/misc1.test index 8573d349b3..f886f896e2 100644 --- a/test/misc1.test +++ b/test/misc1.test @@ -592,4 +592,17 @@ do_test misc1-18.1 { expr {$n>=100} } {1} +# 2014-01-10: In a CREATE TABLE AS, if one or more of the column names +# are an empty string, that is still OK. +# +do_execsql_test misc1-19.1 { + CREATE TABLE t19 AS SELECT 1, 2 AS '', 3; + SELECT * FROM t19; +} {1 2 3} +do_execsql_test misc1-19.2 { + CREATE TABLE t19b AS SELECT 4 AS '', 5 AS '', 6 AS ''; + SELECT * FROM t19b; +} {4 5 6} + + finish_test From 75593d96be4062a3fd6cf86135b945e95367b10b Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Jan 2014 20:46:55 +0000 Subject: [PATCH 17/74] Allow a VALUES clause to be used any place that a SELECT statement can be used. FossilOrigin-Name: c9ea7d199f06a7801ab639e7ac98ebeb98706f24 --- manifest | 18 ++++++------- manifest.uuid | 2 +- src/insert.c | 13 ++++++++- src/parse.y | 70 ++++++++++++++----------------------------------- src/sqliteInt.h | 6 ++--- src/trigger.c | 7 +---- 6 files changed, 46 insertions(+), 70 deletions(-) diff --git a/manifest b/manifest index ef3aae89ea..3ad162dd48 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sCREATE\sTABLE\s...\sAS\sso\sthat\sit\sworks\swith\scolumn\snames\sthat\sare\sempty\nstrings. -D 2014-01-10T20:38:12.815 +C Allow\sa\sVALUES\sclause\sto\sbe\sused\sany\splace\sthat\sa\sSELECT\sstatement\scan\sbe\nused. +D 2014-01-10T20:46:55.338 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -183,7 +183,7 @@ F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c e6b3d7847aaf06170967ff6ea9076717fbc08d3f +F src/insert.c 5ddb48c7f1cb399993744f23f03948989ce1790e F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b @@ -209,7 +209,7 @@ F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968 F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y acee1a9958539e21263362b194594c5255ad2fca +F src/parse.y 0a284e3f2bb78363ddce6f796ca2074e1e265eb7 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b @@ -224,7 +224,7 @@ F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 672ef78e9e6f8b03fc9d37884f9784fb02e6ce2d +F src/sqliteInt.h 0c65967bb807dee3c9eef2bbd17f880eeb28ea30 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -275,7 +275,7 @@ F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c ec4c1a62b890bf1dbcdb966399e140b904c700a4 -F src/trigger.c d84e1f3669e9a217731a14a9d472b1c7b87c87ba +F src/trigger.c 5c1c0b899ac0ce284763dcb8fdbaa38ecf15ef98 F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c e71f19b272f05c8695cf747b4bac1732685f9e5c @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 0484549bb82ca5246488330c8d266e429ccd19b9 -R 00ef541b0e35ee7a66a11f656f6dc3e2 +P 632045f21c553e10f59a14c772d50d7824ca0c2c +R cf996bb200e200afa8de950e78c84a55 U drh -Z 1394adca743ceab6563433db8e16d260 +Z 54c4a7cb2ec6e5258f05fa3eca37aefd diff --git a/manifest.uuid b/manifest.uuid index 7a50b80671..76d9f0f1fb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -632045f21c553e10f59a14c772d50d7824ca0c2c \ No newline at end of file +c9ea7d199f06a7801ab639e7ac98ebeb98706f24 \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index abc34c03ca..038d08a40d 100644 --- a/src/insert.c +++ b/src/insert.c @@ -540,7 +540,6 @@ static int xferOptimization( void sqlite3Insert( Parse *pParse, /* Parser context */ SrcList *pTabList, /* Name of table into which we are inserting */ - ExprList *pList, /* List of values to be inserted */ Select *pSelect, /* A SELECT statement to use as the data source */ IdList *pColumn, /* Column names corresponding to IDLIST. */ int onError /* How to handle constraint errors */ @@ -568,6 +567,7 @@ void sqlite3Insert( Db *pDb; /* The database containing table being inserted into */ int appendFlag = 0; /* True if the insert is likely to be an append */ int withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ + ExprList *pList = 0; /* List of VALUES() to be inserted */ /* Register allocations */ int regFromSelect = 0;/* Base register for data coming from SELECT */ @@ -591,6 +591,17 @@ void sqlite3Insert( goto insert_cleanup; } + /* If the Select object is really just a simple VALUES() list with a + ** single row values (the common case) then keep that one row of values + ** and go ahead and discard the Select object + */ + if( pSelect && (pSelect->selFlags & SF_Values)!=0 && pSelect->pPrior==0 ){ + pList = pSelect->pEList; + pSelect->pEList = 0; + sqlite3SelectDelete(db, pSelect); + pSelect = 0; + } + /* Locate the table into which we will be inserting new information. */ assert( pTabList->nSrc==1 ); diff --git a/src/parse.y b/src/parse.y index ba9feb10fc..00d47cd3ea 100644 --- a/src/parse.y +++ b/src/parse.y @@ -442,6 +442,23 @@ oneselect(A) ::= SELECT distinct(D) selcollist(W) from(X) where_opt(Y) groupby_opt(P) having_opt(Q) orderby_opt(Z) limit_opt(L). { A = sqlite3SelectNew(pParse,W,X,Y,P,Q,Z,D,L.pLimit,L.pOffset); } +oneselect(A) ::= values(X). {A = X;} + +%type values {Select*} +%destructor values {sqlite3SelectDelete(pParse->db, $$);} +values(A) ::= VALUES LP nexprlist(X) RP. { + A = sqlite3SelectNew(pParse,X,0,0,0,0,0,SF_Values,0,0); +} +values(A) ::= values(X) COMMA LP exprlist(Y) RP. { + Select *pRight = sqlite3SelectNew(pParse,Y,0,0,0,0,0,SF_Values,0,0); + if( pRight ){ + pRight->op = TK_ALL; + pRight->pPrior = X; + A = pRight; + }else{ + A = X; + } +} // The "distinct" nonterminal is true (1) if the DISTINCT keyword is // present and false (0) if it is not. @@ -696,58 +713,15 @@ setlist(A) ::= nm(X) EQ expr(Y). { ////////////////////////// The INSERT command ///////////////////////////////// // -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) valuelist(Y). - {sqlite3Insert(pParse, X, Y.pList, Y.pSelect, F, R);} cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). - {sqlite3Insert(pParse, X, 0, S, F, R);} + {sqlite3Insert(pParse, X, S, F, R);} cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. - {sqlite3Insert(pParse, X, 0, 0, F, R);} + {sqlite3Insert(pParse, X, 0, F, R);} %type insert_cmd {u8} insert_cmd(A) ::= INSERT orconf(R). {A = R;} insert_cmd(A) ::= REPLACE. {A = OE_Replace;} -// A ValueList is either a single VALUES clause or a comma-separated list -// of VALUES clauses. If it is a single VALUES clause then the -// ValueList.pList field points to the expression list of that clause. -// If it is a list of VALUES clauses, then those clauses are transformed -// into a set of SELECT statements without FROM clauses and connected by -// UNION ALL and the ValueList.pSelect points to the right-most SELECT in -// that compound. -%type valuelist {struct ValueList} -%destructor valuelist { - sqlite3ExprListDelete(pParse->db, $$.pList); - sqlite3SelectDelete(pParse->db, $$.pSelect); -} -valuelist(A) ::= VALUES LP nexprlist(X) RP. { - A.pList = X; - A.pSelect = 0; -} - -// Since a list of VALUEs is inplemented as a compound SELECT, we have -// to disable the value list option if compound SELECTs are disabled. -%ifndef SQLITE_OMIT_COMPOUND_SELECT -valuelist(A) ::= valuelist(X) COMMA LP exprlist(Y) RP. { - Select *pRight = sqlite3SelectNew(pParse, Y, 0, 0, 0, 0, 0, 0, 0, 0); - if( X.pList ){ - X.pSelect = sqlite3SelectNew(pParse, X.pList, 0, 0, 0, 0, 0, 0, 0, 0); - X.pList = 0; - } - A.pList = 0; - if( X.pSelect==0 || pRight==0 ){ - sqlite3SelectDelete(pParse->db, pRight); - sqlite3SelectDelete(pParse->db, X.pSelect); - A.pSelect = 0; - }else{ - pRight->op = TK_ALL; - pRight->pPrior = X.pSelect; - pRight->selFlags |= SF_Values; - pRight->pPrior->selFlags |= SF_Values; - A.pSelect = pRight; - } -} -%endif SQLITE_OMIT_COMPOUND_SELECT - %type inscollist_opt {IdList*} %destructor inscollist_opt {sqlite3IdListDelete(pParse->db, $$);} %type idlist {IdList*} @@ -1295,12 +1269,8 @@ trigger_cmd(A) ::= { A = sqlite3TriggerUpdateStep(pParse->db, &X, Y, Z, R); } // INSERT -trigger_cmd(A) ::= - insert_cmd(R) INTO trnm(X) inscollist_opt(F) valuelist(Y). - {A = sqlite3TriggerInsertStep(pParse->db, &X, F, Y.pList, Y.pSelect, R);} - trigger_cmd(A) ::= insert_cmd(R) INTO trnm(X) inscollist_opt(F) select(S). - {A = sqlite3TriggerInsertStep(pParse->db, &X, F, 0, S, R);} + {A = sqlite3TriggerInsertStep(pParse->db, &X, F, S, R);} // DELETE trigger_cmd(A) ::= DELETE FROM trnm(X) tridxby where_opt(Y). diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d3e993c135..188a7cacc4 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2483,7 +2483,7 @@ struct TriggerStep { Select *pSelect; /* SELECT statment or RHS of INSERT INTO .. SELECT ... */ Token target; /* Target table for DELETE, UPDATE, INSERT */ Expr *pWhere; /* The WHERE clause for DELETE or UPDATE steps */ - ExprList *pExprList; /* SET clause for UPDATE. VALUES clause for INSERT */ + ExprList *pExprList; /* SET clause for UPDATE. */ IdList *pIdList; /* Column names for INSERT */ TriggerStep *pNext; /* Next in the link-list */ TriggerStep *pLast; /* Last element in link-list. Valid for 1st elem only */ @@ -2898,7 +2898,7 @@ void sqlite3DeleteTable(sqlite3*, Table*); # define sqlite3AutoincrementEnd(X) #endif int sqlite3CodeCoroutine(Parse*, Select*, SelectDest*); -void sqlite3Insert(Parse*, SrcList*, ExprList*, Select*, IdList*, int); +void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int); void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*); int sqlite3IdListIndex(IdList*,const char*); @@ -3036,7 +3036,7 @@ void sqlite3MaterializeView(Parse*, Table*, Expr*, int); void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*); TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*); TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*, - ExprList*,Select*,u8); + Select*,u8); TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8); TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*); void sqlite3DeleteTrigger(sqlite3*, Trigger*); diff --git a/src/trigger.c b/src/trigger.c index 1c68a708dd..fdc2a0388c 100644 --- a/src/trigger.c +++ b/src/trigger.c @@ -397,25 +397,21 @@ TriggerStep *sqlite3TriggerInsertStep( sqlite3 *db, /* The database connection */ Token *pTableName, /* Name of the table into which we insert */ IdList *pColumn, /* List of columns in pTableName to insert into */ - ExprList *pEList, /* The VALUE clause: a list of values to be inserted */ Select *pSelect, /* A SELECT statement that supplies values */ u8 orconf /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ ){ TriggerStep *pTriggerStep; - assert(pEList == 0 || pSelect == 0); - assert(pEList != 0 || pSelect != 0 || db->mallocFailed); + assert(pSelect != 0 || db->mallocFailed); pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName); if( pTriggerStep ){ pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); pTriggerStep->pIdList = pColumn; - pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); pTriggerStep->orconf = orconf; }else{ sqlite3IdListDelete(db, pColumn); } - sqlite3ExprListDelete(db, pEList); sqlite3SelectDelete(db, pSelect); return pTriggerStep; @@ -753,7 +749,6 @@ static int codeTriggerProgram( case TK_INSERT: { sqlite3Insert(pParse, targetSrcList(pParse, pStep), - sqlite3ExprListDup(db, pStep->pExprList, 0), sqlite3SelectDup(db, pStep->pSelect, 0), sqlite3IdListDup(db, pStep->pIdList), pParse->eOrconf From 144ffe7bfb9036ded7e294a0d92320ad97d2f8be Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Jan 2014 20:51:16 +0000 Subject: [PATCH 18/74] Remove unused structure definition from parse.y. FossilOrigin-Name: 7f1e7ae313c7625ef2623d78883dce776eecca30 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/parse.y | 8 -------- 3 files changed, 7 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 3ad162dd48..78da6bddd4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sa\sVALUES\sclause\sto\sbe\sused\sany\splace\sthat\sa\sSELECT\sstatement\scan\sbe\nused. -D 2014-01-10T20:46:55.338 +C Remove\sunused\sstructure\sdefinition\sfrom\sparse.y. +D 2014-01-10T20:51:16.498 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -209,7 +209,7 @@ F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968 F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 0a284e3f2bb78363ddce6f796ca2074e1e265eb7 +F src/parse.y 60baa3aced02c9f91259719d196315bc28580719 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 632045f21c553e10f59a14c772d50d7824ca0c2c -R cf996bb200e200afa8de950e78c84a55 +P c9ea7d199f06a7801ab639e7ac98ebeb98706f24 +R b8682b1b95c61647eceb7cac2c7aa836 U drh -Z 54c4a7cb2ec6e5258f05fa3eca37aefd +Z 2b3078d680e621e51a9cb7dd08f4a256 diff --git a/manifest.uuid b/manifest.uuid index 76d9f0f1fb..2faf2ec7b2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c9ea7d199f06a7801ab639e7ac98ebeb98706f24 \ No newline at end of file +7f1e7ae313c7625ef2623d78883dce776eecca30 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 00d47cd3ea..154929c8d9 100644 --- a/src/parse.y +++ b/src/parse.y @@ -94,14 +94,6 @@ struct TrigEvent { int a; IdList * b; }; */ struct AttachKey { int type; Token key; }; -/* -** One or more VALUES claues -*/ -struct ValueList { - ExprList *pList; - Select *pSelect; -}; - } // end %include // Input is a single SQL command From 898799fa00dfd21887a9870f5262350e495af046 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 10 Jan 2014 23:21:00 +0000 Subject: [PATCH 19/74] Do not use sprintf(), strcpy() or strcat() in the implementation of the lemon parser generator tool, to avoid compiler warnings in OpenBSD. FossilOrigin-Name: e43c522dde01e134f1adc94f534d2b3eda74afc2 --- manifest | 12 ++--- manifest.uuid | 2 +- tool/lemon.c | 125 +++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 115 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 78da6bddd4..edfa17a7d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunused\sstructure\sdefinition\sfrom\sparse.y. -D 2014-01-10T20:51:16.498 +C Do\snot\suse\ssprintf(),\sstrcpy()\sor\sstrcat()\sin\sthe\simplementation\sof\sthe\nlemon\sparser\sgenerator\stool,\sto\savoid\scompiler\swarnings\sin\sOpenBSD. +D 2014-01-10T23:21:00.243 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1109,7 +1109,7 @@ F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c 796930d5fc2036c7636f3f1ee12f9ae03719a2eb +F tool/lemon.c 04f60c891f6c07643221b068bbc2621be46c039a F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c9ea7d199f06a7801ab639e7ac98ebeb98706f24 -R b8682b1b95c61647eceb7cac2c7aa836 +P 7f1e7ae313c7625ef2623d78883dce776eecca30 +R 153a0201d8a4a180b2a188d3abd169a6 U drh -Z 2b3078d680e621e51a9cb7dd08f4a256 +Z c49dfd6bd62bb08d757d02ffce1c49c4 diff --git a/manifest.uuid b/manifest.uuid index 2faf2ec7b2..fdbe371621 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7f1e7ae313c7625ef2623d78883dce776eecca30 \ No newline at end of file +e43c522dde01e134f1adc94f534d2b3eda74afc2 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 58f13880f0..191ab0d88b 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -50,6 +50,97 @@ static char *msort(char*,char**,int(*)(const char*,const char*)); */ #define lemonStrlen(X) ((int)strlen(X)) +/* +** Compilers are starting to complain about the use of sprintf() and strcpy(), +** saying they are unsafe. So we define our own versions of those routines too. +** +** There are three routines here: lemon_sprintf(), lemon_vsprintf(), and +** lemon_addtext(). The first two are replacements for sprintf() and vsprintf(). +** The third is a helper routine for vsnprintf() that adds texts to the end of a +** buffer, making sure the buffer is always zero-terminated. +** +** The string formatter is a minimal subset of stdlib sprintf() supporting only +** a few simply conversions: +** +** %d +** %s +** %.*s +** +*/ +static void lemon_addtext( + char *zBuf, /* The buffer to which text is added */ + int *pnUsed, /* Slots of the buffer used so far */ + const char *zIn, /* Text to add */ + int nIn /* Bytes of text to add. -1 to use strlen() */ +){ + if( nIn<0 ) for(nIn=0; zIn[nIn]; nIn++){} + if( nIn==0 ) return; + memcpy(&zBuf[*pnUsed], zIn, nIn); + *pnUsed += nIn; + zBuf[*pnUsed] = 0; +} +static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ + int i, j, k, c, size; + int nUsed = 0; + const char *z; + char zTemp[50]; + str[0] = 0; + for(i=j=0; (c = zFormat[i])!=0; i++){ + if( c=='%' ){ + lemon_addtext(str, &nUsed, &zFormat[j], i-j); + c = zFormat[++i]; + if( c=='d' ){ + int v = va_arg(ap, int); + if( v<0 ){ + lemon_addtext(str, &nUsed, "-", 1); + v = -v; + }else if( v==0 ){ + lemon_addtext(str, &nUsed, "0", 1); + } + k = 0; + while( v>0 ){ + k++; + zTemp[sizeof(zTemp)-k] = (v%10) + '0'; + v /= 10; + } + lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k); + }else if( c=='s' ){ + z = va_arg(ap, const char*); + lemon_addtext(str, &nUsed, z, -1); + }else if( c=='.' && memcmp(&zFormat[i], ".*s", 3)==0 ){ + i += 2; + k = va_arg(ap, int); + z = va_arg(ap, const char*); + lemon_addtext(str, &nUsed, z, k); + }else if( c=='%' ){ + lemon_addtext(str, &nUsed, "%", 1); + }else{ + fprintf(stderr, "illegal format\n"); + exit(1); + } + j = i+1; + } + } + lemon_addtext(str, &nUsed, &zFormat[j], i-j); + return nUsed; +} +static int lemon_sprintf(char *str, const char *format, ...){ + va_list ap; + int rc; + va_start(ap, format); + rc = lemon_vsprintf(str, format, ap); + va_end(ap); + return rc; +} +static void lemon_strcpy(char *dest, const char *src){ + while( (*(dest++) = *(src++))!=0 ){} +} +static void lemon_strcat(char *dest, const char *src){ + while( *dest ) dest++; + lemon_strcpy(dest, src); +} + + /* a few forward declarations... */ struct rule; struct lemon; @@ -1367,7 +1458,7 @@ static void handle_D_option(char *z){ fprintf(stderr,"out of memory\n"); exit(1); } - strcpy(*paz, z); + lemon_strcpy(*paz, z); for(z=*paz; *z && *z!='='; z++){} *z = 0; } @@ -1378,7 +1469,7 @@ static void handle_T_option(char *z){ if( user_templatename==0 ){ memory_error(); } - strcpy(user_templatename, z); + lemon_strcpy(user_templatename, z); } /* The main program. Parse the command line and do it... */ @@ -2347,7 +2438,7 @@ to follow the previous rule."); for(z=psp->filename, nBack=0; *z; z++){ if( *z=='\\' ) nBack++; } - sprintf(zLine, "#line %d ", psp->tokenlineno); + lemon_sprintf(zLine, "#line %d ", psp->tokenlineno); nLine = lemonStrlen(zLine); n += nLine + lemonStrlen(psp->filename) + nBack; } @@ -2716,10 +2807,10 @@ PRIVATE char *file_makename(struct lemon *lemp, const char *suffix) fprintf(stderr,"Can't allocate space for a filename.\n"); exit(1); } - strcpy(name,lemp->filename); + lemon_strcpy(name,lemp->filename); cp = strrchr(name,'.'); if( cp ) *cp = 0; - strcat(name,suffix); + lemon_strcat(name,suffix); return name; } @@ -2916,7 +3007,7 @@ void ReportOutput(struct lemon *lemp) while( cfp ){ char buf[20]; if( cfp->dot==cfp->rp->nrhs ){ - sprintf(buf,"(%d)",cfp->rp->index); + lemon_sprintf(buf,"(%d)",cfp->rp->index); fprintf(fp," %5s ",buf); }else{ fprintf(fp," "); @@ -2981,7 +3072,7 @@ PRIVATE char *pathsearch(char *argv0, char *name, int modemask) c = *cp; *cp = 0; path = (char *)malloc( lemonStrlen(argv0) + lemonStrlen(name) + 2 ); - if( path ) sprintf(path,"%s/%s",argv0,name); + if( path ) lemon_sprintf(path,"%s/%s",argv0,name); *cp = c; }else{ pathlist = getenv("PATH"); @@ -2990,13 +3081,13 @@ PRIVATE char *pathsearch(char *argv0, char *name, int modemask) path = (char *)malloc( lemonStrlen(pathlist)+lemonStrlen(name)+2 ); if( (pathbuf != 0) && (path!=0) ){ pathbufptr = pathbuf; - strcpy(pathbuf, pathlist); + lemon_strcpy(pathbuf, pathlist); while( *pathbuf ){ cp = strchr(pathbuf,':'); if( cp==0 ) cp = &pathbuf[lemonStrlen(pathbuf)]; c = *cp; *cp = 0; - sprintf(path,"%s/%s",pathbuf,name); + lemon_sprintf(path,"%s/%s",pathbuf,name); *cp = c; if( c==0 ) pathbuf[0] = 0; else pathbuf = &cp[1]; @@ -3087,9 +3178,9 @@ PRIVATE FILE *tplt_open(struct lemon *lemp) cp = strrchr(lemp->filename,'.'); if( cp ){ - sprintf(buf,"%.*s.lt",(int)(cp-lemp->filename),lemp->filename); + lemon_sprintf(buf,"%.*s.lt",(int)(cp-lemp->filename),lemp->filename); }else{ - sprintf(buf,"%s.lt",lemp->filename); + lemon_sprintf(buf,"%s.lt",lemp->filename); } if( access(buf,004)==0 ){ tpltname = buf; @@ -3240,9 +3331,9 @@ PRIVATE char *append_str(const char *zText, int n, int p1, int p2){ while( n-- > 0 ){ c = *(zText++); if( c=='%' && n>0 && zText[0]=='d' ){ - sprintf(zInt, "%d", p1); + lemon_sprintf(zInt, "%d", p1); p1 = p2; - strcpy(&z[used], zInt); + lemon_strcpy(&z[used], zInt); used += lemonStrlen(&z[used]); zText++; n--; @@ -3467,7 +3558,7 @@ void print_stack_union( fprintf(stderr,"Out of memory.\n"); exit(1); } - strcpy(types[hash],stddt); + lemon_strcpy(types[hash],stddt); } } @@ -3856,7 +3947,7 @@ void ReportTable( /* Generate a table containing the symbolic name of every symbol */ for(i=0; insymbol; i++){ - sprintf(line,"\"%s\",",lemp->symbols[i]->name); + lemon_sprintf(line,"\"%s\",",lemp->symbols[i]->name); fprintf(out," %-15s",line); if( (i&3)==3 ){ fprintf(out,"\n"); lineno++; } } @@ -4023,7 +4114,7 @@ void ReportHeader(struct lemon *lemp) if( in ){ int nextChar; for(i=1; interminal && fgets(line,LINESIZE,in); i++){ - sprintf(pattern,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); + lemon_sprintf(pattern,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); if( strcmp(line,pattern) ) break; } nextChar = fgetc(in); @@ -4253,7 +4344,7 @@ const char *Strsafe(const char *y) if( y==0 ) return 0; z = Strsafe_find(y); if( z==0 && (cpy=(char *)malloc( lemonStrlen(y)+1 ))!=0 ){ - strcpy(cpy,y); + lemon_strcpy(cpy,y); z = cpy; Strsafe_insert(z); } From 61f92cd045319f19e12125d2f49bb3d521c0934c Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Jan 2014 03:06:18 +0000 Subject: [PATCH 20/74] In LEMON, fix a bug in the text formatter introduced by the previous commit. Also add the new "%token_class" directive for defining symbolic names that stand any one of a collection of tokens. FossilOrigin-Name: da7890ca6b1d8e511377a469047120220e8c3b2d --- manifest | 12 ++--- manifest.uuid | 2 +- tool/lemon.c | 118 ++++++++++++++++++++++++++++++++++++++------------ 3 files changed, 97 insertions(+), 35 deletions(-) diff --git a/manifest b/manifest index edfa17a7d1..b56fe09692 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\suse\ssprintf(),\sstrcpy()\sor\sstrcat()\sin\sthe\simplementation\sof\sthe\nlemon\sparser\sgenerator\stool,\sto\savoid\scompiler\swarnings\sin\sOpenBSD. -D 2014-01-10T23:21:00.243 +C In\sLEMON,\sfix\sa\sbug\sin\sthe\stext\sformatter\sintroduced\sby\sthe\sprevious\ncommit.\s\sAlso\sadd\sthe\snew\s"%token_class"\sdirective\sfor\sdefining\ssymbolic\nnames\sthat\sstand\sany\sone\sof\sa\scollection\sof\stokens. +D 2014-01-11T03:06:18.172 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1109,7 +1109,7 @@ F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c 04f60c891f6c07643221b068bbc2621be46c039a +F tool/lemon.c 4a3d4a579c5dff6a42785e97d1f2b59789f3b8dd F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7f1e7ae313c7625ef2623d78883dce776eecca30 -R 153a0201d8a4a180b2a188d3abd169a6 +P e43c522dde01e134f1adc94f534d2b3eda74afc2 +R 498b2eca2a4d9bdf3c22bba6f4386191 U drh -Z c49dfd6bd62bb08d757d02ffce1c49c4 +Z 303f39885598d1865b7eb401105ea9ea diff --git a/manifest.uuid b/manifest.uuid index fdbe371621..2d1234c95c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e43c522dde01e134f1adc94f534d2b3eda74afc2 \ No newline at end of file +da7890ca6b1d8e511377a469047120220e8c3b2d \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 191ab0d88b..25787abf8d 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -71,12 +71,15 @@ static void lemon_addtext( char *zBuf, /* The buffer to which text is added */ int *pnUsed, /* Slots of the buffer used so far */ const char *zIn, /* Text to add */ - int nIn /* Bytes of text to add. -1 to use strlen() */ + int nIn, /* Bytes of text to add. -1 to use strlen() */ + int iWidth /* Field width. Negative to left justify */ ){ if( nIn<0 ) for(nIn=0; zIn[nIn]; nIn++){} + while( iWidth>nIn ){ zBuf[*(pnUsed++)] = ' '; iWidth--; } if( nIn==0 ) return; memcpy(&zBuf[*pnUsed], zIn, nIn); *pnUsed += nIn; + while( (-iWidth)>nIn ){ zBuf[*(pnUsed++)] = ' '; iWidth++; } zBuf[*pnUsed] = 0; } static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ @@ -87,15 +90,22 @@ static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ str[0] = 0; for(i=j=0; (c = zFormat[i])!=0; i++){ if( c=='%' ){ - lemon_addtext(str, &nUsed, &zFormat[j], i-j); + int iWidth = 0; + lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); c = zFormat[++i]; + if( isdigit(c) || (c=='-' && isdigit(zFormat[i+1])) ){ + if( c=='-' ) i++; + while( isdigit(zFormat[i]) ) iWidth = iWidth*10 + zFormat[i++] - '0'; + if( c=='-' ) iWidth = -iWidth; + c = zFormat[i]; + } if( c=='d' ){ int v = va_arg(ap, int); if( v<0 ){ - lemon_addtext(str, &nUsed, "-", 1); + lemon_addtext(str, &nUsed, "-", 1, iWidth); v = -v; }else if( v==0 ){ - lemon_addtext(str, &nUsed, "0", 1); + lemon_addtext(str, &nUsed, "0", 1, iWidth); } k = 0; while( v>0 ){ @@ -103,17 +113,17 @@ static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ zTemp[sizeof(zTemp)-k] = (v%10) + '0'; v /= 10; } - lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k); + lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k, iWidth); }else if( c=='s' ){ z = va_arg(ap, const char*); - lemon_addtext(str, &nUsed, z, -1); + lemon_addtext(str, &nUsed, z, -1, iWidth); }else if( c=='.' && memcmp(&zFormat[i], ".*s", 3)==0 ){ i += 2; k = va_arg(ap, int); z = va_arg(ap, const char*); - lemon_addtext(str, &nUsed, z, k); + lemon_addtext(str, &nUsed, z, k, iWidth); }else if( c=='%' ){ - lemon_addtext(str, &nUsed, "%", 1); + lemon_addtext(str, &nUsed, "%", 1, 0); }else{ fprintf(stderr, "illegal format\n"); exit(1); @@ -121,7 +131,7 @@ static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ j = i+1; } } - lemon_addtext(str, &nUsed, &zFormat[j], i-j); + lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0); return nUsed; } static int lemon_sprintf(char *str, const char *format, ...){ @@ -1538,12 +1548,15 @@ int main(int argc, char **argv) } /* Count and index the symbols of the grammar */ - lem.nsymbol = Symbol_count(); Symbol_new("{default}"); + lem.nsymbol = Symbol_count(); lem.symbols = Symbol_arrayof(); - for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i; - qsort(lem.symbols,lem.nsymbol+1,sizeof(struct symbol*), Symbolcmpp); - for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i; + for(i=0; iindex = i; + qsort(lem.symbols,lem.nsymbol,sizeof(struct symbol*), Symbolcmpp); + for(i=0; iindex = i; + while( lem.symbols[i-1]->type==MULTITERMINAL ){ i--; } + assert( strcmp(lem.symbols[i-1]->name,"{default}")==0 ); + lem.nsymbol = i - 1; for(i=1; isupper(lem.symbols[i]->name[0]); i++); lem.nterminal = i; @@ -2031,7 +2044,9 @@ enum e_state { WAITING_FOR_DESTRUCTOR_SYMBOL, WAITING_FOR_DATATYPE_SYMBOL, WAITING_FOR_FALLBACK_ID, - WAITING_FOR_WILDCARD_ID + WAITING_FOR_WILDCARD_ID, + WAITING_FOR_CLASS_ID, + WAITING_FOR_CLASS_TOKEN }; struct pstate { char *filename; /* Name of the input file */ @@ -2041,6 +2056,7 @@ struct pstate { struct lemon *gp; /* Global state vector */ enum e_state state; /* The state of the parser */ struct symbol *fallback; /* The fallback token */ + struct symbol *tkclass; /* Token class symbol */ struct symbol *lhs; /* Left-hand side of current rule */ const char *lhsalias; /* Alias for the LHS */ int nrhs; /* Number of right-hand side symbols seen */ @@ -2345,6 +2361,8 @@ to follow the previous rule."); psp->state = WAITING_FOR_FALLBACK_ID; }else if( strcmp(x,"wildcard")==0 ){ psp->state = WAITING_FOR_WILDCARD_ID; + }else if( strcmp(x,"token_class")==0 ){ + psp->state = WAITING_FOR_CLASS_ID; }else{ ErrorMsg(psp->filename,psp->tokenlineno, "Unknown declaration keyword: \"%%%s\".",x); @@ -2513,6 +2531,40 @@ to follow the previous rule."); } } break; + case WAITING_FOR_CLASS_ID: + if( !islower(x[0]) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class must be followed by an identifier: ", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else if( Symbol_find(x) ){ + ErrorMsg(psp->filename, psp->tokenlineno, + "Symbol \"%s\" already used", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + }else{ + psp->tkclass = Symbol_new(x); + psp->tkclass->type = MULTITERMINAL; + psp->state = WAITING_FOR_CLASS_TOKEN; + } + break; + case WAITING_FOR_CLASS_TOKEN: + if( x[0]=='.' ){ + psp->state = WAITING_FOR_DECL_OR_RULE; + }else if( isupper(x[0]) || ((x[0]=='|' || x[0]=='/') && isupper(x[1])) ){ + struct symbol *msp = psp->tkclass; + msp->nsubsym++; + msp->subsym = (struct symbol **) realloc(msp->subsym, + sizeof(struct symbol*)*msp->nsubsym); + if( !isupper(x[0]) ) x++; + msp->subsym[msp->nsubsym-1] = Symbol_new(x); + }else{ + ErrorMsg(psp->filename, psp->tokenlineno, + "%%token_class argument \"%s\" should be a token", x); + psp->errorcnt++; + psp->state = RESYNC_AFTER_DECL_ERROR; + } + break; case RESYNC_AFTER_RULE_ERROR: /* if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE; ** break; */ @@ -2867,11 +2919,13 @@ void Reprint(struct lemon *lemp) printf(" ::="); for(i=0; inrhs; i++){ sp = rp->rhs[i]; - printf(" %s", sp->name); if( sp->type==MULTITERMINAL ){ + printf(" %s", sp->subsym[0]->name); for(j=1; jnsubsym; j++){ printf("|%s", sp->subsym[j]->name); } + }else{ + printf(" %s", sp->name); } /* if( rp->rhsalias[i] ) printf("(%s)",rp->rhsalias[i]); */ } @@ -2893,11 +2947,13 @@ void ConfigPrint(FILE *fp, struct config *cfp) if( i==cfp->dot ) fprintf(fp," *"); if( i==rp->nrhs ) break; sp = rp->rhs[i]; - fprintf(fp," %s", sp->name); if( sp->type==MULTITERMINAL ){ + fprintf(fp," %s", sp->subsym[0]->name); for(j=1; jnsubsym; j++){ fprintf(fp,"|%s",sp->subsym[j]->name); } + }else{ + fprintf(fp," %s", sp->name); } } } @@ -3644,9 +3700,11 @@ static void writeRuleText(FILE *out, struct rule *rp){ fprintf(out,"%s ::=", rp->lhs->name); for(j=0; jnrhs; j++){ struct symbol *sp = rp->rhs[j]; - fprintf(out," %s", sp->name); - if( sp->type==MULTITERMINAL ){ + if( sp->type!=MULTITERMINAL ){ + fprintf(out," %s", sp->name); + }else{ int k; + fprintf(out," %s", sp->subsym[0]->name); for(k=1; knsubsym; k++){ fprintf(out,"|%s",sp->subsym[k]->name); } @@ -4114,7 +4172,8 @@ void ReportHeader(struct lemon *lemp) if( in ){ int nextChar; for(i=1; interminal && fgets(line,LINESIZE,in); i++){ - lemon_sprintf(pattern,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); + lemon_sprintf(pattern,"#define %s%-30s %3d\n", + prefix,lemp->symbols[i]->name,i); if( strcmp(line,pattern) ) break; } nextChar = fgetc(in); @@ -4127,7 +4186,7 @@ void ReportHeader(struct lemon *lemp) out = file_open(lemp,".h","wb"); if( out ){ for(i=1; interminal; i++){ - fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i); + fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i); } fclose(out); } @@ -4497,11 +4556,15 @@ struct symbol *Symbol_new(const char *x) return sp; } -/* Compare two symbols for working purposes +/* Compare two symbols for sorting purposes. Return negative, +** zero, or positive if a is less then, equal to, or greater +** than b. ** ** Symbols that begin with upper case letters (terminals or tokens) ** must sort before symbols that begin with lower case letters -** (non-terminals). Other than that, the order does not matter. +** (non-terminals). And MULTITERMINAL symbols (created using the +** %token_class directive) must sort at the very end. Other than +** that, the order does not matter. ** ** We find experimentally that leaving the symbols in their original ** order (the order they appeared in the grammar file) gives the @@ -4509,12 +4572,11 @@ struct symbol *Symbol_new(const char *x) */ int Symbolcmpp(const void *_a, const void *_b) { - const struct symbol **a = (const struct symbol **) _a; - const struct symbol **b = (const struct symbol **) _b; - int i1 = (**a).index + 10000000*((**a).name[0]>'Z'); - int i2 = (**b).index + 10000000*((**b).name[0]>'Z'); - assert( i1!=i2 || strcmp((**a).name,(**b).name)==0 ); - return i1-i2; + const struct symbol *a = *(const struct symbol **) _a; + const struct symbol *b = *(const struct symbol **) _b; + int i1 = a->type==MULTITERMINAL ? 3 : a->name[0]>'Z' ? 2 : 1; + int i2 = b->type==MULTITERMINAL ? 3 : b->name[0]>'Z' ? 2 : 1; + return i1==i2 ? a->index - b->index : i1 - i2; } /* There is one instance of the following structure for each From f59b12fbc1db8c734cbe2b659a30155937887122 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Jan 2014 03:54:05 +0000 Subject: [PATCH 21/74] Optimizations to the SQL language grammar that result in a small size reduction and speed increase. FossilOrigin-Name: cb5d1f83e0a33d546d4c0cb817ef1f8440d1f738 --- addopcodes.awk | 1 + manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/parse.y | 44 ++++++++++++++++++++------------------------ src/tokenize.c | 15 +++------------ 5 files changed, 33 insertions(+), 45 deletions(-) diff --git a/addopcodes.awk b/addopcodes.awk index c18c4f3599..dcd31eff84 100644 --- a/addopcodes.awk +++ b/addopcodes.awk @@ -30,4 +30,5 @@ END { printf "#define TK_%-29s %4d\n", "AGG_COLUMN", ++max printf "#define TK_%-29s %4d\n", "UMINUS", ++max printf "#define TK_%-29s %4d\n", "UPLUS", ++max + printf "#define TK_%-29s %4d\n", "REGISTER", ++max } diff --git a/manifest b/manifest index 7c63bcd427..bac5d7c5b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\s"%token_class"\sdirective\sto\sthe\sLEMON\sparser\sgenerator.\s\sThis\sopens\sup\nthe\spossibility\sof\ssimplifying\sthe\sparser.\s\sAlso\sremove\sall\scalls\sto\s\nsprintf(),\sstrcpy(),\sand\sstrcat()\sfrom\sLEMON\sto\savoid\scompiler\swarnings\non\sOpenBSD.\s\s(Aside:\s\sIt\sis\sthis\schange\sto\savoid\sharmless\scompiler\swarnings\nthat\swas\sthe\scause\sof\sthe\sreason\sspat\sof\sbugs.) -D 2014-01-11T03:27:37.624 +C Optimizations\sto\sthe\sSQL\slanguage\sgrammar\sthat\sresult\sin\sa\ssmall\ssize\nreduction\sand\sspeed\sincrease. +D 2014-01-11T03:54:05.594 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -8,7 +8,7 @@ F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION 8ed548d87d0a27fd7d7620476f9e25f9fa742d73 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 -F addopcodes.awk 87ca612393d0f439550634bd2c156ea9ff6195ae +F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 F art/sqlite370.ico af56c1d00fee7cd4753e8631ed60703ed0fc6e90 F art/sqlite370.jpg d512473dae7e378a67e28ff96a34da7cb331def2 @@ -209,7 +209,7 @@ F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968 F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 60baa3aced02c9f91259719d196315bc28580719 +F src/parse.y 3c5384533a8bfce5abd256cc9cb2c38bec05ad61 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b @@ -274,7 +274,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/tokenize.c ec4c1a62b890bf1dbcdb966399e140b904c700a4 +F src/tokenize.c 5d04a1b7d1fe7e18556a869788f5d3e132a586b6 F src/trigger.c 5c1c0b899ac0ce284763dcb8fdbaa38ecf15ef98 F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7f1e7ae313c7625ef2623d78883dce776eecca30 da7890ca6b1d8e511377a469047120220e8c3b2d -R 4ba60b3fb47af12fdb784f3275ccdc2b +P 8eb48c04bd0a14031488b3160fde67307eb8b35d +R 754b7dd57633ea486f18a04af0e67e46 U drh -Z 11bef21bd681893c7b5fd9fec7621236 +Z 0bb5f8caa9532d2db9247df525167cc7 diff --git a/manifest.uuid b/manifest.uuid index f08dc2b5ea..1bd734a197 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8eb48c04bd0a14031488b3160fde67307eb8b35d \ No newline at end of file +cb5d1f83e0a33d546d4c0cb817ef1f8440d1f738 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 154929c8d9..f8e813d115 100644 --- a/src/parse.y +++ b/src/parse.y @@ -194,9 +194,7 @@ columnid(A) ::= nm(X). { // An IDENTIFIER can be a generic identifier, or one of several // keywords. Any non-standard keyword can also be an identifier. // -%type id {Token} -id(A) ::= ID(X). {A = X;} -id(A) ::= INDEXED(X). {A = X;} +%token_class id ID|INDEXED. // The following directive causes tokens ABORT, AFTER, ASC, etc. to // fallback to ID if they will not parse as their original value. @@ -241,8 +239,7 @@ id(A) ::= INDEXED(X). {A = X;} // And "ids" is an identifer-or-string. // -%type ids {Token} -ids(A) ::= ID|STRING(X). {A = X;} +%token_class ids ID|STRING. // The name of a column or table can be any of the following: // @@ -776,22 +773,22 @@ expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). { } term(A) ::= INTEGER|FLOAT|BLOB(X). {spanExpr(&A, pParse, @X, &X);} term(A) ::= STRING(X). {spanExpr(&A, pParse, @X, &X);} -expr(A) ::= REGISTER(X). { - /* When doing a nested parse, one can include terms in an expression - ** that look like this: #1 #2 ... These terms refer to registers - ** in the virtual machine. #N is the N-th register. */ - if( pParse->nested==0 ){ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &X); - A.pExpr = 0; - }else{ - A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &X); - if( A.pExpr ) sqlite3GetInt32(&X.z[1], &A.pExpr->iTable); - } - spanSet(&A, &X, &X); -} expr(A) ::= VARIABLE(X). { - spanExpr(&A, pParse, TK_VARIABLE, &X); - sqlite3ExprAssignVarNumber(pParse, A.pExpr); + if( X.n>=2 && X.z[0]=='#' && sqlite3Isdigit(X.z[1]) ){ + /* When doing a nested parse, one can include terms in an expression + ** that look like this: #1 #2 ... These terms refer to registers + ** in the virtual machine. #N is the N-th register. */ + if( pParse->nested==0 ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &X); + A.pExpr = 0; + }else{ + A.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &X); + if( A.pExpr ) sqlite3GetInt32(&X.z[1], &A.pExpr->iTable); + } + }else{ + spanExpr(&A, pParse, TK_VARIABLE, &X); + sqlite3ExprAssignVarNumber(pParse, A.pExpr); + } spanSet(&A, &X, &X); } expr(A) ::= expr(E) COLLATE ids(C). { @@ -805,7 +802,7 @@ expr(A) ::= CAST(X) LP expr(E) AS typetoken(T) RP(Y). { spanSet(&A,&X,&Y); } %endif SQLITE_OMIT_CAST -expr(A) ::= ID(X) LP distinct(D) exprlist(Y) RP(E). { +expr(A) ::= id(X) LP distinct(D) exprlist(Y) RP(E). { if( Y && Y->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ sqlite3ErrorMsg(pParse, "too many arguments on function %T", &X); } @@ -815,7 +812,7 @@ expr(A) ::= ID(X) LP distinct(D) exprlist(Y) RP(E). { A.pExpr->flags |= EP_Distinct; } } -expr(A) ::= ID(X) LP STAR RP(E). { +expr(A) ::= id(X) LP STAR RP(E). { A.pExpr = sqlite3ExprFunction(pParse, 0, &X); spanSet(&A,&X,&E); } @@ -1165,11 +1162,10 @@ nmnum(A) ::= ON(X). {A = X;} nmnum(A) ::= DELETE(X). {A = X;} nmnum(A) ::= DEFAULT(X). {A = X;} %endif SQLITE_OMIT_PRAGMA +%token_class number INTEGER|FLOAT. plus_num(A) ::= PLUS number(X). {A = X;} plus_num(A) ::= number(X). {A = X;} minus_num(A) ::= MINUS number(X). {A = X;} -number(A) ::= INTEGER|FLOAT(X). {A = X;} - //////////////////////////// The CREATE TRIGGER command ///////////////////// %ifndef SQLITE_OMIT_TRIGGER diff --git a/src/tokenize.c b/src/tokenize.c index f27929ea5b..6e796ef82e 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -303,24 +303,15 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){ for(i=1; sqlite3Isdigit(z[i]); i++){} return i; } - case '#': { - for(i=1; sqlite3Isdigit(z[i]); i++){} - if( i>1 ){ - /* Parameters of the form #NNN (where NNN is a number) are used - ** internally by sqlite3NestedParse. */ - *tokenType = TK_REGISTER; - return i; - } - /* Fall through into the next case if the '#' is not followed by - ** a digit. Try to match #AAAA where AAAA is a parameter name. */ - } #ifndef SQLITE_OMIT_TCL_VARIABLE case '$': #endif case '@': /* For compatibility with MS SQL Server */ + case '#': case ':': { int n = 0; - testcase( z[0]=='$' ); testcase( z[0]=='@' ); testcase( z[0]==':' ); + testcase( z[0]=='$' ); testcase( z[0]=='@' ); + testcase( z[0]==':' ); testcase( z[0]=='#' ); *tokenType = TK_VARIABLE; for(i=1; (c=z[i])!=0; i++){ if( IdChar(c) ){ From 03e1b1f5fffb0d9aa621db37132e6b1cc711238c Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 11 Jan 2014 12:52:25 +0000 Subject: [PATCH 22/74] In LEMON, limit the size of the grammar file to 100MB. This ensures that the program will never experience integer overflow. To be doubly sure, use calloc() instead of malloc() when allocating arrays. FossilOrigin-Name: 29ba458d849ad8864711cbe59fb10447a947e06a --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lemon.c | 31 +++++++++++-------------------- 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index bac5d7c5b5..03003dd0c5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Optimizations\sto\sthe\sSQL\slanguage\sgrammar\sthat\sresult\sin\sa\ssmall\ssize\nreduction\sand\sspeed\sincrease. -D 2014-01-11T03:54:05.594 +C In\sLEMON,\slimit\sthe\ssize\sof\sthe\sgrammar\sfile\sto\s100MB.\s\sThis\sensures\sthat\nthe\sprogram\swill\snever\sexperience\sinteger\soverflow.\s\sTo\sbe\sdoubly\ssure,\nuse\scalloc()\sinstead\sof\smalloc()\swhen\sallocating\sarrays. +D 2014-01-11T12:52:25.201 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1109,7 +1109,7 @@ F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c 624b24c5dc048e09979f88a03e148bc728c70b73 +F tool/lemon.c 6842b2e7af12835f9f6e55808a0b1861cd0696fe F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8eb48c04bd0a14031488b3160fde67307eb8b35d -R 754b7dd57633ea486f18a04af0e67e46 +P cb5d1f83e0a33d546d4c0cb817ef1f8440d1f738 +R 28679f157b50c114aa03e50f74a7a104 U drh -Z 0bb5f8caa9532d2db9247df525167cc7 +Z 608e7b6009060d93ac39bb2434b3c874 diff --git a/manifest.uuid b/manifest.uuid index 1bd734a197..d8b23d3f34 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cb5d1f83e0a33d546d4c0cb817ef1f8440d1f738 \ No newline at end of file +29ba458d849ad8864711cbe59fb10447a947e06a \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index c4adc20864..8dca20c451 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -2659,9 +2659,8 @@ void Parse(struct lemon *gp) filesize = ftell(fp); rewind(fp); filebuf = (char *)malloc( filesize+1 ); - if( filebuf==0 ){ - ErrorMsg(ps.filename,0,"Can't allocate %d of memory to hold this file.", - filesize+1); + if( filesize>100000000 || filebuf==0 ){ + ErrorMsg(ps.filename,0,"Input file too large."); gp->errorcnt++; fclose(fp); return; @@ -4442,8 +4441,7 @@ void Strsafe_init(){ if( x1a ){ x1a->size = 1024; x1a->count = 0; - x1a->tbl = (x1node*)malloc( - (sizeof(x1node) + sizeof(x1node*))*1024 ); + x1a->tbl = (x1node*)calloc(1024, sizeof(x1node) + sizeof(x1node*)); if( x1a->tbl==0 ){ free(x1a); x1a = 0; @@ -4480,8 +4478,7 @@ int Strsafe_insert(const char *data) struct s_x1 array; array.size = size = x1a->size*2; array.count = x1a->count; - array.tbl = (x1node*)malloc( - (sizeof(x1node) + sizeof(x1node*))*size ); + array.tbl = (x1node*)calloc(size, sizeof(x1node) + sizeof(x1node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x1node**)&(array.tbl[size]); for(i=0; isize = 128; x2a->count = 0; - x2a->tbl = (x2node*)malloc( - (sizeof(x2node) + sizeof(x2node*))*128 ); + x2a->tbl = (x2node*)calloc(128, sizeof(x2node) + sizeof(x2node*)); if( x2a->tbl==0 ){ free(x2a); x2a = 0; @@ -4649,8 +4645,7 @@ int Symbol_insert(struct symbol *data, const char *key) struct s_x2 array; array.size = size = x2a->size*2; array.count = x2a->count; - array.tbl = (x2node*)malloc( - (sizeof(x2node) + sizeof(x2node*))*size ); + array.tbl = (x2node*)calloc(size, sizeof(x2node) + sizeof(x2node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x2node**)&(array.tbl[size]); for(i=0; isize = 128; x3a->count = 0; - x3a->tbl = (x3node*)malloc( - (sizeof(x3node) + sizeof(x3node*))*128 ); + x3a->tbl = (x3node*)calloc(128, sizeof(x3node) + sizeof(x3node*)); if( x3a->tbl==0 ){ free(x3a); x3a = 0; @@ -4848,8 +4842,7 @@ int State_insert(struct state *data, struct config *key) struct s_x3 array; array.size = size = x3a->size*2; array.count = x3a->count; - array.tbl = (x3node*)malloc( - (sizeof(x3node) + sizeof(x3node*))*size ); + array.tbl = (x3node*)calloc(size, sizeof(x3node) + sizeof(x3node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x3node**)&(array.tbl[size]); for(i=0; icount; - array = (struct state **)malloc( sizeof(struct state *)*size ); + array = (struct state **)calloc(size, sizeof(struct state *)); if( array ){ for(i=0; itbl[i].data; } @@ -4952,8 +4945,7 @@ void Configtable_init(){ if( x4a ){ x4a->size = 64; x4a->count = 0; - x4a->tbl = (x4node*)malloc( - (sizeof(x4node) + sizeof(x4node*))*64 ); + x4a->tbl = (x4node*)calloc(64, sizeof(x4node) + sizeof(x4node*)); if( x4a->tbl==0 ){ free(x4a); x4a = 0; @@ -4990,8 +4982,7 @@ int Configtable_insert(struct config *data) struct s_x4 array; array.size = size = x4a->size*2; array.count = x4a->count; - array.tbl = (x4node*)malloc( - (sizeof(x4node) + sizeof(x4node*))*size ); + array.tbl = (x4node*)calloc(size, sizeof(x4node) + sizeof(x4node*)); if( array.tbl==0 ) return 0; /* Fail due to malloc failure */ array.ht = (x4node**)&(array.tbl[size]); for(i=0; i Date: Sat, 11 Jan 2014 13:22:17 +0000 Subject: [PATCH 23/74] Parse common table expressions. But do not do anything with them (yet). FossilOrigin-Name: da98b7205eb3d7ec2ddbf8a8e24eee0b2ff499a5 --- manifest | 21 ++++++++++++--------- manifest.uuid | 2 +- src/build.c | 21 +++++++++++++++++++++ src/parse.y | 39 +++++++++++++++++++++++++-------------- src/sqliteInt.h | 4 ++++ tool/mkkeywordhash.c | 7 +++++++ 6 files changed, 70 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 03003dd0c5..16d294c829 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sLEMON,\slimit\sthe\ssize\sof\sthe\sgrammar\sfile\sto\s100MB.\s\sThis\sensures\sthat\nthe\sprogram\swill\snever\sexperience\sinteger\soverflow.\s\sTo\sbe\sdoubly\ssure,\nuse\scalloc()\sinstead\sof\smalloc()\swhen\sallocating\sarrays. -D 2014-01-11T12:52:25.201 +C Parse\scommon\stable\sexpressions.\s\sBut\sdo\snot\sdo\sanything\swith\sthem\s(yet). +D 2014-01-11T13:22:17.016 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,7 +169,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 11e29ef8cf16a42925fde036bcffbeffd9cc82df F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 -F src/build.c 8c56d91447770a746b16d08a6510109c161dbc1a +F src/build.c 198670a78fe748b6e60d6345a8aa1db57794511a F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd @@ -209,7 +209,7 @@ F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968 F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 3c5384533a8bfce5abd256cc9cb2c38bec05ad61 +F src/parse.y 0ccd364dd3670ad5763a5e94dd11196ddc10927a F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b @@ -224,7 +224,7 @@ F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 0c65967bb807dee3c9eef2bbd17f880eeb28ea30 +F src/sqliteInt.h bb093076ba1eb956330cca0ca2a206c91facab4b F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1113,7 +1113,7 @@ F tool/lemon.c 6842b2e7af12835f9f6e55808a0b1861cd0696fe F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 -F tool/mkkeywordhash.c 189d76644e373c7d0864c628deb8ce7b4f403591 +F tool/mkkeywordhash.c c9e05e4a7bcab8fab9f583d5b321fb72f565ad97 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97 @@ -1148,7 +1148,10 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P cb5d1f83e0a33d546d4c0cb817ef1f8440d1f738 -R 28679f157b50c114aa03e50f74a7a104 +P 29ba458d849ad8864711cbe59fb10447a947e06a +R b8f42a0591e43b680c1e09056e8afd6e +T *branch * common-table-expr +T *sym-common-table-expr * +T -sym-trunk * U drh -Z 608e7b6009060d93ac39bb2434b3c874 +Z 9a38087dfe69ad33926bbf7c01c0b22a diff --git a/manifest.uuid b/manifest.uuid index d8b23d3f34..17610fd9e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29ba458d849ad8864711cbe59fb10447a947e06a \ No newline at end of file +da98b7205eb3d7ec2ddbf8a8e24eee0b2ff499a5 \ No newline at end of file diff --git a/src/build.c b/src/build.c index fa82d56cc7..4fb4b4efa9 100644 --- a/src/build.c +++ b/src/build.c @@ -4198,3 +4198,24 @@ KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ } return sqlite3KeyInfoRef(pIdx->pKeyInfo); } + +#ifndef SQLITE_OMIT_CTE +/* This routine is invoked when a single with_query of a +** common-table-expression has been parsed. Record the query. +*/ +void sqlite3CteAdd( + Parse *pParse, /* Parsing context */ + Token *pName, /* Name of the common-table */ + ExprList *pArgList, /* Optional column name list for the table */ + Select *pQuery /* Query used to initialize the table */ +){ + sqlite3ExprListDelete(pParse->db, pArgList); + sqlite3SelectDelete(pParse->db, pQuery); +} + +/* This routine is invoked at the end of the entire WITH clause. +*/ +void sqlite3CteFinish(Parse *pParse, int isRecursive){ + /* TBD */ +} +#endif /* !defined(SQLITE_OMIT_CTE) */ diff --git a/src/parse.y b/src/parse.y index f8e813d115..f4fa4b276d 100644 --- a/src/parse.y +++ b/src/parse.y @@ -204,8 +204,8 @@ columnid(A) ::= nm(X). { ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN - QUERY KEY OF OFFSET PRAGMA RAISE RELEASE REPLACE RESTRICT ROW ROLLBACK - SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITHOUT + QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW + ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT %ifdef SQLITE_OMIT_COMPOUND_SELECT EXCEPT INTERSECT UNION %endif SQLITE_OMIT_COMPOUND_SELECT @@ -396,7 +396,7 @@ cmd ::= DROP VIEW ifexists(E) fullname(X). { //////////////////////// The SELECT statement ///////////////////////////////// // -cmd ::= select(X). { +cmd ::= with select(X). { SelectDest dest = {SRT_Output, 0, 0, 0, 0}; sqlite3Select(pParse, X, &dest); sqlite3ExplainBegin(pParse->pVdbe); @@ -648,7 +648,7 @@ limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y). /////////////////////////// The DELETE statement ///////////////////////////// // %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W) +cmd ::= with DELETE FROM fullname(X) indexed_opt(I) where_opt(W) orderby_opt(O) limit_opt(L). { sqlite3SrcListIndexedBy(pParse, X, &I); W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "DELETE"); @@ -656,7 +656,7 @@ cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W) } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W). { +cmd ::= with DELETE FROM fullname(X) indexed_opt(I) where_opt(W). { sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3DeleteFrom(pParse,X,W); } @@ -671,8 +671,8 @@ where_opt(A) ::= WHERE expr(X). {A = X.pExpr;} ////////////////////////// The UPDATE command //////////////////////////////// // %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) - orderby_opt(O) limit_opt(L). { +cmd ::= with UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) + where_opt(W) orderby_opt(O) limit_opt(L). { sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE"); @@ -680,7 +680,7 @@ cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) +cmd ::= with UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W). { sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); @@ -702,9 +702,9 @@ setlist(A) ::= nm(X) EQ expr(Y). { ////////////////////////// The INSERT command ///////////////////////////////// // -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). +cmd ::= with insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). {sqlite3Insert(pParse, X, S, F, R);} -cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. +cmd ::= with insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. {sqlite3Insert(pParse, X, 0, F, R);} %type insert_cmd {u8} @@ -851,10 +851,8 @@ expr(A) ::= expr(X) STAR|SLASH|REM(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);} %type likeop {struct LikeOp} -likeop(A) ::= LIKE_KW(X). {A.eOperator = X; A.bNot = 0;} -likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.bNot = 1;} -likeop(A) ::= MATCH(X). {A.eOperator = X; A.bNot = 0;} -likeop(A) ::= NOT MATCH(X). {A.eOperator = X; A.bNot = 1;} +likeop(A) ::= LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 0;} +likeop(A) ::= NOT LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 1;} expr(A) ::= expr(X) likeop(OP) expr(Y). [LIKE_KW] { ExprList *pList; pList = sqlite3ExprListAppend(pParse,0, Y.pExpr); @@ -1364,3 +1362,16 @@ anylist ::= . anylist ::= anylist LP anylist RP. anylist ::= anylist ANY. %endif SQLITE_OMIT_VIRTUALTABLE + + +//////////////////////// COMMON TABLE EXPRESSIONS //////////////////////////// +with ::= . +%ifndef SQLITE_OMIT_CTE +with ::= WITH wqlist. {sqlite3CteFinish(pParse, 0);} +with ::= WITH RECURSIVE wqlist. {sqlite3CteFinish(pParse, 1);} +wqlist ::= with_query. +wqlist ::= wqlist COMMA with_query. +with_query ::= nm(X) idxlist_opt(Y) AS LP select(Z) RP. { + sqlite3CteAdd(pParse, &X, Y, Z); +} +%endif SQLITE_OMIT_CTE diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 188a7cacc4..d517789da0 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3329,6 +3329,10 @@ const char *sqlite3JournalModename(int); int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); #endif +#ifndef SQLITE_OMIT_CTE + void sqlite3CteAdd(Parse*,Token*,ExprList*,Select*); + void sqlite3CteFinish(Parse*,int); +#endif /* Declarations for functions in fkey.c. All of these are replaced by ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign diff --git a/tool/mkkeywordhash.c b/tool/mkkeywordhash.c index f3ef73f394..a467931c30 100644 --- a/tool/mkkeywordhash.c +++ b/tool/mkkeywordhash.c @@ -138,6 +138,11 @@ struct Keyword { #else # define AUTOVACUUM 0x00020000 #endif +#ifdef SQLITE_OMIT_CTE +# define CTE 0 +#else +# define CTE 0x00040000 +#endif /* ** These are the keywords @@ -234,6 +239,7 @@ static Keyword aKeywordTable[] = { { "PRIMARY", "TK_PRIMARY", ALWAYS }, { "QUERY", "TK_QUERY", EXPLAIN }, { "RAISE", "TK_RAISE", TRIGGER }, + { "RECURSIVE", "TK_RECURSIVE", CTE }, { "REFERENCES", "TK_REFERENCES", FKEY }, { "REGEXP", "TK_LIKE_KW", ALWAYS }, { "REINDEX", "TK_REINDEX", REINDEX }, @@ -262,6 +268,7 @@ static Keyword aKeywordTable[] = { { "VALUES", "TK_VALUES", ALWAYS }, { "VIEW", "TK_VIEW", VIEW }, { "VIRTUAL", "TK_VIRTUAL", VTAB }, + { "WITH", "TK_WITH", CTE }, { "WITHOUT", "TK_WITHOUT", ALWAYS }, { "WHEN", "TK_WHEN", ALWAYS }, { "WHERE", "TK_WHERE", ALWAYS }, From 7d562dbe023a06bc8b7116c5415c5cb9b6a25cfc Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 11 Jan 2014 19:19:36 +0000 Subject: [PATCH 24/74] Update the parser so that sub-queries and CTEs may have WITH clauses. FossilOrigin-Name: 704d3931b855562a619769955969d439c42ca406 --- manifest | 22 ++++++++++------------ manifest.uuid | 2 +- src/build.c | 18 +++++++++++------- src/parse.y | 30 ++++++++++++++++++++---------- src/sqliteInt.h | 18 ++++++++++++++++-- test/with1.test | 40 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 98 insertions(+), 32 deletions(-) create mode 100644 test/with1.test diff --git a/manifest b/manifest index 16d294c829..c9d03126cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Parse\scommon\stable\sexpressions.\s\sBut\sdo\snot\sdo\sanything\swith\sthem\s(yet). -D 2014-01-11T13:22:17.016 +C Update\sthe\sparser\sso\sthat\ssub-queries\sand\sCTEs\smay\shave\sWITH\sclauses. +D 2014-01-11T19:19:36.147 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,7 +169,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 11e29ef8cf16a42925fde036bcffbeffd9cc82df F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 -F src/build.c 198670a78fe748b6e60d6345a8aa1db57794511a +F src/build.c 1da29e4e93d9774599f1f791d011ba46d841000c F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd @@ -209,7 +209,7 @@ F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968 F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 0ccd364dd3670ad5763a5e94dd11196ddc10927a +F src/parse.y 1e3fd22fc82930d0cce01f2ab90616100c59ee0c F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b @@ -224,7 +224,7 @@ F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h bb093076ba1eb956330cca0ca2a206c91facab4b +F src/sqliteInt.h 2710b3a6c66edda9f193453268f6fe878ff5a0ca F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1091,6 +1091,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d +F test/with1.test e8146198319c3627bb21ef085c58a827f35686a7 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 F test/without_rowid3.test eac3d5c8a1924725b58503a368f2cbd24fd6c8a0 @@ -1148,10 +1149,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 29ba458d849ad8864711cbe59fb10447a947e06a -R b8f42a0591e43b680c1e09056e8afd6e -T *branch * common-table-expr -T *sym-common-table-expr * -T -sym-trunk * -U drh -Z 9a38087dfe69ad33926bbf7c01c0b22a +P da98b7205eb3d7ec2ddbf8a8e24eee0b2ff499a5 +R bbb9d541af414d7b8bb3f418deb50875 +U dan +Z a973a84816d55810384771a899b1092b diff --git a/manifest.uuid b/manifest.uuid index 17610fd9e7..f6a009da18 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -da98b7205eb3d7ec2ddbf8a8e24eee0b2ff499a5 \ No newline at end of file +704d3931b855562a619769955969d439c42ca406 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 4fb4b4efa9..23628f73be 100644 --- a/src/build.c +++ b/src/build.c @@ -4200,22 +4200,26 @@ KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ } #ifndef SQLITE_OMIT_CTE -/* This routine is invoked when a single with_query of a -** common-table-expression has been parsed. Record the query. +/* +** This routine is invoked once per CTE by the parser while parsing a +** WITH clause. */ -void sqlite3CteAdd( +With *sqlite3WithAdd( Parse *pParse, /* Parsing context */ + With *pWith, /* Existing WITH clause, or NULL */ Token *pName, /* Name of the common-table */ - ExprList *pArgList, /* Optional column name list for the table */ + IdList *pArglist, /* Optional column name list for the table */ Select *pQuery /* Query used to initialize the table */ ){ - sqlite3ExprListDelete(pParse->db, pArgList); + sqlite3IdListDelete(pParse->db, pArglist); sqlite3SelectDelete(pParse->db, pQuery); + return 0; } -/* This routine is invoked at the end of the entire WITH clause. +/* +** Free the contents of the With object passed as the second argument. */ -void sqlite3CteFinish(Parse *pParse, int isRecursive){ +void sqlite3WithDelete(sqlite3 *db, With *pWith){ /* TBD */ } #endif /* !defined(SQLITE_OMIT_CTE) */ diff --git a/src/parse.y b/src/parse.y index f4fa4b276d..c3cb2d62aa 100644 --- a/src/parse.y +++ b/src/parse.y @@ -396,7 +396,7 @@ cmd ::= DROP VIEW ifexists(E) fullname(X). { //////////////////////// The SELECT statement ///////////////////////////////// // -cmd ::= with select(X). { +cmd ::= select(X). { SelectDest dest = {SRT_Output, 0, 0, 0, 0}; sqlite3Select(pParse, X, &dest); sqlite3ExplainBegin(pParse->pVdbe); @@ -407,12 +407,16 @@ cmd ::= with select(X). { %type select {Select*} %destructor select {sqlite3SelectDelete(pParse->db, $$);} +%type selectnowith {Select*} +%destructor selectnowith {sqlite3SelectDelete(pParse->db, $$);} %type oneselect {Select*} %destructor oneselect {sqlite3SelectDelete(pParse->db, $$);} -select(A) ::= oneselect(X). {A = X;} +select(A) ::= with selectnowith(X). {A = X;} + +selectnowith(A) ::= oneselect(X). {A = X;} %ifndef SQLITE_OMIT_COMPOUND_SELECT -select(A) ::= select(X) multiselect_op(Y) oneselect(Z). { +selectnowith(A) ::= selectnowith(X) multiselect_op(Y) oneselect(Z). { if( Z ){ Z->op = (u8)Y; Z->pPrior = X; @@ -1365,13 +1369,19 @@ anylist ::= anylist ANY. //////////////////////// COMMON TABLE EXPRESSIONS //////////////////////////// -with ::= . +%type with {With*} +%type wqlist {With*} +%destructor with {sqlite3WithDelete(pParse->db, $$);} + +with(A) ::= . {A = 0;} %ifndef SQLITE_OMIT_CTE -with ::= WITH wqlist. {sqlite3CteFinish(pParse, 0);} -with ::= WITH RECURSIVE wqlist. {sqlite3CteFinish(pParse, 1);} -wqlist ::= with_query. -wqlist ::= wqlist COMMA with_query. -with_query ::= nm(X) idxlist_opt(Y) AS LP select(Z) RP. { - sqlite3CteAdd(pParse, &X, Y, Z); +with(A) ::= WITH wqlist(W). { A = W; } +with(A) ::= WITH RECURSIVE wqlist(W). { A = W; } + +wqlist(A) ::= nm(X) inscollist_opt(Y) AS LP select(Z) RP. { + A = sqlite3WithAdd(pParse, 0, &X, Y, Z); +} +wqlist(A) ::= wqlist(W) COMMA nm(X) inscollist_opt(Y) AS LP select(Z) RP. { + A = sqlite3WithAdd(pParse, W, &X, Y, Z); } %endif SQLITE_OMIT_CTE diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d517789da0..8846ffb04c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -760,6 +760,7 @@ typedef struct VTable VTable; typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; typedef struct WhereInfo WhereInfo; +typedef struct With With; /* ** Defer sourcing vdbe.h and btree.h until after the "u8" and @@ -2631,6 +2632,19 @@ int sqlite3WalkSelectFrom(Walker*, Select*); #define WRC_Prune 1 /* Omit children but continue walking siblings */ #define WRC_Abort 2 /* Abandon the tree walk */ +/* +** An instance of this structure represents a set of CTEs (common table +** expressions) created by a single WITH clause. +*/ +struct With { + int nCte; /* Number of CTEs */ + struct Cte { + const char *zName; /* Name of this CTE */ + IdList *pCol; /* List of explicit column names, or NULL */ + Select *pSelect; /* The contents of the CTE */ + } a[1]; +}; + /* ** Assuming zIn points to the first byte of a UTF-8 character, ** advance zIn to point to the first byte of the next UTF-8 character. @@ -3330,8 +3344,8 @@ const char *sqlite3JournalModename(int); int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); #endif #ifndef SQLITE_OMIT_CTE - void sqlite3CteAdd(Parse*,Token*,ExprList*,Select*); - void sqlite3CteFinish(Parse*,int); + With *sqlite3WithAdd(Parse*,With*,Token*,IdList*,Select*); + void sqlite3WithDelete(sqlite3*,With*); #endif /* Declarations for functions in fkey.c. All of these are replaced by diff --git a/test/with1.test b/test/with1.test new file mode 100644 index 0000000000..80e879ada0 --- /dev/null +++ b/test/with1.test @@ -0,0 +1,40 @@ +# 2014 January 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the WITH clause. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix with1 + +do_execsql_test 1.0 { + CREATE TABLE t1(x INTEGER, y INTEGER); + WITH x(a) AS ( SELECT * FROM t1) SELECT 10 +} {10} + +do_execsql_test 1.1 { + SELECT * FROM ( WITH x AS ( SELECT * FROM t1) SELECT 10 ); +} {10} + +do_execsql_test 1.2 { + WITH x(a) AS ( SELECT * FROM t1) INSERT INTO t1 VALUES(1,2); +} {} + +do_execsql_test 1.3 { + WITH x(a) AS ( SELECT * FROM t1) DELETE FROM t1; +} {} + +do_execsql_test 1.4 { + WITH x(a) AS ( SELECT * FROM t1) UPDATE t1 SET x = y; +} {} + +finish_test From 4e9119d9e8bc3ed4c75b2c77f9ec80c6cf5f5463 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Jan 2014 15:12:23 +0000 Subject: [PATCH 25/74] Add code to handle non-recursive CTEs in the same way as SQL views. FossilOrigin-Name: a26f399ba485e8127c276c5f103ec6c555e11734 --- manifest | 29 ++++++------ manifest.uuid | 2 +- src/build.c | 53 +++++++++++++++++++--- src/expr.c | 19 ++++++++ src/insert.c | 3 +- src/parse.y | 29 +++++++----- src/select.c | 87 ++++++++++++++++++++++++++++++++++-- src/sqliteInt.h | 14 ++++-- src/tokenize.c | 2 + test/pagerfault.test | 1 + test/with1.test | 103 +++++++++++++++++++++++++++++++++++++++++++ test/withM.test | 40 +++++++++++++++++ 12 files changed, 345 insertions(+), 37 deletions(-) create mode 100644 test/withM.test diff --git a/manifest b/manifest index c9d03126cd..010cb6fcfd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sparser\sso\sthat\ssub-queries\sand\sCTEs\smay\shave\sWITH\sclauses. -D 2014-01-11T19:19:36.147 +C Add\scode\sto\shandle\snon-recursive\sCTEs\sin\sthe\ssame\sway\sas\sSQL\sviews. +D 2014-01-13T15:12:23.149 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,13 +169,13 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 11e29ef8cf16a42925fde036bcffbeffd9cc82df F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 -F src/build.c 1da29e4e93d9774599f1f791d011ba46d841000c +F src/build.c 1efdc65020e1f566383b66b30ccf730b4ef6f926 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff -F src/expr.c 4115ad67088cdd55f4fa0ef3ddd22cb8da8f9c94 +F src/expr.c de86bf987c532ec9d63fb1bf8a6eb6ec2cf5ba69 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -183,7 +183,7 @@ F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c 5ddb48c7f1cb399993744f23f03948989ce1790e +F src/insert.c cb4c8ad02b6feb95d34614d94a3c68e0116fbf07 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b @@ -209,7 +209,7 @@ F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968 F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 1e3fd22fc82930d0cce01f2ab90616100c59ee0c +F src/parse.y b433588d7993d6475d9944be0fda2fb3df3cb138 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b @@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 819bb090c9a348d17f69f136cad2bfa9ee9cbb41 +F src/select.c 9dc9177bfb9278a603e3835e828ff60ec68fba6f F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 2710b3a6c66edda9f193453268f6fe878ff5a0ca +F src/sqliteInt.h a350bcf6d62204b6b4720ce1922c95a575a0b5ad F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -274,7 +274,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/tokenize.c 5d04a1b7d1fe7e18556a869788f5d3e132a586b6 +F src/tokenize.c 7dc42e9beb8c3263b79d10c195b3f5264b5f874a F src/trigger.c 5c1c0b899ac0ce284763dcb8fdbaa38ecf15ef98 F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 @@ -728,7 +728,7 @@ F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pager4.test b40ecb4cc7dff957ee7916e41ab21d1ed702a642 -F test/pagerfault.test 7285379906ab2f1108b8e82bbdf2d386cc8ff3ff +F test/pagerfault.test cee8488a935e42a4a2e241e0317dc2814c997783 F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6 @@ -1091,7 +1091,8 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test e8146198319c3627bb21ef085c58a827f35686a7 +F test/with1.test fb8409a35b1314be6e73a87597322f3369b59b2b +F test/withM.test 036349a9ec081fee8b1f72e71ad8a5a5b80f1674 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 F test/without_rowid3.test eac3d5c8a1924725b58503a368f2cbd24fd6c8a0 @@ -1149,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P da98b7205eb3d7ec2ddbf8a8e24eee0b2ff499a5 -R bbb9d541af414d7b8bb3f418deb50875 +P 704d3931b855562a619769955969d439c42ca406 +R fb5ad7a801e83802b92e3793bba4fd6c U dan -Z a973a84816d55810384771a899b1092b +Z 663a5542bd9dc2bb50ed6e3a07d040fb diff --git a/manifest.uuid b/manifest.uuid index f6a009da18..267d8a5499 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -704d3931b855562a619769955969d439c42ca406 \ No newline at end of file +a26f399ba485e8127c276c5f103ec6c555e11734 \ No newline at end of file diff --git a/src/build.c b/src/build.c index 23628f73be..ca184766f9 100644 --- a/src/build.c +++ b/src/build.c @@ -4208,18 +4208,61 @@ With *sqlite3WithAdd( Parse *pParse, /* Parsing context */ With *pWith, /* Existing WITH clause, or NULL */ Token *pName, /* Name of the common-table */ - IdList *pArglist, /* Optional column name list for the table */ + ExprList *pArglist, /* Optional column name list for the table */ Select *pQuery /* Query used to initialize the table */ ){ - sqlite3IdListDelete(pParse->db, pArglist); - sqlite3SelectDelete(pParse->db, pQuery); - return 0; + sqlite3 *db = pParse->db; + With *pNew; + char *zName; + + /* Check that the CTE name is unique within this WITH clause. If + ** not, store an error in the Parse structure. */ + zName = sqlite3NameFromToken(pParse->db, pName); + if( zName && pWith ){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){ + sqlite3ErrorMsg(pParse, "duplicate cte name: %s", zName); + } + } + } + + if( pWith ){ + int nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); + pNew = sqlite3DbRealloc(db, pWith, nByte); + }else{ + pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); + } + assert( zName!=0 || pNew==0 ); + + if( pNew==0 ){ + sqlite3WithDelete(db, pWith); + sqlite3ExprListDelete(db, pArglist); + sqlite3SelectDelete(db, pQuery); + sqlite3DbFree(db, zName); + }else{ + pNew->a[pNew->nCte].pSelect = pQuery; + pNew->a[pNew->nCte].pCols = pArglist; + pNew->a[pNew->nCte].zName = zName; + pNew->nCte++; + } + + return pNew; } /* ** Free the contents of the With object passed as the second argument. */ void sqlite3WithDelete(sqlite3 *db, With *pWith){ - /* TBD */ + if( pWith ){ + int i; + for(i=0; inCte; i++){ + struct Cte *pCte = &pWith->a[i]; + sqlite3ExprListDelete(db, pCte->pCols); + sqlite3SelectDelete(db, pCte->pSelect); + sqlite3DbFree(db, pCte->zName); + } + sqlite3DbFree(db, pWith); + } } #endif /* !defined(SQLITE_OMIT_CTE) */ diff --git a/src/expr.c b/src/expr.c index aad6cd1cda..d0ea280029 100644 --- a/src/expr.c +++ b/src/expr.c @@ -895,6 +895,24 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ return pNew; } +With *withDup(sqlite3 *db, With *p){ + With *pRet = 0; + if( p ){ + int nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + pRet = sqlite3DbMallocZero(db, nByte); + if( pRet ){ + int i; + pRet->nCte = p->nCte; + for(i=0; inCte; i++){ + pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0); + pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0); + pRet->a[i].zName = sqlite3DbStrDup(db, p->a[i].zName); + } + } + } + return pRet; +} + /* ** The following group of routines make deep copies of expressions, ** expression lists, ID lists, and select statements. The copies can @@ -1036,6 +1054,7 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->addrOpenEphm[2] = -1; + pNew->pWith = withDup(db, p->pWith); return pNew; } #else diff --git a/src/insert.c b/src/insert.c index 038d08a40d..b4bd6b7160 100644 --- a/src/insert.c +++ b/src/insert.c @@ -667,7 +667,8 @@ void sqlite3Insert( ** ** This is the 2nd template. */ - if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ + if( pColumn==0 && pParse->pWith==0 + && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ assert( !pTrigger ); assert( pList==0 ); goto insert_end; diff --git a/src/parse.y b/src/parse.y index c3cb2d62aa..1c6465e933 100644 --- a/src/parse.y +++ b/src/parse.y @@ -412,7 +412,7 @@ cmd ::= select(X). { %type oneselect {Select*} %destructor oneselect {sqlite3SelectDelete(pParse->db, $$);} -select(A) ::= with selectnowith(X). {A = X;} +select(A) ::= with(W) selectnowith(X). { if( X ) X->pWith = W; A = X; } selectnowith(A) ::= oneselect(X). {A = X;} %ifndef SQLITE_OMIT_COMPOUND_SELECT @@ -652,15 +652,17 @@ limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y). /////////////////////////// The DELETE statement ///////////////////////////// // %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= with DELETE FROM fullname(X) indexed_opt(I) where_opt(W) +cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W) orderby_opt(O) limit_opt(L). { + sqlite3WithPush(pParse,C); sqlite3SrcListIndexedBy(pParse, X, &I); W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "DELETE"); sqlite3DeleteFrom(pParse,X,W); } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= with DELETE FROM fullname(X) indexed_opt(I) where_opt(W). { +cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). { + sqlite3WithPush(pParse,C); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3DeleteFrom(pParse,X,W); } @@ -684,8 +686,9 @@ cmd ::= with UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) } %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= with UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) +cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W). { + sqlite3WithPush(pParse, C); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); sqlite3Update(pParse,X,Y,W,R); @@ -706,10 +709,15 @@ setlist(A) ::= nm(X) EQ expr(Y). { ////////////////////////// The INSERT command ///////////////////////////////// // -cmd ::= with insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). - {sqlite3Insert(pParse, X, S, F, R);} -cmd ::= with insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. - {sqlite3Insert(pParse, X, 0, F, R);} +cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). { + sqlite3WithPush(pParse, W); + sqlite3Insert(pParse, X, S, F, R); +} +cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. +{ + sqlite3WithPush(pParse, W); + sqlite3Insert(pParse, X, 0, F, R); +} %type insert_cmd {u8} insert_cmd(A) ::= INSERT orconf(R). {A = R;} @@ -1372,16 +1380,17 @@ anylist ::= anylist ANY. %type with {With*} %type wqlist {With*} %destructor with {sqlite3WithDelete(pParse->db, $$);} +%destructor wqlist {sqlite3WithDelete(pParse->db, $$);} with(A) ::= . {A = 0;} %ifndef SQLITE_OMIT_CTE with(A) ::= WITH wqlist(W). { A = W; } with(A) ::= WITH RECURSIVE wqlist(W). { A = W; } -wqlist(A) ::= nm(X) inscollist_opt(Y) AS LP select(Z) RP. { +wqlist(A) ::= nm(X) idxlist_opt(Y) AS LP select(Z) RP. { A = sqlite3WithAdd(pParse, 0, &X, Y, Z); } -wqlist(A) ::= wqlist(W) COMMA nm(X) inscollist_opt(Y) AS LP select(Z) RP. { +wqlist(A) ::= wqlist(W) COMMA nm(X) idxlist_opt(Y) AS LP select(Z) RP. { A = sqlite3WithAdd(pParse, W, &X, Y, Z); } %endif SQLITE_OMIT_CTE diff --git a/src/select.c b/src/select.c index d075116749..b054489b00 100644 --- a/src/select.c +++ b/src/select.c @@ -29,6 +29,7 @@ static void clearSelect(sqlite3 *db, Select *p){ sqlite3SelectDelete(db, p->pPrior); sqlite3ExprDelete(db, p->pLimit); sqlite3ExprDelete(db, p->pOffset); + sqlite3WithDelete(db, p->pWith); } /* @@ -3393,6 +3394,62 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ return WRC_Continue; } +static struct Cte *searchWith(Parse *pParse, struct SrcList_item *p){ + if( p->zDatabase==0 ){ + char *zName = p->zName; + With *pWith; + + for(pWith=pParse->pWith; pWith; pWith=pWith->pOuter){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){ + return &pWith->a[i]; + } + } + } + } + return 0; +} + +void sqlite3WithPush(Parse *pParse, With *pWith){ + if( pWith ){ + pWith->pOuter = pParse->pWith; + pParse->pWith = pWith; + } +} +static void withPop(Parse *pParse, With *pWith){ + if( pWith ){ + assert( pParse->pWith==pWith ); + pParse->pWith = pWith->pOuter; + } +} + +static int ctePush(Parse *pParse, struct Cte *pCte){ + if( pCte ){ + struct Cte *p; + for(p=pParse->pCte; p; p=p->pOuterCte){ + if( p==pCte ){ + sqlite3ErrorMsg( + pParse, "illegal recursive defininition in cte: %s", pCte->zName + ); + return SQLITE_ERROR; + } + } + + pCte->pOuterCte = pParse->pCte; + pParse->pCte = pCte; + } + return SQLITE_OK; +} + +static void ctePop(Parse *pParse, struct Cte *pCte){ + if( pCte ){ + assert( pParse->pCte==pCte ); + pParse->pCte = pCte->pOuterCte; + } +} + + /* ** This routine is a Walker callback for "expanding" a SELECT statement. ** "Expanding" means to do the following: @@ -3447,6 +3504,7 @@ static int selectExpander(Walker *pWalker, Select *p){ ** then create a transient table structure to describe the subquery. */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ + struct Cte *pCte = 0; Table *pTab; if( pFrom->pTab!=0 ){ /* This statement has already been prepared. There is no need @@ -3454,19 +3512,34 @@ static int selectExpander(Walker *pWalker, Select *p){ assert( i==0 ); return WRC_Prune; } - if( pFrom->zName==0 ){ +#ifndef SQLITE_OMIT_CTE + pCte = searchWith(pParse, pFrom); + if( pCte ){ + pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + if( pFrom->pSelect==0 ) return WRC_Abort; + } +#endif + if( pFrom->zName==0 || pCte ){ #ifndef SQLITE_OMIT_SUBQUERY + ExprList *pEList; Select *pSel = pFrom->pSelect; /* A sub-query in the FROM clause of a SELECT */ assert( pSel!=0 ); assert( pFrom->pTab==0 ); + if( ctePush(pParse, pCte) ) return WRC_Abort; sqlite3WalkSelect(pWalker, pSel); + ctePop(pParse, pCte); pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; pTab->nRef = 1; pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } - selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); + if( pCte && pCte->pCols ){ + pEList = pCte->pCols; + }else{ + pEList = pSel->pEList; + } + selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; pTab->nRowEst = 1048576; pTab->tabFlags |= TF_Ephemeral; @@ -3679,6 +3752,14 @@ static int selectExpander(Walker *pWalker, Select *p){ return WRC_Continue; } +static int selectExpanderWith(Walker *pWalker, Select *p){ + int res; + sqlite3WithPush(pWalker->pParse, p->pWith); + res = selectExpander(pWalker, p); + withPop(pWalker->pParse, p->pWith); + return res; +} + /* ** No-op routine for the parse-tree walker. ** @@ -3715,7 +3796,7 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ w.xSelectCallback = convertCompoundSelectToSubquery; sqlite3WalkSelect(&w, pSelect); } - w.xSelectCallback = selectExpander; + w.xSelectCallback = selectExpanderWith; sqlite3WalkSelect(&w, pSelect); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 8846ffb04c..6bd65592b7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2144,6 +2144,7 @@ struct Select { Select *pRightmost; /* Right-most select in a compound select statement */ Expr *pLimit; /* LIMIT expression. NULL means not used. */ Expr *pOffset; /* OFFSET expression. NULL means not used. */ + With *pWith; /* WITH clause attached to this select. Or NULL. */ }; /* @@ -2365,6 +2366,8 @@ struct Parse { #endif Table *pZombieTab; /* List of Table objects to delete after code gen */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ + With *pWith; /* Current WITH clause, or NULL */ + struct Cte *pCte; /* Current CTE, or NULL */ }; /* @@ -2638,10 +2641,12 @@ int sqlite3WalkSelectFrom(Walker*, Select*); */ struct With { int nCte; /* Number of CTEs */ + With *pOuter; /* Containing WITH clause, or NULL */ struct Cte { - const char *zName; /* Name of this CTE */ - IdList *pCol; /* List of explicit column names, or NULL */ + char *zName; /* Name of this CTE */ + ExprList *pCols; /* List of explicit column names, or NULL */ Select *pSelect; /* The contents of the CTE */ + struct Cte *pOuterCte; } a[1]; }; @@ -3344,8 +3349,11 @@ const char *sqlite3JournalModename(int); int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); #endif #ifndef SQLITE_OMIT_CTE - With *sqlite3WithAdd(Parse*,With*,Token*,IdList*,Select*); + With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*); void sqlite3WithDelete(sqlite3*,With*); + void sqlite3WithPush(Parse*, With*); +#else +#define sqlite3WithPush(x,y) #endif /* Declarations for functions in fkey.c. All of these are replaced by diff --git a/src/tokenize.c b/src/tokenize.c index 6e796ef82e..3fd4f78f7a 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -494,6 +494,8 @@ abort_parse: sqlite3DeleteTable(db, pParse->pNewTable); } + assert( pParse->pWith==0 || pParse->pWith->pOuter==0 ); + sqlite3WithDelete(db, pParse->pWith); sqlite3DeleteTrigger(db, pParse->pNewTrigger); for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); sqlite3DbFree(db, pParse->azVar); diff --git a/test/pagerfault.test b/test/pagerfault.test index 796f531c37..1e57c93903 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -63,6 +63,7 @@ do_faultsim_test pagerfault-1 -prep { error "Database content appears incorrect" } } +finish_test #------------------------------------------------------------------------- # Test fault-injection while rolling back a hot-journal file with a diff --git a/test/with1.test b/test/with1.test index 80e879ada0..0b7810ebd3 100644 --- a/test/with1.test +++ b/test/with1.test @@ -37,4 +37,107 @@ do_execsql_test 1.4 { WITH x(a) AS ( SELECT * FROM t1) UPDATE t1 SET x = y; } {} +#-------------------------------------------------------------------------- + +do_execsql_test 2.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(2); + WITH tmp AS ( SELECT * FROM t1 ) SELECT x FROM tmp; +} {1 2} + +do_execsql_test 2.2 { + WITH tmp(a) AS ( SELECT * FROM t1 ) SELECT a FROM tmp; +} {1 2} + +do_execsql_test 2.3 { + SELECT * FROM ( + WITH tmp(a) AS ( SELECT * FROM t1 ) SELECT a FROM tmp + ); +} {1 2} + +do_execsql_test 2.4 { + WITH tmp1(a) AS ( SELECT * FROM t1 ), + tmp2(x) AS ( SELECT * FROM tmp1) + SELECT * FROM tmp2; +} {1 2} + +do_execsql_test 2.5 { + WITH tmp2(x) AS ( SELECT * FROM tmp1), + tmp1(a) AS ( SELECT * FROM t1 ) + SELECT * FROM tmp2; +} {1 2} + +#------------------------------------------------------------------------- +do_catchsql_test 3.1 { + WITH tmp2(x) AS ( SELECT * FROM tmp1), + tmp1(a) AS ( SELECT * FROM tmp2 ) + SELECT * FROM tmp1; +} {1 {illegal recursive defininition in cte: tmp1}} + +do_catchsql_test 3.2 { + CREATE TABLE t2(x INTEGER); + WITH tmp(a) AS (SELECT * FROM t1), + tmp(a) AS (SELECT * FROM t1) + SELECT * FROM tmp; +} {1 {duplicate cte name: tmp}} + +do_execsql_test 3.3 { + CREATE TABLE t3(x); + CREATE TABLE t4(x); + + INSERT INTO t3 VALUES('T3'); + INSERT INTO t4 VALUES('T4'); + + WITH t3(a) AS (SELECT * FROM t4) + SELECT * FROM t3; +} {T4} + +do_execsql_test 3.4 { + WITH tmp AS ( SELECT * FROM t3 ), + tmp2 AS ( WITH tmp AS ( SELECT * FROM t4 ) SELECT * FROM tmp ) + SELECT * FROM tmp2; +} {T4} + +do_execsql_test 3.5 { + WITH tmp AS ( SELECT * FROM t3 ), + tmp2 AS ( WITH xxxx AS ( SELECT * FROM t4 ) SELECT * FROM tmp ) + SELECT * FROM tmp2; +} {T3} + +do_catchsql_test 3.6 { + WITH tmp AS ( SELECT * FROM t3 ), + SELECT * FROM tmp; +} {1 {near "SELECT": syntax error}} + +#------------------------------------------------------------------------- +do_execsql_test 4.1 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(2); + INSERT INTO t1 VALUES(3); + INSERT INTO t1 VALUES(4); + + WITH dset AS ( SELECT 2 UNION ALL SELECT 4 ) + DELETE FROM t1 WHERE x IN dset; + SELECT * FROM t1; +} {1 3} + +do_execsql_test 4.2 { + WITH iset AS ( SELECT 2 UNION ALL SELECT 4 ) + INSERT INTO t1 SELECT * FROM iset; + SELECT * FROM t1; +} {1 3 2 4} + +do_execsql_test 4.3 { + WITH uset(a, b) AS ( SELECT 2, 8 UNION ALL SELECT 4, 9 ) + UPDATE t1 SET x = COALESCE( (SELECT b FROM uset WHERE a=x), x ); + SELECT * FROM t1; +} {1 3 8 9} + finish_test + + + diff --git a/test/withM.test b/test/withM.test new file mode 100644 index 0000000000..de46c85228 --- /dev/null +++ b/test/withM.test @@ -0,0 +1,40 @@ +# 2014 January 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the WITH clause. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set ::testprefix withM + +do_execsql_test 1.0 { + CREATE TABLE t1(x INTEGER, y INTEGER); + INSERT INTO t1 VALUES(123, 456); +} + +do_faultsim_test withM-1 -prep { + sqlite3 db test.db +} -body { + execsql { + WITH tmp AS ( SELECT * FROM t1 ) + SELECT * FROM tmp; + } +} -test { + faultsim_test_result {0 {123 456}} + db close +} + +finish_test + + + From a9f5c13d0c7fd6c718ea8c8811e7824eba78e6e9 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 13 Jan 2014 16:36:40 +0000 Subject: [PATCH 26/74] Fix some memory leaks and crashes that could follow an OOM condition during WITH clause parsing. FossilOrigin-Name: 8839850c44a8938883e493eacd752fa686e542df --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/build.c | 3 ++- src/parse.y | 9 ++++++++- test/withM.test | 18 +++++++++++++++++- 5 files changed, 36 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 010cb6fcfd..a1a39b1a65 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\scode\sto\shandle\snon-recursive\sCTEs\sin\sthe\ssame\sway\sas\sSQL\sviews. -D 2014-01-13T15:12:23.149 +C Fix\ssome\smemory\sleaks\sand\scrashes\sthat\scould\sfollow\san\sOOM\scondition\sduring\sWITH\sclause\sparsing. +D 2014-01-13T16:36:40.215 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,7 +169,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c 11e29ef8cf16a42925fde036bcffbeffd9cc82df F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 -F src/build.c 1efdc65020e1f566383b66b30ccf730b4ef6f926 +F src/build.c 5a2daa6649640711ca60114046903be932348e52 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd @@ -209,7 +209,7 @@ F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968 F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y b433588d7993d6475d9944be0fda2fb3df3cb138 +F src/parse.y e5c4a23f445cde4b30d50948df8f21bc586dd7a4 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b @@ -1092,7 +1092,7 @@ F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/with1.test fb8409a35b1314be6e73a87597322f3369b59b2b -F test/withM.test 036349a9ec081fee8b1f72e71ad8a5a5b80f1674 +F test/withM.test ac3ec7ee0b33a02d0fa15da91214d97ddea64e34 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 F test/without_rowid3.test eac3d5c8a1924725b58503a368f2cbd24fd6c8a0 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 704d3931b855562a619769955969d439c42ca406 -R fb5ad7a801e83802b92e3793bba4fd6c +P a26f399ba485e8127c276c5f103ec6c555e11734 +R 7e06730d3714a8aceb489d831c2bc18d U dan -Z 663a5542bd9dc2bb50ed6e3a07d040fb +Z 780f0749f7ee296248db4bd00ffb23e3 diff --git a/manifest.uuid b/manifest.uuid index 267d8a5499..6382ccbc9d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a26f399ba485e8127c276c5f103ec6c555e11734 \ No newline at end of file +8839850c44a8938883e493eacd752fa686e542df \ No newline at end of file diff --git a/src/build.c b/src/build.c index ca184766f9..c7d956a9b7 100644 --- a/src/build.c +++ b/src/build.c @@ -4234,12 +4234,13 @@ With *sqlite3WithAdd( pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); } assert( zName!=0 || pNew==0 ); + assert( db->mallocFailed==0 || pNew==0 ); if( pNew==0 ){ - sqlite3WithDelete(db, pWith); sqlite3ExprListDelete(db, pArglist); sqlite3SelectDelete(db, pQuery); sqlite3DbFree(db, zName); + pNew = pWith; }else{ pNew->a[pNew->nCte].pSelect = pQuery; pNew->a[pNew->nCte].pCols = pArglist; diff --git a/src/parse.y b/src/parse.y index 1c6465e933..b1992f2415 100644 --- a/src/parse.y +++ b/src/parse.y @@ -412,7 +412,14 @@ cmd ::= select(X). { %type oneselect {Select*} %destructor oneselect {sqlite3SelectDelete(pParse->db, $$);} -select(A) ::= with(W) selectnowith(X). { if( X ) X->pWith = W; A = X; } +select(A) ::= with(W) selectnowith(X). { + if( X ){ + X->pWith = W; + }else{ + sqlite3WithDelete(pParse->db, W); + } + A = X; +} selectnowith(A) ::= oneselect(X). {A = X;} %ifndef SQLITE_OMIT_COMPOUND_SELECT diff --git a/test/withM.test b/test/withM.test index de46c85228..431cca187e 100644 --- a/test/withM.test +++ b/test/withM.test @@ -22,7 +22,7 @@ do_execsql_test 1.0 { INSERT INTO t1 VALUES(123, 456); } -do_faultsim_test withM-1 -prep { +do_faultsim_test withM-1.1 -prep { sqlite3 db test.db } -body { execsql { @@ -34,6 +34,22 @@ do_faultsim_test withM-1 -prep { db close } +do_faultsim_test withM-1.2 -prep { + sqlite3 db test.db +} -body { + execsql { + WITH w1 AS ( SELECT * FROM t1 ), + w2 AS ( + WITH w3 AS ( SELECT * FROM w1 ) + SELECT * FROM w3 + ) + SELECT * FROM w2; + } +} -test { + faultsim_test_result {0 {123 456}} + db close +} + finish_test From 859bc542c6ea1cbca600da863bce8c0207319611 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Jan 2014 20:32:18 +0000 Subject: [PATCH 27/74] For statements of just an unadorned VALUES clause, assign column names as "columnN" for increasing whole numbers N. FossilOrigin-Name: 260587d2727f66d7fd65ef672ee46c92024f1d30 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 03003dd0c5..f27dba4a8e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sLEMON,\slimit\sthe\ssize\sof\sthe\sgrammar\sfile\sto\s100MB.\s\sThis\sensures\sthat\nthe\sprogram\swill\snever\sexperience\sinteger\soverflow.\s\sTo\sbe\sdoubly\ssure,\nuse\scalloc()\sinstead\sof\smalloc()\swhen\sallocating\sarrays. -D 2014-01-11T12:52:25.201 +C For\sstatements\sof\sjust\san\sunadorned\sVALUES\sclause,\sassign\scolumn\snames\nas\s"columnN"\sfor\sincreasing\swhole\snumbers\sN. +D 2014-01-13T20:32:18.451 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 819bb090c9a348d17f69f136cad2bfa9ee9cbb41 +F src/select.c 996d8b88603edbd478aaa70b75d535a3ddea933d F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P cb5d1f83e0a33d546d4c0cb817ef1f8440d1f738 -R 28679f157b50c114aa03e50f74a7a104 +P 29ba458d849ad8864711cbe59fb10447a947e06a +R cfc0f0611de592f2675967818ec833a9 U drh -Z 608e7b6009060d93ac39bb2434b3c874 +Z 9bc6839af49500894d0d88a693460511 diff --git a/manifest.uuid b/manifest.uuid index d8b23d3f34..169cbd691e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -29ba458d849ad8864711cbe59fb10447a947e06a \ No newline at end of file +260587d2727f66d7fd65ef672ee46c92024f1d30 \ No newline at end of file diff --git a/src/select.c b/src/select.c index d075116749..5845897409 100644 --- a/src/select.c +++ b/src/select.c @@ -1363,8 +1363,9 @@ static void generateColumnNames( sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT); } }else{ - sqlite3VdbeSetColName(v, i, COLNAME_NAME, - sqlite3DbStrDup(db, pEList->a[i].zSpan), SQLITE_DYNAMIC); + const char *z = pEList->a[i].zSpan; + z = z==0 ? sqlite3MPrintf(db, "column%d", i+1) : sqlite3DbStrDup(db, z); + sqlite3VdbeSetColName(v, i, COLNAME_NAME, z, SQLITE_DYNAMIC); } } generateColumnTypes(pParse, pTabList, pEList); From c3d6ba49c6b78ca807a68ee6d0e403c073793540 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 13 Jan 2014 20:38:35 +0000 Subject: [PATCH 28/74] In the command-line shell, defend against a NULL-pointer dereference in the case where sqlite3_column_name() returns NULL (as might happen following an OOM error). FossilOrigin-Name: ac15455abcb9bdb88b53129348668a1442f6899f --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index f27dba4a8e..e68fe680fc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sstatements\sof\sjust\san\sunadorned\sVALUES\sclause,\sassign\scolumn\snames\nas\s"columnN"\sfor\sincreasing\swhole\snumbers\sN. -D 2014-01-13T20:32:18.451 +C In\sthe\scommand-line\sshell,\sdefend\sagainst\sa\sNULL-pointer\sdereference\sin\sthe\ncase\swhere\ssqlite3_column_name()\sreturns\sNULL\s(as\smight\shappen\sfollowing\san\nOOM\serror). +D 2014-01-13T20:38:35.227 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -220,7 +220,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 996d8b88603edbd478aaa70b75d535a3ddea933d -F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 +F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 29ba458d849ad8864711cbe59fb10447a947e06a -R cfc0f0611de592f2675967818ec833a9 +P 260587d2727f66d7fd65ef672ee46c92024f1d30 +R 10a3b37371baf3bb8cfb66250579708d U drh -Z 9bc6839af49500894d0d88a693460511 +Z b723d9f207dc3deb542b9214e1e7a09c diff --git a/manifest.uuid b/manifest.uuid index 169cbd691e..ffa41c2bbd 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -260587d2727f66d7fd65ef672ee46c92024f1d30 \ No newline at end of file +ac15455abcb9bdb88b53129348668a1442f6899f \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 00cff6a8e5..b5ce90208c 100644 --- a/src/shell.c +++ b/src/shell.c @@ -597,6 +597,7 @@ static void output_c_string(FILE *out, const char *z){ */ static void output_html_string(FILE *out, const char *z){ int i; + if( z==0 ) z = ""; while( *z ){ for(i=0; z[i] && z[i]!='<' From 866b53eb5a4efabf6a958cbf541bb949b4439b1e Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 14 Jan 2014 10:17:02 +0000 Subject: [PATCH 29/74] For the Win32 VFS, defining winShmMutexHeld should be controlled by NDEBUG, not SQLITE_DEBUG. FossilOrigin-Name: 1e131094b522103a0829f72193b067b04e42ce82 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/os_win.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index e68fe680fc..9bf7f9b689 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\scommand-line\sshell,\sdefend\sagainst\sa\sNULL-pointer\sdereference\sin\sthe\ncase\swhere\ssqlite3_column_name()\sreturns\sNULL\s(as\smight\shappen\sfollowing\san\nOOM\serror). -D 2014-01-13T20:38:35.227 +C For\sthe\sWin32\sVFS,\sdefining\swinShmMutexHeld\sshould\sbe\scontrolled\sby\sNDEBUG,\snot\sSQLITE_DEBUG. +D 2014-01-14T10:17:02.428 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -206,7 +206,7 @@ F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968 -F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07 +F src/os_win.c 1b21af72c5fa6f9e519a5fcab33e80d182b1aedb F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 F src/parse.y 3c5384533a8bfce5abd256cc9cb2c38bec05ad61 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 260587d2727f66d7fd65ef672ee46c92024f1d30 -R 10a3b37371baf3bb8cfb66250579708d -U drh -Z b723d9f207dc3deb542b9214e1e7a09c +P ac15455abcb9bdb88b53129348668a1442f6899f +R 455bba3b3836db70e207929cfbe74107 +U mistachkin +Z 3f9ab398a4d1da777d1daf83b531e5c4 diff --git a/manifest.uuid b/manifest.uuid index ffa41c2bbd..727b07185c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ac15455abcb9bdb88b53129348668a1442f6899f \ No newline at end of file +1e131094b522103a0829f72193b067b04e42ce82 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index faa60569c9..4fb4f02703 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -3224,7 +3224,7 @@ static void winShmEnterMutex(void){ static void winShmLeaveMutex(void){ sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } -#ifdef SQLITE_DEBUG +#ifndef NDEBUG static int winShmMutexHeld(void) { return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); } From 7a429658d7e38e0ff52a9f0591a17fd9cb71929d Mon Sep 17 00:00:00 2001 From: mistachkin Date: Tue, 14 Jan 2014 10:17:21 +0000 Subject: [PATCH 30/74] Fix harmless compiler warning in LEMON. FossilOrigin-Name: f61a70589ac7e05008a362bd9d5b7bde5d07a758 --- manifest | 12 ++++++------ manifest.uuid | 2 +- tool/lemon.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 9bf7f9b689..b7d3993e06 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C For\sthe\sWin32\sVFS,\sdefining\swinShmMutexHeld\sshould\sbe\scontrolled\sby\sNDEBUG,\snot\sSQLITE_DEBUG. -D 2014-01-14T10:17:02.428 +C Fix\sharmless\scompiler\swarning\sin\sLEMON. +D 2014-01-14T10:17:21.693 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1109,7 +1109,7 @@ F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce -F tool/lemon.c 6842b2e7af12835f9f6e55808a0b1861cd0696fe +F tool/lemon.c 07aba6270d5a5016ba8107b09e431eea4ecdc123 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ac15455abcb9bdb88b53129348668a1442f6899f -R 455bba3b3836db70e207929cfbe74107 +P 1e131094b522103a0829f72193b067b04e42ce82 +R fe4a4148cfbc4d588ee2679597656da4 U mistachkin -Z 3f9ab398a4d1da777d1daf83b531e5c4 +Z d589f5ec7c09b4ac8c97497c20166016 diff --git a/manifest.uuid b/manifest.uuid index 727b07185c..d14b8afba5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e131094b522103a0829f72193b067b04e42ce82 \ No newline at end of file +f61a70589ac7e05008a362bd9d5b7bde5d07a758 \ No newline at end of file diff --git a/tool/lemon.c b/tool/lemon.c index 8dca20c451..d7179ad423 100644 --- a/tool/lemon.c +++ b/tool/lemon.c @@ -83,7 +83,7 @@ static void lemon_addtext( zBuf[*pnUsed] = 0; } static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){ - int i, j, k, c, size; + int i, j, k, c; int nUsed = 0; const char *z; char zTemp[50]; From 8ce7184bc26b1330817a68bca751beac4712cbb0 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 14 Jan 2014 20:14:09 +0000 Subject: [PATCH 31/74] Add code to handle recursive CTEs. FossilOrigin-Name: a5c2a54a07d35166911abc792008c05dea897742 --- manifest | 24 +++---- manifest.uuid | 2 +- src/btree.c | 6 +- src/expr.c | 1 + src/select.c | 176 +++++++++++++++++++++++++++++++++++++++++++----- src/sqliteInt.h | 6 +- src/vdbe.c | 47 +++++++++++++ src/where.c | 6 +- test/with1.test | 27 ++++++++ 9 files changed, 262 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index a1a39b1a65..56fe478fb2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\ssome\smemory\sleaks\sand\scrashes\sthat\scould\sfollow\san\sOOM\scondition\sduring\sWITH\sclause\sparsing. -D 2014-01-13T16:36:40.215 +C Add\scode\sto\shandle\srecursive\sCTEs. +D 2014-01-14T20:14:09.053 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 11e29ef8cf16a42925fde036bcffbeffd9cc82df +F src/btree.c c15e1722696b66c4029c487acfb830b0985bf142 F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 F src/build.c 5a2daa6649640711ca60114046903be932348e52 @@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff -F src/expr.c de86bf987c532ec9d63fb1bf8a6eb6ec2cf5ba69 +F src/expr.c fee4b54fdcf5a979e6f5012da3dfb084ac5d3aac F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 9dc9177bfb9278a603e3835e828ff60ec68fba6f +F src/select.c 0f7779b0c1c317dcac0f65f695d036030c16775f F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h a350bcf6d62204b6b4720ce1922c95a575a0b5ad +F src/sqliteInt.h fc7b2516260c4e14bd4342cd9ed47f3d0ca27cc5 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -280,7 +280,7 @@ F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c e71f19b272f05c8695cf747b4bac1732685f9e5c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c b110887e415b5d2af58c2374c4dfdcf774c5d46c +F src/vdbe.c 9b918126afb9a7b7ad232db7453680c7ef5821a4 F src/vdbe.h e6c4c610fcabad4fa80ebb1efc6822a9367e2b26 F src/vdbeInt.h 42db251e9f863401ff847b90d5fe1614c89a6a56 F src/vdbeapi.c ce4e68ea4842cc6081046f533d088dcf01d247ad @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 18f07fd0fd116a5880773c689eb7cd8e60175107 +F src/where.c 830b42f452cfbc4e17582f6c1d388e15b379b833 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1091,7 +1091,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test fb8409a35b1314be6e73a87597322f3369b59b2b +F test/with1.test 12a6661eabcc9ae299020f7b4197a75a9c084748 F test/withM.test ac3ec7ee0b33a02d0fa15da91214d97ddea64e34 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a26f399ba485e8127c276c5f103ec6c555e11734 -R 7e06730d3714a8aceb489d831c2bc18d +P 8839850c44a8938883e493eacd752fa686e542df +R 1d5d9f6beb143e76aa04b4bf9953f5da U dan -Z 780f0749f7ee296248db4bd00ffb23e3 +Z 1616366ca894deb1473f9b1eb1563ebc diff --git a/manifest.uuid b/manifest.uuid index 6382ccbc9d..f994f09e97 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8839850c44a8938883e493eacd752fa686e542df \ No newline at end of file +a5c2a54a07d35166911abc792008c05dea897742 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 20bed057e1..e28a97c846 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7350,6 +7350,7 @@ static int clearDatabasePage( int rc; unsigned char *pCell; int i; + int hdr; assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ @@ -7358,6 +7359,7 @@ static int clearDatabasePage( rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; + hdr = pPage->hdrOffset; for(i=0; inCell; i++){ pCell = findCell(pPage, i); if( !pPage->leaf ){ @@ -7368,7 +7370,7 @@ static int clearDatabasePage( if( rc ) goto cleardatabasepage_out; } if( !pPage->leaf ){ - rc = clearDatabasePage(pBt, get4byte(&pPage->aData[8]), 1, pnChange); + rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); if( rc ) goto cleardatabasepage_out; }else if( pnChange ){ assert( pPage->intKey ); @@ -7377,7 +7379,7 @@ static int clearDatabasePage( if( freePageFlag ){ freePage(pPage, &rc); }else if( (rc = sqlite3PagerWrite(pPage->pDbPage))==0 ){ - zeroPage(pPage, pPage->aData[0] | PTF_LEAF); + zeroPage(pPage, pPage->aData[hdr] | PTF_LEAF); } cleardatabasepage_out: diff --git a/src/expr.c b/src/expr.c index d0ea280029..e271e46796 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1055,6 +1055,7 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->addrOpenEphm[1] = -1; pNew->addrOpenEphm[2] = -1; pNew->pWith = withDup(db, p->pWith); + pNew->pRecurse = p->pRecurse; return pNew; } #else diff --git a/src/select.c b/src/select.c index b054489b00..0db4b01807 100644 --- a/src/select.c +++ b/src/select.c @@ -691,12 +691,26 @@ static void selectInnerLoop( /* Store the result as data using a unique key. */ + case SRT_DistTable: case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); +#ifndef SQLITE_OMIT_CTE + if( eDest==SRT_DistTable ){ + /* If the destination is DistTable, then cursor (iParm+1) is open + ** on an ephemeral index. If the current row is already present + ** in the index, do not write it to the output. If not, add the + ** current row to the index and proceed with writing it to the + ** output table as well. */ + int addr = sqlite3VdbeCurrentAddr(v) + 4; + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); + assert( pOrderBy==0 ); + } +#endif if( pOrderBy ){ pushOntoSorter(pParse, pOrderBy, p, r1); }else{ @@ -1729,6 +1743,7 @@ static int multiSelect( ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. */ assert( p && p->pPrior ); /* Calling function guarantees this much */ + assert( p->pRecurse==0 || p->op==TK_ALL || p->op==TK_UNION ); db = pParse->db; pPrior = p->pPrior; assert( pPrior->pRightmost!=pPrior ); @@ -1774,12 +1789,82 @@ static int multiSelect( goto multi_select_end; } + /* If this is a recursive query, check that there is no ORDER BY or + ** LIMIT clause. Neither of these are supported. */ + assert( p->pOffset==0 || p->pLimit ); + if( p->pRecurse && (p->pOrderBy || p->pLimit) ){ + sqlite3ErrorMsg(pParse, "%s in a recursive query is not allowed", + p->pOrderBy ? "ORDER BY" : "LIMIT" + ); + goto multi_select_end; + } + /* Compound SELECTs that have an ORDER BY clause are handled separately. */ if( p->pOrderBy ){ return multiSelectOrderBy(pParse, p, pDest); } +#ifndef SQLITE_OMIT_CTE + if( p->pRecurse ){ + int nCol = p->pEList->nExpr; + int addrNext; + int addrSwap; + int iCont, iBreak; + int tmp1, tmp2; /* Cursors used to access temporary tables */ + int tmp3 = 0; /* To ensure unique results if UNION */ + int eDest = SRT_Table; + SelectDest tmp2dest; + + iBreak = sqlite3VdbeMakeLabel(v); + iCont = sqlite3VdbeMakeLabel(v); + + tmp1 = pParse->nTab++; + tmp2 = pParse->nTab++; + if( p->op==TK_UNION ){ + eDest = SRT_DistTable; + tmp3 = pParse->nTab++; + } + sqlite3SelectDestInit(&tmp2dest, eDest, tmp2); + + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp1, nCol); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp2, nCol); + if( tmp3 ){ + p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp3, 0); + p->selFlags |= SF_UsesEphemeral; + } + + /* Store the results of the initial SELECT in tmp2. */ + rc = sqlite3Select(pParse, pPrior, &tmp2dest); + if( rc ) goto multi_select_end; + + /* Clear tmp1. Then switch the contents of tmp1 and tmp2. Then teturn + ** the contents of tmp1 to the caller. Or, if tmp1 is empty at this + ** point, the recursive query has finished - jump to address iBreak. */ + addrSwap = sqlite3VdbeAddOp2(v, OP_SwapCursors, tmp1, tmp2); + sqlite3VdbeAddOp2(v, OP_Rewind, tmp1, iBreak); + addrNext = sqlite3VdbeCurrentAddr(v); + selectInnerLoop(pParse, p, p->pEList, tmp1, p->pEList->nExpr, + 0, 0, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, tmp1, addrNext); + + /* Execute the recursive SELECT. Store the results in tmp2. While this + ** SELECT is running, the contents of tmp1 are read by recursive + ** references to the current CTE. */ + p->pPrior = 0; + p->pRecurse->tnum = tmp1; + p->pRecurse->tabFlags |= TF_Recursive; + rc = sqlite3Select(pParse, p, &tmp2dest); + p->pRecurse->tabFlags &= ~TF_Recursive; + p->pPrior = pPrior; + if( rc ) goto multi_select_end; + + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrSwap); + sqlite3VdbeResolveLabel(v, iBreak); + }else +#endif + /* Generate code for the left and right SELECT statements. */ switch( p->op ){ @@ -3449,6 +3534,73 @@ static void ctePop(Parse *pParse, struct Cte *pCte){ } } +static int withExpand( + Walker *pWalker, + struct SrcList_item *pFrom, + struct Cte *pCte +){ + Table *pTab; + Parse *pParse = pWalker->pParse; + sqlite3 *db = pParse->db; + + assert( pFrom->pSelect==0 ); + assert( pFrom->pTab==0 ); + + if( pCte==pParse->pCte && (pTab = pCte->pTab) ){ + /* This is the recursive part of a recursive CTE */ + pFrom->pTab = pTab; + pTab->nRef++; + }else{ + ExprList *pEList; + Select *pSel; + int bRecursive; + + pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return WRC_Abort; + pTab->nRef = 1; + pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); + pTab->iPKey = -1; + pTab->nRowEst = 1048576; + pTab->tabFlags |= TF_Ephemeral; + pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); + if( db->mallocFailed ) return SQLITE_NOMEM; + assert( pFrom->pSelect ); + + if( ctePush(pParse, pCte) ) return WRC_Abort; + pSel = pFrom->pSelect; + bRecursive = (pSel->op==TK_ALL || pSel->op==TK_UNION); + if( bRecursive ){ + assert( pSel->pPrior ); + sqlite3WalkSelect(pWalker, pSel->pPrior); + }else{ + sqlite3WalkSelect(pWalker, pSel); + } + + if( pCte->pCols ){ + pEList = pCte->pCols; + }else{ + Select *pLeft; + for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); + pEList = pLeft->pEList; + } + selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + + if( bRecursive ){ + int nRef = pTab->nRef; + pCte->pTab = pTab; + sqlite3WalkSelect(pWalker, pSel); + pCte->pTab = 0; + if( pTab->nRef > nRef){ + pSel->pRecurse = pTab; + assert( pTab->tnum==0 ); + } + } + + ctePop(pParse, pCte); + } + + return SQLITE_OK; +} /* ** This routine is a Walker callback for "expanding" a SELECT statement. @@ -3515,31 +3667,22 @@ static int selectExpander(Walker *pWalker, Select *p){ #ifndef SQLITE_OMIT_CTE pCte = searchWith(pParse, pFrom); if( pCte ){ - pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); - if( pFrom->pSelect==0 ) return WRC_Abort; - } + if( withExpand(pWalker, pFrom, pCte) ) return WRC_Abort; + }else #endif - if( pFrom->zName==0 || pCte ){ + if( pFrom->zName==0 ){ #ifndef SQLITE_OMIT_SUBQUERY - ExprList *pEList; Select *pSel = pFrom->pSelect; /* A sub-query in the FROM clause of a SELECT */ assert( pSel!=0 ); assert( pFrom->pTab==0 ); - if( ctePush(pParse, pCte) ) return WRC_Abort; sqlite3WalkSelect(pWalker, pSel); - ctePop(pParse, pCte); pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; pTab->nRef = 1; pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); while( pSel->pPrior ){ pSel = pSel->pPrior; } - if( pCte && pCte->pCols ){ - pEList = pCte->pCols; - }else{ - pEList = pSel->pEList; - } - selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + selectColumnsFromExprList(pParse, pSel->pEList, &pTab->nCol, &pTab->aCol); pTab->iPKey = -1; pTab->nRowEst = 1048576; pTab->tabFlags |= TF_Ephemeral; @@ -3831,9 +3974,10 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){ /* A sub-query in the FROM clause of a SELECT */ Select *pSel = pFrom->pSelect; - assert( pSel ); - while( pSel->pPrior ) pSel = pSel->pPrior; - selectAddColumnTypeAndCollation(pParse, pTab, pSel); + if( pSel ){ + while( pSel->pPrior ) pSel = pSel->pPrior; + selectAddColumnTypeAndCollation(pParse, pTab, pSel); + } } } } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 6bd65592b7..c7f0609ba1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1429,7 +1429,7 @@ struct Table { }; /* -** Allowed values for Tabe.tabFlags. +** Allowed values for Table.tabFlags. */ #define TF_Readonly 0x01 /* Read-only system table */ #define TF_Ephemeral 0x02 /* An ephemeral table */ @@ -1437,6 +1437,7 @@ struct Table { #define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ #define TF_Virtual 0x10 /* Is a virtual table */ #define TF_WithoutRowid 0x20 /* No rowid used. PRIMARY KEY is the key */ +#define TF_Recursive 0x40 /* Recursive reference within CTE */ /* @@ -2129,6 +2130,7 @@ struct NameContext { */ struct Select { ExprList *pEList; /* The fields of the result */ + Table *pRecurse; /* Non-NULL for the recursive part of recursive CTE */ u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ u16 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ @@ -2182,6 +2184,7 @@ struct Select { #define SRT_Table 8 /* Store result as data with an automatic rowid */ #define SRT_EphemTab 9 /* Create transient tab and store like SRT_Table */ #define SRT_Coroutine 10 /* Generate a single row of result */ +#define SRT_DistTable 11 /* Like SRT_TABLE, but unique results only */ /* ** An instance of this object describes where to put of the results of @@ -2647,6 +2650,7 @@ struct With { ExprList *pCols; /* List of explicit column names, or NULL */ Select *pSelect; /* The contents of the CTE */ struct Cte *pOuterCte; + Table *pTab; } a[1]; }; diff --git a/src/vdbe.c b/src/vdbe.c index 286bc45ba3..ffb4dd19bd 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3369,6 +3369,53 @@ case OP_OpenEphemeral: { break; } +/* Opcode: OpenEphreader P1 P2 * * * +** +** P2 is a cursor opened by the OpenEphemeral opcode. This opcode opens +** a new read-only cursor named P1 that accesses the same epheremal table +** as P2. +*/ +case OP_OpenEphreader: { + VdbeCursor *pEph; + VdbeCursor *pCx; + Pgno pgno; + + pEph = p->apCsr[pOp->p2]; + pCx = allocateCursor(p, pOp->p1, pEph->nField, -1, 1); + if( pCx==0 ) goto no_mem; + pCx->nullRow = 1; + pCx->pKeyInfo = pEph->pKeyInfo; + pCx->isTable = pEph->isTable; + pCx->isOrdered = pEph->isOrdered; + pgno = MASTER_ROOT + !pCx->isTable; + rc = sqlite3BtreeCursor(pEph->pBt, pgno, 0, pCx->pKeyInfo, pCx->pCursor); + break; +} + +/* Opcode: SwapCursors P1 P2 * * * +** +** Parameters P1 and P2 are both cursors opened by the OpenEphemeral +** opcode. This opcode deletes the contents of epheremal table P1, +** then renames P2 to P1 and P1 to P2. In other words, following this +** opcode cursor P2 is open on an empty table and P1 is open on the +** table that was initially accessed by P2. +*/ +case OP_SwapCursors: { + Mem tmp; + VdbeCursor *pTmp; + + tmp = p->aMem[p->nMem - pOp->p1]; + p->aMem[p->nMem - pOp->p1] = p->aMem[p->nMem - pOp->p2]; + p->aMem[p->nMem - pOp->p2] = tmp; + + pTmp = p->apCsr[pOp->p1]; + p->apCsr[pOp->p1] = p->apCsr[pOp->p2]; + p->apCsr[pOp->p2] = pTmp; + + rc = sqlite3BtreeClearTable(pTmp->pBt, MASTER_ROOT + !pTmp->isTable, 0); + break; +} + /* Opcode: SorterOpen P1 * * P4 * ** ** This opcode works like OP_OpenEphemeral except that it opens diff --git a/src/where.c b/src/where.c index d5444a6054..8827faa839 100644 --- a/src/where.c +++ b/src/where.c @@ -5653,7 +5653,11 @@ WhereInfo *sqlite3WhereBegin( iDb = sqlite3SchemaToIndex(db, pTab->pSchema); pLoop = pLevel->pWLoop; if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ - /* Do nothing */ + if( pTab->tabFlags & TF_Recursive ){ + int iCur = pTabItem->iCursor; + sqlite3VdbeAddOp2(v, OP_OpenEphreader, iCur, pTab->tnum); + } + /* Otherwise do nothing */ }else #ifndef SQLITE_OMIT_VIRTUALTABLE if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ diff --git a/test/with1.test b/test/with1.test index 0b7810ebd3..3dd8c47ea2 100644 --- a/test/with1.test +++ b/test/with1.test @@ -137,6 +137,33 @@ do_execsql_test 4.3 { SELECT * FROM t1; } {1 3 8 9} +#------------------------------------------------------------------------- +# +do_execsql_test 5.1 { + WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i) + SELECT x FROM i LIMIT 10; +} {1 2 3 4 5 6 7 8 9 10} + +do_catchsql_test 5.2 { + WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i ORDER BY 1) + SELECT x FROM i LIMIT 10; +} {1 {ORDER BY in a recursive query is not allowed}} + +do_catchsql_test 5.3 { + WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i LIMIT 10 ) + SELECT x FROM i LIMIT 10; +} {1 {LIMIT in a recursive query is not allowed}} + +do_execsql_test 5.4 { + WITH i(x) AS ( VALUES(1) UNION ALL SELECT (x+1)%10 FROM i) + SELECT x FROM i LIMIT 20; +} {1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0} + +do_execsql_test 5.5 { + WITH i(x) AS ( VALUES(1) UNION SELECT (x+1)%10 FROM i) + SELECT x FROM i LIMIT 20; +} {1 2 3 4 5 6 7 8 9 0} + finish_test From f9db522fee74370e6994fe2c323da2412a097cf9 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Jan 2014 02:40:11 +0000 Subject: [PATCH 32/74] Use the user-supplied table name in WITH RECURSIVE tables as the internal name of the table and the name of the table in VDBE comments. FossilOrigin-Name: a29330238be6366444269a0b1b328475b2d01ae2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 56fe478fb2..3d4a6ad897 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\scode\sto\shandle\srecursive\sCTEs. -D 2014-01-14T20:14:09.053 +C Use\sthe\suser-supplied\stable\sname\sin\sWITH\sRECURSIVE\stables\sas\sthe\sinternal\nname\sof\sthe\stable\sand\sthe\sname\sof\sthe\stable\sin\sVDBE\scomments. +D 2014-01-15T02:40:11.688 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 0f7779b0c1c317dcac0f65f695d036030c16775f +F src/select.c 3d93e3b4736a95507540b708106c1db6d01c41a8 F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8839850c44a8938883e493eacd752fa686e542df -R 1d5d9f6beb143e76aa04b4bf9953f5da -U dan -Z 1616366ca894deb1473f9b1eb1563ebc +P a5c2a54a07d35166911abc792008c05dea897742 +R cc1fc541222b981017b72fc3aee14bde +U drh +Z dcce44e449a071e984cc874f0c8e1e54 diff --git a/manifest.uuid b/manifest.uuid index f994f09e97..1be3de1308 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a5c2a54a07d35166911abc792008c05dea897742 \ No newline at end of file +a29330238be6366444269a0b1b328475b2d01ae2 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0db4b01807..22eaf1b1cb 100644 --- a/src/select.c +++ b/src/select.c @@ -3558,7 +3558,7 @@ static int withExpand( pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; pTab->nRef = 1; - pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); + pTab->zName = sqlite3MPrintf(db, "%s", pCte->zName); pTab->iPKey = -1; pTab->nRowEst = 1048576; pTab->tabFlags |= TF_Ephemeral; From bfe31e7f80a97601c4d68e9dbc359493669ab7e8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Jan 2014 14:17:31 +0000 Subject: [PATCH 33/74] Disable the flattening optimization if the sub-query is a recursive CTE. FossilOrigin-Name: 9472f6d820a7fb233936d9b8f7a39c9d4c4d6d73 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/expr.c | 10 ++++++++-- src/parse.y | 3 ++- src/select.c | 3 +++ test/with1.test | 44 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 68 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 3d4a6ad897..1f819eb388 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sthe\suser-supplied\stable\sname\sin\sWITH\sRECURSIVE\stables\sas\sthe\sinternal\nname\sof\sthe\stable\sand\sthe\sname\sof\sthe\stable\sin\sVDBE\scomments. -D 2014-01-15T02:40:11.688 +C Disable\sthe\sflattening\soptimization\sif\sthe\ssub-query\sis\sa\srecursive\sCTE. +D 2014-01-15T14:17:31.048 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff -F src/expr.c fee4b54fdcf5a979e6f5012da3dfb084ac5d3aac +F src/expr.c 2277282d8938b8eab782e4f5111ca0cd492529b3 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -209,7 +209,7 @@ F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968 F src/os_win.c 16eac0961603182ffc10c02b39fe830126538e07 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y e5c4a23f445cde4b30d50948df8f21bc586dd7a4 +F src/parse.y 824eeb752c026b551bda2b66163889d7664b42e4 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 3d93e3b4736a95507540b708106c1db6d01c41a8 +F src/select.c 51c74176eb949d1ff5797735acedd6c08d19450a F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1091,7 +1091,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test 12a6661eabcc9ae299020f7b4197a75a9c084748 +F test/with1.test 77739a9e5e88873e7655634297d9dc4360334f9a F test/withM.test ac3ec7ee0b33a02d0fa15da91214d97ddea64e34 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a5c2a54a07d35166911abc792008c05dea897742 -R cc1fc541222b981017b72fc3aee14bde -U drh -Z dcce44e449a071e984cc874f0c8e1e54 +P a29330238be6366444269a0b1b328475b2d01ae2 +R baaf1d38169072daf03c7e43cc689f3b +U dan +Z e4d26324d0deafbbb1cfbafb0f22db9f diff --git a/manifest.uuid b/manifest.uuid index 1be3de1308..493b55e692 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a29330238be6366444269a0b1b328475b2d01ae2 \ No newline at end of file +9472f6d820a7fb233936d9b8f7a39c9d4c4d6d73 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index e271e46796..4da9d6263c 100644 --- a/src/expr.c +++ b/src/expr.c @@ -895,7 +895,12 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ return pNew; } -With *withDup(sqlite3 *db, With *p){ +/* +** Create and return a deep copy of the object passed as the second +** argument. If an OOM condition is encountered, NULL is returned +** and the db->mallocFailed flag set. +*/ +static With *withDup(sqlite3 *db, With *p){ With *pRet = 0; if( p ){ int nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); @@ -1055,7 +1060,8 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->addrOpenEphm[1] = -1; pNew->addrOpenEphm[2] = -1; pNew->pWith = withDup(db, p->pWith); - pNew->pRecurse = p->pRecurse; + assert( p->pRecurse==0 ); + pNew->pRecurse = 0; return pNew; } #else diff --git a/src/parse.y b/src/parse.y index b1992f2415..937669eb5c 100644 --- a/src/parse.y +++ b/src/parse.y @@ -684,8 +684,9 @@ where_opt(A) ::= WHERE expr(X). {A = X.pExpr;} ////////////////////////// The UPDATE command //////////////////////////////// // %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT -cmd ::= with UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) +cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) orderby_opt(O) limit_opt(L). { + sqlite3WithPush(pParse, C); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE"); diff --git a/src/select.c b/src/select.c index 22eaf1b1cb..35edcbb03c 100644 --- a/src/select.c +++ b/src/select.c @@ -2844,6 +2844,8 @@ static void substSelect( ** ** Flattening is only attempted if all of the following are true: ** +** (0) The subquery is not a recursive CTE. +** ** (1) The subquery and the outer query do not both use aggregates. ** ** (2) The subquery is not an aggregate or the outer query is not a join. @@ -2968,6 +2970,7 @@ static int flattenSubquery( iParent = pSubitem->iCursor; pSub = pSubitem->pSelect; assert( pSub!=0 ); + if( pSub->pRecurse ) return 0; /* Restriction (0) */ if( isAgg && subqueryIsAgg ) return 0; /* Restriction (1) */ if( subqueryIsAgg && pSrc->nSrc>1 ) return 0; /* Restriction (2) */ pSubSrc = pSub->pSrc; diff --git a/test/with1.test b/test/with1.test index 3dd8c47ea2..480afa8074 100644 --- a/test/with1.test +++ b/test/with1.test @@ -164,6 +164,50 @@ do_execsql_test 5.5 { SELECT x FROM i LIMIT 20; } {1 2 3 4 5 6 7 8 9 0} +#------------------------------------------------------------------------- +# +do_execsql_test 6.1 { + CREATE TABLE f( + id INTEGER PRIMARY KEY, parentid REFERENCES f, name TEXT + ); + + INSERT INTO f VALUES(0, NULL, ''); + INSERT INTO f VALUES(1, 0, 'bin'); + INSERT INTO f VALUES(2, 1, 'true'); + INSERT INTO f VALUES(3, 1, 'false'); + INSERT INTO f VALUES(4, 1, 'ls'); + INSERT INTO f VALUES(5, 1, 'grep'); + INSERT INTO f VALUES(6, 0, 'etc'); + INSERT INTO f VALUES(7, 6, 'rc.d'); + INSERT INTO f VALUES(8, 7, 'rc.apache'); + INSERT INTO f VALUES(9, 7, 'rc.samba'); + INSERT INTO f VALUES(10, 0, 'home'); + INSERT INTO f VALUES(11, 10, 'dan'); + INSERT INTO f VALUES(12, 11, 'public_html'); + INSERT INTO f VALUES(13, 12, 'index.html'); + INSERT INTO f VALUES(14, 13, 'logo.gif'); +} + +do_execsql_test 6.2 { + WITH flat(fid, fpath) AS ( + SELECT id, '' FROM f WHERE parentid IS NULL + UNION ALL + SELECT id, fpath || '/' || name FROM f, flat WHERE +parentid=+fid + ) + SELECT fpath FROM flat WHERE fpath!='' ORDER BY 1; +} { + /bin + /bin/false /bin/grep /bin/ls /bin/true + /etc + /etc/rc.d + /etc/rc.d/rc.apache /etc/rc.d/rc.samba + /home + /home/dan + /home/dan/public_html + /home/dan/public_html/index.html + /home/dan/public_html/index.html/logo.gif +} + finish_test From a379b32f336f4ec5ff35c9a9acf323e95538eec4 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Jan 2014 14:40:41 +0000 Subject: [PATCH 34/74] Don't try to verify the schema of transient table (such as generated inside a WITH clause) when generating code for "IN table" operators. FossilOrigin-Name: 860aa936634a60d68e3954fc408a96a9260394e0 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 8 +++++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 1f819eb388..c853513dbd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\sflattening\soptimization\sif\sthe\ssub-query\sis\sa\srecursive\sCTE. -D 2014-01-15T14:17:31.048 +C Don't\stry\sto\sverify\sthe\sschema\sof\stransient\stable\s(such\sas\sgenerated\sinside\na\sWITH\sclause)\swhen\sgenerating\scode\sfor\s"IN\stable"\soperators. +D 2014-01-15T14:40:41.242 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff -F src/expr.c 2277282d8938b8eab782e4f5111ca0cd492529b3 +F src/expr.c 46ad3b4161aeaf299130305efd0259bc5643bfd6 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a29330238be6366444269a0b1b328475b2d01ae2 -R baaf1d38169072daf03c7e43cc689f3b -U dan -Z e4d26324d0deafbbb1cfbafb0f22db9f +P 9472f6d820a7fb233936d9b8f7a39c9d4c4d6d73 +R 719190df89fa0f9b34e7fe5eb5626240 +U drh +Z 58f028530f5aaa7c16053558b5f5262a diff --git a/manifest.uuid b/manifest.uuid index 493b55e692..cc099a86d2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9472f6d820a7fb233936d9b8f7a39c9d4c4d6d73 \ No newline at end of file +860aa936634a60d68e3954fc408a96a9260394e0 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 4da9d6263c..fca03a8d99 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1581,9 +1581,11 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ iCol = (i16)pExpr->iColumn; /* Code an OP_VerifyCookie and OP_TableLock for . */ - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - sqlite3CodeVerifySchema(pParse, iDb); - sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + if( pTab->pSchema ){ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); + } /* This function is only called from two places. In both cases the vdbe ** has already been allocated. So assume sqlite3GetVdbe() is always From 60e7068d75acc643cfd045a7cb90d635079f3457 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Jan 2014 15:27:51 +0000 Subject: [PATCH 35/74] Return an error if a CTE specifies a different number of columns than its SELECT statement returns. FossilOrigin-Name: 9a514b50e4b01f109fbdb0aabcbfe1ddab129b44 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 14 ++++++++++---- test/with1.test | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index c853513dbd..d314554380 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Don't\stry\sto\sverify\sthe\sschema\sof\stransient\stable\s(such\sas\sgenerated\sinside\na\sWITH\sclause)\swhen\sgenerating\scode\sfor\s"IN\stable"\soperators. -D 2014-01-15T14:40:41.242 +C Return\san\serror\sif\sa\sCTE\sspecifies\sa\sdifferent\snumber\sof\scolumns\sthan\sits\sSELECT\sstatement\sreturns. +D 2014-01-15T15:27:51.337 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 51c74176eb949d1ff5797735acedd6c08d19450a +F src/select.c 7d0d85f5d0a0f35be49230a3b6609fa534980015 F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1091,7 +1091,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test 77739a9e5e88873e7655634297d9dc4360334f9a +F test/with1.test babb3d9c4007596d2d74468e001b1dc02d1ada91 F test/withM.test ac3ec7ee0b33a02d0fa15da91214d97ddea64e34 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9472f6d820a7fb233936d9b8f7a39c9d4c4d6d73 -R 719190df89fa0f9b34e7fe5eb5626240 -U drh -Z 58f028530f5aaa7c16053558b5f5262a +P 860aa936634a60d68e3954fc408a96a9260394e0 +R 0c042289cd11db254af8c2a3a3653738 +U dan +Z d4b370476cdc85ae036fb519ef522aac diff --git a/manifest.uuid b/manifest.uuid index cc099a86d2..2aac3c13b1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -860aa936634a60d68e3954fc408a96a9260394e0 \ No newline at end of file +9a514b50e4b01f109fbdb0aabcbfe1ddab129b44 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 35edcbb03c..34a2297c96 100644 --- a/src/select.c +++ b/src/select.c @@ -3556,6 +3556,7 @@ static int withExpand( }else{ ExprList *pEList; Select *pSel; + Select *pLeft; /* Left-most SELECT statement */ int bRecursive; pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); @@ -3579,13 +3580,18 @@ static int withExpand( sqlite3WalkSelect(pWalker, pSel); } + for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); + pEList = pLeft->pEList; if( pCte->pCols ){ + if( pEList->nExpr!=pCte->pCols->nExpr ){ + sqlite3ErrorMsg(pParse, "cte \"%s\" returns %d values for %d columns", + pCte->zName, pEList->nExpr, pCte->pCols->nExpr + ); + return WRC_Abort; + } pEList = pCte->pCols; - }else{ - Select *pLeft; - for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); - pEList = pLeft->pEList; } + selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); if( bRecursive ){ diff --git a/test/with1.test b/test/with1.test index 480afa8074..dd1d5c0ab0 100644 --- a/test/with1.test +++ b/test/with1.test @@ -164,6 +164,42 @@ do_execsql_test 5.5 { SELECT x FROM i LIMIT 20; } {1 2 3 4 5 6 7 8 9 0} +do_catchsql_test 5.6.1 { + WITH i(x, y) AS ( VALUES(1) ) + SELECT * FROM i; +} {1 {cte "i" returns 1 values for 2 columns}} + +do_catchsql_test 5.6.2 { + WITH i(x) AS ( VALUES(1,2) ) + SELECT * FROM i; +} {1 {cte "i" returns 2 values for 1 columns}} + +do_catchsql_test 5.6.3 { + CREATE TABLE t5(a, b); + WITH i(x) AS ( SELECT * FROM t5 ) + SELECT * FROM i; +} {1 {cte "i" returns 2 values for 1 columns}} + +do_catchsql_test 5.6.4 { + WITH i(x) AS ( SELECT 1, 2 UNION ALL SELECT 1 ) + SELECT * FROM i; +} {1 {cte "i" returns 2 values for 1 columns}} + +do_catchsql_test 5.6.5 { + WITH i(x) AS ( SELECT 1 UNION ALL SELECT 1, 2 ) + SELECT * FROM i; +} {1 {SELECTs to the left and right of UNION ALL do not have the same number of result columns}} + +do_catchsql_test 5.6.6 { + WITH i(x) AS ( SELECT 1 UNION ALL SELECT x+1, x*2 FROM i ) + SELECT * FROM i; +} {1 {SELECTs to the left and right of UNION ALL do not have the same number of result columns}} + +do_catchsql_test 5.6.7 { + WITH i(x) AS ( SELECT 1, 2 UNION SELECT x+1 FROM i ) + SELECT * FROM i; +} {1 {cte "i" returns 2 values for 1 columns}} + #------------------------------------------------------------------------- # do_execsql_test 6.1 { From f43fe6e9f6e8e7e7c2ab43234f46c5767f63b27c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Jan 2014 18:12:00 +0000 Subject: [PATCH 36/74] When resolving names, consider a reference to a recursive CTE column as equivalent to a reference to the outermost name-context. This ensures that correlated sub-queries are correctly identified as such. FossilOrigin-Name: 61be2da0ae623c1572819481508b044e9d32f294 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/expr.c | 1 + src/resolve.c | 8 ++++++++ src/select.c | 2 ++ src/sqliteInt.h | 1 + test/with1.test | 19 +++++++++++++++++++ 7 files changed, 42 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d314554380..7847f8e691 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Return\san\serror\sif\sa\sCTE\sspecifies\sa\sdifferent\snumber\sof\scolumns\sthan\sits\sSELECT\sstatement\sreturns. -D 2014-01-15T15:27:51.337 +C When\sresolving\snames,\sconsider\sa\sreference\sto\sa\srecursive\sCTE\scolumn\sas\sequivalent\sto\sa\sreference\sto\sthe\soutermost\sname-context.\sThis\sensures\sthat\scorrelated\ssub-queries\sare\scorrectly\sidentified\sas\ssuch. +D 2014-01-15T18:12:00.359 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff -F src/expr.c 46ad3b4161aeaf299130305efd0259bc5643bfd6 +F src/expr.c fccff6c8cd170dc1df244fdd4befb2ec783b72b1 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -217,14 +217,14 @@ F src/pragma.c ed409ce4104cf4d9de6ead40ace70974f124853b F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 +F src/resolve.c 41d0cf644aa98131204e6243e108829797f038ab F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 7d0d85f5d0a0f35be49230a3b6609fa534980015 +F src/select.c c6ba9c34f5092aeada10928b93c798a5a262c85a F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h fc7b2516260c4e14bd4342cd9ed47f3d0ca27cc5 +F src/sqliteInt.h 31bcde5190ca666e4873188a9039ac6ff017a0a8 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1091,7 +1091,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test babb3d9c4007596d2d74468e001b1dc02d1ada91 +F test/with1.test 951807f7246215ec26cf5e9946f1b355ba892f89 F test/withM.test ac3ec7ee0b33a02d0fa15da91214d97ddea64e34 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 860aa936634a60d68e3954fc408a96a9260394e0 -R 0c042289cd11db254af8c2a3a3653738 +P 9a514b50e4b01f109fbdb0aabcbfe1ddab129b44 +R 7a326e6d7d898da8a108f7b49c21709d U dan -Z d4b370476cdc85ae036fb519ef522aac +Z a1fae4bd21e81652e55bd538f546aecf diff --git a/manifest.uuid b/manifest.uuid index 2aac3c13b1..7d2b8e9f8a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9a514b50e4b01f109fbdb0aabcbfe1ddab129b44 \ No newline at end of file +61be2da0ae623c1572819481508b044e9d32f294 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index fca03a8d99..9a288b73b2 100644 --- a/src/expr.c +++ b/src/expr.c @@ -998,6 +998,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ pNewItem->regReturn = pOldItem->regReturn; pNewItem->isCorrelated = pOldItem->isCorrelated; pNewItem->viaCoroutine = pOldItem->viaCoroutine; + pNewItem->isRecursive = pOldItem->isRecursive; pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex); pNewItem->notIndexed = pOldItem->notIndexed; pNewItem->pIndex = pOldItem->pIndex; diff --git a/src/resolve.c b/src/resolve.c index b0adb86295..d54442e79c 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -502,6 +502,14 @@ lookupname_end: if( pExpr->op!=TK_AS ){ sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); } + + /* If this expression reads a column value from a recursive CTE + ** reference, then this is equivalent to reading from the outermost + ** available name-context. */ + if( pMatch && pMatch->isRecursive ){ + while( pNC->pNext ) pNC = pNC->pNext; + } + /* Increment the nRef value on all name contexts from TopNC up to ** the point where the name matched. */ for(;;){ diff --git a/src/select.c b/src/select.c index 34a2297c96..3f9f2de551 100644 --- a/src/select.c +++ b/src/select.c @@ -3551,7 +3551,9 @@ static int withExpand( if( pCte==pParse->pCte && (pTab = pCte->pTab) ){ /* This is the recursive part of a recursive CTE */ + assert( pFrom->pTab==0 && pFrom->isRecursive==0 ); pFrom->pTab = pTab; + pFrom->isRecursive = 1; pTab->nRef++; }else{ ExprList *pEList; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c7f0609ba1..c52013f87f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2019,6 +2019,7 @@ struct SrcList { unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ unsigned isCorrelated :1; /* True if sub-query is correlated */ unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ #ifndef SQLITE_OMIT_EXPLAIN u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ #endif diff --git a/test/with1.test b/test/with1.test index dd1d5c0ab0..1f227e8165 100644 --- a/test/with1.test +++ b/test/with1.test @@ -244,6 +244,25 @@ do_execsql_test 6.2 { /home/dan/public_html/index.html/logo.gif } +do_execsql_test 6.3 { + WITH flat(fid, fpath) AS ( + SELECT id, '' FROM f WHERE parentid IS NULL + UNION ALL + SELECT id, fpath || '/' || name FROM f, flat WHERE parentid=+fid + ) + SELECT count(*) FROM flat; +} {15} + +do_execsql_test 6.4 { + WITH x(i) AS ( + SELECT 1 + UNION ALL + SELECT i+1 FROM x WHERE i<10 + ) + SELECT count(*) FROM x +} {10} + + finish_test From 62ba4e418d9105671599b7f9287aa0d3dce031bc Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Jan 2014 18:21:41 +0000 Subject: [PATCH 37/74] Disable automatic indices on recursive CTE references. FossilOrigin-Name: 28aa6db8c878655255dbfb618f8d65be78e3d7e5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 1 + test/with1.test | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 7847f8e691..e2d4787ec4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sresolving\snames,\sconsider\sa\sreference\sto\sa\srecursive\sCTE\scolumn\sas\sequivalent\sto\sa\sreference\sto\sthe\soutermost\sname-context.\sThis\sensures\sthat\scorrelated\ssub-queries\sare\scorrectly\sidentified\sas\ssuch. -D 2014-01-15T18:12:00.359 +C Disable\sautomatic\sindices\son\srecursive\sCTE\sreferences. +D 2014-01-15T18:21:41.108 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 830b42f452cfbc4e17582f6c1d388e15b379b833 +F src/where.c 9448a176c10020a4446a8d5e2cdaecf7526e593d F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1091,7 +1091,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test 951807f7246215ec26cf5e9946f1b355ba892f89 +F test/with1.test fcd45b0128292d57f4435df873b51b7261edc9ba F test/withM.test ac3ec7ee0b33a02d0fa15da91214d97ddea64e34 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9a514b50e4b01f109fbdb0aabcbfe1ddab129b44 -R 7a326e6d7d898da8a108f7b49c21709d +P 61be2da0ae623c1572819481508b044e9d32f294 +R 054a2146ee506f873fefb3d9cec1bd30 U dan -Z a1fae4bd21e81652e55bd538f546aecf +Z 902f9935ce7a37fa7329c31a191dfe9e diff --git a/manifest.uuid b/manifest.uuid index 7d2b8e9f8a..5d6578e47c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -61be2da0ae623c1572819481508b044e9d32f294 \ No newline at end of file +28aa6db8c878655255dbfb618f8d65be78e3d7e5 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 8827faa839..7ac8c57f23 100644 --- a/src/where.c +++ b/src/where.c @@ -4196,6 +4196,7 @@ static int whereLoopAddBtree( && !pSrc->notIndexed && HasRowid(pTab) && !pSrc->isCorrelated + && !pSrc->isRecursive ){ /* Generate auto-index WhereLoops */ WhereTerm *pTerm; diff --git a/test/with1.test b/test/with1.test index 1f227e8165..5c59755ce0 100644 --- a/test/with1.test +++ b/test/with1.test @@ -228,7 +228,7 @@ do_execsql_test 6.2 { WITH flat(fid, fpath) AS ( SELECT id, '' FROM f WHERE parentid IS NULL UNION ALL - SELECT id, fpath || '/' || name FROM f, flat WHERE +parentid=+fid + SELECT id, fpath || '/' || name FROM f, flat WHERE parentid=fid ) SELECT fpath FROM flat WHERE fpath!='' ORDER BY 1; } { @@ -248,7 +248,7 @@ do_execsql_test 6.3 { WITH flat(fid, fpath) AS ( SELECT id, '' FROM f WHERE parentid IS NULL UNION ALL - SELECT id, fpath || '/' || name FROM f, flat WHERE parentid=+fid + SELECT id, fpath || '/' || name FROM f, flat WHERE parentid=fid ) SELECT count(*) FROM flat; } {15} From 60c1a2f0b5aae5e0660306d3f996d826953e117a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Jan 2014 18:23:00 +0000 Subject: [PATCH 38/74] Add a header comment to the searchWith() routine. FossilOrigin-Name: d9ae0f5d9f8230ca7ca10ebed300e2f6635a0614 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 4 ++++ 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index e2d4787ec4..7b7db93b32 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sautomatic\sindices\son\srecursive\sCTE\sreferences. -D 2014-01-15T18:21:41.108 +C Add\sa\sheader\scomment\sto\sthe\ssearchWith()\sroutine. +D 2014-01-15T18:23:00.349 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 41d0cf644aa98131204e6243e108829797f038ab F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c c6ba9c34f5092aeada10928b93c798a5a262c85a +F src/select.c 240e9795df3fbaca15e2fa9c3b0a95796e60e5d7 F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 61be2da0ae623c1572819481508b044e9d32f294 -R 054a2146ee506f873fefb3d9cec1bd30 -U dan -Z 902f9935ce7a37fa7329c31a191dfe9e +P 28aa6db8c878655255dbfb618f8d65be78e3d7e5 +R 19b67b003a2571a98d1dce07a86e95af +U drh +Z 5c4be2977b3d2e13595e058b593ca068 diff --git a/manifest.uuid b/manifest.uuid index 5d6578e47c..3f055fe737 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28aa6db8c878655255dbfb618f8d65be78e3d7e5 \ No newline at end of file +d9ae0f5d9f8230ca7ca10ebed300e2f6635a0614 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 3f9f2de551..3a30031c00 100644 --- a/src/select.c +++ b/src/select.c @@ -3482,6 +3482,10 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ return WRC_Continue; } +/* If the table identified by p is a transient table of +** a common-table-expression (CTE) then return a pointer to the +** CTE that defines table p. If p is not a CTE, then return NULL. +*/ static struct Cte *searchWith(Parse *pParse, struct SrcList_item *p){ if( p->zDatabase==0 ){ char *zName = p->zName; From c49832c208283ee22b107d00246af355585320a9 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Jan 2014 18:35:52 +0000 Subject: [PATCH 39/74] Further comments on WITH-clause processing routines in select.c. FossilOrigin-Name: c948384dfdd9f68a832d5a452af44f35337f66e7 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 13 ++++++++++--- src/sqliteInt.h | 18 +++++++++--------- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 7b7db93b32..b46bf3bc83 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\sheader\scomment\sto\sthe\ssearchWith()\sroutine. -D 2014-01-15T18:23:00.349 +C Further\scomments\son\sWITH-clause\sprocessing\sroutines\sin\sselect.c. +D 2014-01-15T18:35:52.482 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 41d0cf644aa98131204e6243e108829797f038ab F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 240e9795df3fbaca15e2fa9c3b0a95796e60e5d7 +F src/select.c 6bdb90db59e3e8277b8953e4cd3df01ccc253d70 F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 31bcde5190ca666e4873188a9039ac6ff017a0a8 +F src/sqliteInt.h d91991457386b4a1414fa96f51a9a5bfd2b8606d F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 28aa6db8c878655255dbfb618f8d65be78e3d7e5 -R 19b67b003a2571a98d1dce07a86e95af +P d9ae0f5d9f8230ca7ca10ebed300e2f6635a0614 +R de20d9dcfc2fdd09cc5e9c05dc6deb70 U drh -Z 5c4be2977b3d2e13595e058b593ca068 +Z 626093c1115d7d3f00c431f06f508aa1 diff --git a/manifest.uuid b/manifest.uuid index 3f055fe737..fc2d2ece54 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d9ae0f5d9f8230ca7ca10ebed300e2f6635a0614 \ No newline at end of file +c948384dfdd9f68a832d5a452af44f35337f66e7 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 3a30031c00..0984bd30ed 100644 --- a/src/select.c +++ b/src/select.c @@ -3482,9 +3482,9 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ return WRC_Continue; } -/* If the table identified by p is a transient table of +/* If the table identified by FROM clause element p is really ** a common-table-expression (CTE) then return a pointer to the -** CTE that defines table p. If p is not a CTE, then return NULL. +** CTE definition for that table. */ static struct Cte *searchWith(Parse *pParse, struct SrcList_item *p){ if( p->zDatabase==0 ){ @@ -3503,6 +3503,11 @@ static struct Cte *searchWith(Parse *pParse, struct SrcList_item *p){ return 0; } +/* The code generator maintains a stack of active WITH clauses +** with the inner-most WITH clause being at the top of the stack. +** +** These routines push and pull WITH clauses on the stack. +*/ void sqlite3WithPush(Parse *pParse, With *pWith){ if( pWith ){ pWith->pOuter = pParse->pWith; @@ -3516,6 +3521,9 @@ static void withPop(Parse *pParse, With *pWith){ } } +/* Push or pull a CTE on the stack of all CTEs currently being +** coded. +*/ static int ctePush(Parse *pParse, struct Cte *pCte){ if( pCte ){ struct Cte *p; @@ -3533,7 +3541,6 @@ static int ctePush(Parse *pParse, struct Cte *pCte){ } return SQLITE_OK; } - static void ctePop(Parse *pParse, struct Cte *pCte){ if( pCte ){ assert( pParse->pCte==pCte ); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index c52013f87f..3973c02830 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2640,18 +2640,18 @@ int sqlite3WalkSelectFrom(Walker*, Select*); #define WRC_Abort 2 /* Abandon the tree walk */ /* -** An instance of this structure represents a set of CTEs (common table -** expressions) created by a single WITH clause. +** An instance of this structure represents a set of on or more CTEs +** (common table expressions) created by a single WITH clause. */ struct With { - int nCte; /* Number of CTEs */ + int nCte; /* Number of CTEs in the WITH clause */ With *pOuter; /* Containing WITH clause, or NULL */ - struct Cte { - char *zName; /* Name of this CTE */ - ExprList *pCols; /* List of explicit column names, or NULL */ - Select *pSelect; /* The contents of the CTE */ - struct Cte *pOuterCte; - Table *pTab; + struct Cte { /* For each CTE in the WITH clause.... */ + char *zName; /* Name of this CTE */ + ExprList *pCols; /* List of explicit column names, or NULL */ + Select *pSelect; /* The definition of this CTE */ + struct Cte *pOuterCte; /* Next WITH clause in outer context */ + Table *pTab; /* Table object for this CTE */ } a[1]; }; From eede6a538d32de61cc644e09fcb90264496791ac Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 15 Jan 2014 19:42:23 +0000 Subject: [PATCH 40/74] Fixes so that SQLITE_OMIT_CTE builds work. FossilOrigin-Name: 3908e2ea2e7e5f466cbbbffdc27e0fe8dc9751ac --- manifest | 30 ++++++------ manifest.uuid | 2 +- src/expr.c | 4 ++ src/resolve.c | 2 + src/select.c | 121 ++++++++++++++++++++++++++++++---------------- src/sqliteInt.h | 1 + src/test_config.c | 6 +++ src/vdbe.c | 2 + src/where.c | 2 + test/with1.test | 5 ++ test/withM.test | 5 ++ 11 files changed, 123 insertions(+), 57 deletions(-) diff --git a/manifest b/manifest index b46bf3bc83..dae060c0f8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Further\scomments\son\sWITH-clause\sprocessing\sroutines\sin\sselect.c. -D 2014-01-15T18:35:52.482 +C Fixes\sso\sthat\sSQLITE_OMIT_CTE\sbuilds\swork. +D 2014-01-15T19:42:23.894 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff -F src/expr.c fccff6c8cd170dc1df244fdd4befb2ec783b72b1 +F src/expr.c 36c313049a716c3edcdd99d703f0f16ce96a4cc6 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -217,14 +217,14 @@ F src/pragma.c ed409ce4104cf4d9de6ead40ace70974f124853b F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c 41d0cf644aa98131204e6243e108829797f038ab +F src/resolve.c ae278d8ce037883323f677e78c241f64289f12ec F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 6bdb90db59e3e8277b8953e4cd3df01ccc253d70 +F src/select.c b55e726c8dff607790038bf41e6d47925a832b14 F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h d91991457386b4a1414fa96f51a9a5bfd2b8606d +F src/sqliteInt.h 6db445ce172c249c29d8f532af64fd7051fe05dd F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -242,7 +242,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c 10d0e00dd6315879a6d9fac20bd063c7bbbfb8f8 +F src/test_config.c 0336e0bdbe541b4af89d7e3dd0656e8e6b51e585 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -280,7 +280,7 @@ F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c e71f19b272f05c8695cf747b4bac1732685f9e5c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 9b918126afb9a7b7ad232db7453680c7ef5821a4 +F src/vdbe.c b06d79951a3f0da9cb41cb0aae5fdf5612f718a9 F src/vdbe.h e6c4c610fcabad4fa80ebb1efc6822a9367e2b26 F src/vdbeInt.h 42db251e9f863401ff847b90d5fe1614c89a6a56 F src/vdbeapi.c ce4e68ea4842cc6081046f533d088dcf01d247ad @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 9448a176c10020a4446a8d5e2cdaecf7526e593d +F src/where.c 27eb508c4184599a826e2ef3b4319cd28ba2c7af F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1091,8 +1091,8 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test fcd45b0128292d57f4435df873b51b7261edc9ba -F test/withM.test ac3ec7ee0b33a02d0fa15da91214d97ddea64e34 +F test/with1.test 8ac68051fce8f2fe4a7bb6cb7ba62c0e941b1c84 +F test/withM.test 52448ce23e1c2ecba79d10e130ee49ce9f9a2a7a F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 F test/without_rowid3.test eac3d5c8a1924725b58503a368f2cbd24fd6c8a0 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d9ae0f5d9f8230ca7ca10ebed300e2f6635a0614 -R de20d9dcfc2fdd09cc5e9c05dc6deb70 -U drh -Z 626093c1115d7d3f00c431f06f508aa1 +P c948384dfdd9f68a832d5a452af44f35337f66e7 +R 366ade7d6e8c82851813f265ff0c51f5 +U dan +Z a6417c60f97c097fa8d751079d1185b1 diff --git a/manifest.uuid b/manifest.uuid index fc2d2ece54..cab35f4394 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c948384dfdd9f68a832d5a452af44f35337f66e7 \ No newline at end of file +3908e2ea2e7e5f466cbbbffdc27e0fe8dc9751ac \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 9a288b73b2..68bb2fe53e 100644 --- a/src/expr.c +++ b/src/expr.c @@ -900,6 +900,7 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){ ** argument. If an OOM condition is encountered, NULL is returned ** and the db->mallocFailed flag set. */ +#ifndef SQLITE_OMIT_CTE static With *withDup(sqlite3 *db, With *p){ With *pRet = 0; if( p ){ @@ -917,6 +918,9 @@ static With *withDup(sqlite3 *db, With *p){ } return pRet; } +#else +# define withDup(x,y) 0 +#endif /* ** The following group of routines make deep copies of expressions, diff --git a/src/resolve.c b/src/resolve.c index d54442e79c..724b8538fa 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -503,12 +503,14 @@ lookupname_end: sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); } +#ifndef SQLITE_OMIT_CTE /* If this expression reads a column value from a recursive CTE ** reference, then this is equivalent to reading from the outermost ** available name-context. */ if( pMatch && pMatch->isRecursive ){ while( pNC->pNext ) pNC = pNC->pNext; } +#endif /* Increment the nRef value on all name contexts from TopNC up to ** the point where the name matched. */ diff --git a/src/select.c b/src/select.c index 0984bd30ed..7f20dc689a 100644 --- a/src/select.c +++ b/src/select.c @@ -1789,6 +1789,7 @@ static int multiSelect( goto multi_select_end; } +#ifndef SQLITE_OMIT_CTE /* If this is a recursive query, check that there is no ORDER BY or ** LIMIT clause. Neither of these are supported. */ assert( p->pOffset==0 || p->pLimit ); @@ -1799,13 +1800,6 @@ static int multiSelect( goto multi_select_end; } - /* Compound SELECTs that have an ORDER BY clause are handled separately. - */ - if( p->pOrderBy ){ - return multiSelectOrderBy(pParse, p, pDest); - } - -#ifndef SQLITE_OMIT_CTE if( p->pRecurse ){ int nCol = p->pEList->nExpr; int addrNext; @@ -1838,7 +1832,7 @@ static int multiSelect( rc = sqlite3Select(pParse, pPrior, &tmp2dest); if( rc ) goto multi_select_end; - /* Clear tmp1. Then switch the contents of tmp1 and tmp2. Then teturn + /* Clear tmp1. Then switch the contents of tmp1 and tmp2. Then return ** the contents of tmp1 to the caller. Or, if tmp1 is empty at this ** point, the recursive query has finished - jump to address iBreak. */ addrSwap = sqlite3VdbeAddOp2(v, OP_SwapCursors, tmp1, tmp2); @@ -1865,6 +1859,12 @@ static int multiSelect( }else #endif + /* Compound SELECTs that have an ORDER BY clause are handled separately. + */ + if( p->pOrderBy ){ + return multiSelectOrderBy(pParse, p, pDest); + }else + /* Generate code for the left and right SELECT statements. */ switch( p->op ){ @@ -3482,20 +3482,23 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ return WRC_Continue; } -/* If the table identified by FROM clause element p is really -** a common-table-expression (CTE) then return a pointer to the -** CTE definition for that table. +#ifndef SQLITE_OMIT_CTE +/* +** Argument pWith (which may be NULL) points to a linked list of nested +** WITH contexts, from inner to outermost. If the table identified by +** FROM clause element pItem is really a common-table-expression (CTE) +** then return a pointer to the CTE definition for that table. Otherwise +** return NULL. */ -static struct Cte *searchWith(Parse *pParse, struct SrcList_item *p){ - if( p->zDatabase==0 ){ - char *zName = p->zName; - With *pWith; - - for(pWith=pParse->pWith; pWith; pWith=pWith->pOuter){ +static struct Cte *searchWith(With *pWith, struct SrcList_item *pItem){ + if( pItem->zDatabase==0 ){ + const char *zName = pItem->zName; + With *p; + for(p=pWith; p; p=p->pOuter){ int i; - for(i=0; inCte; i++){ - if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){ - return &pWith->a[i]; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){ + return &p->a[i]; } } } @@ -3514,16 +3517,16 @@ void sqlite3WithPush(Parse *pParse, With *pWith){ pParse->pWith = pWith; } } -static void withPop(Parse *pParse, With *pWith){ - if( pWith ){ - assert( pParse->pWith==pWith ); - pParse->pWith = pWith->pOuter; - } -} -/* Push or pull a CTE on the stack of all CTEs currently being -** coded. -*/ +/* +** If argument pCte is not NULL, check if it is already a part of the +** stack of CTEs stored by the parser. If so, this indicates an illegal +** recursive reference in a CTE, set of mutually recursive CTEs. Store +** an error in the parser and return SQLITE_ERROR if this is the case. +** +** Otherwise, if pCte is not already part of the stack of CTEs stored +** in the parser, push it onto the stop of that stack. +*/ static int ctePush(Parse *pParse, struct Cte *pCte){ if( pCte ){ struct Cte *p; @@ -3541,6 +3544,11 @@ static int ctePush(Parse *pParse, struct Cte *pCte){ } return SQLITE_OK; } +/* +** If argument pCte is not NULL, it must be a pointer to the CTE currently +** on top of the stack of CTEs stored in the parser. Remove it from that +** stack. +*/ static void ctePop(Parse *pParse, struct Cte *pCte){ if( pCte ){ assert( pParse->pCte==pCte ); @@ -3548,21 +3556,38 @@ static void ctePop(Parse *pParse, struct Cte *pCte){ } } +/* +** This function checks if argument pFrom refers to a CTE declared by +** a WITH clause on the stack currently maintained by the parser. And, +** if currently processing a CTE expression, if it is a recursive +** reference to the current CTE. +** +** If pFrom falls into either of the two categories above, pFrom->pTab +** and other fields are populated accordingly. The caller should check +** (pFrom->pTab!=0) to determine whether or not a successful match +** was found. +** +** Whether or not a match is found, SQLITE_OK is returned if no error +** occurs. If an error does occur, an error message is stored in the +** parser and some error code other than SQLITE_OK returned. +*/ static int withExpand( Walker *pWalker, - struct SrcList_item *pFrom, - struct Cte *pCte + struct SrcList_item *pFrom ){ Table *pTab; Parse *pParse = pWalker->pParse; sqlite3 *db = pParse->db; + struct Cte *pCte; - assert( pFrom->pSelect==0 ); assert( pFrom->pTab==0 ); - if( pCte==pParse->pCte && (pTab = pCte->pTab) ){ + pCte = searchWith(pParse->pWith, pFrom); + if( pCte==0 ){ + /* no-op */ + }else if( pCte==pParse->pCte && (pTab = pCte->pTab) ){ /* This is the recursive part of a recursive CTE */ - assert( pFrom->pTab==0 && pFrom->isRecursive==0 ); + assert( pFrom->pTab==0 && pFrom->isRecursive==0 && pFrom->pSelect==0 ); pFrom->pTab = pTab; pFrom->isRecursive = 1; pTab->nRef++; @@ -3623,6 +3648,7 @@ static int withExpand( return SQLITE_OK; } +#endif /* ** This routine is a Walker callback for "expanding" a SELECT statement. @@ -3678,7 +3704,6 @@ static int selectExpander(Walker *pWalker, Select *p){ ** then create a transient table structure to describe the subquery. */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - struct Cte *pCte = 0; Table *pTab; if( pFrom->pTab!=0 ){ /* This statement has already been prepared. There is no need @@ -3687,10 +3712,8 @@ static int selectExpander(Walker *pWalker, Select *p){ return WRC_Prune; } #ifndef SQLITE_OMIT_CTE - pCte = searchWith(pParse, pFrom); - if( pCte ){ - if( withExpand(pWalker, pFrom, pCte) ) return WRC_Abort; - }else + if( withExpand(pWalker, pFrom) ) return WRC_Abort; + if( pFrom->pTab ) {} else #endif if( pFrom->zName==0 ){ #ifndef SQLITE_OMIT_SUBQUERY @@ -3917,13 +3940,29 @@ static int selectExpander(Walker *pWalker, Select *p){ return WRC_Continue; } +/* +** Function (or macro) selectExpanderWith is used as the SELECT callback +** by sqlite3SelectExpand(). In builds that do not support CTEs, this +** is equivalent to the selectExpander() function. In CTE-enabled builds, +** any WITH clause associated with the SELECT statement needs to be +** pushed onto the stack before calling selectExpander(), and popped +** off again afterwards. +*/ +#ifndef SQLITE_OMIT_CTE static int selectExpanderWith(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; int res; - sqlite3WithPush(pWalker->pParse, p->pWith); + sqlite3WithPush(pParse, p->pWith); res = selectExpander(pWalker, p); - withPop(pWalker->pParse, p->pWith); + if( p->pWith ){ + assert( pParse->pWith==p->pWith ); + pParse->pWith = p->pWith->pOuter; + } return res; } +#else +#define selectExpanderWith selectExpander +#endif /* ** No-op routine for the parse-tree walker. diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3973c02830..733dad2d8f 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3359,6 +3359,7 @@ const char *sqlite3JournalModename(int); void sqlite3WithPush(Parse*, With*); #else #define sqlite3WithPush(x,y) +#define sqlite3WithDelete(x,y) #endif /* Declarations for functions in fkey.c. All of these are replaced by diff --git a/src/test_config.c b/src/test_config.c index f44be40508..1db8198641 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -225,6 +225,12 @@ static void set_options(Tcl_Interp *interp){ Tcl_SetVar2(interp, "sqlite_options", "check", "1", TCL_GLOBAL_ONLY); #endif +#ifdef SQLITE_OMIT_CTE + Tcl_SetVar2(interp, "sqlite_options", "cte", "0", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "cte", "1", TCL_GLOBAL_ONLY); +#endif + #ifdef SQLITE_ENABLE_COLUMN_METADATA Tcl_SetVar2(interp, "sqlite_options", "columnmetadata", "1", TCL_GLOBAL_ONLY); #else diff --git a/src/vdbe.c b/src/vdbe.c index ffb4dd19bd..301707677d 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3369,6 +3369,7 @@ case OP_OpenEphemeral: { break; } +#ifndef SQLITE_OMIT_CTE /* Opcode: OpenEphreader P1 P2 * * * ** ** P2 is a cursor opened by the OpenEphemeral opcode. This opcode opens @@ -3415,6 +3416,7 @@ case OP_SwapCursors: { rc = sqlite3BtreeClearTable(pTmp->pBt, MASTER_ROOT + !pTmp->isTable, 0); break; } +#endif /* ifndef SQLITE_OMIT_CTE */ /* Opcode: SorterOpen P1 * * P4 * ** diff --git a/src/where.c b/src/where.c index 7ac8c57f23..f1cf29b2e7 100644 --- a/src/where.c +++ b/src/where.c @@ -5654,10 +5654,12 @@ WhereInfo *sqlite3WhereBegin( iDb = sqlite3SchemaToIndex(db, pTab->pSchema); pLoop = pLevel->pWLoop; if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ +#ifndef SQLITE_OMIT_CTE if( pTab->tabFlags & TF_Recursive ){ int iCur = pTabItem->iCursor; sqlite3VdbeAddOp2(v, OP_OpenEphreader, iCur, pTab->tnum); } +#endif /* Otherwise do nothing */ }else #ifndef SQLITE_OMIT_VIRTUALTABLE diff --git a/test/with1.test b/test/with1.test index 5c59755ce0..861c722543 100644 --- a/test/with1.test +++ b/test/with1.test @@ -16,6 +16,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set ::testprefix with1 +ifcapable {!cte} { + finish_test + return +} + do_execsql_test 1.0 { CREATE TABLE t1(x INTEGER, y INTEGER); WITH x(a) AS ( SELECT * FROM t1) SELECT 10 diff --git a/test/withM.test b/test/withM.test index 431cca187e..9bf7ceed3f 100644 --- a/test/withM.test +++ b/test/withM.test @@ -17,6 +17,11 @@ source $testdir/tester.tcl source $testdir/malloc_common.tcl set ::testprefix withM +ifcapable {!cte} { + finish_test + return +} + do_execsql_test 1.0 { CREATE TABLE t1(x INTEGER, y INTEGER); INSERT INTO t1 VALUES(123, 456); From 93c36bb399cee329b8bc0a10bff0af8b3d498f6e Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 15 Jan 2014 20:10:15 +0000 Subject: [PATCH 41/74] Remove an ALWAYS() that is no longer always true. FossilOrigin-Name: c95823cd451f7721174393817a801403647467db --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index dae060c0f8..ddbc250138 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sso\sthat\sSQLITE_OMIT_CTE\sbuilds\swork. -D 2014-01-15T19:42:23.894 +C Remove\san\sALWAYS()\sthat\sis\sno\slonger\salways\strue. +D 2014-01-15T20:10:15.729 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ae278d8ce037883323f677e78c241f64289f12ec F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c b55e726c8dff607790038bf41e6d47925a832b14 +F src/select.c ab437f252415754963b1adfa501120d5364da3ef F src/shell.c a3541193d5fce37e91dad8ef46a9505aa7c9b344 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c948384dfdd9f68a832d5a452af44f35337f66e7 -R 366ade7d6e8c82851813f265ff0c51f5 -U dan -Z a6417c60f97c097fa8d751079d1185b1 +P 3908e2ea2e7e5f466cbbbffdc27e0fe8dc9751ac +R ec33026d9a1a4e4ae1384253da1cbe65 +U drh +Z d40f91394b02fb8b902b74395abcc185 diff --git a/manifest.uuid b/manifest.uuid index cab35f4394..d95e8d3935 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3908e2ea2e7e5f466cbbbffdc27e0fe8dc9751ac \ No newline at end of file +c95823cd451f7721174393817a801403647467db \ No newline at end of file diff --git a/src/select.c b/src/select.c index 7f20dc689a..b4ce7be9ed 100644 --- a/src/select.c +++ b/src/select.c @@ -1217,7 +1217,7 @@ static const char *columnTypeImpl( sNC.pParse = pNC->pParse; zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol, &estWidth); } - }else if( ALWAYS(pTab->pSchema) ){ + }else if( pTab->pSchema ){ /* A real table */ assert( !pS ); if( iCol<0 ) iCol = pTab->iPKey; From 8290c2ad5ad48cffe78f51a25109398ad2b49c41 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Jan 2014 10:58:39 +0000 Subject: [PATCH 42/74] Disable the flattening optimization if the parent query is the recursive part of a recursive CTE and the sub-query is a compound query. FossilOrigin-Name: 6bfa387e82de47ca1f40225fe28d873e29d6f481 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/expr.c | 3 +-- src/select.c | 15 ++++++++++++--- 4 files changed, 22 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index dce447d221..86132841cd 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\schanges.\s\sFix\sa\spossible\sNULL-pointer\sdeference\sin\sWITH\sclause\nname\sresolution. -D 2014-01-16T04:37:13.145 +C Disable\sthe\sflattening\soptimization\sif\sthe\sparent\squery\sis\sthe\srecursive\spart\sof\sa\srecursive\sCTE\sand\sthe\ssub-query\sis\sa\scompound\squery. +D 2014-01-16T10:58:39.954 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff -F src/expr.c 36c313049a716c3edcdd99d703f0f16ce96a4cc6 +F src/expr.c c09e34fa7c58995009135b81f83cb62a676ad181 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ae278d8ce037883323f677e78c241f64289f12ec F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c c9af659146dd1fe924fbfb14a7477f6b2b90f03b +F src/select.c 29976d169853492a20d3201e8a83eeefa39a868f F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c95823cd451f7721174393817a801403647467db f61a70589ac7e05008a362bd9d5b7bde5d07a758 -R 8a379a80b7992f91c6a6d54869d6bc5d -U drh -Z 69979a1b738527f03c61f40bf3b0d3cb +P 7f953b568baa3eede0b9c144be0b9bc86496341a +R 1f82ce2970e79b42a60992cde0f0f626 +U dan +Z da04231961fbad92750efdf2d0f0da36 diff --git a/manifest.uuid b/manifest.uuid index b6146d545b..23deb3b819 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7f953b568baa3eede0b9c144be0b9bc86496341a \ No newline at end of file +6bfa387e82de47ca1f40225fe28d873e29d6f481 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 68bb2fe53e..897da77ed7 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1065,8 +1065,7 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->addrOpenEphm[1] = -1; pNew->addrOpenEphm[2] = -1; pNew->pWith = withDup(db, p->pWith); - assert( p->pRecurse==0 ); - pNew->pRecurse = 0; + pNew->pRecurse = p->pRecurse; return pNew; } #else diff --git a/src/select.c b/src/select.c index 96eae3ff6a..c5fb15cd86 100644 --- a/src/select.c +++ b/src/select.c @@ -1849,9 +1849,11 @@ static int multiSelect( ** references to the current CTE. */ p->pPrior = 0; p->pRecurse->tnum = tmp1; + assert( (p->pRecurse->tabFlags & TF_Recursive)==0 ); p->pRecurse->tabFlags |= TF_Recursive; rc = sqlite3Select(pParse, p, &tmp2dest); p->pRecurse->tabFlags &= ~TF_Recursive; + assert( p->pPrior==0 ); p->pPrior = pPrior; if( rc ) goto multi_select_end; @@ -2845,8 +2847,6 @@ static void substSelect( ** ** Flattening is only attempted if all of the following are true: ** -** (0) The subquery is not a recursive CTE. -** ** (1) The subquery and the outer query do not both use aggregates. ** ** (2) The subquery is not an aggregate or the outer query is not a join. @@ -2930,6 +2930,14 @@ static void substSelect( ** (21) The subquery does not use LIMIT or the outer query is not ** DISTINCT. (See ticket [752e1646fc]). ** +** (22) The subquery is not a recursive CTE. +** +** (23) The parent is not a recursive CTE, or the sub-query is not a +** compound query. This restriction is because transforming the +** parent to a compound query confuses the code that handles +** recursive queries in multiSelect(). +** +** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query ** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates. @@ -2971,7 +2979,6 @@ static int flattenSubquery( iParent = pSubitem->iCursor; pSub = pSubitem->pSelect; assert( pSub!=0 ); - if( pSub->pRecurse ) return 0; /* Restriction (0) */ if( isAgg && subqueryIsAgg ) return 0; /* Restriction (1) */ if( subqueryIsAgg && pSrc->nSrc>1 ) return 0; /* Restriction (2) */ pSubSrc = pSub->pSrc; @@ -3002,6 +3009,8 @@ static int flattenSubquery( if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ return 0; /* Restriction (21) */ } + if( pSub->pRecurse ) return 0; /* Restriction (22) */ + if( p->pRecurse && pSub->pPrior ) return 0; /* Restriction (23) */ /* OBSOLETE COMMENT 1: ** Restriction 3: If the subquery is a join, make sure the subquery is From 6ade453cd844a1cc65648477557d6cb770b60364 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Jan 2014 15:31:41 +0000 Subject: [PATCH 43/74] Always use available indices to optimize LIKE operators even if the pattern of the LIKE operator has a COLLATE modifier. This fixes an ineffiency that was introduced into 3.7.15 by check-in [8542e6180d4] on 2012-12-08. FossilOrigin-Name: 16bd54783a3f5531c55564ddefdada657c078eb0 --- manifest | 16 +++++++-------- manifest.uuid | 2 +- src/where.c | 2 +- test/like.test | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b7d3993e06..1aa2634f3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarning\sin\sLEMON. -D 2014-01-14T10:17:21.693 +C Always\suse\savailable\sindices\sto\soptimize\sLIKE\soperators\seven\sif\sthe\spattern\nof\sthe\sLIKE\soperator\shas\sa\sCOLLATE\smodifier.\s\sThis\sfixes\san\sineffiency\sthat\nwas\sintroduced\sinto\s3.7.15\sby\scheck-in\s[8542e6180d4]\son\s2012-12-08. +D 2014-01-16T15:31:41.806 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 18f07fd0fd116a5880773c689eb7cd8e60175107 +F src/where.c 81cec50fe73633144b0730de477e141c53485862 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -646,7 +646,7 @@ F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa F test/keyword1.test a2400977a2e4fde43bf33754c2929fda34dbca05 F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 -F test/like.test 935fb4f608e3ea126891496a6e99b9468372bf5c +F test/like.test e191e536d0fcd722a6b965e7cd1ee0bfd12a5991 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da F test/limit.test cc0ab63385239b63c72452b0e93700bf5e8f0b99 F test/loadext.test 92e6dfefd1229c3ef4aaabd87419efd8fa57a7a5 @@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1e131094b522103a0829f72193b067b04e42ce82 -R fe4a4148cfbc4d588ee2679597656da4 -U mistachkin -Z d589f5ec7c09b4ac8c97497c20166016 +P f61a70589ac7e05008a362bd9d5b7bde5d07a758 +R c3393c71f3ebd7292d430bc7115b905d +U drh +Z 57df34b30b12b159eb99c0c8be54bec6 diff --git a/manifest.uuid b/manifest.uuid index d14b8afba5..b734040749 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f61a70589ac7e05008a362bd9d5b7bde5d07a758 \ No newline at end of file +16bd54783a3f5531c55564ddefdada657c078eb0 \ No newline at end of file diff --git a/src/where.c b/src/where.c index d5444a6054..a9f527313a 100644 --- a/src/where.c +++ b/src/where.c @@ -667,7 +667,7 @@ static int isLikeOrGlob( } assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ - pRight = pList->a[0].pExpr; + pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); op = pRight->op; if( op==TK_VARIABLE ){ Vdbe *pReprepare = pParse->pReprepare; diff --git a/test/like.test b/test/like.test index 230dc74fed..923272cfb2 100644 --- a/test/like.test +++ b/test/like.test @@ -893,5 +893,60 @@ do_test like-11.10 { } } {abc abcd sort {} t11cb} +# A COLLATE clause on the pattern does not change the result of a +# LIKE operator. +# +do_execsql_test like-12.1 { + CREATE TABLE t12nc(id INTEGER, x TEXT UNIQUE COLLATE nocase); + INSERT INTO t12nc VALUES(1,'abcde'),(2,'uvwxy'),(3,'ABCDEF'); + CREATE TABLE t12b(id INTEGER, x TEXT UNIQUE COLLATE binary); + INSERT INTO t12b VALUES(1,'abcde'),(2,'uvwxy'),(3,'ABCDEF'); + SELECT id FROM t12nc WHERE x LIKE 'abc%' ORDER BY +id; +} {1 3} +do_execsql_test like-12.2 { + SELECT id FROM t12b WHERE x LIKE 'abc%' ORDER BY +id; +} {1 3} +do_execsql_test like-12.3 { + SELECT id FROM t12nc WHERE x LIKE 'abc%' COLLATE binary ORDER BY +id; +} {1 3} +do_execsql_test like-12.4 { + SELECT id FROM t12b WHERE x LIKE 'abc%' COLLATE binary ORDER BY +id; +} {1 3} +do_execsql_test like-12.5 { + SELECT id FROM t12nc WHERE x LIKE 'abc%' COLLATE nocase ORDER BY +id; +} {1 3} +do_execsql_test like-12.6 { + SELECT id FROM t12b WHERE x LIKE 'abc%' COLLATE nocase ORDER BY +id; +} {1 3} + +# Adding a COLLATE clause to the pattern of a LIKE operator does nothing +# to change the suitability of using an index to satisfy that LIKE +# operator. +# +do_execsql_test like-12.11 { + EXPLAIN QUERY PLAN + SELECT id FROM t12nc WHERE x LIKE 'abc%' ORDER BY +id; +} {/SEARCH/} +do_execsql_test like-12.12 { + EXPLAIN QUERY PLAN + SELECT id FROM t12b WHERE x LIKE 'abc%' ORDER BY +id; +} {/SCAN/} +do_execsql_test like-12.13 { + EXPLAIN QUERY PLAN + SELECT id FROM t12nc WHERE x LIKE 'abc%' COLLATE nocase ORDER BY +id; +} {/SEARCH/} +do_execsql_test like-12.14 { + EXPLAIN QUERY PLAN + SELECT id FROM t12b WHERE x LIKE 'abc%' COLLATE nocase ORDER BY +id; +} {/SCAN/} +do_execsql_test like-12.15 { + EXPLAIN QUERY PLAN + SELECT id FROM t12nc WHERE x LIKE 'abc%' COLLATE binary ORDER BY +id; +} {/SEARCH/} +do_execsql_test like-12.16 { + EXPLAIN QUERY PLAN + SELECT id FROM t12b WHERE x LIKE 'abc%' COLLATE binary ORDER BY +id; +} {/SCAN/} + finish_test From eae73fbfb90594c3a56f16a55f69270f18d61bab Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Jan 2014 18:34:33 +0000 Subject: [PATCH 44/74] Allow only a single recursive reference in a recursive CTE. Also require that this reference is not part of a sub-query. FossilOrigin-Name: a296b73360d34c9364eceb2cc09a9a92adc4abb8 --- manifest | 20 +++++------ manifest.uuid | 2 +- src/expr.c | 1 - src/select.c | 88 +++++++++++++++++++++++++++---------------------- src/sqliteInt.h | 5 ++- src/vdbe.c | 23 ------------- src/where.c | 8 +---- 7 files changed, 62 insertions(+), 85 deletions(-) diff --git a/manifest b/manifest index 86132841cd..817e0abe3d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\sflattening\soptimization\sif\sthe\sparent\squery\sis\sthe\srecursive\spart\sof\sa\srecursive\sCTE\sand\sthe\ssub-query\sis\sa\scompound\squery. -D 2014-01-16T10:58:39.954 +C Allow\sonly\sa\ssingle\srecursive\sreference\sin\sa\srecursive\sCTE.\sAlso\srequire\sthat\sthis\sreference\sis\snot\spart\sof\sa\ssub-query. +D 2014-01-16T18:34:33.041 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff -F src/expr.c c09e34fa7c58995009135b81f83cb62a676ad181 +F src/expr.c e239763d8b43356fa1f46f1cf41d62a076f7f72e F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ae278d8ce037883323f677e78c241f64289f12ec F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 29976d169853492a20d3201e8a83eeefa39a868f +F src/select.c 2d2da29fd4c877646109f7d2fba26b5cc454cd37 F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 6db445ce172c249c29d8f532af64fd7051fe05dd +F src/sqliteInt.h b30bd95081be525b32551f180cf43ccfaff9f5bc F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -280,7 +280,7 @@ F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c e71f19b272f05c8695cf747b4bac1732685f9e5c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c b06d79951a3f0da9cb41cb0aae5fdf5612f718a9 +F src/vdbe.c ccc8594e89751966022642464ec2b5c5fa7840a2 F src/vdbe.h e6c4c610fcabad4fa80ebb1efc6822a9367e2b26 F src/vdbeInt.h 42db251e9f863401ff847b90d5fe1614c89a6a56 F src/vdbeapi.c ce4e68ea4842cc6081046f533d088dcf01d247ad @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 -F src/where.c 27eb508c4184599a826e2ef3b4319cd28ba2c7af +F src/where.c 369b0259fabfb22644d197736ae622f762cbaba8 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7f953b568baa3eede0b9c144be0b9bc86496341a -R 1f82ce2970e79b42a60992cde0f0f626 +P 6bfa387e82de47ca1f40225fe28d873e29d6f481 +R d0f7d170eccd6b8c29b55960683e8b9a U dan -Z da04231961fbad92750efdf2d0f0da36 +Z 617b83d8e92edcc7afd63e19b605afe5 diff --git a/manifest.uuid b/manifest.uuid index 23deb3b819..8513fb5092 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6bfa387e82de47ca1f40225fe28d873e29d6f481 \ No newline at end of file +a296b73360d34c9364eceb2cc09a9a92adc4abb8 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 897da77ed7..0614be1cf5 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1065,7 +1065,6 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->addrOpenEphm[1] = -1; pNew->addrOpenEphm[2] = -1; pNew->pWith = withDup(db, p->pWith); - pNew->pRecurse = p->pRecurse; return pNew; } #else diff --git a/src/select.c b/src/select.c index c5fb15cd86..d70d815a2a 100644 --- a/src/select.c +++ b/src/select.c @@ -1744,7 +1744,7 @@ static int multiSelect( ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. */ assert( p && p->pPrior ); /* Calling function guarantees this much */ - assert( p->pRecurse==0 || p->op==TK_ALL || p->op==TK_UNION ); + assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION ); db = pParse->db; pPrior = p->pPrior; assert( pPrior->pRightmost!=pPrior ); @@ -1794,27 +1794,36 @@ static int multiSelect( /* If this is a recursive query, check that there is no ORDER BY or ** LIMIT clause. Neither of these are supported. */ assert( p->pOffset==0 || p->pLimit ); - if( p->pRecurse && (p->pOrderBy || p->pLimit) ){ + if( (p->selFlags & SF_Recursive) && (p->pOrderBy || p->pLimit) ){ sqlite3ErrorMsg(pParse, "%s in a recursive query is not allowed", p->pOrderBy ? "ORDER BY" : "LIMIT" ); goto multi_select_end; } - if( p->pRecurse ){ + if( p->selFlags & SF_Recursive ){ + SrcList *pSrc = p->pSrc; int nCol = p->pEList->nExpr; int addrNext; int addrSwap; int iCont, iBreak; - int tmp1, tmp2; /* Cursors used to access temporary tables */ + int tmp1; /* Intermediate table */ + int tmp2; /* Next intermediate table */ int tmp3 = 0; /* To ensure unique results if UNION */ int eDest = SRT_Table; SelectDest tmp2dest; + int i; iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); - tmp1 = pParse->nTab++; + for(i=0; ALWAYS(inSrc); i++){ + if( pSrc->a[i].isRecursive ){ + tmp1 = pSrc->a[i].iCursor; + break; + } + } + tmp2 = pParse->nTab++; if( p->op==TK_UNION ){ eDest = SRT_DistTable; @@ -1848,11 +1857,7 @@ static int multiSelect( ** SELECT is running, the contents of tmp1 are read by recursive ** references to the current CTE. */ p->pPrior = 0; - p->pRecurse->tnum = tmp1; - assert( (p->pRecurse->tabFlags & TF_Recursive)==0 ); - p->pRecurse->tabFlags |= TF_Recursive; rc = sqlite3Select(pParse, p, &tmp2dest); - p->pRecurse->tabFlags &= ~TF_Recursive; assert( p->pPrior==0 ); p->pPrior = pPrior; if( rc ) goto multi_select_end; @@ -3009,8 +3014,8 @@ static int flattenSubquery( if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ return 0; /* Restriction (21) */ } - if( pSub->pRecurse ) return 0; /* Restriction (22) */ - if( p->pRecurse && pSub->pPrior ) return 0; /* Restriction (23) */ + if( pSub->selFlags & SF_Recursive ) return 0; /* Restriction (22) */ + if( (p->selFlags & SF_Recursive) && pSub->pPrior ) return 0; /* (23) */ /* OBSOLETE COMMENT 1: ** Restriction 3: If the subquery is a join, make sure the subquery is @@ -3593,19 +3598,10 @@ static int withExpand( assert( pFrom->pTab==0 ); pCte = searchWith(pParse->pWith, pFrom); - if( pCte==0 ){ - /* no-op */ - }else if( pCte==pParse->pCte && (pTab = pCte->pTab) ){ - /* This is the recursive part of a recursive CTE */ - assert( pFrom->pTab==0 && pFrom->isRecursive==0 && pFrom->pSelect==0 ); - pFrom->pTab = pTab; - pFrom->isRecursive = 1; - pTab->nRef++; - }else{ + if( pCte ){ ExprList *pEList; Select *pSel; Select *pLeft; /* Left-most SELECT statement */ - int bRecursive; pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; @@ -3618,16 +3614,37 @@ static int withExpand( if( db->mallocFailed ) return SQLITE_NOMEM; assert( pFrom->pSelect ); - if( ctePush(pParse, pCte) ) return WRC_Abort; + /* Check if this is a recursive CTE. */ pSel = pFrom->pSelect; - bRecursive = (pSel->op==TK_ALL || pSel->op==TK_UNION); - if( bRecursive ){ - assert( pSel->pPrior ); - sqlite3WalkSelect(pWalker, pSel->pPrior); - }else{ - sqlite3WalkSelect(pWalker, pSel); + if( pSel->op==TK_ALL || pSel->op==TK_UNION ){ + int i; + SrcList *pSrc = pFrom->pSelect->pSrc; + for(i=0; inSrc; i++){ + struct SrcList_item *pItem = &pSrc->a[i]; + if( pItem->zDatabase==0 + && pItem->zName!=0 + && 0==sqlite3StrICmp(pItem->zName, pCte->zName) + ){ + pItem->pTab = pTab; + pItem->isRecursive = 1; + pTab->nRef++; + pSel->selFlags |= SF_Recursive; + } + } } + /* Only one recursive reference is permitted. */ + if( pTab->nRef>2 ){ + sqlite3ErrorMsg( + pParse, "multiple recursive references in cte: %s", pCte->zName + ); + return WRC_Abort; + } + assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); + + if( ctePush(pParse, pCte) ) return WRC_Abort; + sqlite3WalkSelect(pWalker, pTab->nRef==2 ? pSel->pPrior : pSel); + for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); pEList = pLeft->pEList; if( pCte->pCols ){ @@ -3639,20 +3656,9 @@ static int withExpand( } pEList = pCte->pCols; } - selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); - if( bRecursive ){ - int nRef = pTab->nRef; - pCte->pTab = pTab; - sqlite3WalkSelect(pWalker, pSel); - pCte->pTab = 0; - if( pTab->nRef > nRef){ - pSel->pRecurse = pTab; - assert( pTab->tnum==0 ); - } - } - + if( pSel->selFlags & SF_Recursive ) sqlite3WalkSelect(pWalker, pSel); ctePop(pParse, pCte); } @@ -3715,6 +3721,8 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; + assert( pFrom->isRecursive==0 || pFrom->pTab ); + if( pFrom->isRecursive ) continue; if( pFrom->pTab!=0 ){ /* This statement has already been prepared. There is no need ** to go further. */ diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 733dad2d8f..43a1a57b0a 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2131,7 +2131,6 @@ struct NameContext { */ struct Select { ExprList *pEList; /* The fields of the result */ - Table *pRecurse; /* Non-NULL for the recursive part of recursive CTE */ u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ u16 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ @@ -2165,6 +2164,7 @@ struct Select { #define SF_Materialize 0x0100 /* Force materialization of views */ #define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ #define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ +#define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ /* @@ -2640,7 +2640,7 @@ int sqlite3WalkSelectFrom(Walker*, Select*); #define WRC_Abort 2 /* Abandon the tree walk */ /* -** An instance of this structure represents a set of on or more CTEs +** An instance of this structure represents a set of one or more CTEs ** (common table expressions) created by a single WITH clause. */ struct With { @@ -2651,7 +2651,6 @@ struct With { ExprList *pCols; /* List of explicit column names, or NULL */ Select *pSelect; /* The definition of this CTE */ struct Cte *pOuterCte; /* Next WITH clause in outer context */ - Table *pTab; /* Table object for this CTE */ } a[1]; }; diff --git a/src/vdbe.c b/src/vdbe.c index 301707677d..9af4a62d7e 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3370,29 +3370,6 @@ case OP_OpenEphemeral: { } #ifndef SQLITE_OMIT_CTE -/* Opcode: OpenEphreader P1 P2 * * * -** -** P2 is a cursor opened by the OpenEphemeral opcode. This opcode opens -** a new read-only cursor named P1 that accesses the same epheremal table -** as P2. -*/ -case OP_OpenEphreader: { - VdbeCursor *pEph; - VdbeCursor *pCx; - Pgno pgno; - - pEph = p->apCsr[pOp->p2]; - pCx = allocateCursor(p, pOp->p1, pEph->nField, -1, 1); - if( pCx==0 ) goto no_mem; - pCx->nullRow = 1; - pCx->pKeyInfo = pEph->pKeyInfo; - pCx->isTable = pEph->isTable; - pCx->isOrdered = pEph->isOrdered; - pgno = MASTER_ROOT + !pCx->isTable; - rc = sqlite3BtreeCursor(pEph->pBt, pgno, 0, pCx->pKeyInfo, pCx->pCursor); - break; -} - /* Opcode: SwapCursors P1 P2 * * * ** ** Parameters P1 and P2 are both cursors opened by the OpenEphemeral diff --git a/src/where.c b/src/where.c index f1cf29b2e7..52bf46a74e 100644 --- a/src/where.c +++ b/src/where.c @@ -5654,13 +5654,7 @@ WhereInfo *sqlite3WhereBegin( iDb = sqlite3SchemaToIndex(db, pTab->pSchema); pLoop = pLevel->pWLoop; if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ -#ifndef SQLITE_OMIT_CTE - if( pTab->tabFlags & TF_Recursive ){ - int iCur = pTabItem->iCursor; - sqlite3VdbeAddOp2(v, OP_OpenEphreader, iCur, pTab->tnum); - } -#endif - /* Otherwise do nothing */ + /* Do nothing */ }else #ifndef SQLITE_OMIT_VIRTUALTABLE if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ){ From f2655fe8b68be84a471389f0face897c82b02082 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 16 Jan 2014 21:02:02 +0000 Subject: [PATCH 45/74] Improve the error messages used to report illegal recursive cte references. FossilOrigin-Name: 54eee9fe99290e59469bd3e1a66bb749887d37ee --- manifest | 18 +++++++------- manifest.uuid | 2 +- src/build.c | 1 + src/select.c | 66 +++++++++++++++++-------------------------------- src/sqliteInt.h | 3 +-- test/with1.test | 62 ++++++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 95 insertions(+), 57 deletions(-) diff --git a/manifest b/manifest index 817e0abe3d..9f4c9c003b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Allow\sonly\sa\ssingle\srecursive\sreference\sin\sa\srecursive\sCTE.\sAlso\srequire\sthat\sthis\sreference\sis\snot\spart\sof\sa\ssub-query. -D 2014-01-16T18:34:33.041 +C Improve\sthe\serror\smessages\sused\sto\sreport\sillegal\srecursive\scte\sreferences. +D 2014-01-16T21:02:02.206 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,7 +169,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c c15e1722696b66c4029c487acfb830b0985bf142 F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 -F src/build.c 5a2daa6649640711ca60114046903be932348e52 +F src/build.c a6b9ba918f9194e955d198402076fe3203ff2567 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd @@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ae278d8ce037883323f677e78c241f64289f12ec F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 2d2da29fd4c877646109f7d2fba26b5cc454cd37 +F src/select.c 1b058f7ef37e2667fd33921531480a616a04bca2 F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h b30bd95081be525b32551f180cf43ccfaff9f5bc +F src/sqliteInt.h d49c0bea5282f15c1eb1eb9d705770f70d19c1e2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1091,7 +1091,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test 8ac68051fce8f2fe4a7bb6cb7ba62c0e941b1c84 +F test/with1.test f1602892682cd9bfa5b3b0a5d04b2283b61e7982 F test/withM.test 52448ce23e1c2ecba79d10e130ee49ce9f9a2a7a F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 6bfa387e82de47ca1f40225fe28d873e29d6f481 -R d0f7d170eccd6b8c29b55960683e8b9a +P a296b73360d34c9364eceb2cc09a9a92adc4abb8 +R 20dd6a1609afa0cf73fe04c795b9e1e4 U dan -Z 617b83d8e92edcc7afd63e19b605afe5 +Z 6bd9745179436f1c3b8a75548e64bd7b diff --git a/manifest.uuid b/manifest.uuid index 8513fb5092..2e74fdae17 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a296b73360d34c9364eceb2cc09a9a92adc4abb8 \ No newline at end of file +54eee9fe99290e59469bd3e1a66bb749887d37ee \ No newline at end of file diff --git a/src/build.c b/src/build.c index c7d956a9b7..353c938074 100644 --- a/src/build.c +++ b/src/build.c @@ -4245,6 +4245,7 @@ With *sqlite3WithAdd( pNew->a[pNew->nCte].pSelect = pQuery; pNew->a[pNew->nCte].pCols = pArglist; pNew->a[pNew->nCte].zName = zName; + pNew->a[pNew->nCte].zErr = 0; pNew->nCte++; } diff --git a/src/select.c b/src/select.c index d70d815a2a..7b6c5e9c2c 100644 --- a/src/select.c +++ b/src/select.c @@ -3533,44 +3533,6 @@ void sqlite3WithPush(Parse *pParse, With *pWith){ } } -/* -** If argument pCte is not NULL, check if it is already a part of the -** stack of CTEs stored by the parser. If so, this indicates an illegal -** recursive reference in a CTE, set of mutually recursive CTEs. Store -** an error in the parser and return SQLITE_ERROR if this is the case. -** -** Otherwise, if pCte is not already part of the stack of CTEs stored -** in the parser, push it onto the stop of that stack. -*/ -static int ctePush(Parse *pParse, struct Cte *pCte){ - if( pCte ){ - struct Cte *p; - for(p=pParse->pCte; p; p=p->pOuterCte){ - if( p==pCte ){ - sqlite3ErrorMsg( - pParse, "illegal recursive defininition in cte: %s", pCte->zName - ); - return SQLITE_ERROR; - } - } - - pCte->pOuterCte = pParse->pCte; - pParse->pCte = pCte; - } - return SQLITE_OK; -} -/* -** If argument pCte is not NULL, it must be a pointer to the CTE currently -** on top of the stack of CTEs stored in the parser. Remove it from that -** stack. -*/ -static void ctePop(Parse *pParse, struct Cte *pCte){ - if( pCte ){ - assert( pParse->pCte==pCte ); - pParse->pCte = pCte->pOuterCte; - } -} - /* ** This function checks if argument pFrom refers to a CTE declared by ** a WITH clause on the stack currently maintained by the parser. And, @@ -3602,6 +3564,16 @@ static int withExpand( ExprList *pEList; Select *pSel; Select *pLeft; /* Left-most SELECT statement */ + int bMayRecursive; /* True if compound joined by UNION [ALL] */ + + /* If pCte->zErr is non-NULL at this point, then this is an illegal + ** recursive reference to CTE pCte. Leave an error in pParse and return + ** early. If pCte->zErr is NULL, then this is not a recursive reference. + ** In this case, proceed. */ + if( pCte->zErr ){ + sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName); + return WRC_Abort; + } pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; @@ -3616,7 +3588,8 @@ static int withExpand( /* Check if this is a recursive CTE. */ pSel = pFrom->pSelect; - if( pSel->op==TK_ALL || pSel->op==TK_UNION ){ + bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); + if( bMayRecursive ){ int i; SrcList *pSrc = pFrom->pSelect->pSrc; for(i=0; inSrc; i++){ @@ -3642,8 +3615,8 @@ static int withExpand( } assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); - if( ctePush(pParse, pCte) ) return WRC_Abort; - sqlite3WalkSelect(pWalker, pTab->nRef==2 ? pSel->pPrior : pSel); + pCte->zErr = "circular reference to cte: %s"; + sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); pEList = pLeft->pEList; @@ -3658,8 +3631,15 @@ static int withExpand( } selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); - if( pSel->selFlags & SF_Recursive ) sqlite3WalkSelect(pWalker, pSel); - ctePop(pParse, pCte); + if( bMayRecursive ){ + if( pSel->selFlags & SF_Recursive ){ + pCte->zErr = "multiple recursive references in cte: %s"; + }else{ + pCte->zErr = "recursive reference may not appear in sub-query: %s"; + } + sqlite3WalkSelect(pWalker, pSel); + } + pCte->zErr = 0; } return SQLITE_OK; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 43a1a57b0a..3f29f58d4e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2371,7 +2371,6 @@ struct Parse { Table *pZombieTab; /* List of Table objects to delete after code gen */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ With *pWith; /* Current WITH clause, or NULL */ - struct Cte *pCte; /* Current CTE, or NULL */ }; /* @@ -2650,7 +2649,7 @@ struct With { char *zName; /* Name of this CTE */ ExprList *pCols; /* List of explicit column names, or NULL */ Select *pSelect; /* The definition of this CTE */ - struct Cte *pOuterCte; /* Next WITH clause in outer context */ + const char *zErr; /* Error message for circular references */ } a[1]; }; diff --git a/test/with1.test b/test/with1.test index 861c722543..4ad6260b78 100644 --- a/test/with1.test +++ b/test/with1.test @@ -76,10 +76,10 @@ do_execsql_test 2.5 { #------------------------------------------------------------------------- do_catchsql_test 3.1 { - WITH tmp2(x) AS ( SELECT * FROM tmp1), + WITH tmp2(x) AS ( SELECT * FROM tmp1 ), tmp1(a) AS ( SELECT * FROM tmp2 ) SELECT * FROM tmp1; -} {1 {illegal recursive defininition in cte: tmp1}} +} {1 {circular reference to cte: tmp1}} do_catchsql_test 3.2 { CREATE TABLE t2(x INTEGER); @@ -268,6 +268,64 @@ do_execsql_test 6.4 { } {10} +#------------------------------------------------------------------------- + +do_execsql_test 7.1 { + CREATE TABLE tree(i, p); + INSERT INTO tree VALUES(1, NULL); + INSERT INTO tree VALUES(2, 1); + INSERT INTO tree VALUES(3, 1); + INSERT INTO tree VALUES(4, 2); + INSERT INTO tree VALUES(5, 4); +} + +do_execsql_test 7.2 { + WITH t(id, path) AS ( + SELECT i, '' FROM tree WHERE p IS NULL + UNION ALL + SELECT i, path || '/' || i FROM tree, t WHERE p = id + ) + SELECT path FROM t; +} {{} /2 /3 /2/4 /2/4/5} + +do_execsql_test 7.3 { + WITH t(id) AS ( + VALUES(2) + UNION ALL + SELECT i FROM tree, t WHERE p = id + ) + SELECT id FROM t; +} {2 4 5} + +do_catchsql_test 7.4 { + WITH t(id) AS ( + VALUES(2) + UNION ALL + SELECT i FROM tree WHERE p IN (SELECT id FROM t) + ) + SELECT id FROM t; +} {1 {recursive reference may not appear in sub-query: t}} + +do_catchsql_test 7.5 { + WITH t(id) AS ( + VALUES(2) + UNION ALL + SELECT i FROM tree, t WHERE p = id AND p IN (SELECT id FROM t) + ) + SELECT id FROM t; +} {1 {multiple recursive references in cte: t}} + +do_catchsql_test 7.6 { + WITH t(id) AS ( + SELECT i FROM tree WHERE 2 IN (SELECT id FROM t) + UNION ALL + SELECT i FROM tree, t WHERE p = id + ) + SELECT id FROM t; +} {1 {circular reference to cte: t}} + + + finish_test From 727a99f1e313adecf162cf7c3199cc496d87099e Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Jan 2014 21:59:51 +0000 Subject: [PATCH 46/74] Tweaks to error message text. FossilOrigin-Name: 090a77d97808b86d1e9f5c63c743a2b159a15f5d --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/build.c | 2 +- src/select.c | 10 +++++----- test/with1.test | 23 ++++++++++------------- 5 files changed, 26 insertions(+), 29 deletions(-) diff --git a/manifest b/manifest index 9f4c9c003b..f942fc379a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Improve\sthe\serror\smessages\sused\sto\sreport\sillegal\srecursive\scte\sreferences. -D 2014-01-16T21:02:02.206 +C Tweaks\sto\serror\smessage\stext. +D 2014-01-16T21:59:51.988 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -169,7 +169,7 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 F src/btree.c c15e1722696b66c4029c487acfb830b0985bf142 F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 -F src/build.c a6b9ba918f9194e955d198402076fe3203ff2567 +F src/build.c 7e6c275ab1731510d6f793d0f88373ab3e858e69 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ae278d8ce037883323f677e78c241f64289f12ec F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 1b058f7ef37e2667fd33921531480a616a04bca2 +F src/select.c d75733ab2ad5e9f0d79fb4ab9f45d3d3d3675a3d F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1091,7 +1091,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test f1602892682cd9bfa5b3b0a5d04b2283b61e7982 +F test/with1.test 90490c75e98e1914d84b7cef9e636b48917a020f F test/withM.test 52448ce23e1c2ecba79d10e130ee49ce9f9a2a7a F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a296b73360d34c9364eceb2cc09a9a92adc4abb8 -R 20dd6a1609afa0cf73fe04c795b9e1e4 -U dan -Z 6bd9745179436f1c3b8a75548e64bd7b +P 54eee9fe99290e59469bd3e1a66bb749887d37ee +R 7385964692340c50e255f43b779c83ed +U drh +Z 3d3568f983c7b2bd0c47dd0558e3c02c diff --git a/manifest.uuid b/manifest.uuid index 2e74fdae17..bdfa111f79 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -54eee9fe99290e59469bd3e1a66bb749887d37ee \ No newline at end of file +090a77d97808b86d1e9f5c63c743a2b159a15f5d \ No newline at end of file diff --git a/src/build.c b/src/build.c index 353c938074..fa7364c3da 100644 --- a/src/build.c +++ b/src/build.c @@ -4222,7 +4222,7 @@ With *sqlite3WithAdd( int i; for(i=0; inCte; i++){ if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){ - sqlite3ErrorMsg(pParse, "duplicate cte name: %s", zName); + sqlite3ErrorMsg(pParse, "duplicate WITH table name: %s", zName); } } } diff --git a/src/select.c b/src/select.c index 7b6c5e9c2c..ae2e0ce655 100644 --- a/src/select.c +++ b/src/select.c @@ -3609,20 +3609,20 @@ static int withExpand( /* Only one recursive reference is permitted. */ if( pTab->nRef>2 ){ sqlite3ErrorMsg( - pParse, "multiple recursive references in cte: %s", pCte->zName + pParse, "multiple references to recursive table: %s", pCte->zName ); return WRC_Abort; } assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); - pCte->zErr = "circular reference to cte: %s"; + pCte->zErr = "circular reference: %s"; sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); pEList = pLeft->pEList; if( pCte->pCols ){ if( pEList->nExpr!=pCte->pCols->nExpr ){ - sqlite3ErrorMsg(pParse, "cte \"%s\" returns %d values for %d columns", + sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns", pCte->zName, pEList->nExpr, pCte->pCols->nExpr ); return WRC_Abort; @@ -3633,9 +3633,9 @@ static int withExpand( if( bMayRecursive ){ if( pSel->selFlags & SF_Recursive ){ - pCte->zErr = "multiple recursive references in cte: %s"; + pCte->zErr = "multiple recursive references: %s"; }else{ - pCte->zErr = "recursive reference may not appear in sub-query: %s"; + pCte->zErr = "recursive reference in a subquery: %s"; } sqlite3WalkSelect(pWalker, pSel); } diff --git a/test/with1.test b/test/with1.test index 4ad6260b78..db0d7bc36e 100644 --- a/test/with1.test +++ b/test/with1.test @@ -79,14 +79,14 @@ do_catchsql_test 3.1 { WITH tmp2(x) AS ( SELECT * FROM tmp1 ), tmp1(a) AS ( SELECT * FROM tmp2 ) SELECT * FROM tmp1; -} {1 {circular reference to cte: tmp1}} +} {1 {circular reference: tmp1}} do_catchsql_test 3.2 { CREATE TABLE t2(x INTEGER); WITH tmp(a) AS (SELECT * FROM t1), tmp(a) AS (SELECT * FROM t1) SELECT * FROM tmp; -} {1 {duplicate cte name: tmp}} +} {1 {duplicate WITH table name: tmp}} do_execsql_test 3.3 { CREATE TABLE t3(x); @@ -172,23 +172,23 @@ do_execsql_test 5.5 { do_catchsql_test 5.6.1 { WITH i(x, y) AS ( VALUES(1) ) SELECT * FROM i; -} {1 {cte "i" returns 1 values for 2 columns}} +} {1 {table i has 1 values for 2 columns}} do_catchsql_test 5.6.2 { WITH i(x) AS ( VALUES(1,2) ) SELECT * FROM i; -} {1 {cte "i" returns 2 values for 1 columns}} +} {1 {table i has 2 values for 1 columns}} do_catchsql_test 5.6.3 { CREATE TABLE t5(a, b); WITH i(x) AS ( SELECT * FROM t5 ) SELECT * FROM i; -} {1 {cte "i" returns 2 values for 1 columns}} +} {1 {table i has 2 values for 1 columns}} do_catchsql_test 5.6.4 { WITH i(x) AS ( SELECT 1, 2 UNION ALL SELECT 1 ) SELECT * FROM i; -} {1 {cte "i" returns 2 values for 1 columns}} +} {1 {table i has 2 values for 1 columns}} do_catchsql_test 5.6.5 { WITH i(x) AS ( SELECT 1 UNION ALL SELECT 1, 2 ) @@ -203,7 +203,7 @@ do_catchsql_test 5.6.6 { do_catchsql_test 5.6.7 { WITH i(x) AS ( SELECT 1, 2 UNION SELECT x+1 FROM i ) SELECT * FROM i; -} {1 {cte "i" returns 2 values for 1 columns}} +} {1 {table i has 2 values for 1 columns}} #------------------------------------------------------------------------- # @@ -304,7 +304,7 @@ do_catchsql_test 7.4 { SELECT i FROM tree WHERE p IN (SELECT id FROM t) ) SELECT id FROM t; -} {1 {recursive reference may not appear in sub-query: t}} +} {1 {recursive reference in a subquery: t}} do_catchsql_test 7.5 { WITH t(id) AS ( @@ -313,7 +313,7 @@ do_catchsql_test 7.5 { SELECT i FROM tree, t WHERE p = id AND p IN (SELECT id FROM t) ) SELECT id FROM t; -} {1 {multiple recursive references in cte: t}} +} {1 {multiple recursive references: t}} do_catchsql_test 7.6 { WITH t(id) AS ( @@ -322,11 +322,8 @@ do_catchsql_test 7.6 { SELECT i FROM tree, t WHERE p = id ) SELECT id FROM t; -} {1 {circular reference to cte: t}} +} {1 {circular reference: t}} finish_test - - - From 65a2aaa633f1e8ab5a17be2af5e69dc90a0ea951 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 16 Jan 2014 22:40:02 +0000 Subject: [PATCH 47/74] Add the ability for the authorizer callback to disallow recursive queries. FossilOrigin-Name: 9efc120a1548c03f3d8aabbadf1050ff2a119c31 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/select.c | 3 +++ src/sqlite.h.in | 1 + src/tclsqlite.c | 1 + test/auth.test | 36 ++++++++++++++++++++++++++++++++++++ 6 files changed, 51 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index f942fc379a..4d2bec4250 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Tweaks\sto\serror\smessage\stext. -D 2014-01-16T21:59:51.988 +C Add\sthe\sability\sfor\sthe\sauthorizer\scallback\sto\sdisallow\srecursive\nqueries. +D 2014-01-16T22:40:02.405 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,16 +219,16 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c ae278d8ce037883323f677e78c241f64289f12ec F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c d75733ab2ad5e9f0d79fb4ab9f45d3d3d3675a3d +F src/select.c fc7499ac90fd4d49782e0a16372d3a5efde2aa3b F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 -F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9 +F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc F src/sqliteInt.h d49c0bea5282f15c1eb1eb9d705770f70d19c1e2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e -F src/tclsqlite.c c43379f77f90399802b0e215faa71c0adc3a4d2e +F src/tclsqlite.c 46073db71011b6542fde1f234c56a076d5ff23f9 F src/test1.c db16ba651453b15001c7f2838c446284dde4ecaf F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c @@ -327,7 +327,7 @@ F test/attach2.test 0ec5defa340363de6cd50fd595046465e9aaba2d F test/attach3.test 359eb65d00102cdfcef6fa4e81dc1648f8f80b27 F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 -F test/auth.test 9bea29041871807d9f289ee679d05d3ed103642f +F test/auth.test 5bdf154eb28c0e4bbc0473f335858c0d96171768 F test/auth2.test c3b415b76c033bedb81292118fb7c01f5f10cbcd F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 54eee9fe99290e59469bd3e1a66bb749887d37ee -R 7385964692340c50e255f43b779c83ed +P 090a77d97808b86d1e9f5c63c743a2b159a15f5d +R f01d2088e8545f61e859d9aad946ffb1 U drh -Z 3d3568f983c7b2bd0c47dd0558e3c02c +Z b5ce4714b23c3d6e2db0cb8e18c5a93d diff --git a/manifest.uuid b/manifest.uuid index bdfa111f79..dfccdb311f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -090a77d97808b86d1e9f5c63c743a2b159a15f5d \ No newline at end of file +9efc120a1548c03f3d8aabbadf1050ff2a119c31 \ No newline at end of file diff --git a/src/select.c b/src/select.c index ae2e0ce655..66eb4b3367 100644 --- a/src/select.c +++ b/src/select.c @@ -1814,6 +1814,9 @@ static int multiSelect( SelectDest tmp2dest; int i; + if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ){ + goto multi_select_end; + } iBreak = sqlite3VdbeMakeLabel(v); iCont = sqlite3VdbeMakeLabel(v); diff --git a/src/sqlite.h.in b/src/sqlite.h.in index 5012f864a5..51c864c5e2 100644 --- a/src/sqlite.h.in +++ b/src/sqlite.h.in @@ -2561,6 +2561,7 @@ int sqlite3_set_authorizer( #define SQLITE_FUNCTION 31 /* NULL Function Name */ #define SQLITE_SAVEPOINT 32 /* Operation Savepoint Name */ #define SQLITE_COPY 0 /* No longer used */ +#define SQLITE_RECURSIVE 33 /* NULL NULL */ /* ** CAPI3REF: Tracing And Profiling Functions diff --git a/src/tclsqlite.c b/src/tclsqlite.c index e3e5628b17..1e81912526 100644 --- a/src/tclsqlite.c +++ b/src/tclsqlite.c @@ -914,6 +914,7 @@ static int auth_callback( case SQLITE_DROP_VTABLE : zCode="SQLITE_DROP_VTABLE"; break; case SQLITE_FUNCTION : zCode="SQLITE_FUNCTION"; break; case SQLITE_SAVEPOINT : zCode="SQLITE_SAVEPOINT"; break; + case SQLITE_RECURSIVE : zCode="SQLITE_RECURSIVE"; break; default : zCode="????"; break; } Tcl_DStringInit(&str); diff --git a/test/auth.test b/test/auth.test index 5e91b33eaa..43e53ef2e3 100644 --- a/test/auth.test +++ b/test/auth.test @@ -2080,6 +2080,42 @@ ifcapable {altertable} { execsql {DROP TABLE t5} } ;# ifcapable altertable +ifcapable {cte} { + do_test auth-1.310 { + proc auth {code arg1 arg2 arg3 arg4} { + if {$code=="SQLITE_RECURSIVE"} { + return SQLITE_DENY + } + return SQLITE_OK + } + db eval { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a,b); + INSERT INTO t1 VALUES(1,2),(3,4),(5,6); + } + } {} + do_catchsql_test auth-1.311 { + WITH + auth1311(x,y) AS (SELECT a+b, b-a FROM t1) + SELECT * FROM auth1311 ORDER BY x; + } {0 {3 1 7 1 11 1}} + do_catchsql_test auth-1.312 { + WITH RECURSIVE + auth1312(x,y) AS (SELECT a+b, b-a FROM t1) + SELECT x, y FROM auth1312 ORDER BY x; + } {0 {3 1 7 1 11 1}} + do_catchsql_test auth-1.313 { + WITH RECURSIVE + auth1313(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM auth1313 WHERE x<5) + SELECT * FROM t1; + } {0 {1 2 3 4 5 6}} + do_catchsql_test auth-1.314 { + WITH RECURSIVE + auth1314(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM auth1314 WHERE x<5) + SELECT * FROM t1 LEFT JOIN auth1314; + } {1 {not authorized}} +} ;# ifcapable cte + do_test auth-2.1 { proc auth {code arg1 arg2 arg3 arg4} { if {$code=="SQLITE_READ" && $arg1=="t3" && $arg2=="x"} { From 2d4dc5fc60b30abb406d7f5efcf5a1003b3fe300 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 17 Jan 2014 11:48:24 +0000 Subject: [PATCH 48/74] Remove some code from resolve.c that was only required for recursive cte references in sub-queries. Also a stray "finish_test" command in pagerfault.test. FossilOrigin-Name: f68c6c4d36481526a9348244adc571ea282dc9eb --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/resolve.c | 10 ---------- src/select.c | 22 +++++++++++----------- test/pagerfault.test | 1 - 5 files changed, 21 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index 4d2bec4250..5d2226d267 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sability\sfor\sthe\sauthorizer\scallback\sto\sdisallow\srecursive\nqueries. -D 2014-01-16T22:40:02.405 +C Remove\ssome\scode\sfrom\sresolve.c\sthat\swas\sonly\srequired\sfor\srecursive\scte\sreferences\sin\ssub-queries.\sAlso\sa\sstray\s"finish_test"\scommand\sin\spagerfault.test. +D 2014-01-17T11:48:24.294 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,9 +217,9 @@ F src/pragma.c ed409ce4104cf4d9de6ead40ace70974f124853b F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece -F src/resolve.c ae278d8ce037883323f677e78c241f64289f12ec +F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c fc7499ac90fd4d49782e0a16372d3a5efde2aa3b +F src/select.c 65c13f22edfd6af04829439955c7cf5749ea4e87 F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -728,7 +728,7 @@ F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f F test/pager4.test b40ecb4cc7dff957ee7916e41ab21d1ed702a642 -F test/pagerfault.test cee8488a935e42a4a2e241e0317dc2814c997783 +F test/pagerfault.test 7285379906ab2f1108b8e82bbdf2d386cc8ff3ff F test/pagerfault2.test caf4c7facb914fd3b03a17b31ae2b180c8d6ca1f F test/pagerfault3.test 1003fcda009bf48a8e22a516e193b6ef0dd1bbd8 F test/pageropt.test 6b8f6a123a5572c195ad4ae40f2987007923bbd6 @@ -1150,7 +1150,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 090a77d97808b86d1e9f5c63c743a2b159a15f5d -R f01d2088e8545f61e859d9aad946ffb1 -U drh -Z b5ce4714b23c3d6e2db0cb8e18c5a93d +P 9efc120a1548c03f3d8aabbadf1050ff2a119c31 +R 175ad2c9f66168be6fabd0b7fdb44bd2 +U dan +Z 6e3919f2cc39eeea18a32475675bdfc0 diff --git a/manifest.uuid b/manifest.uuid index dfccdb311f..6e22de7d56 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9efc120a1548c03f3d8aabbadf1050ff2a119c31 \ No newline at end of file +f68c6c4d36481526a9348244adc571ea282dc9eb \ No newline at end of file diff --git a/src/resolve.c b/src/resolve.c index 724b8538fa..b0adb86295 100644 --- a/src/resolve.c +++ b/src/resolve.c @@ -502,16 +502,6 @@ lookupname_end: if( pExpr->op!=TK_AS ){ sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); } - -#ifndef SQLITE_OMIT_CTE - /* If this expression reads a column value from a recursive CTE - ** reference, then this is equivalent to reading from the outermost - ** available name-context. */ - if( pMatch && pMatch->isRecursive ){ - while( pNC->pNext ) pNC = pNC->pNext; - } -#endif - /* Increment the nRef value on all name contexts from TopNC up to ** the point where the name matched. */ for(;;){ diff --git a/src/select.c b/src/select.c index 66eb4b3367..dc784b3b10 100644 --- a/src/select.c +++ b/src/select.c @@ -1791,16 +1791,6 @@ static int multiSelect( } #ifndef SQLITE_OMIT_CTE - /* If this is a recursive query, check that there is no ORDER BY or - ** LIMIT clause. Neither of these are supported. */ - assert( p->pOffset==0 || p->pLimit ); - if( (p->selFlags & SF_Recursive) && (p->pOrderBy || p->pLimit) ){ - sqlite3ErrorMsg(pParse, "%s in a recursive query is not allowed", - p->pOrderBy ? "ORDER BY" : "LIMIT" - ); - goto multi_select_end; - } - if( p->selFlags & SF_Recursive ){ SrcList *pSrc = p->pSrc; int nCol = p->pEList->nExpr; @@ -1814,6 +1804,16 @@ static int multiSelect( SelectDest tmp2dest; int i; + /* Check that there is no ORDER BY or LIMIT clause. Neither of these + ** are supported on recursive queries. */ + assert( p->pOffset==0 || p->pLimit ); + if( p->pOrderBy || p->pLimit ){ + sqlite3ErrorMsg(pParse, "%s in a recursive query is not allowed", + p->pOrderBy ? "ORDER BY" : "LIMIT" + ); + goto multi_select_end; + } + if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ){ goto multi_select_end; } @@ -3581,7 +3581,7 @@ static int withExpand( pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; pTab->nRef = 1; - pTab->zName = sqlite3MPrintf(db, "%s", pCte->zName); + pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; pTab->nRowEst = 1048576; pTab->tabFlags |= TF_Ephemeral; diff --git a/test/pagerfault.test b/test/pagerfault.test index 1e57c93903..796f531c37 100644 --- a/test/pagerfault.test +++ b/test/pagerfault.test @@ -63,7 +63,6 @@ do_faultsim_test pagerfault-1 -prep { error "Database content appears incorrect" } } -finish_test #------------------------------------------------------------------------- # Test fault-injection while rolling back a hot-journal file with a From b290f1177553b866b806e5b987c060b9eed99758 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 17 Jan 2014 14:59:27 +0000 Subject: [PATCH 49/74] Fix some problems to do with WITH clauses and name resolution. FossilOrigin-Name: 6a549187ed8b5ed50daefa676ff666ae2ed43346 --- manifest | 21 +++++++++-------- manifest.uuid | 2 +- src/parse.y | 12 +++++----- src/select.c | 61 ++++++++++++++++++++++++------------------------- src/sqliteInt.h | 7 +++--- src/tokenize.c | 3 +-- src/walker.c | 23 ++++++++++--------- test/with2.test | 56 +++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 121 insertions(+), 64 deletions(-) create mode 100644 test/with2.test diff --git a/manifest b/manifest index 5d2226d267..586f354eac 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\ssome\scode\sfrom\sresolve.c\sthat\swas\sonly\srequired\sfor\srecursive\scte\sreferences\sin\ssub-queries.\sAlso\sa\sstray\s"finish_test"\scommand\sin\spagerfault.test. -D 2014-01-17T11:48:24.294 +C Fix\ssome\sproblems\sto\sdo\swith\sWITH\sclauses\sand\sname\sresolution. +D 2014-01-17T14:59:27.898 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -209,7 +209,7 @@ F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968 F src/os_win.c 1b21af72c5fa6f9e519a5fcab33e80d182b1aedb F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 824eeb752c026b551bda2b66163889d7664b42e4 +F src/parse.y 475896cb883bbf4782e98abda42efbbdcbdb75f5 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b @@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 65c13f22edfd6af04829439955c7cf5749ea4e87 +F src/select.c c77955f93121adc8b4b43a98add62fbaa2b48132 F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h d49c0bea5282f15c1eb1eb9d705770f70d19c1e2 +F src/sqliteInt.h 9600eeb486c274fbdb815d040e4a7f262b7317e1 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -274,7 +274,7 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 -F src/tokenize.c 7dc42e9beb8c3263b79d10c195b3f5264b5f874a +F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7 F src/trigger.c 5c1c0b899ac0ce284763dcb8fdbaa38ecf15ef98 F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 @@ -292,7 +292,7 @@ F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 -F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74 +F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 F src/where.c 369b0259fabfb22644d197736ae622f762cbaba8 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 @@ -1092,6 +1092,7 @@ F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/with1.test 90490c75e98e1914d84b7cef9e636b48917a020f +F test/with2.test 790c4b7ab3f4eb6984a3bbdae8d4ab429ebe9259 F test/withM.test 52448ce23e1c2ecba79d10e130ee49ce9f9a2a7a F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1150,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9efc120a1548c03f3d8aabbadf1050ff2a119c31 -R 175ad2c9f66168be6fabd0b7fdb44bd2 +P f68c6c4d36481526a9348244adc571ea282dc9eb +R d73fd639096ae3d6dc6dd6a7018ffc11 U dan -Z 6e3919f2cc39eeea18a32475675bdfc0 +Z 6339477b00dd8c67441747d0ac03cfea diff --git a/manifest.uuid b/manifest.uuid index 6e22de7d56..71dd74abb9 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f68c6c4d36481526a9348244adc571ea282dc9eb \ No newline at end of file +6a549187ed8b5ed50daefa676ff666ae2ed43346 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 937669eb5c..0805407920 100644 --- a/src/parse.y +++ b/src/parse.y @@ -661,7 +661,7 @@ limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y). %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W) orderby_opt(O) limit_opt(L). { - sqlite3WithPush(pParse,C); + sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "DELETE"); sqlite3DeleteFrom(pParse,X,W); @@ -669,7 +669,7 @@ cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W) %endif %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). { - sqlite3WithPush(pParse,C); + sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3DeleteFrom(pParse,X,W); } @@ -686,7 +686,7 @@ where_opt(A) ::= WHERE expr(X). {A = X.pExpr;} %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W) orderby_opt(O) limit_opt(L). { - sqlite3WithPush(pParse, C); + sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE"); @@ -696,7 +696,7 @@ cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W). { - sqlite3WithPush(pParse, C); + sqlite3WithPush(pParse, C, 1); sqlite3SrcListIndexedBy(pParse, X, &I); sqlite3ExprListCheckLength(pParse,Y,"set list"); sqlite3Update(pParse,X,Y,W,R); @@ -718,12 +718,12 @@ setlist(A) ::= nm(X) EQ expr(Y). { ////////////////////////// The INSERT command ///////////////////////////////// // cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). { - sqlite3WithPush(pParse, W); + sqlite3WithPush(pParse, W, 1); sqlite3Insert(pParse, X, S, F, R); } cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES. { - sqlite3WithPush(pParse, W); + sqlite3WithPush(pParse, W, 1); sqlite3Insert(pParse, X, 0, F, R); } diff --git a/src/select.c b/src/select.c index dc784b3b10..d9625cba43 100644 --- a/src/select.c +++ b/src/select.c @@ -3527,12 +3527,19 @@ static struct Cte *searchWith(With *pWith, struct SrcList_item *pItem){ /* The code generator maintains a stack of active WITH clauses ** with the inner-most WITH clause being at the top of the stack. ** -** These routines push and pull WITH clauses on the stack. +** This routine pushes the WITH clause passed as the second argument +** onto the top of the stack. If argument bFree is true, then this +** WITH clause will never be popped from the stack. In this case it +** should be freed along with the Parse object. In other cases, when +** bFree==0, the With object will be freed along with the SELECT +** statement with which it is associated. */ -void sqlite3WithPush(Parse *pParse, With *pWith){ +void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ + assert( bFree==0 || pParse->pWith==0 ); if( pWith ){ pWith->pOuter = pParse->pWith; pParse->pWith = pWith; + pParse->bFreeWith = bFree; } } @@ -3649,6 +3656,19 @@ static int withExpand( } #endif +#ifndef SQLITE_OMIT_CTE +static void selectPopWith(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + if( p->pWith ){ + assert( pParse->pWith==p->pWith ); + pParse->pWith = p->pWith->pOuter; + } + return WRC_Continue; +} +#else +#define selectPopWith 0 +#endif + /* ** This routine is a Walker callback for "expanding" a SELECT statement. ** "Expanding" means to do the following: @@ -3692,6 +3712,7 @@ static int selectExpander(Walker *pWalker, Select *p){ } pTabList = p->pSrc; pEList = p->pEList; + sqlite3WithPush(pParse, p->pWith, 0); /* Make sure cursor numbers have been assigned to all entries in ** the FROM clause of the SELECT statement. @@ -3710,6 +3731,9 @@ static int selectExpander(Walker *pWalker, Select *p){ /* This statement has already been prepared. There is no need ** to go further. */ assert( i==0 ); +#ifndef SQLITE_OMIT_CTE + selectPopWith(pWalker, p); +#endif return WRC_Prune; } #ifndef SQLITE_OMIT_CTE @@ -3941,30 +3965,6 @@ static int selectExpander(Walker *pWalker, Select *p){ return WRC_Continue; } -/* -** Function (or macro) selectExpanderWith is used as the SELECT callback -** by sqlite3SelectExpand(). In builds that do not support CTEs, this -** is equivalent to the selectExpander() function. In CTE-enabled builds, -** any WITH clause associated with the SELECT statement needs to be -** pushed onto the stack before calling selectExpander(), and popped -** off again afterwards. -*/ -#ifndef SQLITE_OMIT_CTE -static int selectExpanderWith(Walker *pWalker, Select *p){ - Parse *pParse = pWalker->pParse; - int res; - sqlite3WithPush(pParse, p->pWith); - res = selectExpander(pWalker, p); - if( p->pWith ){ - assert( pParse->pWith==p->pWith ); - pParse->pWith = p->pWith->pOuter; - } - return res; -} -#else -#define selectExpanderWith selectExpander -#endif - /* ** No-op routine for the parse-tree walker. ** @@ -4001,7 +4001,8 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ w.xSelectCallback = convertCompoundSelectToSubquery; sqlite3WalkSelect(&w, pSelect); } - w.xSelectCallback = selectExpanderWith; + w.xSelectCallback = selectExpander; + w.xSelectCallback2 = selectPopWith; sqlite3WalkSelect(&w, pSelect); } @@ -4020,7 +4021,7 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ ** at that point because identifiers had not yet been resolved. This ** routine is called after identifier resolution. */ -static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ +static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Parse *pParse; int i; SrcList *pTabList; @@ -4043,7 +4044,6 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ } } } - return WRC_Continue; } #endif @@ -4059,10 +4059,9 @@ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ #ifndef SQLITE_OMIT_SUBQUERY Walker w; memset(&w, 0, sizeof(w)); - w.xSelectCallback = selectAddSubqueryTypeInfo; + w.xSelectCallback2 = selectAddSubqueryTypeInfo; w.xExprCallback = exprWalkNoop; w.pParse = pParse; - w.bSelectDepthFirst = 1; sqlite3WalkSelect(&w, pSelect); #endif } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3f29f58d4e..1cdaf09dc7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2371,6 +2371,7 @@ struct Parse { Table *pZombieTab; /* List of Table objects to delete after code gen */ TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ With *pWith; /* Current WITH clause, or NULL */ + u8 bFreeWith; /* True if pWith should be freed with parser */ }; /* @@ -2612,9 +2613,9 @@ struct Sqlite3Config { struct Walker { int (*xExprCallback)(Walker*, Expr*); /* Callback for expressions */ int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ + void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ Parse *pParse; /* Parser context. */ int walkerDepth; /* Number of subqueries */ - u8 bSelectDepthFirst; /* Do subqueries first */ union { /* Extra data for callback */ NameContext *pNC; /* Naming context */ int i; /* Integer value */ @@ -3354,9 +3355,9 @@ const char *sqlite3JournalModename(int); #ifndef SQLITE_OMIT_CTE With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*); void sqlite3WithDelete(sqlite3*,With*); - void sqlite3WithPush(Parse*, With*); + void sqlite3WithPush(Parse*, With*, u8); #else -#define sqlite3WithPush(x,y) +#define sqlite3WithPush(x,y,z) #define sqlite3WithDelete(x,y) #endif diff --git a/src/tokenize.c b/src/tokenize.c index 3fd4f78f7a..87553e25b0 100644 --- a/src/tokenize.c +++ b/src/tokenize.c @@ -494,8 +494,7 @@ abort_parse: sqlite3DeleteTable(db, pParse->pNewTable); } - assert( pParse->pWith==0 || pParse->pWith->pOuter==0 ); - sqlite3WithDelete(db, pParse->pWith); + if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith); sqlite3DeleteTrigger(db, pParse->pNewTrigger); for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); sqlite3DbFree(db, pParse->azVar); diff --git a/src/walker.c b/src/walker.c index cde34ad780..016ae77a92 100644 --- a/src/walker.c +++ b/src/walker.c @@ -113,9 +113,12 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ /* ** Call sqlite3WalkExpr() for every expression in Select statement p. ** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and -** on the compound select chain, p->pPrior. Invoke the xSelectCallback() -** either before or after the walk of expressions and FROM clause, depending -** on whether pWalker->bSelectDepthFirst is false or true, respectively. +** on the compound select chain, p->pPrior. +** +** If it is not NULL, the xSelectCallback() callback is invoked before +** the walk of the expressions and FROM clause. The xSelectCallback2() +** method, if it is not NULL, is invoked following the walk of the +** expressions and FROM clause. ** ** Return WRC_Continue under normal conditions. Return WRC_Abort if ** there is an abort request. @@ -125,11 +128,13 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ */ int sqlite3WalkSelect(Walker *pWalker, Select *p){ int rc; - if( p==0 || pWalker->xSelectCallback==0 ) return WRC_Continue; + if( p==0 || (pWalker->xSelectCallback==0 && pWalker->xSelectCallback2==0) ){ + return WRC_Continue; + } rc = WRC_Continue; pWalker->walkerDepth++; while( p ){ - if( !pWalker->bSelectDepthFirst ){ + if( pWalker->xSelectCallback ){ rc = pWalker->xSelectCallback(pWalker, p); if( rc ) break; } @@ -139,12 +144,8 @@ int sqlite3WalkSelect(Walker *pWalker, Select *p){ pWalker->walkerDepth--; return WRC_Abort; } - if( pWalker->bSelectDepthFirst ){ - rc = pWalker->xSelectCallback(pWalker, p); - /* Depth-first search is currently only used for - ** selectAddSubqueryTypeInfo() and that routine always returns - ** WRC_Continue (0). So the following branch is never taken. */ - if( NEVER(rc) ) break; + if( pWalker->xSelectCallback2 ){ + pWalker->xSelectCallback2(pWalker, p); } p = p->pPrior; } diff --git a/test/with2.test b/test/with2.test new file mode 100644 index 0000000000..83b3b73646 --- /dev/null +++ b/test/with2.test @@ -0,0 +1,56 @@ +# 2014 January 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the WITH clause. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix with2 + +do_execsql_test 1.0 { + CREATE TABLE t1(a); + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(2); +} + +do_execsql_test 1.1 { + WITH x1 AS (SELECT * FROM t1) + SELECT sum(a) FROM x1; +} {3} + +do_execsql_test 1.2 { + WITH x1 AS (SELECT * FROM t1) + SELECT (SELECT sum(a) FROM x1); +} {3} + +do_execsql_test 1.3 { + WITH x1 AS (SELECT * FROM t1) + SELECT (SELECT sum(a) FROM x1); +} {3} + +do_execsql_test 1.4 { + CREATE TABLE t2(i); + INSERT INTO t2 VALUES(2); + INSERT INTO t2 VALUES(3); + INSERT INTO t2 VALUES(5); + + WITH x1 AS (SELECT i FROM t2), + i(a) AS ( + SELECT min(i)-1 FROM x1 UNION SELECT a+1 FROM i WHERE a<10 + ) + SELECT a FROM i WHERE a NOT IN x1 +} {1 4 6 7 8 9 10} + +finish_test + + + From 6785bcca586fca4c1dec27ede9f2f246de5d9fa9 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Jan 2014 15:27:00 +0000 Subject: [PATCH 50/74] Fix a compiler warning in selectPopWith(). FossilOrigin-Name: c8eb11635a356182611ce2ccb8f358b6c453486e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 1 - 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 61c516e9dd..ab69979816 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\ssupport\sfor\scommon\stable\sexpressions\s(WITH\sclauses). -D 2014-01-17T15:15:10.270 +C Fix\sa\scompiler\swarning\sin\sselectPopWith(). +D 2014-01-17T15:27:00.626 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 231079b8b07ea6f3cf4663e9c0ef2315abe2236c +F src/select.c b17dd5c3ef4647190bd80f51b56e366c7b12f6e5 F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1151,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 16bd54783a3f5531c55564ddefdada657c078eb0 6a549187ed8b5ed50daefa676ff666ae2ed43346 -R 3149ee3ed72aa52234b32eaa7e696309 -U dan -Z e17f48449fd6868168f4a2f61958f3af +P 0171e3bb4f663a9414b0e8b64c87b5d0683855b5 +R e6bcc1e354d62890d139633299faf33a +U drh +Z 11b7b8a08f3ff8f2c5eec014f3cfaffd diff --git a/manifest.uuid b/manifest.uuid index 5d482e8fa7..7b5cce4842 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0171e3bb4f663a9414b0e8b64c87b5d0683855b5 \ No newline at end of file +c8eb11635a356182611ce2ccb8f358b6c453486e \ No newline at end of file diff --git a/src/select.c b/src/select.c index b3a6daf170..956015840c 100644 --- a/src/select.c +++ b/src/select.c @@ -3671,7 +3671,6 @@ static void selectPopWith(Walker *pWalker, Select *p){ assert( pParse->pWith==p->pWith ); pParse->pWith = p->pWith->pOuter; } - return WRC_Continue; } #else #define selectPopWith 0 From a026b98500999f34506b2c4cf0ecb727b326c619 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Jan 2014 16:19:01 +0000 Subject: [PATCH 51/74] Add tests that verify that keywords WITH, WITHOUT, and RECURSIVE can still be used as table and column names. FossilOrigin-Name: 9ca18a0191db6a9b0763e2f0b3b35d23099fb71b --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/keyword1.test | 3 +++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index ab69979816..984a0a27fe 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scompiler\swarning\sin\sselectPopWith(). -D 2014-01-17T15:27:00.626 +C Add\stests\sthat\sverify\sthat\skeywords\sWITH,\sWITHOUT,\sand\sRECURSIVE\scan\sstill\nbe\sused\sas\stable\sand\scolumn\snames. +D 2014-01-17T16:19:01.842 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -643,7 +643,7 @@ F test/journal3.test ff8af941f9e06161d3db1b46bb9f965ff0e7f307 F test/jrnlmode.test 9ee3a78f53d52cca737db69293d15dc41c0cbd36 F test/jrnlmode2.test 81610545a4e6ed239ea8fa661891893385e23a1d F test/jrnlmode3.test 556b447a05be0e0963f4311e95ab1632b11c9eaa -F test/keyword1.test a2400977a2e4fde43bf33754c2929fda34dbca05 +F test/keyword1.test 37ef6bba5d2ed5b07ecdd6810571de2956599dff F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test e191e536d0fcd722a6b965e7cd1ee0bfd12a5991 @@ -1151,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 0171e3bb4f663a9414b0e8b64c87b5d0683855b5 -R e6bcc1e354d62890d139633299faf33a +P c8eb11635a356182611ce2ccb8f358b6c453486e +R c791b577950f1652718e7fda0d0264df U drh -Z 11b7b8a08f3ff8f2c5eec014f3cfaffd +Z 3da1c453f8a3e6f09e0b3d8bdad77daa diff --git a/manifest.uuid b/manifest.uuid index 7b5cce4842..4cfd423b13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c8eb11635a356182611ce2ccb8f358b6c453486e \ No newline at end of file +9ca18a0191db6a9b0763e2f0b3b35d23099fb71b \ No newline at end of file diff --git a/test/keyword1.test b/test/keyword1.test index 94831201ce..2bc1ff5494 100644 --- a/test/keyword1.test +++ b/test/keyword1.test @@ -65,6 +65,7 @@ set kwlist { pragma query raise + recursive regexp reindex release @@ -80,6 +81,8 @@ set kwlist { vacuum view virtual + with + without }; set exprkw { cast From 98f45e53a74e4522f129c11d03993b3d17785559 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 17 Jan 2014 17:40:46 +0000 Subject: [PATCH 52/74] Resolve table names within CTEs in the context in which the CTE is declared, not the context in which it is used. FossilOrigin-Name: a7323838bbd354a1c2f339e5e0f164f0eada47b3 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 30 ++++++++++++++++++++++-------- test/with2.test | 21 +++++++++++++++++++-- 4 files changed, 50 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 984a0a27fe..7dbf068c5b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sthat\sverify\sthat\skeywords\sWITH,\sWITHOUT,\sand\sRECURSIVE\scan\sstill\nbe\sused\sas\stable\sand\scolumn\snames. -D 2014-01-17T16:19:01.842 +C Resolve\stable\snames\swithin\sCTEs\sin\sthe\scontext\sin\swhich\sthe\sCTE\sis\sdeclared,\snot\sthe\scontext\sin\swhich\sit\sis\sused. +D 2014-01-17T17:40:46.168 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c b17dd5c3ef4647190bd80f51b56e366c7b12f6e5 +F src/select.c a13cf4a450534bcbdf1cc840d695e259cd4a1409 F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1092,7 +1092,7 @@ F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/with1.test 90490c75e98e1914d84b7cef9e636b48917a020f -F test/with2.test 790c4b7ab3f4eb6984a3bbdae8d4ab429ebe9259 +F test/with2.test 21057990b59eb652a0a30c6a421fac9daad4412d F test/withM.test 52448ce23e1c2ecba79d10e130ee49ce9f9a2a7a F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1151,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c8eb11635a356182611ce2ccb8f358b6c453486e -R c791b577950f1652718e7fda0d0264df -U drh -Z 3da1c453f8a3e6f09e0b3d8bdad77daa +P 9ca18a0191db6a9b0763e2f0b3b35d23099fb71b +R b5a3302763f84d4efb48b14faba25e2d +U dan +Z c42b677f74f4986c260cd54897199c85 diff --git a/manifest.uuid b/manifest.uuid index 4cfd423b13..6be4f2395e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ca18a0191db6a9b0763e2f0b3b35d23099fb71b \ No newline at end of file +a7323838bbd354a1c2f339e5e0f164f0eada47b3 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 956015840c..b3f16e6113 100644 --- a/src/select.c +++ b/src/select.c @@ -3507,8 +3507,15 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ ** FROM clause element pItem is really a common-table-expression (CTE) ** then return a pointer to the CTE definition for that table. Otherwise ** return NULL. +** +** If a non-NULL value is returned, set *ppContext to point to the With +** object that the returned CTE belongs to. */ -static struct Cte *searchWith(With *pWith, struct SrcList_item *pItem){ +static struct Cte *searchWith( + With *pWith, /* Current outermost WITH clause */ + struct SrcList_item *pItem, /* FROM clause element to resolve */ + With **ppContext /* OUT: WITH clause return value belongs to */ +){ const char *zName; if( pItem->zDatabase==0 && (zName = pItem->zName)!=0 ){ With *p; @@ -3516,6 +3523,7 @@ static struct Cte *searchWith(With *pWith, struct SrcList_item *pItem){ int i; for(i=0; inCte; i++){ if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){ + *ppContext = p; return &p->a[i]; } } @@ -3562,19 +3570,21 @@ static int withExpand( Walker *pWalker, struct SrcList_item *pFrom ){ - Table *pTab; Parse *pParse = pWalker->pParse; sqlite3 *db = pParse->db; - struct Cte *pCte; + struct Cte *pCte; /* Matched CTE (or NULL if no match) */ + With *pWith; /* WITH clause that pCte belongs to */ assert( pFrom->pTab==0 ); - pCte = searchWith(pParse->pWith, pFrom); + pCte = searchWith(pParse->pWith, pFrom, &pWith); if( pCte ){ + Table *pTab; ExprList *pEList; Select *pSel; Select *pLeft; /* Left-most SELECT statement */ int bMayRecursive; /* True if compound joined by UNION [ALL] */ + With *pSavedWith; /* Initial value of pParse->pWith */ /* If pCte->zErr is non-NULL at this point, then this is an illegal ** recursive reference to CTE pCte. Leave an error in pParse and return @@ -3582,7 +3592,7 @@ static int withExpand( ** In this case, proceed. */ if( pCte->zErr ){ sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName); - return WRC_Abort; + return SQLITE_ERROR; } pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); @@ -3621,11 +3631,13 @@ static int withExpand( sqlite3ErrorMsg( pParse, "multiple references to recursive table: %s", pCte->zName ); - return WRC_Abort; + return SQLITE_ERROR; } assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); pCte->zErr = "circular reference: %s"; + pSavedWith = pParse->pWith; + pParse->pWith = pWith; sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); @@ -3635,12 +3647,13 @@ static int withExpand( sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns", pCte->zName, pEList->nExpr, pCte->pCols->nExpr ); - return WRC_Abort; + pParse->pWith = pSavedWith; + return SQLITE_ERROR; } pEList = pCte->pCols; } - selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); + selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol); if( bMayRecursive ){ if( pSel->selFlags & SF_Recursive ){ pCte->zErr = "multiple recursive references: %s"; @@ -3650,6 +3663,7 @@ static int withExpand( sqlite3WalkSelect(pWalker, pSel); } pCte->zErr = 0; + pParse->pWith = pSavedWith; } return SQLITE_OK; diff --git a/test/with2.test b/test/with2.test index 83b3b73646..08ef7bf6f7 100644 --- a/test/with2.test +++ b/test/with2.test @@ -50,7 +50,24 @@ do_execsql_test 1.4 { SELECT a FROM i WHERE a NOT IN x1 } {1 4 6 7 8 9 10} +do_execsql_test 1.5 { + WITH x1 AS (SELECT a FROM t1), + x2 AS (SELECT i FROM t2), + x3 AS (SELECT * FROM x1, x2 WHERE x1.a IN x2 AND x2.i IN x1) + SELECT * FROM x3 +} {2 2} + +do_execsql_test 1.6 { + CREATE TABLE t3 AS SELECT 3 AS x; + CREATE TABLE t4 AS SELECT 4 AS x; + + WITH x1 AS (SELECT * FROM t3), + x2 AS ( + WITH t3 AS (SELECT * FROM t4) + SELECT * FROM x1 + ) + SELECT * FROM x2; +} {3} + finish_test - - From c59731c4aebeb878b45ad598537eed220363c4b9 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 17 Jan 2014 18:34:28 +0000 Subject: [PATCH 53/74] Minor simplification of error message text for a couple of errors associated with WITH clause processing. FossilOrigin-Name: 2031004d960526d6426d50d7b732f37b281534e2 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 2 +- test/with1.test | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 7dbf068c5b..f0167a7f8a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Resolve\stable\snames\swithin\sCTEs\sin\sthe\scontext\sin\swhich\sthe\sCTE\sis\sdeclared,\snot\sthe\scontext\sin\swhich\sit\sis\sused. -D 2014-01-17T17:40:46.168 +C Minor\ssimplification\sof\serror\smessage\stext\sfor\sa\scouple\sof\serrors\sassociated\nwith\sWITH\sclause\sprocessing. +D 2014-01-17T18:34:28.795 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c a13cf4a450534bcbdf1cc840d695e259cd4a1409 +F src/select.c 170a9f9e2c89b3d336d8c39d67690f3c7a06aaaf F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1091,7 +1091,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test 90490c75e98e1914d84b7cef9e636b48917a020f +F test/with1.test 6e49c7841abb5603425f8f1316ab077f6a9bbb49 F test/with2.test 21057990b59eb652a0a30c6a421fac9daad4412d F test/withM.test 52448ce23e1c2ecba79d10e130ee49ce9f9a2a7a F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 @@ -1151,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9ca18a0191db6a9b0763e2f0b3b35d23099fb71b -R b5a3302763f84d4efb48b14faba25e2d -U dan -Z c42b677f74f4986c260cd54897199c85 +P a7323838bbd354a1c2f339e5e0f164f0eada47b3 +R eb3aa3bb26890774162c9a927560c5c6 +U drh +Z f6f02c349d009e02c31d866af981661b diff --git a/manifest.uuid b/manifest.uuid index 6be4f2395e..86d2f4327b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a7323838bbd354a1c2f339e5e0f164f0eada47b3 \ No newline at end of file +2031004d960526d6426d50d7b732f37b281534e2 \ No newline at end of file diff --git a/src/select.c b/src/select.c index b3f16e6113..0ac13ea0a0 100644 --- a/src/select.c +++ b/src/select.c @@ -1808,7 +1808,7 @@ static int multiSelect( ** are supported on recursive queries. */ assert( p->pOffset==0 || p->pLimit ); if( p->pOrderBy || p->pLimit ){ - sqlite3ErrorMsg(pParse, "%s in a recursive query is not allowed", + sqlite3ErrorMsg(pParse, "%s in a recursive query", p->pOrderBy ? "ORDER BY" : "LIMIT" ); goto multi_select_end; diff --git a/test/with1.test b/test/with1.test index db0d7bc36e..d5b86209e6 100644 --- a/test/with1.test +++ b/test/with1.test @@ -152,12 +152,12 @@ do_execsql_test 5.1 { do_catchsql_test 5.2 { WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i ORDER BY 1) SELECT x FROM i LIMIT 10; -} {1 {ORDER BY in a recursive query is not allowed}} +} {1 {ORDER BY in a recursive query}} do_catchsql_test 5.3 { WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i LIMIT 10 ) SELECT x FROM i LIMIT 10; -} {1 {LIMIT in a recursive query is not allowed}} +} {1 {LIMIT in a recursive query}} do_execsql_test 5.4 { WITH i(x) AS ( VALUES(1) UNION ALL SELECT (x+1)%10 FROM i) From 7c82932723fc9e7d7531408db34f8ce67c2bbe81 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 17 Jan 2014 20:36:17 +0000 Subject: [PATCH 54/74] Add extra tests to with2.test. FossilOrigin-Name: eecc325afd72e37d7d565787c8cea68aad6d7a5c --- manifest | 14 +++---- manifest.uuid | 2 +- test/with2.test | 106 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index f0167a7f8a..1e4ac0fde0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\ssimplification\sof\serror\smessage\stext\sfor\sa\scouple\sof\serrors\sassociated\nwith\sWITH\sclause\sprocessing. -D 2014-01-17T18:34:28.795 +C Add\sextra\stests\sto\swith2.test. +D 2014-01-17T20:36:17.628 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1092,7 +1092,7 @@ F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/with1.test 6e49c7841abb5603425f8f1316ab077f6a9bbb49 -F test/with2.test 21057990b59eb652a0a30c6a421fac9daad4412d +F test/with2.test eaafa54aa7aec8dd824de7d166a2452a63156737 F test/withM.test 52448ce23e1c2ecba79d10e130ee49ce9f9a2a7a F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1151,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a7323838bbd354a1c2f339e5e0f164f0eada47b3 -R eb3aa3bb26890774162c9a927560c5c6 -U drh -Z f6f02c349d009e02c31d866af981661b +P 2031004d960526d6426d50d7b732f37b281534e2 +R 18aca364b17ce09cd2d25269185d295f +U dan +Z d7942df5ea7f32fd101e4ac65d1e6fab diff --git a/manifest.uuid b/manifest.uuid index 86d2f4327b..317857e6c4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2031004d960526d6426d50d7b732f37b281534e2 \ No newline at end of file +eecc325afd72e37d7d565787c8cea68aad6d7a5c \ No newline at end of file diff --git a/test/with2.test b/test/with2.test index 08ef7bf6f7..6035ee2b36 100644 --- a/test/with2.test +++ b/test/with2.test @@ -69,5 +69,111 @@ do_execsql_test 1.6 { SELECT * FROM x2; } {3} +do_execsql_test 1.7 { + WITH x1 AS (SELECT * FROM t1) + SELECT (SELECT sum(a) FROM x1), (SELECT max(a) FROM x1); +} {3 2} + +do_execsql_test 1.8 { + WITH x1 AS (SELECT * FROM t1) + SELECT (SELECT sum(a) FROM x1), (SELECT max(a) FROM x1), a FROM x1; +} {3 2 1 3 2 2} + +do_execsql_test 1.9 { + WITH + i(x) AS ( + WITH + j(x) AS ( SELECT * FROM i ), + i(x) AS ( SELECT * FROM t1 ) + SELECT * FROM j + ) + SELECT * FROM i; +} {1 2} + +#--------------------------------------------------------------------------- +# Check that variables can be used in CTEs. +# +set ::min [expr 3] +set ::max [expr 9] +do_execsql_test 2.1 { + WITH i(x) AS ( + VALUES($min) UNION ALL SELECT x+1 FROM i WHERE x < $max + ) + SELECT * FROM i; +} {3 4 5 6 7 8 9} + +do_execsql_test 2.2 { + WITH i(x) AS ( + VALUES($min) UNION ALL SELECT x+1 FROM i WHERE x < $max + ) + SELECT x FROM i JOIN i AS j USING (x); +} {3 4 5 6 7 8 9} + +#--------------------------------------------------------------------------- +# Check that circular references are rejected. +# +do_catchsql_test 3.1 { + WITH i(x, y) AS ( VALUES(1, (SELECT x FROM i)) ) + SELECT * FROM i; +} {1 {circular reference: i}} + +do_catchsql_test 3.2 { + WITH + i(x) AS ( SELECT * FROM j ), + j(x) AS ( SELECT * FROM k ), + k(x) AS ( SELECT * FROM i ) + SELECT * FROM i; +} {1 {circular reference: i}} + +do_catchsql_test 3.3 { + WITH + i(x) AS ( SELECT * FROM (SELECT * FROM j) ), + j(x) AS ( SELECT * FROM (SELECT * FROM i) ) + SELECT * FROM i; +} {1 {circular reference: i}} + +do_catchsql_test 3.4 { + WITH + i(x) AS ( SELECT * FROM (SELECT * FROM j) ), + j(x) AS ( SELECT * FROM (SELECT * FROM i) ) + SELECT * FROM j; +} {1 {circular reference: j}} + +do_catchsql_test 3.5 { + WITH + i(x) AS ( + WITH j(x) AS ( SELECT * FROM i ) + SELECT * FROM j + ) + SELECT * FROM i; +} {1 {circular reference: i}} + +#--------------------------------------------------------------------------- +# Try empty and very long column lists. +# +do_catchsql_test 4.1 { + WITH x() AS ( SELECT 1,2,3 ) + SELECT * FROM x; +} {1 {near ")": syntax error}} + +proc genstmt {n} { + for {set i 1} {$i<=$n} {incr i} { + lappend cols "c$i" + lappend vals $i + } + return " + WITH x([join $cols ,]) AS (SELECT [join $vals ,]) + SELECT (c$n == $n) FROM x + " +} + +do_execsql_test 4.2 [genstmt 10] 1 +do_execsql_test 4.3 [genstmt 100] 1 +do_execsql_test 4.4 [genstmt 255] 1 +set nLimit [sqlite3_limit db SQLITE_LIMIT_COLUMN -1] +do_execsql_test 4.5 [genstmt [expr $nLimit-1]] 1 +do_execsql_test 4.6 [genstmt $nLimit] 1 +do_catchsql_test 4.7 [genstmt [expr $nLimit+1]] {1 {too many columns in index}} + finish_test From ebbf08a012317c757808ae48e85ffd93a0a9953c Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 18 Jan 2014 08:27:02 +0000 Subject: [PATCH 55/74] Avoid spurious "no such table" errors in statements of the form "INSERT INTO tbl WITH xxx AS (...) SELECT * FROM xxx". FossilOrigin-Name: cccff8a0b427feb05cc8952a765b829e731394fd --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/insert.c | 9 +++++++-- test/with2.test | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 1e4ac0fde0..efb724704c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests\sto\swith2.test. -D 2014-01-17T20:36:17.628 +C Avoid\sspurious\s"no\ssuch\stable"\serrors\sin\sstatements\sof\sthe\sform\s"INSERT\sINTO\stbl\sWITH\sxxx\sAS\s(...)\sSELECT\s*\sFROM\sxxx". +D 2014-01-18T08:27:02.507 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -183,7 +183,7 @@ F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08 -F src/insert.c cb4c8ad02b6feb95d34614d94a3c68e0116fbf07 +F src/insert.c a4450f0c46a9f221622e6551ab0953b03c4f8ee8 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b @@ -1092,7 +1092,7 @@ F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/with1.test 6e49c7841abb5603425f8f1316ab077f6a9bbb49 -F test/with2.test eaafa54aa7aec8dd824de7d166a2452a63156737 +F test/with2.test 5f7ea3453c998a6b9ed3456f251c80d94722f22a F test/withM.test 52448ce23e1c2ecba79d10e130ee49ce9f9a2a7a F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1151,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2031004d960526d6426d50d7b732f37b281534e2 -R 18aca364b17ce09cd2d25269185d295f +P eecc325afd72e37d7d565787c8cea68aad6d7a5c +R 2a1f6d87654c7bd1758f1648d991f83e U dan -Z d7942df5ea7f32fd101e4ac65d1e6fab +Z bdd16ade1b5f6c0b188978b71304d4f1 diff --git a/manifest.uuid b/manifest.uuid index 317857e6c4..b0c2aeeecb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eecc325afd72e37d7d565787c8cea68aad6d7a5c \ No newline at end of file +cccff8a0b427feb05cc8952a765b829e731394fd \ No newline at end of file diff --git a/src/insert.c b/src/insert.c index b4bd6b7160..a2366360c7 100644 --- a/src/insert.c +++ b/src/insert.c @@ -667,8 +667,7 @@ void sqlite3Insert( ** ** This is the 2nd template. */ - if( pColumn==0 && pParse->pWith==0 - && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ + if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ assert( !pTrigger ); assert( pList==0 ); goto insert_end; @@ -1859,6 +1858,12 @@ static int xferOptimization( if( pSelect==0 ){ return 0; /* Must be of the form INSERT INTO ... SELECT ... */ } + if( pParse->pWith || pSelect->pWith ){ + /* Do not attempt to process this query if there are an WITH clauses + ** attached to it. Proceeding may generate a false "no such table: xxx" + ** error if pSelect reads from a CTE named "xxx". */ + return 0; + } if( sqlite3TriggerList(pParse, pDest) ){ return 0; /* tab1 must not have triggers */ } diff --git a/test/with2.test b/test/with2.test index 6035ee2b36..de199c2e4b 100644 --- a/test/with2.test +++ b/test/with2.test @@ -175,5 +175,40 @@ do_execsql_test 4.5 [genstmt [expr $nLimit-1]] 1 do_execsql_test 4.6 [genstmt $nLimit] 1 do_catchsql_test 4.7 [genstmt [expr $nLimit+1]] {1 {too many columns in index}} +#--------------------------------------------------------------------------- +# Check that adding a WITH clause to an INSERT disables the xfer +# optimization. +# +proc do_xfer_test {tn bXfer sql {res {}}} { + set ::sqlite3_xferopt_count 0 + uplevel [list do_test $tn [subst -nocommands { + set dres [db eval {$sql}] + list [set ::sqlite3_xferopt_count] [set dres] + }] [list $bXfer $res]] +} + +do_execsql_test 5.1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); +} + +do_xfer_test 5.2 1 { INSERT INTO t1 SELECT * FROM t2 } +do_xfer_test 5.3 0 { INSERT INTO t1 SELECT a, b FROM t2 } +do_xfer_test 5.4 0 { INSERT INTO t1 SELECT b, a FROM t2 } +do_xfer_test 5.5 0 { + WITH x AS (SELECT a, b FROM t2) INSERT INTO t1 SELECT * FROM x +} +do_xfer_test 5.6 0 { + WITH x AS (SELECT a, b FROM t2) INSERT INTO t1 SELECT * FROM t2 +} +do_xfer_test 5.7 0 { + INSERT INTO t1 WITH x AS ( SELECT * FROM t2 ) SELECT * FROM x +} +do_xfer_test 5.8 0 { + INSERT INTO t1 WITH x(a,b) AS ( SELECT * FROM t2 ) SELECT * FROM x +} + finish_test From 75303a2c683fff538603e9f8a69263d2db4c0c5f Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Jan 2014 15:22:53 +0000 Subject: [PATCH 56/74] Add asserts() for a couple of unreachable conditions. Add the Mandelbrot Set query as a test case. FossilOrigin-Name: 2ad4583c0cc7988f0dfe78fd0a2eb0fdb92d835a --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 2 +- src/vdbe.c | 3 ++- test/with1.test | 43 ++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 55 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index efb724704c..f3a6dea836 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sspurious\s"no\ssuch\stable"\serrors\sin\sstatements\sof\sthe\sform\s"INSERT\sINTO\stbl\sWITH\sxxx\sAS\s(...)\sSELECT\s*\sFROM\sxxx". -D 2014-01-18T08:27:02.507 +C Add\sasserts()\sfor\sa\scouple\sof\sunreachable\sconditions.\s\sAdd\sthe\sMandelbrot\sSet\nquery\sas\sa\stest\scase. +D 2014-01-18T15:22:53.229 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff -F src/expr.c e239763d8b43356fa1f46f1cf41d62a076f7f72e +F src/expr.c 8c7e482bc8f7982333f046851a610ccdb8a1ba94 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -280,7 +280,7 @@ F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c e71f19b272f05c8695cf747b4bac1732685f9e5c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c ccc8594e89751966022642464ec2b5c5fa7840a2 +F src/vdbe.c 98d96d04d9a2bef78ca850be1053dc91d031338a F src/vdbe.h e6c4c610fcabad4fa80ebb1efc6822a9367e2b26 F src/vdbeInt.h 42db251e9f863401ff847b90d5fe1614c89a6a56 F src/vdbeapi.c ce4e68ea4842cc6081046f533d088dcf01d247ad @@ -1091,7 +1091,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test 6e49c7841abb5603425f8f1316ab077f6a9bbb49 +F test/with1.test be21f7efdc08fe6ad182dc943d42704300f0fcdb F test/with2.test 5f7ea3453c998a6b9ed3456f251c80d94722f22a F test/withM.test 52448ce23e1c2ecba79d10e130ee49ce9f9a2a7a F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 @@ -1151,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eecc325afd72e37d7d565787c8cea68aad6d7a5c -R 2a1f6d87654c7bd1758f1648d991f83e -U dan -Z bdd16ade1b5f6c0b188978b71304d4f1 +P cccff8a0b427feb05cc8952a765b829e731394fd +R 538a9377a8d02c8e21002b1bc024f5ac +U drh +Z 8baad4729759494d05074bc32769b912 diff --git a/manifest.uuid b/manifest.uuid index b0c2aeeecb..34f13530bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cccff8a0b427feb05cc8952a765b829e731394fd \ No newline at end of file +2ad4583c0cc7988f0dfe78fd0a2eb0fdb92d835a \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 0614be1cf5..5f11dec420 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1584,7 +1584,7 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ iCol = (i16)pExpr->iColumn; /* Code an OP_VerifyCookie and OP_TableLock for
. */ - if( pTab->pSchema ){ + if( ALWAYS(pTab->pSchema) ){ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); diff --git a/src/vdbe.c b/src/vdbe.c index 9af4a62d7e..86aae3c65d 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3390,7 +3390,8 @@ case OP_SwapCursors: { p->apCsr[pOp->p1] = p->apCsr[pOp->p2]; p->apCsr[pOp->p2] = pTmp; - rc = sqlite3BtreeClearTable(pTmp->pBt, MASTER_ROOT + !pTmp->isTable, 0); + assert( pTmp->isTable ); + rc = sqlite3BtreeClearTable(pTmp->pBt, MASTER_ROOT, 0); break; } #endif /* ifndef SQLITE_OMIT_CTE */ diff --git a/test/with1.test b/test/with1.test index d5b86209e6..54e0f0aa91 100644 --- a/test/with1.test +++ b/test/with1.test @@ -324,6 +324,47 @@ do_catchsql_test 7.6 { SELECT id FROM t; } {1 {circular reference: t}} - +# Compute the mandelbrot set using a recursive query +# +do_execsql_test 8.1 { + WITH RECURSIVE + xaxis(x) AS (VALUES(-2.0) UNION ALL SELECT x+0.05 FROM xaxis WHERE x<1.2), + yaxis(y) AS (VALUES(-1.0) UNION ALL SELECT y+0.1 FROM yaxis WHERE y<1.0), + m(iter, cx, cy, x, y) AS ( + SELECT 0, x, y, 0.0, 0.0 FROM xaxis, yaxis + UNION ALL + SELECT iter+1, cx, cy, x*x-y*y + cx, 2.0*x*y + cy FROM m + WHERE (x*x + y*y) < 4.0 AND iter<28 + ), + m2(iter, cx, cy) AS ( + SELECT max(iter), cx, cy FROM m GROUP BY cx, cy + ), + a(t) AS ( + SELECT group_concat( substr(' .+*#', 1+min(iter/7,4), 1), '') + FROM m2 GROUP BY cy + ) + SELECT group_concat(rtrim(t),x'0a') FROM a; +} {{ ....# + ..#*.. + ..+####+. + .......+####.... + + ..##+*##########+.++++ + .+.##################+. + .............+###################+.+ + ..++..#.....*#####################+. + ...+#######++#######################. + ....+*################################. + #############################################... + ....+*################################. + ...+#######++#######################. + ..++..#.....*#####################+. + .............+###################+.+ + .+.##################+. + ..##+*##########+.++++ + .......+####.... + + ..+####+. + ..#*.. + ....# + +.}} finish_test From 1fe3c4b52668f261a893a9abe05829e40cad5d1e Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 18 Jan 2014 15:59:35 +0000 Subject: [PATCH 57/74] Add extra test cases. No changes to code. FossilOrigin-Name: d38d485e581dab99a3ee6b348da8ddaf9b379ff2 --- manifest | 16 ++--- manifest.uuid | 2 +- test/with2.test | 176 +++++++++++++++++++++++++++++++++++++++++++++++- test/withM.test | 16 +++++ 4 files changed, 199 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f3a6dea836..522f30a10d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sasserts()\sfor\sa\scouple\sof\sunreachable\sconditions.\s\sAdd\sthe\sMandelbrot\sSet\nquery\sas\sa\stest\scase. -D 2014-01-18T15:22:53.229 +C Add\sextra\stest\scases.\sNo\schanges\sto\scode. +D 2014-01-18T15:59:35.958 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1092,8 +1092,8 @@ F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/with1.test be21f7efdc08fe6ad182dc943d42704300f0fcdb -F test/with2.test 5f7ea3453c998a6b9ed3456f251c80d94722f22a -F test/withM.test 52448ce23e1c2ecba79d10e130ee49ce9f9a2a7a +F test/with2.test 67a3347f2d78618db9434a248fb5003af8f04bc1 +F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 F test/without_rowid3.test eac3d5c8a1924725b58503a368f2cbd24fd6c8a0 @@ -1151,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P cccff8a0b427feb05cc8952a765b829e731394fd -R 538a9377a8d02c8e21002b1bc024f5ac -U drh -Z 8baad4729759494d05074bc32769b912 +P 2ad4583c0cc7988f0dfe78fd0a2eb0fdb92d835a +R 71fcd9d8c24b81a08237ce7106794f40 +U dan +Z 1a2b2ad5eb21bdc784c4eae4f3f812b5 diff --git a/manifest.uuid b/manifest.uuid index 34f13530bf..e5d4cbf4df 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2ad4583c0cc7988f0dfe78fd0a2eb0fdb92d835a \ No newline at end of file +d38d485e581dab99a3ee6b348da8ddaf9b379ff2 \ No newline at end of file diff --git a/test/with2.test b/test/with2.test index de199c2e4b..90a64a4755 100644 --- a/test/with2.test +++ b/test/with2.test @@ -70,16 +70,32 @@ do_execsql_test 1.6 { } {3} do_execsql_test 1.7 { + WITH x2 AS ( + WITH t3 AS (SELECT * FROM t4) + SELECT * FROM t3 + ) + SELECT * FROM x2; +} {4} + +do_execsql_test 1.8 { + WITH x2 AS ( + WITH t3 AS (SELECT * FROM t4) + SELECT * FROM main.t3 + ) + SELECT * FROM x2; +} {3} + +do_execsql_test 1.9 { WITH x1 AS (SELECT * FROM t1) SELECT (SELECT sum(a) FROM x1), (SELECT max(a) FROM x1); } {3 2} -do_execsql_test 1.8 { +do_execsql_test 1.10 { WITH x1 AS (SELECT * FROM t1) SELECT (SELECT sum(a) FROM x1), (SELECT max(a) FROM x1), a FROM x1; } {3 2 1 3 2 2} -do_execsql_test 1.9 { +do_execsql_test 1.11 { WITH i(x) AS ( WITH @@ -90,6 +106,62 @@ do_execsql_test 1.9 { SELECT * FROM i; } {1 2} +do_execsql_test 1.12 { + WITH r(i) AS ( + VALUES('.') + UNION ALL + SELECT i || '.' FROM r, ( + SELECT x FROM x INTERSECT SELECT y FROM y + ) WHERE length(i) < 10 + ), + x(x) AS ( VALUES(1) UNION ALL VALUES(2) UNION ALL VALUES(3) ), + y(y) AS ( VALUES(2) UNION ALL VALUES(4) UNION ALL VALUES(6) ) + + SELECT * FROM r; +} {. .. ... .... ..... ...... ....... ........ ......... ..........} + +do_execsql_test 1.13 { + WITH r(i) AS ( + VALUES('.') + UNION ALL + SELECT i || '.' FROM r, ( SELECT x FROM x WHERE x=2 ) WHERE length(i) < 10 + ), + x(x) AS ( VALUES(1) UNION ALL VALUES(2) UNION ALL VALUES(3) ) + + SELECT * FROM r ORDER BY length(i) DESC; +} {.......... ......... ........ ....... ...... ..... .... ... .. .} + +do_execsql_test 1.14 { + WITH + t4(x) AS ( + VALUES(4) + UNION ALL + SELECT x+1 FROM t4 WHERE x<10 + ) + SELECT * FROM t4; +} {4 5 6 7 8 9 10} + +do_execsql_test 1.15 { + WITH + t4(x) AS ( + VALUES(4) + UNION ALL + SELECT x+1 FROM main.t4 WHERE x<10 + ) + SELECT * FROM t4; +} {4 5} + +do_catchsql_test 1.16 { + WITH + t4(x) AS ( + VALUES(4) + UNION ALL + SELECT x+1 FROM t4, main.t4, t4 WHERE x<10 + ) + SELECT * FROM t4; +} {1 {multiple references to recursive table: t4}} + + #--------------------------------------------------------------------------- # Check that variables can be used in CTEs. # @@ -210,5 +282,105 @@ do_xfer_test 5.8 0 { INSERT INTO t1 WITH x(a,b) AS ( SELECT * FROM t2 ) SELECT * FROM x } +#--------------------------------------------------------------------------- +# Check that syntax (and other) errors in statements with WITH clauses +# attached to them do not cause problems (e.g. memory leaks). +# +do_execsql_test 6.1 { + DROP TABLE IF EXISTS t1; + DROP TABLE IF EXISTS t2; + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); +} + +do_catchsql_test 6.2 { + WITH x AS (SELECT * FROM t1) + INSERT INTO t2 VALUES(1, 2,); +} {1 {near ")": syntax error}} + +do_catchsql_test 6.3 { + WITH x AS (SELECT * FROM t1) + INSERT INTO t2 SELECT a, b, FROM t1; +} {1 {near "FROM": syntax error}} + +do_catchsql_test 6.3 { + WITH x AS (SELECT * FROM t1) + INSERT INTO t2 SELECT a, b FROM abc; +} {1 {no such table: abc}} + +do_catchsql_test 6.4 { + WITH x AS (SELECT * FROM t1) + INSERT INTO t2 SELECT a, b, FROM t1 a a a; +} {1 {near "FROM": syntax error}} + +do_catchsql_test 6.5 { + WITH x AS (SELECT * FROM t1) + DELETE FROM t2 WHERE; +} {1 {near ";": syntax error}} + +do_catchsql_test 6.6 { + WITH x AS (SELECT * FROM t1) DELETE FROM t2 WHERE +} {/1 {near .* syntax error}/} + +do_catchsql_test 6.7 { + WITH x AS (SELECT * FROM t1) DELETE FROM t2 WHRE 1; +} {/1 {near .* syntax error}/} + +do_catchsql_test 6.8 { + WITH x AS (SELECT * FROM t1) UPDATE t2 SET a = 10, b = ; +} {/1 {near .* syntax error}/} + +do_catchsql_test 6.9 { + WITH x AS (SELECT * FROM t1) UPDATE t2 SET a = 10, b = 1 WHERE a===b; +} {/1 {near .* syntax error}/} + +do_catchsql_test 6.10 { + WITH x(a,b) AS ( + SELECT 1, 1 + UNION ALL + SELECT a*b,a+b FROM x WHERE c=2 + ) + SELECT * FROM x +} {1 {no such column: c}} + +#------------------------------------------------------------------------- +# Recursive queries in IN(...) expressions. +# +do_execsql_test 7.1 { + CREATE TABLE t5(x INTEGER); + CREATE TABLE t6(y INTEGER); + + WITH s(x) AS ( VALUES(7) UNION ALL SELECT x+7 FROM s WHERE x<49 ) + INSERT INTO t5 + SELECT * FROM s; + + INSERT INTO t6 + WITH s(x) AS ( VALUES(2) UNION ALL SELECT x+2 FROM s WHERE x<49 ) + SELECT * FROM s; +} + +do_execsql_test 7.2 { + SELECT * FROM t6 WHERE y IN (SELECT x FROM t5) +} {14 28 42} + +do_execsql_test 7.3 { + WITH ss AS (SELECT x FROM t5) + SELECT * FROM t6 WHERE y IN (SELECT x FROM ss) +} {14 28 42} + +do_execsql_test 7.4 { + WITH ss(x) AS ( VALUES(7) UNION ALL SELECT x+7 FROM ss WHERE x<49 ) + SELECT * FROM t6 WHERE y IN (SELECT x FROM ss) +} {14 28 42} + +do_execsql_test 7.5 { + SELECT * FROM t6 WHERE y IN ( + WITH ss(x) AS ( VALUES(7) UNION ALL SELECT x+7 FROM ss WHERE x<49 ) + SELECT x FROM ss + ) +} {14 28 42} + + + finish_test diff --git a/test/withM.test b/test/withM.test index 9bf7ceed3f..c1650d9576 100644 --- a/test/withM.test +++ b/test/withM.test @@ -55,6 +55,22 @@ do_faultsim_test withM-1.2 -prep { db close } +do_faultsim_test withM-1.3 -prep { + sqlite3 db test.db +} -body { + execsql { + WITH w1(a,b) AS ( + SELECT 1, 1 + UNION ALL + SELECT a+1, b + 2*a + 1 FROM w1 + ) + SELECT * FROM w1 LIMIT 5; + } +} -test { + faultsim_test_result {0 {1 1 2 4 3 9 4 16 5 25}} + db close +} + finish_test From 717c09c4a94406e27148dc75f18afd02e023b845 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 18 Jan 2014 18:33:44 +0000 Subject: [PATCH 58/74] Add a sudoku solver to the recursive query tests in with1.test. FossilOrigin-Name: 679eff8759aa25368b977c0d26b78a9fcd9486f5 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/with1.test | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 522f30a10d..7463d67494 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stest\scases.\sNo\schanges\sto\scode. -D 2014-01-18T15:59:35.958 +C Add\sa\ssudoku\ssolver\sto\sthe\srecursive\squery\stests\sin\swith1.test. +D 2014-01-18T18:33:44.994 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1091,7 +1091,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test be21f7efdc08fe6ad182dc943d42704300f0fcdb +F test/with1.test cec63b56797a70842afa8929c241dfdb3d864283 F test/with2.test 67a3347f2d78618db9434a248fb5003af8f04bc1 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 @@ -1151,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2ad4583c0cc7988f0dfe78fd0a2eb0fdb92d835a -R 71fcd9d8c24b81a08237ce7106794f40 -U dan -Z 1a2b2ad5eb21bdc784c4eae4f3f812b5 +P d38d485e581dab99a3ee6b348da8ddaf9b379ff2 +R 65511deacff317a7a6d96d8a97a06cc3 +U drh +Z 1781cdebe197572ff0a729675feb42d9 diff --git a/manifest.uuid b/manifest.uuid index e5d4cbf4df..8e6cce16db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d38d485e581dab99a3ee6b348da8ddaf9b379ff2 \ No newline at end of file +679eff8759aa25368b977c0d26b78a9fcd9486f5 \ No newline at end of file diff --git a/test/with1.test b/test/with1.test index 54e0f0aa91..fcf59267f2 100644 --- a/test/with1.test +++ b/test/with1.test @@ -367,4 +367,41 @@ do_execsql_test 8.1 { ....# +.}} +# Solve a sudoku puzzle using a recursive query +# +do_execsql_test 8.2 { + WITH RECURSIVE + input(sud) AS ( + VALUES('53..7....6..195....98....6.8...6...34..8.3..17...2...6.6....28....419..5....8..79') + ), + + /* A table filled with digits 1..9, inclusive. */ + digits(z, lp) AS ( + VALUES('1', 1) + UNION ALL SELECT + CAST(lp+1 AS TEXT), lp+1 FROM digits WHERE lp<9 + ), + + /* The tricky bit. */ + x(s, ind) AS ( + SELECT sud, instr(sud, '.') FROM input + UNION ALL + SELECT + substr(s, 1, ind-1) || z || substr(s, ind+1), + instr( substr(s, 1, ind-1) || z || substr(s, ind+1), '.' ) + FROM x, digits AS z + WHERE ind>0 + AND NOT EXISTS ( + SELECT 1 + FROM digits AS lp + WHERE z.z = substr(s, ((ind-1)/9)*9 + lp, 1) + OR z.z = substr(s, ((ind-1)%9) + (lp-1)*9 + 1, 1) + OR z.z = substr(s, (((ind-1)/3) % 3) * 3 + + ((ind-1)/27) * 27 + lp + + ((lp-1) / 3) * 6, 1) + ) + ) + SELECT s FROM x WHERE ind=0; +} {534678912672195348198342567859761423426853791713924856961537284287419635345286179} + finish_test From 7f3068aa83c4593dead9ee0827b2413d5a1f1bb6 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Jan 2014 14:17:08 +0000 Subject: [PATCH 59/74] Do not run the tests in with2.test with SQLITE_OMIT_CTE builds. FossilOrigin-Name: 8a973912e98c9b1bb9d3f914527d35c1e7f2011a --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/with2.test | 5 +++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 7463d67494..39ddd14f77 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\ssudoku\ssolver\sto\sthe\srecursive\squery\stests\sin\swith1.test. -D 2014-01-18T18:33:44.994 +C Do\snot\srun\sthe\stests\sin\swith2.test\swith\sSQLITE_OMIT_CTE\sbuilds. +D 2014-01-20T14:17:08.203 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1092,7 +1092,7 @@ F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/with1.test cec63b56797a70842afa8929c241dfdb3d864283 -F test/with2.test 67a3347f2d78618db9434a248fb5003af8f04bc1 +F test/with2.test 2fe78fcd8deef2a0f9cfc49bfc755911d0b3fd64 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1151,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d38d485e581dab99a3ee6b348da8ddaf9b379ff2 -R 65511deacff317a7a6d96d8a97a06cc3 -U drh -Z 1781cdebe197572ff0a729675feb42d9 +P 679eff8759aa25368b977c0d26b78a9fcd9486f5 +R 4f2132a97fe1dbb0dda85feb1706370a +U dan +Z 185238e927f243f40ec02c5fcc28d90b diff --git a/manifest.uuid b/manifest.uuid index 8e6cce16db..8f8eb075b5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -679eff8759aa25368b977c0d26b78a9fcd9486f5 \ No newline at end of file +8a973912e98c9b1bb9d3f914527d35c1e7f2011a \ No newline at end of file diff --git a/test/with2.test b/test/with2.test index 90a64a4755..d702f8c962 100644 --- a/test/with2.test +++ b/test/with2.test @@ -16,6 +16,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set ::testprefix with2 +ifcapable {!cte} { + finish_test + return +} + do_execsql_test 1.0 { CREATE TABLE t1(a); INSERT INTO t1 VALUES(1); From c25e2ebc01a94bb18959d61237a47a758c17d267 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 20 Jan 2014 14:58:55 +0000 Subject: [PATCH 60/74] Remove an unused #define and add an assert(), both associated with WITH logic. FossilOrigin-Name: a06235e0f6aa1e8fefa3f2873ee035eac9dac750 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 1 + src/sqliteInt.h | 1 - 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 39ddd14f77..6691d6c5ff 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Do\snot\srun\sthe\stests\sin\swith2.test\swith\sSQLITE_OMIT_CTE\sbuilds. -D 2014-01-20T14:17:08.203 +C Remove\san\sunused\s#define\sand\sadd\san\sassert(),\sboth\sassociated\swith\sWITH\slogic. +D 2014-01-20T14:58:55.056 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 170a9f9e2c89b3d336d8c39d67690f3c7a06aaaf +F src/select.c a27ac21844df3123b7c1e89d79cd7034d4eb0e8e F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 9600eeb486c274fbdb815d040e4a7f262b7317e1 +F src/sqliteInt.h 99fd628541e420b98fc52072635e8ba431706250 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1151,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 679eff8759aa25368b977c0d26b78a9fcd9486f5 -R 4f2132a97fe1dbb0dda85feb1706370a -U dan -Z 185238e927f243f40ec02c5fcc28d90b +P 8a973912e98c9b1bb9d3f914527d35c1e7f2011a +R bcd03400ea7bd2e959020317f5125b8b +U drh +Z f5ee43186682e26d4e29624d49787797 diff --git a/manifest.uuid b/manifest.uuid index 8f8eb075b5..9e604f5cb8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8a973912e98c9b1bb9d3f914527d35c1e7f2011a \ No newline at end of file +a06235e0f6aa1e8fefa3f2873ee035eac9dac750 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0ac13ea0a0..dd58ed22e6 100644 --- a/src/select.c +++ b/src/select.c @@ -3595,6 +3595,7 @@ static int withExpand( return SQLITE_ERROR; } + assert( pFrom->pTab==0 ); pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return WRC_Abort; pTab->nRef = 1; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 1cdaf09dc7..317e9eb227 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1437,7 +1437,6 @@ struct Table { #define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ #define TF_Virtual 0x10 /* Is a virtual table */ #define TF_WithoutRowid 0x20 /* No rowid used. PRIMARY KEY is the key */ -#define TF_Recursive 0x40 /* Recursive reference within CTE */ /* From 7df42aba12a1ef6607ed8718202eb434ddc514c1 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Jan 2014 18:25:44 +0000 Subject: [PATCH 61/74] Handle a few obscure problems that could manifest if a database corrupted in a certain way was written by a connection in the middle of a SELECT statement on the same db. FossilOrigin-Name: eba8a564e62f84a9620008beead80081fe90a1b7 --- manifest | 15 ++--- manifest.uuid | 2 +- src/btree.c | 35 +++++------ test/corruptH.test | 150 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 176 insertions(+), 26 deletions(-) create mode 100644 test/corruptH.test diff --git a/manifest b/manifest index 6691d6c5ff..23358e38e5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunused\s#define\sand\sadd\san\sassert(),\sboth\sassociated\swith\sWITH\slogic. -D 2014-01-20T14:58:55.056 +C Handle\sa\sfew\sobscure\sproblems\sthat\scould\smanifest\sif\sa\sdatabase\scorrupted\sin\sa\scertain\sway\swas\swritten\sby\sa\sconnection\sin\sthe\smiddle\sof\sa\sSELECT\sstatement\son\sthe\ssame\sdb. +D 2014-01-20T18:25:44.841 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -166,7 +166,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c c15e1722696b66c4029c487acfb830b0985bf142 +F src/btree.c 02e1a4e71d8fc37e9fd5216c15d989d148a77c87 F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0 F src/build.c 7e6c275ab1731510d6f793d0f88373ab3e858e69 @@ -406,6 +406,7 @@ F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test c150f156dace653c00a121ad0f5772a0568c41ba +F test/corruptH.test 0a247f3dc8a8f3578db5f639d86c6bb4d520207f F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1151,7 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 8a973912e98c9b1bb9d3f914527d35c1e7f2011a -R bcd03400ea7bd2e959020317f5125b8b -U drh -Z f5ee43186682e26d4e29624d49787797 +P a06235e0f6aa1e8fefa3f2873ee035eac9dac750 +R 05b4cba80caa92c648883df94b9ec5d5 +U dan +Z 37a297a819c7d62ae4b0e647e3e25c85 diff --git a/manifest.uuid b/manifest.uuid index 9e604f5cb8..ad1f20fbcf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a06235e0f6aa1e8fefa3f2873ee035eac9dac750 \ No newline at end of file +eba8a564e62f84a9620008beead80081fe90a1b7 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index e28a97c846..032f3d8c78 100644 --- a/src/btree.c +++ b/src/btree.c @@ -3754,7 +3754,7 @@ int sqlite3BtreeCloseCursor(BtCursor *pCur){ int iPage = pCur->iPage; memset(&info, 0, sizeof(info)); btreeParseCell(pCur->apPage[iPage], pCur->aiIdx[iPage], &info); - assert( memcmp(&info, &pCur->info, sizeof(info))==0 ); + assert( CORRUPT_DB || memcmp(&info, &pCur->info, sizeof(info))==0 ); } #else #define assertCellInfo(x) @@ -4390,26 +4390,24 @@ static int moveToRoot(BtCursor *pCur){ return rc; } pCur->iPage = 0; - - /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor - ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is - ** NULL, the caller expects a table b-tree. If this is not the case, - ** return an SQLITE_CORRUPT error. */ - assert( pCur->apPage[0]->intKey==1 || pCur->apPage[0]->intKey==0 ); - if( (pCur->pKeyInfo==0)!=pCur->apPage[0]->intKey ){ - return SQLITE_CORRUPT_BKPT; - } } - - /* Assert that the root page is of the correct type. This must be the - ** case as the call to this function that loaded the root-page (either - ** this call or a previous invocation) would have detected corruption - ** if the assumption were not true, and it is not possible for the flags - ** byte to have been modified while this cursor is holding a reference - ** to the page. */ pRoot = pCur->apPage[0]; assert( pRoot->pgno==pCur->pgnoRoot ); - assert( pRoot->isInit && (pCur->pKeyInfo==0)==pRoot->intKey ); + + /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor + ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is + ** NULL, the caller expects a table b-tree. If this is not the case, + ** return an SQLITE_CORRUPT error. + ** + ** Earlier versions of SQLite assumed that this test could not fail + ** if the root page was already loaded when this function was called (i.e. + ** if pCur->iPage>=0). But this is not so if the database is corrupted + ** in such a way that page pRoot is linked into a second b-tree table + ** (or the freelist). */ + assert( pRoot->intKey==1 || pRoot->intKey==0 ); + if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){ + return SQLITE_CORRUPT_BKPT; + } pCur->aiIdx[0] = 0; pCur->info.nSize = 0; @@ -5251,6 +5249,7 @@ end_allocate_page: if( rc==SQLITE_OK ){ if( sqlite3PagerPageRefcount((*ppPage)->pDbPage)>1 ){ releasePage(*ppPage); + *ppPage = 0; return SQLITE_CORRUPT_BKPT; } (*ppPage)->isInit = 0; diff --git a/test/corruptH.test b/test/corruptH.test new file mode 100644 index 0000000000..23f80632b1 --- /dev/null +++ b/test/corruptH.test @@ -0,0 +1,150 @@ +# 2014-01-20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix corruptH + +# Do not use a codec for tests in this file, as the database file is +# manipulated directly using tcl scripts (using the [hexio_write] command). +# +do_not_use_codec +database_may_be_corrupt + +# Initialize the database. +# +do_execsql_test 1.1 { + PRAGMA page_size=1024; + + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + + CREATE TABLE t2(x); + INSERT INTO t2 VALUES(randomblob(200)); + INSERT INTO t2 SELECT randomblob(200) FROM t2; + INSERT INTO t2 SELECT randomblob(200) FROM t2; + INSERT INTO t2 SELECT randomblob(200) FROM t2; + INSERT INTO t2 SELECT randomblob(200) FROM t2; + INSERT INTO t2 SELECT randomblob(200) FROM t2; + INSERT INTO t2 SELECT randomblob(200) FROM t2; +} {} + +# Corrupt the file so that the root page of t1 is also linked into t2 as +# a leaf page. +# +do_test 1.2 { + db eval { SELECT name, rootpage FROM sqlite_master } { + set r($name) $rootpage + } + db close + hexio_write test.db [expr {($r(t2)-1)*1024 + 11}] [format %.2X $r(t1)] + sqlite3 db test.db +} {} + +do_test 1.3 { + db eval { PRAGMA secure_delete=1 } + list [catch { + db eval { SELECT * FROM t1 WHERE a IN (1, 2) } { + db eval { DELETE FROM t2 } + } + } msg] $msg +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db + +# Initialize the database. +# +do_execsql_test 2.1 { + PRAGMA page_size=1024; + + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + + CREATE TABLE t3(x); + + CREATE TABLE t2(x PRIMARY KEY) WITHOUT ROWID; + INSERT INTO t2 VALUES(randomblob(100)); + + DROP TABLE t3; +} {} + +do_test 2.2 { + db eval { SELECT name, rootpage FROM sqlite_master } { + set r($name) $rootpage + } + db close + set fl [hexio_get_int [hexio_read test.db 32 4]] + + hexio_write test.db [expr {($fl-1) * 1024 + 0}] 00000000 + hexio_write test.db [expr {($fl-1) * 1024 + 4}] 00000001 + hexio_write test.db [expr {($fl-1) * 1024 + 8}] [format %.8X $r(t1)] + hexio_write test.db 36 00000002 + + sqlite3 db test.db +} {} + +do_test 2.3 { + list [catch { + db eval { SELECT * FROM t1 WHERE a IN (1, 2) } { + db eval { + INSERT INTO t2 SELECT randomblob(100) FROM t2; + INSERT INTO t2 SELECT randomblob(100) FROM t2; + INSERT INTO t2 SELECT randomblob(100) FROM t2; + INSERT INTO t2 SELECT randomblob(100) FROM t2; + INSERT INTO t2 SELECT randomblob(100) FROM t2; + } + } + } msg] $msg +} {1 {database disk image is malformed}} + +#------------------------------------------------------------------------- +reset_db + +# Initialize the database. +# +do_execsql_test 3.1 { + PRAGMA page_size=1024; + + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'one'); + INSERT INTO t1 VALUES(2, 'two'); + + CREATE TABLE t2(c INTEGER PRAGMA KEY, d); + INSERT INTO t2 VALUES(1, randomblob(1100)); +} {} + +do_test 3.2 { + db eval { SELECT name, rootpage FROM sqlite_master } { + set r($name) $rootpage + } + db close + + hexio_write test.db [expr {($r(t2)-1) * 1024 + 1020}] 00000002 + + sqlite3 db test.db +} {} + +do_test 3.3 { + list [catch { + db eval { SELECT * FROM t1 WHERE a IN (1, 2) } { + db eval { + DELETE FROM t2 WHERE c=1; + } + } + } msg] $msg +} {1 {database disk image is malformed}} + +finish_test + From a4ff825095456e0fe12dd4d9c848043fd6b91697 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 20 Jan 2014 19:55:33 +0000 Subject: [PATCH 62/74] In where.c, do not allocate space in sqlite3_index_info structures for the internal WHERE clause "terms" generated to record column equivalencies. FossilOrigin-Name: 7d9e22187daaa3160b875a1df17b924969bf718e --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 6 ++++-- test/vtab1.test | 19 +++++++++++++++++++ 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 23358e38e5..7c7eeb389c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\sa\sfew\sobscure\sproblems\sthat\scould\smanifest\sif\sa\sdatabase\scorrupted\sin\sa\scertain\sway\swas\swritten\sby\sa\sconnection\sin\sthe\smiddle\sof\sa\sSELECT\sstatement\son\sthe\ssame\sdb. -D 2014-01-20T18:25:44.841 +C In\swhere.c,\sdo\snot\sallocate\sspace\sin\ssqlite3_index_info\sstructures\sfor\sthe\sinternal\sWHERE\sclause\s"terms"\sgenerated\sto\srecord\scolumn\sequivalencies. +D 2014-01-20T19:55:33.120 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 5e11de480a94e6ff8f9922e3a04a31b56d5f33b5 +F src/where.c 56f85486bc8d0cb57fc15e5db2a58d1dfa1114cf F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1027,7 +1027,7 @@ F test/vacuum4.test d3f8ecff345f166911568f397d2432c16d2867d9 F test/varint.test ab7b110089a08b9926ed7390e7e97bdefeb74102 F test/veryquick.test 57ab846bacf7b90cf4e9a672721ea5c5b669b661 F test/view.test 4057630287bfa5955628fe90a13d4c225d1c7352 -F test/vtab1.test 45ddde57764659c0ec90874bcb6c4831f1004c06 +F test/vtab1.test b631d147b198cfd7903ab5fed028eb2a3d321dc6 F test/vtab2.test 7bcffc050da5c68f4f312e49e443063e2d391c0d F test/vtab3.test baad99fd27217f5d6db10660522e0b7192446de1 F test/vtab4.test 942f8b8280b3ea8a41dae20e7822d065ca1cb275 @@ -1152,7 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a06235e0f6aa1e8fefa3f2873ee035eac9dac750 -R 05b4cba80caa92c648883df94b9ec5d5 +P eba8a564e62f84a9620008beead80081fe90a1b7 +R c774305f58a7e7fff71d714265811a28 U dan -Z 37a297a819c7d62ae4b0e647e3e25c85 +Z 8e05aa582afc8018a3754b6d40d97300 diff --git a/manifest.uuid b/manifest.uuid index ad1f20fbcf..1199189ccc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eba8a564e62f84a9620008beead80081fe90a1b7 \ No newline at end of file +7d9e22187daaa3160b875a1df17b924969bf718e \ No newline at end of file diff --git a/src/where.c b/src/where.c index 30c8597408..86376c4955 100644 --- a/src/where.c +++ b/src/where.c @@ -1751,7 +1751,8 @@ static sqlite3_index_info *allocateIndexInfo( assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); testcase( pTerm->eOperator & WO_IN ); testcase( pTerm->eOperator & WO_ISNULL ); - if( pTerm->eOperator & (WO_ISNULL) ) continue; + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; nTerm++; } @@ -1803,7 +1804,8 @@ static sqlite3_index_info *allocateIndexInfo( assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); testcase( pTerm->eOperator & WO_IN ); testcase( pTerm->eOperator & WO_ISNULL ); - if( pTerm->eOperator & (WO_ISNULL) ) continue; + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV))==0 ) continue; if( pTerm->wtFlags & TERM_VNULL ) continue; pIdxCons[j].iColumn = pTerm->u.leftColumn; pIdxCons[j].iTermOffset = i; diff --git a/test/vtab1.test b/test/vtab1.test index 3eb37357d9..0542ee6fdd 100644 --- a/test/vtab1.test +++ b/test/vtab1.test @@ -1376,4 +1376,23 @@ do_execsql_test 20.4 { ORDER BY 1, 2; } {5 5 6 6 11 11 12 12} +#------------------------------------------------------------------------- +# +do_execsql_test 21.1 { + CREATE TABLE t9(a,b,c); + CREATE VIRTUAL TABLE t9v USING echo(t9); + + INSERT INTO t9 VALUES(1,2,3); + INSERT INTO t9 VALUES(3,2,1); + INSERT INTO t9 VALUES(2,2,2); +} + +do_execsql_test 21.2 { + SELECT * FROM t9v WHERE a Date: Tue, 21 Jan 2014 15:04:47 +0000 Subject: [PATCH 63/74] Remove the undocumented requirement for applications that use an SQLITE_ENABLE_SQLLOG build to define a sqlite3_init_sqllog() function. FossilOrigin-Name: 5e43bf013253921e4dfbe71de11ee7ed4b3e7eae --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/main.c | 7 ------- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 7c7eeb389c..9793a62dd6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\swhere.c,\sdo\snot\sallocate\sspace\sin\ssqlite3_index_info\sstructures\sfor\sthe\sinternal\sWHERE\sclause\s"terms"\sgenerated\sto\srecord\scolumn\sequivalencies. -D 2014-01-20T19:55:33.120 +C Remove\sthe\sundocumented\srequirement\sfor\sapplications\sthat\suse\san\sSQLITE_ENABLE_SQLLOG\sbuild\sto\sdefine\sa\ssqlite3_init_sqllog()\sfunction. +D 2014-01-21T15:04:47.807 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -188,7 +188,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c 37f55de2000f6c882414ad3e1bcdb751c531fddd +F src/main.c 4a05a9706579c7649d7eebb0094586728eb53fcb F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -1152,7 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eba8a564e62f84a9620008beead80081fe90a1b7 -R c774305f58a7e7fff71d714265811a28 +P 7d9e22187daaa3160b875a1df17b924969bf718e +R 72db5d0300da6235ddfc19ebd0af857f U dan -Z 8e05aa582afc8018a3754b6d40d97300 +Z 21763f37f81635b2fe9a520f9fd3a652 diff --git a/manifest.uuid b/manifest.uuid index 1199189ccc..89ddacefff 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7d9e22187daaa3160b875a1df17b924969bf718e \ No newline at end of file +5e43bf013253921e4dfbe71de11ee7ed4b3e7eae \ No newline at end of file diff --git a/src/main.c b/src/main.c index 13ad2b9081..98579105c7 100644 --- a/src/main.c +++ b/src/main.c @@ -135,13 +135,6 @@ int sqlite3_initialize(void){ */ if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; -#ifdef SQLITE_ENABLE_SQLLOG - { - extern void sqlite3_init_sqllog(void); - sqlite3_init_sqllog(); - } -#endif - /* Make sure the mutex subsystem is initialized. If unable to ** initialize the mutex subsystem, return early with the error. ** If the system is so sick that we are unable to allocate a mutex, From e73f059093bbf6710688d90bc854358be27bade8 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 21 Jan 2014 22:25:45 +0000 Subject: [PATCH 64/74] Change the recursive common table expression algorithm to use a queue instead of a pair of tables. Runs about 25% faster on the sudoku solver query. The OP_SwapCursors opcode is no longer required. The current implementation uses just a fifo, but the plan is to change it into a queue that will support ORDER BY and LIMIT in a recursive query. FossilOrigin-Name: b2671e1133d2f1fbd36e7cd4b86d6cc7b528aa97 --- manifest | 23 +++++++------ manifest.uuid | 2 +- src/select.c | 91 +++++++++++++++++++++++++++++---------------------- src/shell.c | 6 ++-- src/vdbe.c | 28 ---------------- src/where.c | 12 ++++--- 6 files changed, 76 insertions(+), 86 deletions(-) diff --git a/manifest b/manifest index 9793a62dd6..955bc3f8a9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sundocumented\srequirement\sfor\sapplications\sthat\suse\san\sSQLITE_ENABLE_SQLLOG\sbuild\sto\sdefine\sa\ssqlite3_init_sqllog()\sfunction. -D 2014-01-21T15:04:47.807 +C Change\sthe\srecursive\scommon\stable\sexpression\salgorithm\sto\suse\sa\squeue\sinstead\nof\sa\spair\sof\stables.\s\sRuns\sabout\s25%\sfaster\son\sthe\ssudoku\ssolver\squery.\s\nThe\sOP_SwapCursors\sopcode\sis\sno\slonger\srequired.\s\sThe\scurrent\simplementation\nuses\sjust\sa\sfifo,\sbut\sthe\splan\sis\sto\schange\sit\sinto\sa\squeue\sthat\swill\ssupport\s\nORDER\sBY\sand\sLIMIT\sin\sa\srecursive\squery. +D 2014-01-21T22:25:45.721 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,8 +219,8 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c a27ac21844df3123b7c1e89d79cd7034d4eb0e8e -F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48 +F src/select.c f7b1558aae71d4f6ff48ad91122185d065730ba6 +F src/shell.c 24722d24d4ea8ca93db35e44db7308de786767ca F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -280,7 +280,7 @@ F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c e71f19b272f05c8695cf747b4bac1732685f9e5c F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 98d96d04d9a2bef78ca850be1053dc91d031338a +F src/vdbe.c dede894c2990329f8bc5a70da7de44ce8c3c6bf5 F src/vdbe.h e6c4c610fcabad4fa80ebb1efc6822a9367e2b26 F src/vdbeInt.h 42db251e9f863401ff847b90d5fe1614c89a6a56 F src/vdbeapi.c ce4e68ea4842cc6081046f533d088dcf01d247ad @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 56f85486bc8d0cb57fc15e5db2a58d1dfa1114cf +F src/where.c d908f4e9e45b567e87a890959ebef01187fab46f F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1152,7 +1152,10 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7d9e22187daaa3160b875a1df17b924969bf718e -R 72db5d0300da6235ddfc19ebd0af857f -U dan -Z 21763f37f81635b2fe9a520f9fd3a652 +P 5e43bf013253921e4dfbe71de11ee7ed4b3e7eae +R ead1532b7d5d88209851a3d527991f22 +T *branch * cte-via-queue +T *sym-cte-via-queue * +T -sym-trunk * +U drh +Z 386bc5602f3047b62d7a78b288244cbd diff --git a/manifest.uuid b/manifest.uuid index 89ddacefff..76656dfd03 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5e43bf013253921e4dfbe71de11ee7ed4b3e7eae \ No newline at end of file +b2671e1133d2f1fbd36e7cd4b86d6cc7b528aa97 \ No newline at end of file diff --git a/src/select.c b/src/select.c index dd58ed22e6..e6dd0e7a0a 100644 --- a/src/select.c +++ b/src/select.c @@ -1792,20 +1792,21 @@ static int multiSelect( #ifndef SQLITE_OMIT_CTE if( p->selFlags & SF_Recursive ){ - SrcList *pSrc = p->pSrc; - int nCol = p->pEList->nExpr; - int addrNext; - int addrSwap; - int iCont, iBreak; - int tmp1; /* Intermediate table */ - int tmp2; /* Next intermediate table */ - int tmp3 = 0; /* To ensure unique results if UNION */ - int eDest = SRT_Table; - SelectDest tmp2dest; - int i; + SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */ + int nCol = p->pEList->nExpr; /* Number of columns in the CTE */ + int addrTop; /* Top of the loop */ + int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ + int iCurrent; /* The Current table */ + int regCurrent; /* Register holding Current table */ + int iQueue; /* The Queue table */ + int iDistinct; /* To ensure unique results if UNION */ + int eDest; /* How to write to Queue */ + SelectDest destQueue; /* SelectDest targetting the Queue table */ + int i; /* Loop counter */ /* Check that there is no ORDER BY or LIMIT clause. Neither of these - ** are supported on recursive queries. */ + ** are currently supported on recursive queries. + */ assert( p->pOffset==0 || p->pLimit ); if( p->pOrderBy || p->pLimit ){ sqlite3ErrorMsg(pParse, "%s in a recursive query", @@ -1817,56 +1818,66 @@ static int multiSelect( if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ){ goto multi_select_end; } - iBreak = sqlite3VdbeMakeLabel(v); - iCont = sqlite3VdbeMakeLabel(v); + addrBreak = sqlite3VdbeMakeLabel(v); + addrCont = sqlite3VdbeMakeLabel(v); + /* Locate the cursor number of the Current table */ for(i=0; ALWAYS(inSrc); i++){ if( pSrc->a[i].isRecursive ){ - tmp1 = pSrc->a[i].iCursor; + iCurrent = pSrc->a[i].iCursor; break; } } - tmp2 = pParse->nTab++; + /* Allocate cursors for Queue and Distinct. The cursor number for + ** the Distinct table must be exactly one greater than Queue in order + ** for the SRT_DistTable destination to work. */ + iQueue = pParse->nTab++; if( p->op==TK_UNION ){ eDest = SRT_DistTable; - tmp3 = pParse->nTab++; + iDistinct = pParse->nTab++; + }else{ + eDest = SRT_Table; + iDistinct = 0; } - sqlite3SelectDestInit(&tmp2dest, eDest, tmp2); + sqlite3SelectDestInit(&destQueue, eDest, iQueue); - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp1, nCol); - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp2, nCol); - if( tmp3 ){ - p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp3, 0); + /* Allocate cursors for Current, Queue, and iDistinct. */ + regCurrent = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol); + if( iDistinct ){ + p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); p->selFlags |= SF_UsesEphemeral; } - /* Store the results of the initial SELECT in tmp2. */ - rc = sqlite3Select(pParse, pPrior, &tmp2dest); + /* Store the results of the initial SELECT in Queue. */ + rc = sqlite3Select(pParse, pPrior, &destQueue); if( rc ) goto multi_select_end; - /* Clear tmp1. Then switch the contents of tmp1 and tmp2. Then return - ** the contents of tmp1 to the caller. Or, if tmp1 is empty at this - ** point, the recursive query has finished - jump to address iBreak. */ - addrSwap = sqlite3VdbeAddOp2(v, OP_SwapCursors, tmp1, tmp2); - sqlite3VdbeAddOp2(v, OP_Rewind, tmp1, iBreak); - addrNext = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, p->pEList, tmp1, p->pEList->nExpr, - 0, 0, &dest, iCont, iBreak); - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, tmp1, addrNext); + /* Find the next row in the Queue and output that row */ + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); + selectInnerLoop(pParse, p, p->pEList, iQueue, p->pEList->nExpr, + 0, 0, &dest, addrCont, addrBreak); + sqlite3VdbeResolveLabel(v, addrCont); - /* Execute the recursive SELECT. Store the results in tmp2. While this - ** SELECT is running, the contents of tmp1 are read by recursive - ** references to the current CTE. */ + /* Transfer the next row in Queue over to Current */ + sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */ + sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent); + sqlite3VdbeAddOp1(v, OP_Delete, iQueue); + + /* Execute the recursive SELECT taking the single row in Current as + ** the value for the CTE. Store the results in the Queue. + */ p->pPrior = 0; - rc = sqlite3Select(pParse, p, &tmp2dest); + rc = sqlite3Select(pParse, p, &destQueue); assert( p->pPrior==0 ); p->pPrior = pPrior; if( rc ) goto multi_select_end; - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrSwap); - sqlite3VdbeResolveLabel(v, iBreak); + /* Keep running the loop until the Queue is empty */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeResolveLabel(v, addrBreak); }else #endif diff --git a/src/shell.c b/src/shell.c index b5ce90208c..1c4c4ad3e3 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1177,7 +1177,7 @@ static int str_in_array(const char *zStr, const char **azArray){ ** ** * For each "Goto", if the jump destination is earlier in the program ** and ends on one of: -** Yield SeekGt SeekLt RowSetRead +** Yield SeekGt SeekLt RowSetRead Rewind ** then indent all opcodes between the earlier instruction ** and "Goto" by 2 spaces. */ @@ -1189,7 +1189,7 @@ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ int iOp; /* Index of operation in p->aiIndent[] */ const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", 0 }; - const char *azYield[] = { "Yield", "SeekLt", "SeekGt", "RowSetRead", 0 }; + const char *azYield[] = { "Yield", "SeekLt", "SeekGt", "RowSetRead", "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; /* Try to figure out if this is really an EXPLAIN statement. If this @@ -1226,7 +1226,7 @@ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ for(i=p2op; iaiIndent[i] += 2; } if( str_in_array(zOp, azGoto) && p2opnIndent && abYield[p2op] ){ - for(i=p2op; iaiIndent[i] += 2; + for(i=p2op+1; iaiIndent[i] += 2; } } diff --git a/src/vdbe.c b/src/vdbe.c index 86aae3c65d..10a70750e3 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -3369,33 +3369,6 @@ case OP_OpenEphemeral: { break; } -#ifndef SQLITE_OMIT_CTE -/* Opcode: SwapCursors P1 P2 * * * -** -** Parameters P1 and P2 are both cursors opened by the OpenEphemeral -** opcode. This opcode deletes the contents of epheremal table P1, -** then renames P2 to P1 and P1 to P2. In other words, following this -** opcode cursor P2 is open on an empty table and P1 is open on the -** table that was initially accessed by P2. -*/ -case OP_SwapCursors: { - Mem tmp; - VdbeCursor *pTmp; - - tmp = p->aMem[p->nMem - pOp->p1]; - p->aMem[p->nMem - pOp->p1] = p->aMem[p->nMem - pOp->p2]; - p->aMem[p->nMem - pOp->p2] = tmp; - - pTmp = p->apCsr[pOp->p1]; - p->apCsr[pOp->p1] = p->apCsr[pOp->p2]; - p->apCsr[pOp->p2] = pTmp; - - assert( pTmp->isTable ); - rc = sqlite3BtreeClearTable(pTmp->pBt, MASTER_ROOT, 0); - break; -} -#endif /* ifndef SQLITE_OMIT_CTE */ - /* Opcode: SorterOpen P1 * * P4 * ** ** This opcode works like OP_OpenEphemeral except that it opens @@ -4393,7 +4366,6 @@ case OP_NullRow: { pC->nullRow = 1; pC->rowidIsValid = 0; pC->cacheStatus = CACHE_STALE; - assert( pC->pCursor || pC->pVtabCursor ); if( pC->pCursor ){ sqlite3BtreeClearCursor(pC->pCursor); } diff --git a/src/where.c b/src/where.c index 86376c4955..f2d72760d3 100644 --- a/src/where.c +++ b/src/where.c @@ -3410,10 +3410,14 @@ static Bitmask codeOneLoopStart( static const u8 aStep[] = { OP_Next, OP_Prev }; static const u8 aStart[] = { OP_Rewind, OP_Last }; assert( bRev==0 || bRev==1 ); - pLevel->op = aStep[bRev]; - pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); - pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + if( pTabItem->isRecursive ){ + pLevel->op = OP_Noop; + }else{ + pLevel->op = aStep[bRev]; + pLevel->p1 = iCur; + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; + } } /* Insert code to test every subexpression that can be completely From 340309fd69aecc620dcd23a396a0f75d1bd5da0b Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Jan 2014 00:23:49 +0000 Subject: [PATCH 65/74] Remove an unnecessary parameter from selectInnerLoop(). Clean up comments. FossilOrigin-Name: 5e6c4a55f6df30da9dbaa8170f3223613cc86f65 --- manifest | 19 ++++----- manifest.uuid | 2 +- src/select.c | 110 +++++++++++++----------------------------------- src/sqliteInt.h | 54 +++++++++++++++++++++++- src/where.c | 2 + 5 files changed, 93 insertions(+), 94 deletions(-) diff --git a/manifest b/manifest index 955bc3f8a9..744daa026d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\srecursive\scommon\stable\sexpression\salgorithm\sto\suse\sa\squeue\sinstead\nof\sa\spair\sof\stables.\s\sRuns\sabout\s25%\sfaster\son\sthe\ssudoku\ssolver\squery.\s\nThe\sOP_SwapCursors\sopcode\sis\sno\slonger\srequired.\s\sThe\scurrent\simplementation\nuses\sjust\sa\sfifo,\sbut\sthe\splan\sis\sto\schange\sit\sinto\sa\squeue\sthat\swill\ssupport\s\nORDER\sBY\sand\sLIMIT\sin\sa\srecursive\squery. -D 2014-01-21T22:25:45.721 +C Remove\san\sunnecessary\sparameter\sfrom\sselectInnerLoop().\s\sClean\sup\scomments. +D 2014-01-22T00:23:49.966 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c f7b1558aae71d4f6ff48ad91122185d065730ba6 +F src/select.c c2021c75911362ac2fa4d9390741c889101bca65 F src/shell.c 24722d24d4ea8ca93db35e44db7308de786767ca F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 99fd628541e420b98fc52072635e8ba431706250 +F src/sqliteInt.h 73074ee596a978bc4431746c417527d03a3bb510 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c d908f4e9e45b567e87a890959ebef01187fab46f +F src/where.c 943bb821042c8775672f2d0021d963b7243e53a9 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1152,10 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5e43bf013253921e4dfbe71de11ee7ed4b3e7eae -R ead1532b7d5d88209851a3d527991f22 -T *branch * cte-via-queue -T *sym-cte-via-queue * -T -sym-trunk * +P b2671e1133d2f1fbd36e7cd4b86d6cc7b528aa97 +R 8aa16044970c1d470244a1583e8ab6b5 U drh -Z 386bc5602f3047b62d7a78b288244cbd +Z 0567013342859692eded4e80186919d9 diff --git a/manifest.uuid b/manifest.uuid index 76656dfd03..06282ce842 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b2671e1133d2f1fbd36e7cd4b86d6cc7b528aa97 \ No newline at end of file +5e6c4a55f6df30da9dbaa8170f3223613cc86f65 \ No newline at end of file diff --git a/src/select.c b/src/select.c index e6dd0e7a0a..a6eee2735d 100644 --- a/src/select.c +++ b/src/select.c @@ -543,17 +543,16 @@ struct DistinctCtx { ** This routine generates the code for the inside of the inner loop ** of a SELECT. ** -** If srcTab and nColumn are both zero, then the pEList expressions -** are evaluated in order to get the data for this row. If nColumn>0 -** then data is pulled from srcTab and pEList is used only to get the -** datatypes for each column. +** If srcTab is negative, then the pEList expressions +** are evaluated in order to get the data for this row. If srcTab is +** zero or more, then data is pulled from srcTab and pEList is used only +** to get number columns and the datatype for each column. */ static void selectInnerLoop( Parse *pParse, /* The parser context */ Select *p, /* The complete select statement being coded */ ExprList *pEList, /* List of values being extracted */ int srcTab, /* Pull data from this table */ - int nColumn, /* Number of columns in the source table */ ExprList *pOrderBy, /* If not NULL, sort results using this key */ DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ @@ -569,7 +568,6 @@ static void selectInnerLoop( int nResultCol; /* Number of result columns */ assert( v ); - if( NEVER(v==0) ) return; assert( pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; if( pOrderBy==0 && !hasDistinct ){ @@ -578,11 +576,7 @@ static void selectInnerLoop( /* Pull the requested columns. */ - if( nColumn>0 ){ - nResultCol = nColumn; - }else{ - nResultCol = pEList->nExpr; - } + nResultCol = pEList->nExpr; if( pDest->iSdst==0 ){ pDest->iSdst = pParse->nMem+1; pDest->nSdst = nResultCol; @@ -591,9 +585,10 @@ static void selectInnerLoop( assert( pDest->nSdst==nResultCol ); } regResult = pDest->iSdst; - if( nColumn>0 ){ - for(i=0; i=0 ){ + for(i=0; ia[i].zName)); } }else if( eDest!=SRT_Exists ){ /* If the destination is an EXISTS(...) expression, the actual @@ -602,15 +597,12 @@ static void selectInnerLoop( sqlite3ExprCodeExprList(pParse, pEList, regResult, (eDest==SRT_Output)?SQLITE_ECEL_DUP:0); } - nColumn = nResultCol; /* If the DISTINCT keyword was present on the SELECT statement ** and this row has been seen before, then do not make this row ** part of the result. */ if( hasDistinct ){ - assert( pEList!=0 ); - assert( pEList->nExpr==nColumn ); switch( pDistinct->eTnctType ){ case WHERE_DISTINCT_ORDERED: { VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ @@ -619,7 +611,7 @@ static void selectInnerLoop( /* Allocate space for the previous row */ regPrev = pParse->nMem+1; - pParse->nMem += nColumn; + pParse->nMem += nResultCol; /* Change the OP_OpenEphemeral coded earlier to an OP_Null ** sets the MEM_Cleared bit on the first register of the @@ -633,10 +625,10 @@ static void selectInnerLoop( pOp->p1 = 1; pOp->p2 = regPrev; - iJump = sqlite3VdbeCurrentAddr(v) + nColumn; - for(i=0; ia[i].pExpr); - if( ieTnctType==WHERE_DISTINCT_UNORDERED ); - codeDistinct(pParse, pDistinct->tabTnct, iContinue, nColumn, regResult); + codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, regResult); break; } } @@ -673,7 +665,7 @@ static void selectInnerLoop( case SRT_Union: { int r1; r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); break; @@ -684,7 +676,7 @@ static void selectInnerLoop( ** the temporary table iParm. */ case SRT_Except: { - sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nColumn); + sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); break; } #endif @@ -697,7 +689,7 @@ static void selectInnerLoop( int r1 = sqlite3GetTempReg(pParse); testcase( eDest==SRT_Table ); testcase( eDest==SRT_EphemTab ); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistTable ){ /* If the destination is DistTable, then cursor (iParm+1) is open @@ -730,7 +722,7 @@ static void selectInnerLoop( ** item into the set table with bogus data. */ case SRT_Set: { - assert( nColumn==1 ); + assert( nResultCol==1 ); pDest->affSdst = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pOrderBy ){ @@ -762,7 +754,7 @@ static void selectInnerLoop( ** of the scan loop. */ case SRT_Mem: { - assert( nColumn==1 ); + assert( nResultCol==1 ); if( pOrderBy ){ pushOntoSorter(pParse, pOrderBy, p, regResult); }else{ @@ -783,14 +775,14 @@ static void selectInnerLoop( testcase( eDest==SRT_Output ); if( pOrderBy ){ int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); pushOntoSorter(pParse, pOrderBy, p, r1); sqlite3ReleaseTempReg(pParse, r1); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ - sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nColumn); - sqlite3ExprCacheAffinityChange(pParse, regResult, nColumn); + sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol); + sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); } break; } @@ -1857,7 +1849,7 @@ static int multiSelect( /* Find the next row in the Queue and output that row */ addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); - selectInnerLoop(pParse, p, p->pEList, iQueue, p->pEList->nExpr, + selectInnerLoop(pParse, p, p->pEList, iQueue, 0, 0, &dest, addrCont, addrBreak); sqlite3VdbeResolveLabel(v, addrCont); @@ -2020,7 +2012,7 @@ static int multiSelect( computeLimitRegisters(pParse, p, iBreak); sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); iStart = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, p->pEList, unionTab, p->pEList->nExpr, + selectInnerLoop(pParse, p, p->pEList, unionTab, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); @@ -2098,7 +2090,7 @@ static int multiSelect( iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); sqlite3ReleaseTempReg(pParse, r1); - selectInnerLoop(pParse, p, p->pEList, tab1, p->pEList->nExpr, + selectInnerLoop(pParse, p, p->pEList, tab1, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); @@ -4299,50 +4291,8 @@ static void explainSimpleCount( /* ** Generate code for the SELECT statement given in the p argument. ** -** The results are distributed in various ways depending on the -** contents of the SelectDest structure pointed to by argument pDest -** as follows: -** -** pDest->eDest Result -** ------------ ------------------------------------------- -** SRT_Output Generate a row of output (using the OP_ResultRow -** opcode) for each row in the result set. -** -** SRT_Mem Only valid if the result is a single column. -** Store the first column of the first result row -** in register pDest->iSDParm then abandon the rest -** of the query. This destination implies "LIMIT 1". -** -** SRT_Set The result must be a single column. Store each -** row of result as the key in table pDest->iSDParm. -** Apply the affinity pDest->affSdst before storing -** results. Used to implement "IN (SELECT ...)". -** -** SRT_Union Store results as a key in a temporary table -** identified by pDest->iSDParm. -** -** SRT_Except Remove results from the temporary table pDest->iSDParm. -** -** SRT_Table Store results in temporary table pDest->iSDParm. -** This is like SRT_EphemTab except that the table -** is assumed to already be open. -** -** SRT_EphemTab Create an temporary table pDest->iSDParm and store -** the result there. The cursor is left open after -** returning. This is like SRT_Table except that -** this destination uses OP_OpenEphemeral to create -** the table first. -** -** SRT_Coroutine Generate a co-routine that returns a new row of -** results each time it is invoked. The entry point -** of the co-routine is stored in register pDest->iSDParm. -** -** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result -** set is not empty. -** -** SRT_Discard Throw the results away. This is used by SELECT -** statements within triggers whose only purpose is -** the side-effects of functions. +** The results are returned according to the SelectDest structure. +** See comments in sqliteInt.h for further information. ** ** This routine returns the number of errors. If any errors are ** encountered, then an appropriate error message is left in @@ -4683,7 +4633,7 @@ int sqlite3Select( } /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, 0, 0, pOrderBy, &sDistinct, pDest, + selectInnerLoop(pParse, p, pEList, -1, pOrderBy, &sDistinct, pDest, sqlite3WhereContinueLabel(pWInfo), sqlite3WhereBreakLabel(pWInfo)); @@ -4955,7 +4905,7 @@ int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, pOrderBy, + selectInnerLoop(pParse, p, p->pEList, -1, pOrderBy, &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); @@ -5098,7 +5048,7 @@ int sqlite3Select( pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, 0, 0, 0, 0, + selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, pDest, addrEnd, addrEnd); sqlite3ExprListDelete(db, pDel); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 317e9eb227..f2c08ad296 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2167,8 +2167,58 @@ struct Select { /* -** The results of a select can be distributed in several ways. The -** "SRT" prefix means "SELECT Result Type". +** The results of a SELECT can be distributed in several ways, as defined +** by one of the following macros. The "SRT" prefix means "SELECT Result +** Type". +** +** SRT_Union Store results as a key in a temporary index +** identified by pDest->iSDParm. +** +** SRT_Except Remove results from the temporary index pDest->iSDParm. +** +** SRT_Exists Store a 1 in memory cell pDest->iSDParm if the result +** set is not empty. +** +** SRT_Discard Throw the results away. This is used by SELECT +** statements within triggers whose only purpose is +** the side-effects of functions. +** +** All of the above are free to ignore their ORDER BY clause. Those that +** follow must honor the ORDER BY clause. +** +** SRT_Output Generate a row of output (using the OP_ResultRow +** opcode) for each row in the result set. +** +** SRT_Mem Only valid if the result is a single column. +** Store the first column of the first result row +** in register pDest->iSDParm then abandon the rest +** of the query. This destination implies "LIMIT 1". +** +** SRT_Set The result must be a single column. Store each +** row of result as the key in table pDest->iSDParm. +** Apply the affinity pDest->affSdst before storing +** results. Used to implement "IN (SELECT ...)". +** +** SRT_Table Store results in temporary table pDest->iSDParm. +** This is like SRT_EphemTab except that the table +** is assumed to already be open. +** +** SRT_EphemTab Create an temporary table pDest->iSDParm and store +** the result there. The cursor is left open after +** returning. This is like SRT_Table except that +** this destination uses OP_OpenEphemeral to create +** the table first. +** +** SRT_Coroutine Generate a co-routine that returns a new row of +** results each time it is invoked. The entry point +** of the co-routine is stored in register pDest->iSDParm +** and the result row is stored in pDest->nDest registers +** starting with pDest->iSdst. +** +** SRT_DistTable Store results in a temporary table pDest->iSDParm. +** But also use temporary table pDest->iSDParm+1 as +** a record of all prior results and ignore any duplicate +** rows. Name means: "Distinct Table". */ #define SRT_Union 1 /* Store result as keys in an index */ #define SRT_Except 2 /* Remove result from a UNION index */ diff --git a/src/where.c b/src/where.c index f2d72760d3..51223aaa93 100644 --- a/src/where.c +++ b/src/where.c @@ -3411,6 +3411,8 @@ static Bitmask codeOneLoopStart( static const u8 aStart[] = { OP_Rewind, OP_Last }; assert( bRev==0 || bRev==1 ); if( pTabItem->isRecursive ){ + /* Tables marked isRecursive have only a single row that is stored in + ** a pseudo-cursor. Need need to Rewind or Next such cursors. */ pLevel->op = OP_Noop; }else{ pLevel->op = aStep[bRev]; From 41028151beebfc5dcb1e936f797a141abd7ab11c Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 22 Jan 2014 10:22:25 +0000 Subject: [PATCH 66/74] Fix a typo in a comment. No changes to code or tests. FossilOrigin-Name: cceacc0e79c4e54682daddf2056c6bb8e88d9484 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 744daa026d..bd186f12e4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\san\sunnecessary\sparameter\sfrom\sselectInnerLoop().\s\sClean\sup\scomments. -D 2014-01-22T00:23:49.966 +C Fix\sa\stypo\sin\sa\scomment.\sNo\schanges\sto\scode\sor\stests. +D 2014-01-22T10:22:25.214 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -293,7 +293,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 943bb821042c8775672f2d0021d963b7243e53a9 +F src/where.c 67ae3b5e97ecff36c70cb61ccc7d74cf228f1596 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1152,7 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b2671e1133d2f1fbd36e7cd4b86d6cc7b528aa97 -R 8aa16044970c1d470244a1583e8ab6b5 -U drh -Z 0567013342859692eded4e80186919d9 +P 5e6c4a55f6df30da9dbaa8170f3223613cc86f65 +R d64224deb212538f78a2f080c518a12f +U dan +Z 6bf281c516c82d71cce682fb71803759 diff --git a/manifest.uuid b/manifest.uuid index 06282ce842..bcd373500c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5e6c4a55f6df30da9dbaa8170f3223613cc86f65 \ No newline at end of file +cceacc0e79c4e54682daddf2056c6bb8e88d9484 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 51223aaa93..6fac5e5ed1 100644 --- a/src/where.c +++ b/src/where.c @@ -3412,7 +3412,7 @@ static Bitmask codeOneLoopStart( assert( bRev==0 || bRev==1 ); if( pTabItem->isRecursive ){ /* Tables marked isRecursive have only a single row that is stored in - ** a pseudo-cursor. Need need to Rewind or Next such cursors. */ + ** a pseudo-cursor. No need to Rewind or Next such cursors. */ pLevel->op = OP_Noop; }else{ pLevel->op = aStep[bRev]; From 781def29c7e5fec282f4b69d94a4508b93e4891f Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Jan 2014 13:35:53 +0000 Subject: [PATCH 67/74] Add new SelectDest codes, SRT_Queue and SRT_DistQueue in anticipation of adding ORDER BY support on recursive queries. Factor out the recursive query code generator into a separate procedure. FossilOrigin-Name: 3eb5f9f8d6ac1ee145cb4119087c516f66fe1456 --- manifest | 16 ++-- manifest.uuid | 2 +- src/select.c | 245 +++++++++++++++++++++++++++++++----------------- src/sqliteInt.h | 26 +++-- 4 files changed, 184 insertions(+), 105 deletions(-) diff --git a/manifest b/manifest index bd186f12e4..1f03409943 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\stypo\sin\sa\scomment.\sNo\schanges\sto\scode\sor\stests. -D 2014-01-22T10:22:25.214 +C Add\snew\sSelectDest\scodes,\sSRT_Queue\sand\sSRT_DistQueue\sin\santicipation\sof\sadding\nORDER\sBY\ssupport\son\srecursive\squeries.\s\sFactor\sout\sthe\srecursive\squery\ncode\sgenerator\sinto\sa\sseparate\sprocedure. +D 2014-01-22T13:35:53.476 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c c2021c75911362ac2fa4d9390741c889101bca65 +F src/select.c 11c02c82a6f3cb8a491452fc7474a568a48c64ef F src/shell.c 24722d24d4ea8ca93db35e44db7308de786767ca F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 73074ee596a978bc4431746c417527d03a3bb510 +F src/sqliteInt.h 16c73326604e5603c3c10c97ac1e63473590d4ff F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1152,7 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 5e6c4a55f6df30da9dbaa8170f3223613cc86f65 -R d64224deb212538f78a2f080c518a12f -U dan -Z 6bf281c516c82d71cce682fb71803759 +P cceacc0e79c4e54682daddf2056c6bb8e88d9484 +R 3e53ba84b5d9e416a6e5f351b0fd4313 +U drh +Z aa06559c4dc503939fd80ca5aa152df9 diff --git a/manifest.uuid b/manifest.uuid index bcd373500c..2b7addbf9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cceacc0e79c4e54682daddf2056c6bb8e88d9484 \ No newline at end of file +3eb5f9f8d6ac1ee145cb4119087c516f66fe1456 \ No newline at end of file diff --git a/src/select.c b/src/select.c index a6eee2735d..0152db8b25 100644 --- a/src/select.c +++ b/src/select.c @@ -581,8 +581,10 @@ static void selectInnerLoop( pDest->iSdst = pParse->nMem+1; pDest->nSdst = nResultCol; pParse->nMem += nResultCol; + if( eDest==SRT_Queue ) pParse->nMem++; }else{ assert( pDest->nSdst==nResultCol ); + assert( eDest!=SRT_Queue ); } regResult = pDest->iSdst; if( srcTab>=0 ){ @@ -662,10 +664,30 @@ static void selectInnerLoop( ** table iParm. */ #ifndef SQLITE_OMIT_COMPOUND_SELECT +#ifndef SQLITE_OMIT_CTE + case SRT_Queue: { + sqlite3VdbeAddOp2(v, OP_Sequence, iParm, regResult+nResultCol); + nResultCol++; + /* Fall through into SRT_Union */ + } + case SRT_DistQueue: +#endif /* SQLITE_OMIT_CTE */ case SRT_Union: { int r1; r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); +#ifndef SQLITE_OMIT_CTE + if( eDest==SRT_DistQueue ){ + /* If the destination is DistQueue, then cursor (iParm+1) is open + ** on a second ephemeral index that holds all values every previously + ** added to the queue. Only add this new value if it has never before + ** been added */ + int addr = sqlite3VdbeCurrentAddr(v) + 3; + sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); + assert( pOrderBy==0 ); + } +#endif /* SQLITE_OMIT_CTE */ sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); break; @@ -679,7 +701,7 @@ static void selectInnerLoop( sqlite3VdbeAddOp3(v, OP_IdxDelete, iParm, regResult, nResultCol); break; } -#endif +#endif /* SQLITE_OMIT_COMPOUND_SELECT */ /* Store the result as data using a unique key. */ @@ -1676,7 +1698,139 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ } #endif /* SQLITE_OMIT_COMPOUND_SELECT */ -/* Forward reference */ +#ifndef SQLITE_OMIT_CTE +/* +** This routine generates VDBE code to compute the content of a WITH RECURSIVE +** query of the form: +** +** AS ( UNION [ALL] ) +** \___________/ \_______________/ +** p->pPrior p +** +** +** There is exactly one reference to the recursive-table in the FROM clause +** of recursive-query, marked with the SrcList->a[].isRecursive flag. +** +** The setup-query runs once to generate an initial set of rows that go +** into a Queue table. Rows are extracted from the Queue table one by +** one. Each row extracted from iQueue is output to pDest. Then the single +** extracted row (now the iCurrent table) becomes the content of the +** recursive-table and recursive-query is run. The output of the recursive-query +** is added back into the Queue table. Then another row is extracted from Queue +** and the iteration continues until the Queue table is empty. +** +** If the compound query operator is UNION then no duplicate rows are ever +** inserted into the Queue table. The iDistinct table keeps a copy of all rows +** that have ever been inserted into Queue and causes duplicates to be +** discarded. If the operator is UNION ALL, then duplicates are allowed. +** +** If the query has an ORDER BY, then entries in the Queue table are kept in +** ORDER BY order and the first entry is extracted for each cycle. Without +** an ORDER BY, the Queue table is just a FIFO. +** +** If a LIMIT clause is provided, then the iteration stops after LIMIT rows +** have been output to pDest. A LIMIT of zero means to output no rows and a +** negative LIMIT means to output all rows. If there is also an OFFSET clause +** with a positive value, then the first OFFSET outputs are discarded rather +** than being sent to pDest. The LIMIT count does not begin until after OFFSET +** rows have been skipped. +*/ +static void generateWithRecursiveQuery( + Parse *pParse, /* Parsing context */ + Select *p, /* The recursive SELECT to be coded */ + SelectDest *pDest /* What to do with query results */ +){ + SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */ + int nCol = p->pEList->nExpr; /* Number of columns in the recursive table */ + Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + Select *pSetup = p->pPrior; /* The setup query */ + int addrTop; /* Top of the loop */ + int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ + int iCurrent; /* The Current table */ + int regCurrent; /* Register holding Current table */ + int iQueue; /* The Queue table */ + int iDistinct = 0; /* To ensure unique results if UNION */ + int eDest = SRT_Table; /* How to write to Queue */ + SelectDest destQueue; /* SelectDest targetting the Queue table */ + int i; /* Loop counter */ + int rc; /* Result code */ + + /* Obtain authorization to do a recursive query */ + if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return; + addrBreak = sqlite3VdbeMakeLabel(v); + addrCont = sqlite3VdbeMakeLabel(v); + + + /* Check that there is no ORDER BY or LIMIT clause. Neither of these + ** are currently supported on recursive queries. + */ + assert( p->pOffset==0 || p->pLimit ); + if( p->pOrderBy || p->pLimit ){ + sqlite3ErrorMsg(pParse, "%s in a recursive query", + p->pOrderBy ? "ORDER BY" : "LIMIT" + ); + return; + } + + /* Locate the cursor number of the Current table */ + for(i=0; ALWAYS(inSrc); i++){ + if( pSrc->a[i].isRecursive ){ + iCurrent = pSrc->a[i].iCursor; + break; + } + } + + /* Allocate cursors for Queue and Distinct. The cursor number for + ** the Distinct table must be exactly one greater than Queue in order + ** for the SRT_DistTable destination to work. */ + iQueue = pParse->nTab++; + if( p->op==TK_UNION ){ + assert( SRT_Table+1==SRT_DistTable ); + assert( SRT_Queue+1==SRT_DistQueue ); + eDest++; + iDistinct = pParse->nTab++; + } + sqlite3SelectDestInit(&destQueue, eDest, iQueue); + + /* Allocate cursors for Current, Queue, and Distinct. */ + regCurrent = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol); + if( iDistinct ){ + p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); + p->selFlags |= SF_UsesEphemeral; + } + + /* Store the results of the setup-query in Queue. */ + rc = sqlite3Select(pParse, pSetup, &destQueue); + if( rc ) return; + + /* Find the next row in the Queue and output that row */ + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); + selectInnerLoop(pParse, p, p->pEList, iQueue, + 0, 0, pDest, addrCont, addrBreak); + sqlite3VdbeResolveLabel(v, addrCont); + + /* Transfer the next row in Queue over to Current */ + sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */ + sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent); + sqlite3VdbeAddOp1(v, OP_Delete, iQueue); + + /* Execute the recursive SELECT taking the single row in Current as + ** the value for the recursive-table. Store the results in the Queue. + */ + p->pPrior = 0; + sqlite3Select(pParse, p, &destQueue); + assert( p->pPrior==0 ); + p->pPrior = pSetup; + + /* Keep running the loop until the Queue is empty */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); + sqlite3VdbeResolveLabel(v, addrBreak); +} +#endif + +/* Forward references */ static int multiSelectOrderBy( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ @@ -1784,92 +1938,7 @@ static int multiSelect( #ifndef SQLITE_OMIT_CTE if( p->selFlags & SF_Recursive ){ - SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */ - int nCol = p->pEList->nExpr; /* Number of columns in the CTE */ - int addrTop; /* Top of the loop */ - int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ - int iCurrent; /* The Current table */ - int regCurrent; /* Register holding Current table */ - int iQueue; /* The Queue table */ - int iDistinct; /* To ensure unique results if UNION */ - int eDest; /* How to write to Queue */ - SelectDest destQueue; /* SelectDest targetting the Queue table */ - int i; /* Loop counter */ - - /* Check that there is no ORDER BY or LIMIT clause. Neither of these - ** are currently supported on recursive queries. - */ - assert( p->pOffset==0 || p->pLimit ); - if( p->pOrderBy || p->pLimit ){ - sqlite3ErrorMsg(pParse, "%s in a recursive query", - p->pOrderBy ? "ORDER BY" : "LIMIT" - ); - goto multi_select_end; - } - - if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ){ - goto multi_select_end; - } - addrBreak = sqlite3VdbeMakeLabel(v); - addrCont = sqlite3VdbeMakeLabel(v); - - /* Locate the cursor number of the Current table */ - for(i=0; ALWAYS(inSrc); i++){ - if( pSrc->a[i].isRecursive ){ - iCurrent = pSrc->a[i].iCursor; - break; - } - } - - /* Allocate cursors for Queue and Distinct. The cursor number for - ** the Distinct table must be exactly one greater than Queue in order - ** for the SRT_DistTable destination to work. */ - iQueue = pParse->nTab++; - if( p->op==TK_UNION ){ - eDest = SRT_DistTable; - iDistinct = pParse->nTab++; - }else{ - eDest = SRT_Table; - iDistinct = 0; - } - sqlite3SelectDestInit(&destQueue, eDest, iQueue); - - /* Allocate cursors for Current, Queue, and iDistinct. */ - regCurrent = ++pParse->nMem; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol); - if( iDistinct ){ - p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); - p->selFlags |= SF_UsesEphemeral; - } - - /* Store the results of the initial SELECT in Queue. */ - rc = sqlite3Select(pParse, pPrior, &destQueue); - if( rc ) goto multi_select_end; - - /* Find the next row in the Queue and output that row */ - addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); - selectInnerLoop(pParse, p, p->pEList, iQueue, - 0, 0, &dest, addrCont, addrBreak); - sqlite3VdbeResolveLabel(v, addrCont); - - /* Transfer the next row in Queue over to Current */ - sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */ - sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent); - sqlite3VdbeAddOp1(v, OP_Delete, iQueue); - - /* Execute the recursive SELECT taking the single row in Current as - ** the value for the CTE. Store the results in the Queue. - */ - p->pPrior = 0; - rc = sqlite3Select(pParse, p, &destQueue); - assert( p->pPrior==0 ); - p->pPrior = pPrior; - if( rc ) goto multi_select_end; - - /* Keep running the loop until the Queue is empty */ - sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); - sqlite3VdbeResolveLabel(v, addrBreak); + generateWithRecursiveQuery(pParse, p, &dest); }else #endif diff --git a/src/sqliteInt.h b/src/sqliteInt.h index f2c08ad296..ac48291d80 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2199,10 +2199,6 @@ struct Select { ** Apply the affinity pDest->affSdst before storing ** results. Used to implement "IN (SELECT ...)". ** -** SRT_Table Store results in temporary table pDest->iSDParm. -** This is like SRT_EphemTab except that the table -** is assumed to already be open. -** ** SRT_EphemTab Create an temporary table pDest->iSDParm and store ** the result there. The cursor is left open after ** returning. This is like SRT_Table except that @@ -2215,10 +2211,22 @@ struct Select { ** and the result row is stored in pDest->nDest registers ** starting with pDest->iSdst. ** +** SRT_Table Store results in temporary table pDest->iSDParm. +** This is like SRT_EphemTab except that the table +** is assumed to already be open. +** ** SRT_DistTable Store results in a temporary table pDest->iSDParm. ** But also use temporary table pDest->iSDParm+1 as ** a record of all prior results and ignore any duplicate ** rows. Name means: "Distinct Table". +** +** SRT_Queue Store results in priority queue pDest->iSDParm (really +** an index). Append a sequence number so that all entries +** are distinct. +** +** SRT_DistQueue Store results in priority queue pDest->iSDParm only if +** the same record has never been stored before. The +** index at pDest->iSDParm+1 hold all prior stores. */ #define SRT_Union 1 /* Store result as keys in an index */ #define SRT_Except 2 /* Remove result from a UNION index */ @@ -2231,10 +2239,12 @@ struct Select { #define SRT_Output 5 /* Output each row of result */ #define SRT_Mem 6 /* Store result in a memory cell */ #define SRT_Set 7 /* Store results as keys in an index */ -#define SRT_Table 8 /* Store result as data with an automatic rowid */ -#define SRT_EphemTab 9 /* Create transient tab and store like SRT_Table */ -#define SRT_Coroutine 10 /* Generate a single row of result */ -#define SRT_DistTable 11 /* Like SRT_TABLE, but unique results only */ +#define SRT_EphemTab 8 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 9 /* Generate a single row of result */ +#define SRT_Table 10 /* Store result as data with an automatic rowid */ +#define SRT_DistTable 11 /* Like SRT_Table, but unique results only */ +#define SRT_Queue 12 /* Store result in an queue */ +#define SRT_DistQueue 13 /* Like SRT_Queue, but unique results only */ /* ** An instance of this object describes where to put of the results of From fe1c6bb9c291e2a670236b2d33fb74b587bf8b16 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Jan 2014 17:28:35 +0000 Subject: [PATCH 68/74] Get ORDER BY working for recursive queries. FossilOrigin-Name: 37b343b01841b338954ddfa9b76d92aa50037aec --- manifest | 16 +++--- manifest.uuid | 2 +- src/select.c | 135 ++++++++++++++++++++++++++++++++---------------- src/sqliteInt.h | 11 ++-- test/with1.test | 39 +++++++++++++- 5 files changed, 143 insertions(+), 60 deletions(-) diff --git a/manifest b/manifest index 1f03409943..896f5cf947 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\snew\sSelectDest\scodes,\sSRT_Queue\sand\sSRT_DistQueue\sin\santicipation\sof\sadding\nORDER\sBY\ssupport\son\srecursive\squeries.\s\sFactor\sout\sthe\srecursive\squery\ncode\sgenerator\sinto\sa\sseparate\sprocedure. -D 2014-01-22T13:35:53.476 +C Get\sORDER\sBY\sworking\sfor\srecursive\squeries. +D 2014-01-22T17:28:35.279 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,12 +219,12 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 11c02c82a6f3cb8a491452fc7474a568a48c64ef +F src/select.c b5430b99c0339dcfe9d06a4c251548e650e386e0 F src/shell.c 24722d24d4ea8ca93db35e44db7308de786767ca F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 16c73326604e5603c3c10c97ac1e63473590d4ff +F src/sqliteInt.h 87a90ad4818ac5d68d3463eb7fe3ed96e5209b25 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1092,7 +1092,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test cec63b56797a70842afa8929c241dfdb3d864283 +F test/with1.test f4aa699a3712207e02f13945cd2b67d01a9d9f57 F test/with2.test 2fe78fcd8deef2a0f9cfc49bfc755911d0b3fd64 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 @@ -1152,7 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P cceacc0e79c4e54682daddf2056c6bb8e88d9484 -R 3e53ba84b5d9e416a6e5f351b0fd4313 +P 3eb5f9f8d6ac1ee145cb4119087c516f66fe1456 +R a615c5dcec438a7db4f9b5a3e0983f82 U drh -Z aa06559c4dc503939fd80ca5aa152df9 +Z 4ad026d26a172a85d439a21b0ddb47e4 diff --git a/manifest.uuid b/manifest.uuid index 2b7addbf9a..87e23ab961 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3eb5f9f8d6ac1ee145cb4119087c516f66fe1456 \ No newline at end of file +37b343b01841b338954ddfa9b76d92aa50037aec \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0152db8b25..13e9562ba2 100644 --- a/src/select.c +++ b/src/select.c @@ -581,10 +581,8 @@ static void selectInnerLoop( pDest->iSdst = pParse->nMem+1; pDest->nSdst = nResultCol; pParse->nMem += nResultCol; - if( eDest==SRT_Queue ) pParse->nMem++; }else{ assert( pDest->nSdst==nResultCol ); - assert( eDest!=SRT_Queue ); } regResult = pDest->iSdst; if( srcTab>=0 ){ @@ -664,30 +662,10 @@ static void selectInnerLoop( ** table iParm. */ #ifndef SQLITE_OMIT_COMPOUND_SELECT -#ifndef SQLITE_OMIT_CTE - case SRT_Queue: { - sqlite3VdbeAddOp2(v, OP_Sequence, iParm, regResult+nResultCol); - nResultCol++; - /* Fall through into SRT_Union */ - } - case SRT_DistQueue: -#endif /* SQLITE_OMIT_CTE */ case SRT_Union: { int r1; r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); -#ifndef SQLITE_OMIT_CTE - if( eDest==SRT_DistQueue ){ - /* If the destination is DistQueue, then cursor (iParm+1) is open - ** on a second ephemeral index that holds all values every previously - ** added to the queue. Only add this new value if it has never before - ** been added */ - int addr = sqlite3VdbeCurrentAddr(v) + 3; - sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); - assert( pOrderBy==0 ); - } -#endif /* SQLITE_OMIT_CTE */ sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); sqlite3ReleaseTempReg(pParse, r1); break; @@ -809,6 +787,51 @@ static void selectInnerLoop( break; } +#ifndef SQLITE_OMIT_CTE + /* Write the results into a priority queue that is order according to + ** pDest->pOrderBy (in pSO). pDest->iSDParm (in iParm) is the cursor for an + ** index with pSO->nExpr+2 columns. Build a key using pSO for the first + ** pSO->nExpr columns, then make sure all keys are unique by adding a + ** final OP_Sequence column. The last column is the record as a blob. + */ + case SRT_DistQueue: + case SRT_Queue: { + int nKey; + int r1, r2, r3; + int addrTest = 0; + ExprList *pSO; + pSO = pDest->pOrderBy; + assert( pSO ); + nKey = pSO->nExpr; + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempRange(pParse, nKey+2); + r3 = r2+nKey+1; + sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r3); + if( eDest==SRT_DistQueue ){ + /* If the destination is DistQueue, then cursor (iParm+1) is open + ** on a second ephemeral index that holds all values every previously + ** added to the queue. Only add this new value if it has never before + ** been added */ + addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, r3, 0); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); + } + for(i=0; ia[i].u.x.iOrderByCol - 1, + r2+i); + } + sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); + sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + if( addrTest ) sqlite3VdbeJumpHere(v, addrTest); + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempRange(pParse, r2, nKey+2); + break; + } +#endif /* SQLITE_OMIT_CTE */ + + + #if !defined(SQLITE_OMIT_TRIGGER) /* Discard the results. This is used for SELECT statements inside ** the body of a TRIGGER. The purpose of such selects is to call @@ -897,7 +920,7 @@ int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } ** function is responsible for seeing that this structure is eventually ** freed. */ -static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ +static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList, int nExtra){ int nExpr; KeyInfo *pInfo; struct ExprList_item *pItem; @@ -905,7 +928,7 @@ static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList){ int i; nExpr = pList->nExpr; - pInfo = sqlite3KeyInfoAlloc(db, nExpr, 1); + pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra, 1); if( pInfo ){ assert( sqlite3KeyInfoIsWriteable(pInfo) ); for(i=0, pItem=pList->a; ipOffset==0 || p->pLimit ); - if( p->pOrderBy || p->pLimit ){ + if( /*p->pOrderBy ||*/ p->pLimit ){ sqlite3ErrorMsg(pParse, "%s in a recursive query", p->pOrderBy ? "ORDER BY" : "LIMIT" ); @@ -1780,22 +1802,34 @@ static void generateWithRecursiveQuery( } } - /* Allocate cursors for Queue and Distinct. The cursor number for + /* Detach the ORDER BY clause from the compound SELECT */ + pOrderBy = p->pOrderBy; + p->pOrderBy = 0; + + /* Allocate cursors numbers for Queue and Distinct. The cursor number for ** the Distinct table must be exactly one greater than Queue in order - ** for the SRT_DistTable destination to work. */ + ** for the SRT_DistTable and SRT_DistQueue destinations to work. */ iQueue = pParse->nTab++; if( p->op==TK_UNION ){ - assert( SRT_Table+1==SRT_DistTable ); - assert( SRT_Queue+1==SRT_DistQueue ); - eDest++; + eDest = pOrderBy ? SRT_DistQueue : SRT_DistTable; iDistinct = pParse->nTab++; + }else{ + eDest = pOrderBy ? SRT_Queue : SRT_Table; } sqlite3SelectDestInit(&destQueue, eDest, iQueue); /* Allocate cursors for Current, Queue, and Distinct. */ regCurrent = ++pParse->nMem; sqlite3VdbeAddOp3(v, OP_OpenPseudo, iCurrent, regCurrent, nCol); - sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol); + if( pOrderBy ){ + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pOrderBy, 1); + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, iQueue, pOrderBy->nExpr+2, 0, + (char*)pKeyInfo, P4_KEYINFO); + destQueue.pOrderBy = pOrderBy; + }else{ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iQueue, nCol); + } + VdbeComment((v, "Queue table")); if( iDistinct ){ p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iDistinct, 0); p->selFlags |= SF_UsesEphemeral; @@ -1803,19 +1837,26 @@ static void generateWithRecursiveQuery( /* Store the results of the setup-query in Queue. */ rc = sqlite3Select(pParse, pSetup, &destQueue); - if( rc ) return; + if( rc ) goto end_of_recursive_query; /* Find the next row in the Queue and output that row */ addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iQueue, addrBreak); - selectInnerLoop(pParse, p, p->pEList, iQueue, - 0, 0, pDest, addrCont, addrBreak); - sqlite3VdbeResolveLabel(v, addrCont); /* Transfer the next row in Queue over to Current */ sqlite3VdbeAddOp1(v, OP_NullRow, iCurrent); /* To reset column cache */ - sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent); + if( pOrderBy ){ + sqlite3VdbeAddOp3(v, OP_Column, iQueue, pOrderBy->nExpr+1, regCurrent); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iQueue, regCurrent); + } sqlite3VdbeAddOp1(v, OP_Delete, iQueue); + /* Output the single row in Current */ + addrCont = sqlite3VdbeMakeLabel(v); + selectInnerLoop(pParse, p, p->pEList, iCurrent, + 0, 0, pDest, addrCont, addrBreak); + sqlite3VdbeResolveLabel(v, addrCont); + /* Execute the recursive SELECT taking the single row in Current as ** the value for the recursive-table. Store the results in the Queue. */ @@ -1827,6 +1868,10 @@ static void generateWithRecursiveQuery( /* Keep running the loop until the Queue is empty */ sqlite3VdbeAddOp2(v, OP_Goto, 0, addrTop); sqlite3VdbeResolveLabel(v, addrBreak); + +end_of_recursive_query: + p->pOrderBy = pOrderBy; + return; } #endif @@ -4227,7 +4272,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ "argument"); pFunc->iDistinct = -1; }else{ - KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList); + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); } @@ -4636,7 +4681,7 @@ int sqlite3Select( */ if( pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, pOrderBy); + pKeyInfo = keyInfoFromExprList(pParse, pOrderBy, 0); pOrderBy->iECursor = pParse->nTab++; p->addrOpenEphm[2] = addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, @@ -4668,7 +4713,7 @@ int sqlite3Select( sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList), + (char*)keyInfoFromExprList(pParse, p->pEList, 0), P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; @@ -4792,7 +4837,7 @@ int sqlite3Select( ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy); + pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0); addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index ac48291d80..c1fa7b69ad 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2251,11 +2251,12 @@ struct Select { ** a SELECT statement. */ struct SelectDest { - u8 eDest; /* How to dispose of the results. On of SRT_* above. */ - char affSdst; /* Affinity used when eDest==SRT_Set */ - int iSDParm; /* A parameter used by the eDest disposal method */ - int iSdst; /* Base register where results are written */ - int nSdst; /* Number of registers allocated */ + u8 eDest; /* How to dispose of the results. On of SRT_* above. */ + char affSdst; /* Affinity used when eDest==SRT_Set */ + int iSDParm; /* A parameter used by the eDest disposal method */ + int iSdst; /* Base register where results are written */ + int nSdst; /* Number of registers allocated */ + ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */ }; /* diff --git a/test/with1.test b/test/with1.test index fcf59267f2..0895312b0c 100644 --- a/test/with1.test +++ b/test/with1.test @@ -152,7 +152,44 @@ do_execsql_test 5.1 { do_catchsql_test 5.2 { WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i ORDER BY 1) SELECT x FROM i LIMIT 10; -} {1 {ORDER BY in a recursive query}} +} {0 {1 2 3 4 5 6 7 8 9 10}} + +do_execsql_test 5.2.1 { + CREATE TABLE edge(xfrom, xto, seq, PRIMARY KEY(xfrom, xto)) WITHOUT ROWID; + INSERT INTO edge VALUES(0, 1, 10); + INSERT INTO edge VALUES(1, 2, 20); + INSERT INTO edge VALUES(0, 3, 30); + INSERT INTO edge VALUES(2, 4, 40); + INSERT INTO edge VALUES(3, 4, 40); + INSERT INTO edge VALUES(2, 5, 50); + INSERT INTO edge VALUES(3, 6, 60); + INSERT INTO edge VALUES(5, 7, 70); + INSERT INTO edge VALUES(3, 7, 70); + INSERT INTO edge VALUES(4, 8, 80); + INSERT INTO edge VALUES(7, 8, 80); + INSERT INTO edge VALUES(8, 9, 90); + + WITH RECURSIVE + ancest(id, mtime) AS + (VALUES(0, 0) + UNION + SELECT edge.xto, edge.seq FROM edge, ancest + WHERE edge.xfrom=ancest.id + ORDER BY 2 + ) + SELECT * FROM ancest; +} {0 0 1 10 2 20 3 30 4 40 5 50 6 60 7 70 8 80 9 90} +do_execsql_test 5.2.2 { + WITH RECURSIVE + ancest(id, mtime) AS + (VALUES(0, 0) + UNION ALL + SELECT edge.xto, edge.seq FROM edge, ancest + WHERE edge.xfrom=ancest.id + ORDER BY 2 + ) + SELECT * FROM ancest; +} {0 0 1 10 2 20 3 30 4 40 4 40 5 50 6 60 7 70 7 70 8 80 8 80 8 80 8 80 9 90 9 90 9 90 9 90} do_catchsql_test 5.3 { WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i LIMIT 10 ) From a8a0617e06534c751e4079f0b9c5a47fc4ca07af Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 22 Jan 2014 17:43:16 +0000 Subject: [PATCH 69/74] =?UTF-8?q?Update=20the=20spellfix=20virtual=20table?= =?UTF-8?q?=20to=20optimize=20queries=20of=20the=20form=20"SELECT=20...=20?= =?UTF-8?q?FROM=20tbl=20WHERE=20rowid=3D=3F".?= FossilOrigin-Name: a0ba55ff0596c5f15e9cdb254c68ef50df2dfaad --- ext/misc/spellfix.c | 32 ++++++++++++++++++++++++++++---- manifest | 14 +++++++------- manifest.uuid | 2 +- test/spellfix.test | 45 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 81 insertions(+), 12 deletions(-) diff --git a/ext/misc/spellfix.c b/ext/misc/spellfix.c index 768ea5753c..4be73bb39e 100644 --- a/ext/misc/spellfix.c +++ b/ext/misc/spellfix.c @@ -2051,6 +2051,7 @@ static int spellfix1Close(sqlite3_vtab_cursor *cur){ ** (D) scope = $scope ** (E) distance < $distance ** (F) distance <= $distance +** (G) rowid = $rowid ** ** The plan number is a bit mask formed with these bits: ** @@ -2060,8 +2061,9 @@ static int spellfix1Close(sqlite3_vtab_cursor *cur){ ** 0x08 (D) is found ** 0x10 (E) is found ** 0x20 (F) is found +** 0x40 (G) is found ** -** filter.argv[*] values contains $str, $langid, $top, and $scope, +** filter.argv[*] values contains $str, $langid, $top, $scope and $rowid ** if specified and in that order. */ static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ @@ -2070,6 +2072,7 @@ static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int iTopTerm = -1; int iScopeTerm = -1; int iDistTerm = -1; + int iRowidTerm = -1; int i; const struct sqlite3_index_constraint *pConstraint; pConstraint = pIdxInfo->aConstraint; @@ -2122,6 +2125,15 @@ static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ iPlan |= pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ? 16 : 32; iDistTerm = i; } + + /* Terms of the form: distance < $dist or distance <= $dist */ + if( (iPlan & 64)==0 + && pConstraint->iColumn<0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 64; + iRowidTerm = i; + } } if( iPlan&1 ){ int idx = 2; @@ -2149,6 +2161,11 @@ static int spellfix1BestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ pIdxInfo->aConstraintUsage[iDistTerm].omit = 1; } pIdxInfo->estimatedCost = 1e5; + }else if( (iPlan & 64) ){ + pIdxInfo->idxNum = 64; + pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1; + pIdxInfo->estimatedCost = 5; }else{ pIdxInfo->idxNum = 0; pIdxInfo->estimatedCost = 1e50; @@ -2465,16 +2482,23 @@ static int spellfix1FilterForFullScan( int argc, sqlite3_value **argv ){ - int rc; + int rc = SQLITE_OK; char *zSql; spellfix1_vtab *pVTab = pCur->pVTab; spellfix1ResetCursor(pCur); + assert( idxNum==0 || idxNum==64 ); zSql = sqlite3_mprintf( - "SELECT word, rank, NULL, langid, id FROM \"%w\".\"%w_vocab\"", - pVTab->zDbName, pVTab->zTableName); + "SELECT word, rank, NULL, langid, id FROM \"%w\".\"%w_vocab\"%s", + pVTab->zDbName, pVTab->zTableName, + ((idxNum & 64) ? " WHERE rowid=?" : "") + ); if( zSql==0 ) return SQLITE_NOMEM; rc = sqlite3_prepare_v2(pVTab->db, zSql, -1, &pCur->pFullScan, 0); sqlite3_free(zSql); + if( rc==SQLITE_OK && (idxNum & 64) ){ + assert( argc==1 ); + rc = sqlite3_bind_value(pCur->pFullScan, 1, argv[0]); + } pCur->nRow = pCur->iRow = 0; if( rc==SQLITE_OK ){ rc = sqlite3_step(pCur->pFullScan); diff --git a/manifest b/manifest index 9793a62dd6..336d04f480 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sundocumented\srequirement\sfor\sapplications\sthat\suse\san\sSQLITE_ENABLE_SQLLOG\sbuild\sto\sdefine\sa\ssqlite3_init_sqllog()\sfunction. -D 2014-01-21T15:04:47.807 +C Update\sthe\sspellfix\svirtual\stable\sto\soptimize\squeries\sof\sthe\sform\s"SELECT\s...\sFROM\stbl\sWHERE\srowid=?". +D 2014-01-22T17:43:16.604 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -114,7 +114,7 @@ F ext/misc/nextchar.c 35c8b8baacb96d92abbb34a83a997b797075b342 F ext/misc/percentile.c bcbee3c061b884eccb80e21651daaae8e1e43c63 F ext/misc/regexp.c af92cdaa5058fcec1451e49becc7ba44dba023dc F ext/misc/rot13.c 1ac6f95f99b575907b9b09c81a349114cf9be45a -F ext/misc/spellfix.c 76578f2c56ceaa23d22032338d90db79c68490fb +F ext/misc/spellfix.c adfc569fafef7a1eb8f21528e5277686b358c3ce F ext/misc/totype.c 4a167594e791abeed95e0a8db028822b5e8fe512 F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e @@ -825,7 +825,7 @@ F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b F test/speedtest1.c 7130d2cb6db45baa553a4ab2f715116c71c2d9f4 -F test/spellfix.test 8c40b169b104086d8795781f670ba3c786d6d8be +F test/spellfix.test 674db5da8b16d2b54939b68ccc0ac31ca53d9977 F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9 @@ -1152,7 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7d9e22187daaa3160b875a1df17b924969bf718e -R 72db5d0300da6235ddfc19ebd0af857f +P 5e43bf013253921e4dfbe71de11ee7ed4b3e7eae +R d2ae80bd066cad79603d23bc83a10a19 U dan -Z 21763f37f81635b2fe9a520f9fd3a652 +Z 90dc7fd11fe48035054a3857f5aa405e diff --git a/manifest.uuid b/manifest.uuid index 89ddacefff..3b856a138c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5e43bf013253921e4dfbe71de11ee7ed4b3e7eae \ No newline at end of file +a0ba55ff0596c5f15e9cdb254c68ef50df2dfaad \ No newline at end of file diff --git a/test/spellfix.test b/test/spellfix.test index 9eb7a45f5d..3e6fdf7eec 100644 --- a/test/spellfix.test +++ b/test/spellfix.test @@ -221,6 +221,51 @@ foreach {tn word res} { } $res } +#------------------------------------------------------------------------- +# Try some queries by rowid. +# +do_execsql_test 6.1.1 { + SELECT word FROM t3 WHERE rowid = 10; +} {keener} +do_execsql_test 6.1.2 { + SELECT word, distance FROM t3 WHERE rowid = 10; +} {keener {}} +do_execsql_test 6.1.3 { + SELECT word, distance FROM t3 WHERE rowid = 10 AND word MATCH 'kiiner'; +} {keener 300} + +proc trace_callback {sql} { + if {[string range $sql 0 2] == "-- "} { + lappend ::trace [string range $sql 3 end] + } +} + +proc do_tracesql_test {tn sql {res {}}} { + set ::trace [list] + uplevel [list do_test $tn [subst -nocommands { + set vals [execsql {$sql}] + concat [set vals] [set ::trace] + }] [list {*}$res]] +} + +db trace trace_callback +do_tracesql_test 6.2.1 { + SELECT word FROM t3 WHERE rowid = 10; +} {keener + {SELECT word, rank, NULL, langid, id FROM "main"."t3_vocab" WHERE rowid=?} +} +do_tracesql_test 6.2.2 { + SELECT word, distance FROM t3 WHERE rowid = 10; +} {keener {} + {SELECT word, rank, NULL, langid, id FROM "main"."t3_vocab" WHERE rowid=?} +} +do_tracesql_test 6.2.3 { + SELECT word, distance FROM t3 WHERE rowid = 10 AND word MATCH 'kiiner'; +} {keener 300 + {SELECT id, word, rank, k1 FROM "main"."t3_vocab" WHERE langid=0 AND k2>=?1 AND k2 Date: Wed, 22 Jan 2014 18:07:04 +0000 Subject: [PATCH 70/74] Add support for LIMIT and OFFSET in a recursive query. FossilOrigin-Name: 1945484e6b9769c1943f750f5b09860417fb190a --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/select.c | 55 ++++++++++++++++++++++++++++--------------------- test/with1.test | 17 ++++++++++++--- 4 files changed, 54 insertions(+), 34 deletions(-) diff --git a/manifest b/manifest index 896f5cf947..aa1a8aadb5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Get\sORDER\sBY\sworking\sfor\srecursive\squeries. -D 2014-01-22T17:28:35.279 +C Add\ssupport\sfor\sLIMIT\sand\sOFFSET\sin\sa\srecursive\squery. +D 2014-01-22T18:07:04.473 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c b5430b99c0339dcfe9d06a4c251548e650e386e0 +F src/select.c f6d84f3a109d3e43d38089da6a4f131a5ce4c6ef F src/shell.c 24722d24d4ea8ca93db35e44db7308de786767ca F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1092,7 +1092,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test f4aa699a3712207e02f13945cd2b67d01a9d9f57 +F test/with1.test 97166cc72de5327bbae782aece707c45ee40e41b F test/with2.test 2fe78fcd8deef2a0f9cfc49bfc755911d0b3fd64 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 @@ -1152,7 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 3eb5f9f8d6ac1ee145cb4119087c516f66fe1456 -R a615c5dcec438a7db4f9b5a3e0983f82 +P 37b343b01841b338954ddfa9b76d92aa50037aec +R f6fa1b44c6431db29f11962241b14895 U drh -Z 4ad026d26a172a85d439a21b0ddb47e4 +Z a802989c6813792648a40d2b5db727ef diff --git a/manifest.uuid b/manifest.uuid index 87e23ab961..adee55d513 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -37b343b01841b338954ddfa9b76d92aa50037aec \ No newline at end of file +1945484e6b9769c1943f750f5b09860417fb190a \ No newline at end of file diff --git a/src/select.c b/src/select.c index 13e9562ba2..891bbf7604 100644 --- a/src/select.c +++ b/src/select.c @@ -462,13 +462,13 @@ static void pushOntoSorter( */ static void codeOffset( Vdbe *v, /* Generate code into this VM */ - Select *p, /* The SELECT statement being coded */ + int iOffset, /* Register holding the offset counter */ int iContinue /* Jump here to skip the current record */ ){ - if( p->iOffset && iContinue!=0 ){ + if( iOffset>0 && iContinue!=0 ){ int addr; - sqlite3VdbeAddOp2(v, OP_AddImm, p->iOffset, -1); - addr = sqlite3VdbeAddOp1(v, OP_IfNeg, p->iOffset); + sqlite3VdbeAddOp2(v, OP_AddImm, iOffset, -1); + addr = sqlite3VdbeAddOp1(v, OP_IfNeg, iOffset); sqlite3VdbeAddOp2(v, OP_Goto, 0, iContinue); VdbeComment((v, "skip OFFSET records")); sqlite3VdbeJumpHere(v, addr); @@ -571,7 +571,7 @@ static void selectInnerLoop( assert( pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; if( pOrderBy==0 && !hasDistinct ){ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); } /* Pull the requested columns. @@ -653,7 +653,7 @@ static void selectInnerLoop( } } if( pOrderBy==0 ){ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); } } @@ -1069,13 +1069,13 @@ static void generateSortTail( int ptab2 = pParse->nTab++; sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); - codeOffset(v, p, addrContinue); + codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); sqlite3VdbeAddOp3(v, OP_Column, ptab2, pOrderBy->nExpr+1, regRow); sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); - codeOffset(v, p, addrContinue); + codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr+1, regRow); } switch( eDest ){ @@ -1639,8 +1639,13 @@ Vdbe *sqlite3GetVdbe(Parse *pParse){ ** ** This routine changes the values of iLimit and iOffset only if ** a limit or offset is defined by pLimit and pOffset. iLimit and -** iOffset should have been preset to appropriate default values -** (usually but not always -1) prior to calling this routine. +** iOffset should have been preset to appropriate default values (zero) +** prior to calling this routine. +** +** The iOffset register (if it exists) is initialized to the value +** of the OFFSET. The iLimit register is initialized to LIMIT. Register +** iOffset+1 is initialized to LIMIT+OFFSET. +** ** Only if pLimit!=0 or pOffset!=0 do the limit registers get ** redefined. The UNION ALL operator uses this property to force ** the reuse of the same limit and offset registers across multiple @@ -1664,7 +1669,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ if( p->pLimit ){ p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return; /* VDBE should have already been allocated */ + assert( v!=0 ); if( sqlite3ExprIsInteger(p->pLimit, &n) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); @@ -1778,21 +1783,21 @@ static void generateWithRecursiveQuery( int i; /* Loop counter */ int rc; /* Result code */ ExprList *pOrderBy; /* The ORDER BY clause */ + Expr *pLimit, *pOffset; /* Saved LIMIT and OFFSET */ + int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */ /* Obtain authorization to do a recursive query */ if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return; - addrBreak = sqlite3VdbeMakeLabel(v); - /* Check that there is no ORDER BY or LIMIT clause. Neither of these - ** are currently supported on recursive queries. - */ - assert( p->pOffset==0 || p->pLimit ); - if( /*p->pOrderBy ||*/ p->pLimit ){ - sqlite3ErrorMsg(pParse, "%s in a recursive query", - p->pOrderBy ? "ORDER BY" : "LIMIT" - ); - return; - } + /* Process the LIMIT and OFFSET clauses, if they exist */ + addrBreak = sqlite3VdbeMakeLabel(v); + computeLimitRegisters(pParse, p, addrBreak); + pLimit = p->pLimit; + pOffset = p->pOffset; + regLimit = p->iLimit; + regOffset = p->iOffset; + p->pLimit = p->pOffset = 0; + p->iLimit = p->iOffset = 0; /* Locate the cursor number of the Current table */ for(i=0; ALWAYS(inSrc); i++){ @@ -1853,8 +1858,10 @@ static void generateWithRecursiveQuery( /* Output the single row in Current */ addrCont = sqlite3VdbeMakeLabel(v); + codeOffset(v, regOffset, addrCont); selectInnerLoop(pParse, p, p->pEList, iCurrent, 0, 0, pDest, addrCont, addrBreak); + if( regLimit ) sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); sqlite3VdbeResolveLabel(v, addrCont); /* Execute the recursive SELECT taking the single row in Current as @@ -1871,6 +1878,8 @@ static void generateWithRecursiveQuery( end_of_recursive_query: p->pOrderBy = pOrderBy; + p->pLimit = pLimit; + p->pOffset = pOffset; return; } #endif @@ -2326,7 +2335,7 @@ static int generateOutputSubroutine( /* Suppress the first OFFSET entries if there is an OFFSET clause */ - codeOffset(v, p, iContinue); + codeOffset(v, p->iOffset, iContinue); switch( pDest->eDest ){ /* Store the result as data using a unique key. diff --git a/test/with1.test b/test/with1.test index 0895312b0c..fcd1929277 100644 --- a/test/with1.test +++ b/test/with1.test @@ -190,11 +190,22 @@ do_execsql_test 5.2.2 { ) SELECT * FROM ancest; } {0 0 1 10 2 20 3 30 4 40 4 40 5 50 6 60 7 70 7 70 8 80 8 80 8 80 8 80 9 90 9 90 9 90 9 90} +do_execsql_test 5.2.3 { + WITH RECURSIVE + ancest(id, mtime) AS + (VALUES(0, 0) + UNION ALL + SELECT edge.xto, edge.seq FROM edge, ancest + WHERE edge.xfrom=ancest.id + ORDER BY 2 LIMIT 4 OFFSET 2 + ) + SELECT * FROM ancest; +} {2 20 3 30 4 40 4 40} do_catchsql_test 5.3 { - WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i LIMIT 10 ) - SELECT x FROM i LIMIT 10; -} {1 {LIMIT in a recursive query}} + WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i LIMIT 5) + SELECT x FROM i; +} {0 {1 2 3 4 5}} do_execsql_test 5.4 { WITH i(x) AS ( VALUES(1) UNION ALL SELECT (x+1)%10 FROM i) From edf83d1e3d13058d0b3ea87e88817a73ef35ee84 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 22 Jan 2014 18:31:27 +0000 Subject: [PATCH 71/74] Fix harmless compiler warnings. FossilOrigin-Name: dea2ca6a159d5dcfd8deceedf1c2a73fb4ac1cfc --- manifest | 17 ++++++++--------- manifest.uuid | 2 +- src/expr.c | 8 +++----- src/parse.y | 2 +- src/select.c | 6 +++--- 5 files changed, 16 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 1fd0e675c0..5418051033 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sWITH\sRECURSIVE\simplementation\sto\suse\sa\squeue\sinstead\sof\sa\spair\sof\ntables.\s\sAdd\ssupport\sfor\sORDER\sBY,\sLIMIT,\sand\sOFFSET\son\srecursive\squeries. -D 2014-01-22T18:16:27.891 +C Fix\sharmless\scompiler\swarnings. +D 2014-01-22T18:31:27.905 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -175,7 +175,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff -F src/expr.c 8c7e482bc8f7982333f046851a610ccdb8a1ba94 +F src/expr.c 61f9105820d6702d7153dfb6ca3d58e751a5e95a F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19 @@ -209,7 +209,7 @@ F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968 F src/os_win.c 1b21af72c5fa6f9e519a5fcab33e80d182b1aedb F src/pager.c efa923693e958696eee69b205a20bfbc402c8480 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 475896cb883bbf4782e98abda42efbbdcbdb75f5 +F src/parse.y bd51bc17cbfe7549adb4ca3747b1c3d384645065 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c f6d84f3a109d3e43d38089da6a4f131a5ce4c6ef +F src/select.c a4ac380e9c07135b4613f9021c93e9252a1586d8 F src/shell.c 24722d24d4ea8ca93db35e44db7308de786767ca F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1152,8 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a0ba55ff0596c5f15e9cdb254c68ef50df2dfaad 1945484e6b9769c1943f750f5b09860417fb190a -R 0105f6ff7a9dec4a544ebd63835ab3ca -T +closed 1945484e6b9769c1943f750f5b09860417fb190a +P b6cea42006910d590373e8f9e296d7672edb114b +R 1ba001345e8bb7e9c10a92447c5b79e1 U drh -Z c6c3dc0c740728b9edf5697c95361334 +Z 3fb7fa817c3bd22cb870eb237b846ff7 diff --git a/manifest.uuid b/manifest.uuid index 6b5e670c10..ef71cf8a80 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b6cea42006910d590373e8f9e296d7672edb114b \ No newline at end of file +dea2ca6a159d5dcfd8deceedf1c2a73fb4ac1cfc \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 5f11dec420..2e8079eb8f 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1584,11 +1584,9 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ iCol = (i16)pExpr->iColumn; /* Code an OP_VerifyCookie and OP_TableLock for
. */ - if( ALWAYS(pTab->pSchema) ){ - iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - sqlite3CodeVerifySchema(pParse, iDb); - sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - } + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); /* This function is only called from two places. In both cases the vdbe ** has already been allocated. So assume sqlite3GetVdbe() is always diff --git a/src/parse.y b/src/parse.y index 0805407920..d0ec821d7a 100644 --- a/src/parse.y +++ b/src/parse.y @@ -397,7 +397,7 @@ cmd ::= DROP VIEW ifexists(E) fullname(X). { //////////////////////// The SELECT statement ///////////////////////////////// // cmd ::= select(X). { - SelectDest dest = {SRT_Output, 0, 0, 0, 0}; + SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; sqlite3Select(pParse, X, &dest); sqlite3ExplainBegin(pParse->pVdbe); sqlite3ExplainSelect(pParse->pVdbe, X); diff --git a/src/select.c b/src/select.c index 891bbf7604..0fb1cf8362 100644 --- a/src/select.c +++ b/src/select.c @@ -1774,7 +1774,7 @@ static void generateWithRecursiveQuery( Select *pSetup = p->pPrior; /* The setup query */ int addrTop; /* Top of the loop */ int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ - int iCurrent; /* The Current table */ + int iCurrent = 0; /* The Current table */ int regCurrent; /* Register holding Current table */ int iQueue; /* The Queue table */ int iDistinct = 0; /* To ensure unique results if UNION */ @@ -1936,8 +1936,8 @@ static int multiSelect( Select *pDelete = 0; /* Chain of simple selects to delete */ sqlite3 *db; /* Database connection */ #ifndef SQLITE_OMIT_EXPLAIN - int iSub1; /* EQP id of left-hand query */ - int iSub2; /* EQP id of right-hand query */ + int iSub1 = 0; /* EQP id of left-hand query */ + int iSub2 = 0; /* EQP id of right-hand query */ #endif /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only From cfe24586a8866fcbe1bb9d9f45b267fdb23a8207 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 22 Jan 2014 19:23:30 +0000 Subject: [PATCH 72/74] Avoid an extra seek when inserting records into the epheremal index used to ensure that rows returned by UNION recursive queries are unique. FossilOrigin-Name: 72c4b3f07a3faacb5b62d5bc374b4e125a0bd8b3 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 1 + 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5418051033..0452c399e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2014-01-22T18:31:27.905 +C Avoid\san\sextra\sseek\swhen\sinserting\srecords\sinto\sthe\sepheremal\sindex\sused\sto\sensure\sthat\srows\sreturned\sby\sUNION\srecursive\squeries\sare\sunique. +D 2014-01-22T19:23:30.685 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -219,7 +219,7 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c a4ac380e9c07135b4613f9021c93e9252a1586d8 +F src/select.c 93764e0d81946c070e2c7f1127f35e21efabbcc3 F src/shell.c 24722d24d4ea8ca93db35e44db7308de786767ca F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1152,7 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b6cea42006910d590373e8f9e296d7672edb114b -R 1ba001345e8bb7e9c10a92447c5b79e1 -U drh -Z 3fb7fa817c3bd22cb870eb237b846ff7 +P dea2ca6a159d5dcfd8deceedf1c2a73fb4ac1cfc +R 94fca679783d65c2e2622ed1059ebe94 +U dan +Z f354f4f3e5d1356fcb7db559f2782a7e diff --git a/manifest.uuid b/manifest.uuid index ef71cf8a80..93493f97eb 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -dea2ca6a159d5dcfd8deceedf1c2a73fb4ac1cfc \ No newline at end of file +72c4b3f07a3faacb5b62d5bc374b4e125a0bd8b3 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 0fb1cf8362..98ed526e08 100644 --- a/src/select.c +++ b/src/select.c @@ -814,6 +814,7 @@ static void selectInnerLoop( ** been added */ addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, r3, 0); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r3); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); } for(i=0; i Date: Thu, 23 Jan 2014 14:44:08 +0000 Subject: [PATCH 73/74] Modifications to test files to omit any tests that intentionally access out-of-bounds locations in clang -fsanitize=address builds. FossilOrigin-Name: f4a701d55f5c4e1e62ed64b779ad4fff89dd31b7 --- manifest | 24 ++++++++-------- manifest.uuid | 2 +- src/test1.c | 24 +++++++++++++++- test/capi3.test | 42 ++++++++++++++++------------ test/capi3c.test | 36 ++++++++++++++---------- test/e_fkey.test | 71 ++++++++++++++++++++++++++--------------------- test/misc7.test | 8 ++++-- test/misuse.test | 61 +++++++++++++++++++++------------------- test/select7.test | 30 ++++++++++---------- 9 files changed, 173 insertions(+), 125 deletions(-) diff --git a/manifest b/manifest index 0452c399e9..eff8df8471 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\san\sextra\sseek\swhen\sinserting\srecords\sinto\sthe\sepheremal\sindex\sused\sto\sensure\sthat\srows\sreturned\sby\sUNION\srecursive\squeries\sare\sunique. -D 2014-01-22T19:23:30.685 +C Modifications\sto\stest\sfiles\sto\somit\sany\stests\sthat\sintentionally\saccess\sout-of-bounds\slocations\sin\sclang\s-fsanitize=address\sbuilds. +D 2014-01-23T14:44:08.207 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -229,7 +229,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 46073db71011b6542fde1f234c56a076d5ff23f9 -F src/test1.c db16ba651453b15001c7f2838c446284dde4ecaf +F src/test1.c 2401eee14a4309a7cfe2aeb2f30ad517a1d9c299 F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -364,9 +364,9 @@ F test/btreefault.test c2bcb542685eea44621275cfedbd8a13f65201e3 F test/busy.test 76b4887f8b9160ba903c1ac22e8ff406ad6ae2f0 F test/cache.test 13bc046b26210471ca6f2889aceb1ea52dc717de F test/capi2.test 011c16da245fdc0106a2785035de6b242c05e738 -F test/capi3.test 56ab450125ead38846cbae7e5b6a216686c3cffa +F test/capi3.test 6cdd49656bd62a296924f4d2fcfd05cd2a298369 F test/capi3b.test efb2b9cfd127efa84433cd7a2d72ce0454ae0dc4 -F test/capi3c.test 93d24621c9ff84da9da060f30431e0453db1cdb0 +F test/capi3c.test a21869e4d50d5dbb7e566e328fc0bc7c2efa6a32 F test/capi3d.test 6d0fc0a86d73f42dd19a7d8b7761ab9bc02277d0 F test/capi3e.test ad90088b18b0367125ff2d4b5400153fd2f99aab F test/cast.test 4c275cbdc8202d6f9c54a3596701719868ac7dc3 @@ -439,7 +439,7 @@ F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a F test/e_droptrigger.test 3cd080807622c13e5bbb61fc9a57bd7754da2412 F test/e_dropview.test 0c9f7f60989164a70a67a9d9c26d1083bc808306 F test/e_expr.test 5c71d183fbf519a4769fd2e2124afdc70b5b1f42 -F test/e_fkey.test d83a04478bb9c02d2c513518548a69f818869f41 +F test/e_fkey.test 630597377549af579d34faaf64c6959a5a68ef76 F test/e_fts3.test 5c02288842e4f941896fd44afdef564dd5fc1459 F test/e_insert.test 1e44f84d2abe44d66e4fbf198be4b20e3cc724a0 F test/e_reindex.test 396b7b4f0a66863b4e95116a67d93b227193e589 @@ -700,8 +700,8 @@ F test/misc3.test cf3dda47d5dda3e53fc5804a100d3c82be736c9d F test/misc4.test 9c078510fbfff05a9869a0b6d8b86a623ad2c4f6 F test/misc5.test 528468b26d03303b1f047146e5eefc941b9069f5 F test/misc6.test 953cc693924d88e6117aeba16f46f0bf5abede91 -F test/misc7.test 1265eb98c2e22a446a13fdef754118b272716684 -F test/misuse.test ba4fb5d1a6101d1c171ea38b3c613d0661c83054 +F test/misc7.test edd0b63e2ee29a256900b0514f6fff27e19e9bb2 +F test/misuse.test 3c34719944ba045cc6c188a4852ba04680728912 F test/mmap1.test 93d167b328255cbe6679fe1e1a23be1b1197d07b F test/mmap2.test 9d6dd9ddb4ad2379f29cc78f38ce1e63ed418022 F test/mmap3.test c92273e16eb8d23c1d55c9815b446bb72ef0512e @@ -783,7 +783,7 @@ F test/select3.test 2ce595f8fb8e2ac10071d3b4e424cadd4634a054 F test/select4.test 00179be44e531fe04c1c3f15df216439dff2519d F test/select5.test e758b8ef94f69b111df4cb819008856655dcd535 F test/select6.test e76bd10a56988f15726c097a5d5a7966fe82d3b2 -F test/select7.test dad6f00f0d49728a879d6eb6451d4752db0b0abe +F test/select7.test 7fd2ef598cfabb6b9ff6ac13973b91d0527df49d F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95 F test/selectA.test 99cf21df033b93033ea4f34aba14a500f48f04fe @@ -1152,7 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P dea2ca6a159d5dcfd8deceedf1c2a73fb4ac1cfc -R 94fca679783d65c2e2622ed1059ebe94 +P 72c4b3f07a3faacb5b62d5bc374b4e125a0bd8b3 +R 1a965aa9944d1df45f2b3d9d29de01da U dan -Z f354f4f3e5d1356fcb7db559f2782a7e +Z 5e3d3ac89b5b8048d3858093fe9eb7e9 diff --git a/manifest.uuid b/manifest.uuid index 93493f97eb..d96db9705f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -72c4b3f07a3faacb5b62d5bc374b4e125a0bd8b3 \ No newline at end of file +f4a701d55f5c4e1e62ed64b779ad4fff89dd31b7 \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 109f90143f..3000288c7d 100644 --- a/src/test1.c +++ b/src/test1.c @@ -242,7 +242,28 @@ static int test_io_trace( return TCL_OK; } - +/* +** Usage: clang_sanitize_address +** +** Returns true if the program was compiled using clang with the +** -fsanitize=address switch on the command line. False otherwise. +*/ +static int clang_sanitize_address( + void *NotUsed, + Tcl_Interp *interp, /* The TCL interpreter that invoked this command */ + int argc, /* Number of arguments */ + char **argv /* Text of each argument */ +){ + int res = 0; +#if defined(__has_feature) +# if __has_feature(address_sanitizer) + res = 1; +# endif +#endif + Tcl_SetObjResult(interp, Tcl_NewIntObj(res)); + return TCL_OK; +} + /* ** Usage: sqlite3_exec_printf DB FORMAT STRING ** @@ -6323,6 +6344,7 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "sqlite3_busy_timeout", (Tcl_CmdProc*)test_busy_timeout }, { "printf", (Tcl_CmdProc*)test_printf }, { "sqlite3IoTrace", (Tcl_CmdProc*)test_io_trace }, + { "clang_sanitize_address", (Tcl_CmdProc*)clang_sanitize_address }, }; static struct { char *zName; diff --git a/test/capi3.test b/test/capi3.test index 9d7434d25d..f0234e8897 100644 --- a/test/capi3.test +++ b/test/capi3.test @@ -179,16 +179,18 @@ do_test capi3-3.4 { do_test capi3-3.5 { sqlite3_close $db2 } {SQLITE_OK} -do_test capi3-3.6.1-misuse { - sqlite3_close $db2 -} {SQLITE_MISUSE} -do_test capi3-3.6.2-misuse { - sqlite3_errmsg $db2 -} {library routine called out of sequence} -ifcapable {utf16} { - do_test capi3-3.6.3-misuse { - utf8 [sqlite3_errmsg16 $db2] +if {[clang_sanitize_address]==0} { + do_test capi3-3.6.1-misuse { + sqlite3_close $db2 + } {SQLITE_MISUSE} + do_test capi3-3.6.2-misuse { + sqlite3_errmsg $db2 } {library routine called out of sequence} + ifcapable {utf16} { + do_test capi3-3.6.3-misuse { + utf8 [sqlite3_errmsg16 $db2] + } {library routine called out of sequence} + } } do_test capi3-3.7 { @@ -661,10 +663,12 @@ do_test capi3-6.3 { sqlite3_finalize $STMT } {SQLITE_OK} -do_test capi3-6.4-misuse { - db cache flush - sqlite3_close $DB -} {SQLITE_OK} +if {[clang_sanitize_address]==0} { + do_test capi3-6.4-misuse { + db cache flush + sqlite3_close $DB + } {SQLITE_OK} +} db close # This procedure sets the value of the file-format in file 'test.db' @@ -1060,11 +1064,13 @@ if {[llength [info commands sqlite3_sleep]]>0} { } # Ticket #1219: Make sure binding APIs can handle a NULL pointer. -# -do_test capi3-14.1-misuse { - set rc [catch {sqlite3_bind_text 0 1 hello 5} msg] - lappend rc $msg -} {1 SQLITE_MISUSE} +# +if {[clang_sanitize_address]==0} { + do_test capi3-14.1-misuse { + set rc [catch {sqlite3_bind_text 0 1 hello 5} msg] + lappend rc $msg + } {1 SQLITE_MISUSE} +} # Ticket #1650: Honor the nBytes parameter to sqlite3_prepare. # diff --git a/test/capi3c.test b/test/capi3c.test index 14545c0a68..6388717e00 100644 --- a/test/capi3c.test +++ b/test/capi3c.test @@ -169,16 +169,18 @@ do_test capi3c-3.4 { do_test capi3c-3.5 { sqlite3_close $db2 } {SQLITE_OK} -do_test capi3c-3.6.1-misuse { - sqlite3_close $db2 -} {SQLITE_MISUSE} -do_test capi3c-3.6.2-misuse { - sqlite3_errmsg $db2 -} {library routine called out of sequence} -ifcapable {utf16} { - do_test capi3c-3.6.3-misuse { - utf8 [sqlite3_errmsg16 $db2] +if {[clang_sanitize_address]==0} { + do_test capi3c-3.6.1-misuse { + sqlite3_close $db2 + } {SQLITE_MISUSE} + do_test capi3c-3.6.2-misuse { + sqlite3_errmsg $db2 } {library routine called out of sequence} + ifcapable {utf16} { + do_test capi3c-3.6.3-misuse { + utf8 [sqlite3_errmsg16 $db2] + } {library routine called out of sequence} + } } # rename sqlite3_open "" @@ -627,13 +629,17 @@ check_data $STMT capi3c-6.3 {INTEGER} {1} {1.0} {1} do_test capi3c-6.3 { sqlite3_finalize $STMT } {SQLITE_OK} -do_test capi3c-6.4 { - db cache flush - sqlite3_close $DB -} {SQLITE_OK} -do_test capi3c-6.99-misuse { +if {[clang_sanitize_address]==0} { + do_test capi3c-6.4 { + db cache flush + sqlite3_close $DB + } {SQLITE_OK} + do_test capi3c-6.99-misuse { + db close + } {} +} else { db close -} {} +} # This procedure sets the value of the file-format in file 'test.db' # to $newval. Also, the schema cookie is incremented. diff --git a/test/e_fkey.test b/test/e_fkey.test index 022611695d..921e967176 100644 --- a/test/e_fkey.test +++ b/test/e_fkey.test @@ -2946,38 +2946,45 @@ proc test_on_update_recursion {limit} { " } -do_test e_fkey-63.1.1 { - test_on_delete_recursion $SQLITE_MAX_TRIGGER_DEPTH -} {0 0} -do_test e_fkey-63.1.2 { - test_on_delete_recursion [expr $SQLITE_MAX_TRIGGER_DEPTH+1] -} {1 {too many levels of trigger recursion}} -do_test e_fkey-63.1.3 { - sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 5 - test_on_delete_recursion 5 -} {0 0} -do_test e_fkey-63.1.4 { - test_on_delete_recursion 6 -} {1 {too many levels of trigger recursion}} -do_test e_fkey-63.1.5 { - sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 1000000 -} {5} -do_test e_fkey-63.2.1 { - test_on_update_recursion $SQLITE_MAX_TRIGGER_DEPTH -} {0 0} -do_test e_fkey-63.2.2 { - test_on_update_recursion [expr $SQLITE_MAX_TRIGGER_DEPTH+1] -} {1 {too many levels of trigger recursion}} -do_test e_fkey-63.2.3 { - sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 5 - test_on_update_recursion 5 -} {0 0} -do_test e_fkey-63.2.4 { - test_on_update_recursion 6 -} {1 {too many levels of trigger recursion}} -do_test e_fkey-63.2.5 { - sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 1000000 -} {5} +# If the current build was created using clang with the -fsanitize=address +# switch, then the library uses considerably more stack space than usual. +# So much more, that some of the following tests cause stack overflows +# if they are run under this configuration. +# +if {[clang_sanitize_address]==0} { + do_test e_fkey-63.1.1 { + test_on_delete_recursion $SQLITE_MAX_TRIGGER_DEPTH + } {0 0} + do_test e_fkey-63.1.2 { + test_on_delete_recursion [expr $SQLITE_MAX_TRIGGER_DEPTH+1] + } {1 {too many levels of trigger recursion}} + do_test e_fkey-63.1.3 { + sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 5 + test_on_delete_recursion 5 + } {0 0} + do_test e_fkey-63.1.4 { + test_on_delete_recursion 6 + } {1 {too many levels of trigger recursion}} + do_test e_fkey-63.1.5 { + sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 1000000 + } {5} + do_test e_fkey-63.2.1 { + test_on_update_recursion $SQLITE_MAX_TRIGGER_DEPTH + } {0 0} + do_test e_fkey-63.2.2 { + test_on_update_recursion [expr $SQLITE_MAX_TRIGGER_DEPTH+1] + } {1 {too many levels of trigger recursion}} + do_test e_fkey-63.2.3 { + sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 5 + test_on_update_recursion 5 + } {0 0} + do_test e_fkey-63.2.4 { + test_on_update_recursion 6 + } {1 {too many levels of trigger recursion}} + do_test e_fkey-63.2.5 { + sqlite3_limit db SQLITE_LIMIT_TRIGGER_DEPTH 1000000 + } {5} +} #------------------------------------------------------------------------- # The setting of the recursive_triggers pragma does not affect foreign diff --git a/test/misc7.test b/test/misc7.test index ec042ff0ee..8fd5fe7546 100644 --- a/test/misc7.test +++ b/test/misc7.test @@ -15,9 +15,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -do_test misc7-1-misuse { - c_misuse_test -} {} +if {[clang_sanitize_address]==0} { + do_test misc7-1-misuse { + c_misuse_test + } {} +} do_test misc7-2 { c_realloc_test diff --git a/test/misuse.test b/test/misuse.test index 71ee0118c8..d5d836cbfb 100644 --- a/test/misuse.test +++ b/test/misuse.test @@ -171,37 +171,40 @@ do_test misuse-4.3 { } msg] lappend v $msg $r } {0 {} SQLITE_BUSY} -do_test misuse-4.4 { + +if {[clang_sanitize_address]==0} { + do_test misuse-4.4 { # Flush the TCL statement cache here, otherwise the sqlite3_close() will # fail because there are still un-finalized() VDBEs. - db cache flush - sqlite3_close $::DB - catchsql2 {SELECT * FROM t1} -} {1 {library routine called out of sequence}} -do_test misuse-4.5 { - catchsql { - SELECT * FROM t1 - } -} {1 {library routine called out of sequence}} + db cache flush + sqlite3_close $::DB + catchsql2 {SELECT * FROM t1} + } {1 {library routine called out of sequence}} + do_test misuse-4.5 { + catchsql { + SELECT * FROM t1 + } + } {1 {library routine called out of sequence}} -# Attempt to use a database after it has been closed. -# -do_test misuse-5.1 { - db close - sqlite3 db test2.db; set ::DB [sqlite3_connection_pointer db] - execsql { - SELECT * FROM t1 - } -} {1 2} -do_test misuse-5.2 { - catchsql2 {SELECT * FROM t1} -} {0 {a b 1 2}} -do_test misuse-5.3 { - db close - set r [catch { - sqlite3_prepare $::DB {SELECT * FROM t1} -1 TAIL - } msg] - lappend r $msg -} {1 {(21) library routine called out of sequence}} + # Attempt to use a database after it has been closed. + # + do_test misuse-5.1 { + db close + sqlite3 db test2.db; set ::DB [sqlite3_connection_pointer db] + execsql { + SELECT * FROM t1 + } + } {1 2} + do_test misuse-5.2 { + catchsql2 {SELECT * FROM t1} + } {0 {a b 1 2}} + do_test misuse-5.3 { + db close + set r [catch { + sqlite3_prepare $::DB {SELECT * FROM t1} -1 TAIL + } msg] + lappend r $msg + } {1 {(21) library routine called out of sequence}} +} finish_test diff --git a/test/select7.test b/test/select7.test index e8fc440006..6816b9fcb9 100644 --- a/test/select7.test +++ b/test/select7.test @@ -138,21 +138,23 @@ ifcapable {subquery && compound} { # Verify that an error occurs if you have too many terms on a # compound select statement. # -ifcapable compound { - if {$SQLITE_MAX_COMPOUND_SELECT>0} { - set sql {SELECT 0} - set result 0 - for {set i 1} {$i<$SQLITE_MAX_COMPOUND_SELECT} {incr i} { - append sql " UNION ALL SELECT $i" - lappend result $i +if {[clang_sanitize_address]==0} { + ifcapable compound { + if {$SQLITE_MAX_COMPOUND_SELECT>0} { + set sql {SELECT 0} + set result 0 + for {set i 1} {$i<$SQLITE_MAX_COMPOUND_SELECT} {incr i} { + append sql " UNION ALL SELECT $i" + lappend result $i + } + do_test select7-6.1 { + catchsql $sql + } [list 0 $result] + append sql { UNION ALL SELECT 99999999} + do_test select7-6.2 { + catchsql $sql + } {1 {too many terms in compound SELECT}} } - do_test select7-6.1 { - catchsql $sql - } [list 0 $result] - append sql { UNION ALL SELECT 99999999} - do_test select7-6.2 { - catchsql $sql - } {1 {too many terms in compound SELECT}} } } From b090352b5a46884806d1c1d99dbcd336b103fe67 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 24 Jan 2014 11:16:01 +0000 Subject: [PATCH 74/74] Add test cases showing the use of ORDER BY on a recursive query to control depth-first versus breath-first search of a tree. FossilOrigin-Name: 83b0b2916589db0184435dbd4c304387f393ed60 --- manifest | 14 +++--- manifest.uuid | 2 +- test/with1.test | 123 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 129 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index eff8df8471..fb401ca29f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modifications\sto\stest\sfiles\sto\somit\sany\stests\sthat\sintentionally\saccess\sout-of-bounds\slocations\sin\sclang\s-fsanitize=address\sbuilds. -D 2014-01-23T14:44:08.207 +C Add\stest\scases\sshowing\sthe\suse\sof\sORDER\sBY\son\sa\srecursive\squery\sto\scontrol\ndepth-first\sversus\sbreath-first\ssearch\sof\sa\stree. +D 2014-01-24T11:16:01.884 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1092,7 +1092,7 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d -F test/with1.test 97166cc72de5327bbae782aece707c45ee40e41b +F test/with1.test 9d3537372c8cf6d5e0a5e9af037a52f3375fb704 F test/with2.test 2fe78fcd8deef2a0f9cfc49bfc755911d0b3fd64 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8 @@ -1152,7 +1152,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 72c4b3f07a3faacb5b62d5bc374b4e125a0bd8b3 -R 1a965aa9944d1df45f2b3d9d29de01da -U dan -Z 5e3d3ac89b5b8048d3858093fe9eb7e9 +P f4a701d55f5c4e1e62ed64b779ad4fff89dd31b7 +R 15c1d48ec007e0818a79a87f9c81fe3d +U drh +Z a61c87789af8810b68d544f76885410e diff --git a/manifest.uuid b/manifest.uuid index d96db9705f..1c78fab271 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f4a701d55f5c4e1e62ed64b779ad4fff89dd31b7 \ No newline at end of file +83b0b2916589db0184435dbd4c304387f393ed60 \ No newline at end of file diff --git a/test/with1.test b/test/with1.test index fcd1929277..6dd67d1690 100644 --- a/test/with1.test +++ b/test/with1.test @@ -374,7 +374,7 @@ do_catchsql_test 7.6 { # Compute the mandelbrot set using a recursive query # -do_execsql_test 8.1 { +do_execsql_test 8.1-mandelbrot { WITH RECURSIVE xaxis(x) AS (VALUES(-2.0) UNION ALL SELECT x+0.05 FROM xaxis WHERE x<1.2), yaxis(y) AS (VALUES(-1.0) UNION ALL SELECT y+0.1 FROM yaxis WHERE y<1.0), @@ -417,7 +417,7 @@ do_execsql_test 8.1 { # Solve a sudoku puzzle using a recursive query # -do_execsql_test 8.2 { +do_execsql_test 8.2-soduko { WITH RECURSIVE input(sud) AS ( VALUES('53..7....6..195....98....6.8...6...34..8.3..17...2...6.6....28....419..5....8..79') @@ -452,4 +452,123 @@ do_execsql_test 8.2 { SELECT s FROM x WHERE ind=0; } {534678912672195348198342567859761423426853791713924856961537284287419635345286179} + +# Test cases to illustrate on the ORDER BY clause on a recursive query can be +# used to control depth-first versus breath-first search in a tree. +# +do_execsql_test 9.1 { + CREATE TABLE org( + name TEXT PRIMARY KEY, + boss TEXT REFERENCES org + ) WITHOUT ROWID; + INSERT INTO org VALUES('Alice',NULL); + INSERT INTO org VALUES('Bob','Alice'); + INSERT INTO org VALUES('Cindy','Alice'); + INSERT INTO org VALUES('Dave','Bob'); + INSERT INTO org VALUES('Emma','Bob'); + INSERT INTO org VALUES('Fred','Cindy'); + INSERT INTO org VALUES('Gail','Cindy'); + INSERT INTO org VALUES('Harry','Dave'); + INSERT INTO org VALUES('Ingrid','Dave'); + INSERT INTO org VALUES('Jim','Emma'); + INSERT INTO org VALUES('Kate','Emma'); + INSERT INTO org VALUES('Lanny','Fred'); + INSERT INTO org VALUES('Mary','Fred'); + INSERT INTO org VALUES('Noland','Gail'); + INSERT INTO org VALUES('Olivia','Gail'); + -- The above are all under Alice. Add a few more records for people + -- not in Alice's group, just to prove that they won't be selected. + INSERT INTO org VALUES('Xaviar',NULL); + INSERT INTO org VALUES('Xia','Xaviar'); + INSERT INTO org VALUES('Xerxes','Xaviar'); + INSERT INTO org VALUES('Xena','Xia'); + -- Find all members of Alice's group, breath-first order + WITH RECURSIVE + under_alice(name,level) AS ( + VALUES('Alice','0') + UNION ALL + SELECT org.name, under_alice.level+1 + FROM org, under_alice + WHERE org.boss=under_alice.name + ORDER BY 2 + ) + SELECT group_concat(substr('...............',1,level*3) || name,x'0a') + FROM under_alice; +} {{Alice +...Bob +...Cindy +......Dave +......Emma +......Fred +......Gail +.........Harry +.........Ingrid +.........Jim +.........Kate +.........Lanny +.........Mary +.........Noland +.........Olivia}} + +# The previous query used "ORDER BY level" to yield a breath-first search. +# Change that to "ORDER BY level DESC" for a depth-first search. +# +do_execsql_test 9.2 { + WITH RECURSIVE + under_alice(name,level) AS ( + VALUES('Alice','0') + UNION ALL + SELECT org.name, under_alice.level+1 + FROM org, under_alice + WHERE org.boss=under_alice.name + ORDER BY 2 DESC + ) + SELECT group_concat(substr('...............',1,level*3) || name,x'0a') + FROM under_alice; +} {{Alice +...Bob +......Dave +.........Harry +.........Ingrid +......Emma +.........Jim +.........Kate +...Cindy +......Fred +.........Lanny +.........Mary +......Gail +.........Noland +.........Olivia}} + +# Without an ORDER BY clause, the recursive query should use a FIFO, +# resulting in a breath-first search. +# +do_execsql_test 9.3 { + WITH RECURSIVE + under_alice(name,level) AS ( + VALUES('Alice','0') + UNION ALL + SELECT org.name, under_alice.level+1 + FROM org, under_alice + WHERE org.boss=under_alice.name + ) + SELECT group_concat(substr('...............',1,level*3) || name,x'0a') + FROM under_alice; +} {{Alice +...Bob +...Cindy +......Dave +......Emma +......Fred +......Gail +.........Harry +.........Ingrid +.........Jim +.........Kate +.........Lanny +.........Mary +.........Noland +.........Olivia}} + finish_test