From 87af14a639aa677047f75e008aa7d107343a7510 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Mar 2014 12:44:46 +0000 Subject: [PATCH 01/49] Prevent the rtree module from reading sqlite_stat1 data from the wrong database. FossilOrigin-Name: 7ce03c1b5552d830300575c5b41a874db7a2ec77 --- ext/rtree/rtree.c | 36 ++++++++++-------- ext/rtree/rtreeC.test | 86 +++++++++++++++++++++++++++++++++++++++++++ manifest | 19 ++++------ manifest.uuid | 2 +- 4 files changed, 116 insertions(+), 27 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index 577e19d4c6..cd652a3936 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -2947,26 +2947,32 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ ** on sqlite_stat1 data. Otherwise, use RTREE_DEFAULT_ROWEST. */ static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ - const char *zSql = "SELECT stat FROM sqlite_stat1 WHERE tbl= ? || '_rowid'"; + const char *zFmt = "SELECT stat FROM %Q.sqlite_stat1 WHERE tbl = '%q_rowid'"; + char *zSql; sqlite3_stmt *p; int rc; i64 nRow = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); - if( rc==SQLITE_OK ){ - sqlite3_bind_text(p, 1, pRtree->zName, -1, SQLITE_STATIC); - if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); - rc = sqlite3_finalize(p); - }else if( rc!=SQLITE_NOMEM ){ - rc = SQLITE_OK; - } - - if( rc==SQLITE_OK ){ - if( nRow==0 ){ - pRtree->nRowEst = RTREE_DEFAULT_ROWEST; - }else{ - pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &p, 0); + if( rc==SQLITE_OK ){ + if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); + rc = sqlite3_finalize(p); + }else if( rc!=SQLITE_NOMEM ){ + rc = SQLITE_OK; } + + if( rc==SQLITE_OK ){ + if( nRow==0 ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + }else{ + pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); + } + } + sqlite3_free(zSql); } return rc; diff --git a/ext/rtree/rtreeC.test b/ext/rtree/rtreeC.test index 2e5bedec96..b72007245f 100644 --- a/ext/rtree/rtreeC.test +++ b/ext/rtree/rtreeC.test @@ -158,5 +158,91 @@ do_execsql_test 4.3 { SELECT b, a FROM t2 LEFT JOIN t1 ON (+a = +b); } {1 1 3 {}} +#-------------------------------------------------------------------- +# Test that the sqlite_stat1 data is used correctly. +# +reset_db +do_execsql_test 5.1 { + CREATE TABLE t1(x PRIMARY KEY, y); + CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2); + + INSERT INTO t1(x) VALUES(1); + INSERT INTO t1(x) SELECT x+1 FROM t1; -- 2 + INSERT INTO t1(x) SELECT x+2 FROM t1; -- 4 + INSERT INTO t1(x) SELECT x+4 FROM t1; -- 8 + INSERT INTO t1(x) SELECT x+8 FROM t1; -- 16 + INSERT INTO t1(x) SELECT x+16 FROM t1; -- 32 + INSERT INTO t1(x) SELECT x+32 FROM t1; -- 64 + INSERT INTO t1(x) SELECT x+64 FROM t1; -- 128 + INSERT INTO t1(x) SELECT x+128 FROM t1; -- 256 + INSERT INTO t1(x) SELECT x+256 FROM t1; -- 512 + INSERT INTO t1(x) SELECT x+512 FROM t1; --1024 + + INSERT INTO rt SELECT x, x, x+1 FROM t1 WHERE x<=5; +} + +# First test a query with no ANALYZE data at all. The outer loop is +# real table "t1". +# +do_eqp_test 5.2 { + SELECT * FROM t1, rt WHERE x==id; +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:} +} + +# Now create enough ANALYZE data to tell SQLite that virtual table "rt" +# contains very few rows. This causes it to move "rt" to the outer loop. +# +do_execsql_test 5.3 { + ANALYZE; + DELETE FROM sqlite_stat1 WHERE tbl='t1'; +} +db close +sqlite3 db test.db +do_eqp_test 5.4 { + SELECT * FROM t1, rt WHERE x==id; +} { + 0 0 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 2:} + 0 1 0 {SEARCH TABLE t1 USING INDEX sqlite_autoindex_t1_1 (x=?)} +} + +# Delete the ANALYZE data. "t1" should be the outer loop again. +# +do_execsql_test 5.5 { DROP TABLE sqlite_stat1; } +db close +sqlite3 db test.db +do_eqp_test 5.6 { + SELECT * FROM t1, rt WHERE x==id; +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:} +} + +# This time create and attach a database that contains ANALYZE data for +# tables of the same names as those used internally by virtual table +# "rt". Check that the rtree module is not fooled into using this data. +# Table "t1" should remain the outer loop. +# +do_test 5.7 { + db backup test.db2 + sqlite3 db2 test.db2 + db2 eval { + ANALYZE; + DELETE FROM sqlite_stat1 WHERE tbl='t1'; + } + db2 close + db close + sqlite3 db test.db + execsql { ATTACH 'test.db2' AS aux; } +} {} +do_eqp_test 5.8 { + SELECT * FROM t1, rt WHERE x==id; +} { + 0 0 0 {SCAN TABLE t1} + 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:} +} + + finish_test diff --git a/manifest b/manifest index 219399a93b..8dd646f40a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.4.1 -D 2014-03-11T15:27:36.923 +C Prevent\sthe\srtree\smodule\sfrom\sreading\ssqlite_stat1\sdata\sfrom\sthe\swrong\sdatabase. +D 2014-03-12T12:44:46.777 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -120,7 +120,7 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c a4e18b2c150adad20aecbeb3408cd235a0a57441 +F ext/rtree/rtree.c d963b961f7ebb3842a6dbc959ed8c3f3534206c4 F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test cf679265ecafff494a768ac9c2f43a70915a6290 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -133,7 +133,7 @@ F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf F ext/rtree/rtreeB.test 983e567b49b5dca165940f66b87e161aa30e82b2 -F ext/rtree/rtreeC.test 03975565f40a0bee165f613143e4dec716dd5a59 +F ext/rtree/rtreeC.test afe930ac948c79fd12e48691c478d3f3bfc763d8 F ext/rtree/rtreeD.test 636630357638f5983701550b37f0f5867130d2ca F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea @@ -1156,10 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 33f5694fa42d9a521ca081de4ddd1e9dc8952b16 -R 581994aed374f15c1c6bac5b0a9a5df5 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.4.1 * -U drh -Z c1ec45dc8a92b80bfa16f4c74ccfeec8 +P 018d317b1257ce68a92908b05c9c7cf1494050d0 +R 771148b6f32577641865e57d520a643c +U dan +Z 84dfe23ccda2d3e1d846c6ac2ce685a0 diff --git a/manifest.uuid b/manifest.uuid index 1f0b42178b..b36a223a45 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -018d317b1257ce68a92908b05c9c7cf1494050d0 \ No newline at end of file +7ce03c1b5552d830300575c5b41a874db7a2ec77 \ No newline at end of file From 82d25da5b08cc1358b1b19fdb625575dc923879e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Mar 2014 19:20:36 +0000 Subject: [PATCH 02/49] Changes to FTS to ensure that it does not access the database from within the xConnect method. FossilOrigin-Name: c67a52c356416cc200f51f9168d942e78db11d34 --- ext/fts3/fts3.c | 44 ++++++++++++++++++++++++++++++++++++------- ext/fts3/fts3Int.h | 2 +- ext/fts3/fts3_write.c | 4 ++++ manifest | 18 +++++++++--------- manifest.uuid | 2 +- test/fts3d.test | 11 +++++++++++ 6 files changed, 63 insertions(+), 18 deletions(-) diff --git a/ext/fts3/fts3.c b/ext/fts3/fts3.c index 44b7f431df..f7b2232902 100644 --- a/ext/fts3/fts3.c +++ b/ext/fts3/fts3.c @@ -1410,10 +1410,7 @@ static int fts3InitVtab( ** addition of a %_stat table so that it can use incremental merge. */ if( !isFts4 && !isCreate ){ - int rc2 = SQLITE_OK; - fts3DbExec(&rc2, db, "SELECT 1 FROM %Q.'%q_stat' WHERE id=2", - p->zDb, p->zName); - if( rc2==SQLITE_OK ) p->bHasStat = 1; + p->bHasStat = 2; } /* Figure out the page-size for the database. This is required in order to @@ -3320,7 +3317,34 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ } /* -** Implementation of xBegin() method. This is a no-op. +** If it is currently unknown whether or not the FTS table has an %_stat +** table (if p->bHasStat==2), attempt to determine this (set p->bHasStat +** to 0 or 1). Return SQLITE_OK if successful, or an SQLite error code +** if an error occurs. +*/ +static int fts3SetHasStat(Fts3Table *p){ + int rc = SQLITE_OK; + if( p->bHasStat==2 ){ + const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; + char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); + if( zSql ){ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) p->bHasStat = bHasStat; + } + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + return rc; +} + +/* +** Implementation of xBegin() method. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; @@ -3331,7 +3355,7 @@ static int fts3BeginMethod(sqlite3_vtab *pVtab){ TESTONLY( p->inTransaction = 1 ); TESTONLY( p->mxSavepoint = -1; ); p->nLeafAdd = 0; - return SQLITE_OK; + return fts3SetHasStat(p); } /* @@ -3580,6 +3604,10 @@ static int fts3RenameMethod( sqlite3 *db = p->db; /* Database connection */ int rc; /* Return Code */ + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + rc = fts3SetHasStat(p); + /* As it happens, the pending terms table is always empty here. This is ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction ** always opens a savepoint transaction. And the xSavepoint() method @@ -3587,7 +3615,9 @@ static int fts3RenameMethod( ** PendingTermsFlush() in in case that changes. */ assert( p->nPendingData==0 ); - rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts3PendingTermsFlush(p); + } if( p->zContentTbl==0 ){ fts3DbExec(&rc, db, diff --git a/ext/fts3/fts3Int.h b/ext/fts3/fts3Int.h index 7f71736888..a5bb2f0041 100644 --- a/ext/fts3/fts3Int.h +++ b/ext/fts3/fts3Int.h @@ -223,7 +223,7 @@ struct Fts3Table { int nNodeSize; /* Soft limit for node size */ u8 bFts4; /* True for FTS4, false for FTS3 */ - u8 bHasStat; /* True if %_stat table exists */ + u8 bHasStat; /* True if %_stat table exists (2==unknown) */ u8 bHasDocsize; /* True if %_docsize table exists */ u8 bDescIdx; /* True if doclists are in reverse order */ u8 bIgnoreSavepoint; /* True to ignore xSavepoint invocations */ diff --git a/ext/fts3/fts3_write.c b/ext/fts3/fts3_write.c index 3564cf8206..90d1609226 100644 --- a/ext/fts3/fts3_write.c +++ b/ext/fts3/fts3_write.c @@ -5271,6 +5271,10 @@ int sqlite3Fts3UpdateMethod( int nChng = 0; /* Net change in number of documents */ int bInsertDone = 0; + /* At this point it must be known if the %_stat table exists or not. + ** So bHasStat may not be 2. */ + assert( p->bHasStat==0 || p->bHasStat==1 ); + assert( p->pSegments==0 ); assert( nArg==1 /* DELETE operations */ diff --git a/manifest b/manifest index 8dd646f40a..313f580784 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Prevent\sthe\srtree\smodule\sfrom\sreading\ssqlite_stat1\sdata\sfrom\sthe\swrong\sdatabase. -D 2014-03-12T12:44:46.777 +C Changes\sto\sFTS\sto\sensure\sthat\sit\sdoes\snot\saccess\sthe\sdatabase\sfrom\swithin\sthe\sxConnect\smethod. +D 2014-03-12T19:20:36.904 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -78,9 +78,9 @@ F ext/fts3/README.content fdc666a70d5257a64fee209f97cf89e0e6e32b51 F ext/fts3/README.syntax a19711dc5458c20734b8e485e75fb1981ec2427a F ext/fts3/README.tokenizers e0a8b81383ea60d0334d274fadf305ea14a8c314 F ext/fts3/README.txt 8c18f41574404623b76917b9da66fcb0ab38328d -F ext/fts3/fts3.c 3fe91e36a0304ad4b35020f0e22ff37e95873166 +F ext/fts3/fts3.c 41b1920b9a8657963f09cb93b208c2671c5568db F ext/fts3/fts3.h 3a10a0af180d502cecc50df77b1b22df142817fe -F ext/fts3/fts3Int.h eb5f8029589f3d8f1dc7fd50c773326a640388b1 +F ext/fts3/fts3Int.h bdeb9015405e8facffb8fc7e09174521a2a780f4 F ext/fts3/fts3_aux.c 5c211e17a64885faeb16b9ba7772f9d5445c2365 F ext/fts3/fts3_expr.c 5165c365cb5a035f5be8bb296f7aa3211d43e4ac F ext/fts3/fts3_hash.c 29b986e43f4e9dd40110eafa377dc0d63c422c60 @@ -96,7 +96,7 @@ F ext/fts3/fts3_tokenizer.h 64c6ef6c5272c51ebe60fc607a896e84288fcbc3 F ext/fts3/fts3_tokenizer1.c 5c98225a53705e5ee34824087478cf477bdb7004 F ext/fts3/fts3_unicode.c 92391b4b4fb043564c6539ea9b8661e3bcba47b9 F ext/fts3/fts3_unicode2.c 0113d3acf13429e6dc38e0647d1bc71211c31a4d -F ext/fts3/fts3_write.c cdebe72171a217b1465032dec103f8d17f7dad4d +F ext/fts3/fts3_write.c 74c00329006c3ed6325ba4e5ab7c9b5fc99c8934 F ext/fts3/fts3speed.tcl b54caf6a18d38174f1a6e84219950d85e98bb1e9 F ext/fts3/mkfts3amal.tcl 252ecb7fe6467854f2aa237bf2c390b74e71f100 F ext/fts3/tool/fts3view.c 6cfc5b67a5f0e09c0d698f9fd012c784bfaa9197 @@ -535,7 +535,7 @@ F test/fts3conf.test ee8500c86dd58ec075e8831a1e216a79989436de F test/fts3corrupt.test 2710b77983cc7789295ddbffea52c1d3b7506dbb F test/fts3corrupt2.test 6d96efae2f8a6af3eeaf283aba437e6d0e5447ba F test/fts3cov.test e0fb00d8b715ddae4a94c305992dfc3ef70353d7 -F test/fts3d.test c8a193513a269ec4c205ca09645e26e0bc71b860 +F test/fts3d.test 597b0b76e41f0d672e2731c4d7b631d628efd13f F test/fts3defer.test 0be4440b73a2e651fc1e472066686d6ada4b9963 F test/fts3defer2.test a3b6cbeabaf28c9398652a4d101ea224d9358479 F test/fts3defer3.test dd53fc13223c6d8264a98244e9b19abd35ed71cd @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 018d317b1257ce68a92908b05c9c7cf1494050d0 -R 771148b6f32577641865e57d520a643c +P 7ce03c1b5552d830300575c5b41a874db7a2ec77 +R deefc2d49a0ad1f14f370fa09b6f4e09 U dan -Z 84dfe23ccda2d3e1d846c6ac2ce685a0 +Z 287a2bd2233940be3dc25c35e9cb956e diff --git a/manifest.uuid b/manifest.uuid index b36a223a45..1a5538a0a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ce03c1b5552d830300575c5b41a874db7a2ec77 \ No newline at end of file +c67a52c356416cc200f51f9168d942e78db11d34 \ No newline at end of file diff --git a/test/fts3d.test b/test/fts3d.test index 818456bfae..2914818d4e 100644 --- a/test/fts3d.test +++ b/test/fts3d.test @@ -354,6 +354,17 @@ do_test fts3d-6.5 { SELECT name FROM sqlite_master WHERE name GLOB '???_*' ORDER BY 1; } } {xyz_content xyz_segdir xyz_segments} + +# ALTER TABLE RENAME on an FTS3 table following an incr-merge op. +# +do_test fts3d-6.6 { + execsql { INSERT INTO xyz(xyz) VALUES('merge=2,2') } + sqlite3 db test.db + execsql { + ALTER TABLE xyz RENAME TO ott; + SELECT name FROM sqlite_master WHERE name GLOB '???_*' ORDER BY 1; + } +} {ott_content ott_segdir ott_segments ott_stat} finish_test From d88e521f591a447cc13b377fcbd078ed435ebdb8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 12 Mar 2014 19:38:38 +0000 Subject: [PATCH 03/49] Fix an obscure bug causing sqlite3_close() to fail if there are virtual tables on the disconnect list when it is called. FossilOrigin-Name: 6504aa47a8ebb13827be017c4cb4b2dc3c4c55f4 --- ext/rtree/rtree.c | 2 ++ ext/rtree/rtreeC.test | 26 ++++++++++++++++++++++++ manifest | 20 +++++++++--------- manifest.uuid | 2 +- src/main.c | 1 + test/fts3ao.test | 28 ++++++++++++++++++++++++++ test/vtab_shared.test | 47 +++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 115 insertions(+), 11 deletions(-) diff --git a/ext/rtree/rtree.c b/ext/rtree/rtree.c index cd652a3936..cefb9a8b2a 100644 --- a/ext/rtree/rtree.c +++ b/ext/rtree/rtree.c @@ -3241,6 +3241,8 @@ static int rtreeInit( if( rc==SQLITE_OK ){ *ppVtab = (sqlite3_vtab *)pRtree; }else{ + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); rtreeRelease(pRtree); } return rc; diff --git a/ext/rtree/rtreeC.test b/ext/rtree/rtreeC.test index b72007245f..23dc607841 100644 --- a/ext/rtree/rtreeC.test +++ b/ext/rtree/rtreeC.test @@ -243,6 +243,32 @@ do_eqp_test 5.8 { 0 1 1 {SCAN TABLE rt VIRTUAL TABLE INDEX 1:} } +#-------------------------------------------------------------------- +# Test that having a second connection drop the sqlite_stat1 table +# before it is required by rtreeConnect() does not cause problems. +# +ifcapable rtree { + reset_db + do_execsql_test 6.1 { + CREATE TABLE t1(x); + CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2); + INSERT INTO t1 VALUES(1); + INSERT INTO rt VALUES(1,2,3); + ANALYZE; + } + db close + sqlite3 db test.db + do_execsql_test 6.2 { SELECT * FROM t1 } {1} + + do_test 6.3 { + sqlite3 db2 test.db + db2 eval { DROP TABLE sqlite_stat1 } + db2 close + execsql { SELECT * FROM rt } + } {1 2.0 3.0} + db close +} + finish_test diff --git a/manifest b/manifest index 313f580784..66dfec84b5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Changes\sto\sFTS\sto\sensure\sthat\sit\sdoes\snot\saccess\sthe\sdatabase\sfrom\swithin\sthe\sxConnect\smethod. -D 2014-03-12T19:20:36.904 +C Fix\san\sobscure\sbug\scausing\ssqlite3_close()\sto\sfail\sif\sthere\sare\svirtual\stables\son\sthe\sdisconnect\slist\swhen\sit\sis\scalled. +D 2014-03-12T19:38:38.004 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -120,7 +120,7 @@ F ext/misc/vfslog.c fe40fab5c077a40477f7e5eba994309ecac6cc95 F ext/misc/vtshim.c babb0dc2bf116029e3e7c9a618b8a1377045303e F ext/misc/wholenumber.c 784b12543d60702ebdd47da936e278aa03076212 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761 -F ext/rtree/rtree.c d963b961f7ebb3842a6dbc959ed8c3f3534206c4 +F ext/rtree/rtree.c 2d9f95da404d850474e628c720c5ce15d29b47de F ext/rtree/rtree.h 834dbcb82dc85b2481cde6a07cdadfddc99e9b9e F ext/rtree/rtree1.test cf679265ecafff494a768ac9c2f43a70915a6290 F ext/rtree/rtree2.test acbb3a4ce0f4fbc2c304d2b4b784cfa161856bba @@ -133,7 +133,7 @@ F ext/rtree/rtree8.test db79c812f9e4a11f9b1f3f9934007884610a713a F ext/rtree/rtree9.test d86ebf08ff6328895613ed577dd8a2a37c472c34 F ext/rtree/rtreeA.test ace05e729a36e342d40cf94e9efc7b4723d9dcdf F ext/rtree/rtreeB.test 983e567b49b5dca165940f66b87e161aa30e82b2 -F ext/rtree/rtreeC.test afe930ac948c79fd12e48691c478d3f3bfc763d8 +F ext/rtree/rtreeC.test 16d7aa86ecb6a876d2a38cf590a1471a41b3a46d F ext/rtree/rtreeD.test 636630357638f5983701550b37f0f5867130d2ca F ext/rtree/rtree_perf.tcl 6c18c1f23cd48e0f948930c98dfdd37dfccb5195 F ext/rtree/rtree_util.tcl 06aab2ed5b826545bf215fff90ecb9255a8647ea @@ -186,7 +186,7 @@ F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b F src/loadext.c 867c7b330b740c6c917af9956b13b81d0a048303 -F src/main.c e054917b1beb3081b0f23e8bdd3d6c0e12933dd3 +F src/main.c 691b25754bef596108fe60ff1bcbe8445369c9db F src/malloc.c 0203ebce9152c6a0e5de520140b8ba65187350be F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b @@ -523,7 +523,7 @@ F test/fts3ak.test bd14deafe9d1586e8e9bf032411026ac4f8c925d F test/fts3al.test 07d64326e79bbdbab20ee87fc3328fbf01641c9f F test/fts3am.test 218aa6ba0dfc50c7c16b2022aac5c6be593d08d8 F test/fts3an.test a49ccadc07a2f7d646ec1b81bc09da2d85a85b18 -F test/fts3ao.test 71b0675e3df5c512a5a03daaa95ee1916de23dda +F test/fts3ao.test 3e4e3d5e75c076520341d0bdf4eb17c00e8cbde2 F test/fts3atoken.test fca30fd86db9241d571c637751e9a8a2f50f1451 F test/fts3auto.test b981fea19b132b4e6878f50d7c1f369b28f68eb9 F test/fts3aux1.test f8f287a4a73f381f8fa15b6a70f36245f903d221 @@ -1047,7 +1047,7 @@ F test/vtabE.test 7c4693638d7797ce2eda17af74292b97e705cc61 F test/vtabF.test fd5ad376f5a34fe0891df1f3cddb4fe7c3eb077e F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5 F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8 -F test/vtab_shared.test 6acafaae7126c9f49be72c2f57eb8bef3024f1cb +F test/vtab_shared.test ea8778d5b0df200adef2ca7c00c3c37d4375f772 F test/wal.test 40073e54359d43354925b2b700b7eebd5e207285 F test/wal2.test a8e3963abf6b232cf0b852b09b53665ef34007af F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7ce03c1b5552d830300575c5b41a874db7a2ec77 -R deefc2d49a0ad1f14f370fa09b6f4e09 +P c67a52c356416cc200f51f9168d942e78db11d34 +R 675a28e896dc8931c7ccad371f04b002 U dan -Z 287a2bd2233940be3dc25c35e9cb956e +Z 867ad6b65f1e7902a3a764dd985f4266 diff --git a/manifest.uuid b/manifest.uuid index 1a5538a0a1..7a533b2f8c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c67a52c356416cc200f51f9168d942e78db11d34 \ No newline at end of file +6504aa47a8ebb13827be017c4cb4b2dc3c4c55f4 \ No newline at end of file diff --git a/src/main.c b/src/main.c index 8b7495dd49..9e83d4963a 100644 --- a/src/main.c +++ b/src/main.c @@ -800,6 +800,7 @@ static void disconnectAllVtab(sqlite3 *db){ } } } + sqlite3VtabUnlockList(db); sqlite3BtreeLeaveAll(db); #else UNUSED_PARAMETER(db); diff --git a/test/fts3ao.test b/test/fts3ao.test index 91693094db..60f0aa7d90 100644 --- a/test/fts3ao.test +++ b/test/fts3ao.test @@ -219,4 +219,32 @@ do_execsql_test 5.2 { SELECT count(*) FROM sqlite_master WHERE name LIKE 't8%'; } {0 6} +# At one point this was causing a memory leak. +# +foreach {tn sql} { + 1 {} + 2 { INSERT INTO ft(ft) VALUES('merge=2,2'); } +} { + reset_db + do_execsql_test 6.$tn.1 " + CREATE TABLE t1(x); + CREATE VIRTUAL TABLE ft USING fts3; + INSERT INTO ft VALUES('hello world'); + $sql + " + + db close + sqlite3 db test.db + do_execsql_test 6.$tn.2 { SELECT * FROM t1 } {} + + do_test 6.$tn.3 { + sqlite3 db2 test.db + db2 eval { DROP TABLE t1 } + db2 close + set stmt [sqlite3_prepare db { SELECT * FROM ft } -1 dummy] + sqlite3_finalize $stmt + } {SQLITE_OK} + db close +} + finish_test diff --git a/test/vtab_shared.test b/test/vtab_shared.test index 13237a8d20..3473992900 100644 --- a/test/vtab_shared.test +++ b/test/vtab_shared.test @@ -15,6 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +set testprefix vtab_shared ifcapable !vtab||!shared_cache { finish_test @@ -228,5 +229,51 @@ do_test vtab_shared_1.15.3 { db close db2 close + +#--------------------------------------------------------------- +# Test calling sqlite3_close() with vtabs on the disconnect list. +# +ifcapable rtree { + reset_db + do_test 2.1.1 { + sqlite3 db test.db + sqlite3 db2 test.db + + # Create a virtual table using [db]. + execsql { + CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2); + INSERT INTO rt VALUES(1, 2 ,3); + SELECT * FROM rt; + } + + # Drop the virtual table using [db2]. The sqlite3_vtab object belonging + # to [db] is moved to the sqlite3.pDisconnect list. + execsql { DROP TABLE rt } db2 + + # Immediately close [db]. At one point this would fail due to the + # unfinalized statements held by the un-xDisconnect()ed sqlite3_vtab. + db close + } {} + db2 close +} + +ifcapable fts3 { + # Same test as above, except using fts3 instead of rtree. + reset_db + do_test 2.2.1 { + sqlite3 db test.db + sqlite3 db2 test.db + execsql { + CREATE VIRTUAL TABLE ft USING fts3; + INSERT INTO ft VALUES('hello world'); + SELECT * FROM ft; + } + execsql { DROP TABLE ft } db2 + db close + } {} + db2 close +} + sqlite3_enable_shared_cache 0 finish_test + From 18c7e40ca3de1a1d10548f07d7cdb73db8abbbf1 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 14 Mar 2014 11:46:10 +0000 Subject: [PATCH 04/49] Fix a harmless compiler warning that crops up with SQLITE_MAX_MMAP_SIZE=0. FossilOrigin-Name: 1277932b7e8bb36c7070ffafbf110a8e128c267b --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 2 ++ src/btree.h | 4 +++- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 66dfec84b5..60f7a4b93f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sobscure\sbug\scausing\ssqlite3_close()\sto\sfail\sif\sthere\sare\svirtual\stables\son\sthe\sdisconnect\slist\swhen\sit\sis\scalled. -D 2014-03-12T19:38:38.004 +C Fix\sa\sharmless\scompiler\swarning\sthat\scrops\sup\swith\sSQLITE_MAX_MMAP_SIZE=0. +D 2014-03-14T11:46:10.508 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,8 +164,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 4d28fb15543f0e071b1780b2af8cd2ee489de32d -F src/btree.h 9e0f97c01b972f779eb7655cfb4f8727fd6dc26f +F src/btree.c 2a0e73f26c329f49e426237e71a879ffd205d778 +F src/btree.h da1b69b441ecee21f8b34ba73f0ae339540c4025 F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c67a52c356416cc200f51f9168d942e78db11d34 -R 675a28e896dc8931c7ccad371f04b002 -U dan -Z 867ad6b65f1e7902a3a764dd985f4266 +P 6504aa47a8ebb13827be017c4cb4b2dc3c4c55f4 +R afd560ffe96625fe4d1d6971544be730 +U drh +Z 413087b4f5e9f622f8abe560a87a95ed diff --git a/manifest.uuid b/manifest.uuid index 7a533b2f8c..04c2fe3e5e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6504aa47a8ebb13827be017c4cb4b2dc3c4c55f4 \ No newline at end of file +1277932b7e8bb36c7070ffafbf110a8e128c267b \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 29ead1c67a..9ca600e8cb 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2161,6 +2161,7 @@ int sqlite3BtreeSetCacheSize(Btree *p, int mxPage){ return SQLITE_OK; } +#if SQLITE_MAX_MMAP_SIZE>0 /* ** Change the limit on the amount of the database file that may be ** memory mapped. @@ -2173,6 +2174,7 @@ int sqlite3BtreeSetMmapLimit(Btree *p, sqlite3_int64 szMmap){ sqlite3BtreeLeave(p); return SQLITE_OK; } +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ /* ** Change the way data is synced to disk in order to increase or decrease diff --git a/src/btree.h b/src/btree.h index eda7bef70a..0c3fdfad63 100644 --- a/src/btree.h +++ b/src/btree.h @@ -63,7 +63,9 @@ int sqlite3BtreeOpen( int sqlite3BtreeClose(Btree*); int sqlite3BtreeSetCacheSize(Btree*,int); -int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); +#if SQLITE_MAX_MMAP_SIZE>0 + int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); +#endif int sqlite3BtreeSetPagerFlags(Btree*,unsigned); int sqlite3BtreeSyncDisabled(Btree*); int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); From a70a073b3fb789be80ec0c6bbbaeba2e67ee9a22 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Mar 2014 14:24:27 +0000 Subject: [PATCH 05/49] Performance enhancements and size reduction for sqlite3VXPrintf() FossilOrigin-Name: eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 --- manifest | 12 ++-- manifest.uuid | 2 +- src/printf.c | 155 ++++++++++++++++++++++++++------------------------ 3 files changed, 89 insertions(+), 80 deletions(-) diff --git a/manifest b/manifest index 60f7a4b93f..7a2c6867be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning\sthat\scrops\sup\swith\sSQLITE_MAX_MMAP_SIZE=0. -D 2014-03-14T11:46:10.508 +C Performance\senhancements\sand\ssize\sreduction\sfor\ssqlite3VXPrintf() +D 2014-03-17T14:24:27.523 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c F src/pragma.c e78b4bf2a267de2c17ee09f90b6807cf8d40e6a3 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 -F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269 +F src/printf.c d95ea1b8a1e825a2f4756322f2ce35f91289fa1a F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 6504aa47a8ebb13827be017c4cb4b2dc3c4c55f4 -R afd560ffe96625fe4d1d6971544be730 +P 1277932b7e8bb36c7070ffafbf110a8e128c267b +R 6d92934790c660d014aba176737edbd6 U drh -Z 413087b4f5e9f622f8abe560a87a95ed +Z c25cfd9c876d8aae4bc2be28c38a9b40 diff --git a/manifest.uuid b/manifest.uuid index 04c2fe3e5e..5f5b8ba0b3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1277932b7e8bb36c7070ffafbf110a8e128c267b \ No newline at end of file +eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index 8cfa542b41..a5a3fbbe0c 100644 --- a/src/printf.c +++ b/src/printf.c @@ -134,20 +134,6 @@ static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ } #endif /* SQLITE_OMIT_FLOATING_POINT */ -/* -** Append N space characters to the given string buffer. -*/ -void sqlite3AppendSpace(StrAccum *pAccum, int N){ - static const char zSpaces[] = " "; - while( N>=(int)sizeof(zSpaces)-1 ){ - sqlite3StrAccumAppend(pAccum, zSpaces, sizeof(zSpaces)-1); - N -= sizeof(zSpaces)-1; - } - if( N>0 ){ - sqlite3StrAccumAppend(pAccum, zSpaces, N); - } -} - /* ** Set the StrAccum object to an error mode. */ @@ -237,11 +223,9 @@ void sqlite3VXPrintf( } for(; (c=(*fmt))!=0; ++fmt){ if( c!='%' ){ - int amt; bufpt = (char *)fmt; - amt = 1; - while( (c=(*++fmt))!='%' && c!=0 ) amt++; - sqlite3StrAccumAppend(pAccum, bufpt, amt); + while( (c=(*++fmt))!='%' && c!=0 ){}; + sqlite3StrAccumAppend(pAccum, bufpt, (int)(fmt - bufpt)); if( c==0 ) break; } if( (c=(*++fmt))==0 ){ @@ -729,29 +713,89 @@ void sqlite3VXPrintf( ** "length" characters long. The field width is "width". Do ** the output. */ - if( !flag_leftjustify ){ - register int nspace; - nspace = width-length; - if( nspace>0 ){ - sqlite3AppendSpace(pAccum, nspace); - } - } - if( length>0 ){ - sqlite3StrAccumAppend(pAccum, bufpt, length); - } - if( flag_leftjustify ){ - register int nspace; - nspace = width-length; - if( nspace>0 ){ - sqlite3AppendSpace(pAccum, nspace); - } - } + width -= length; + if( width>0 && !flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + sqlite3StrAccumAppend(pAccum, bufpt, length); + if( width>0 && flag_leftjustify ) sqlite3AppendSpace(pAccum, width); + if( zExtra ) sqlite3_free(zExtra); }/* End for loop over the format string */ } /* End of function */ /* -** Append N bytes of text from z to the StrAccum object. +** Enlarge the memory allocation on a StrAccum object so that it is +** able to accept at least N more bytes of text. +** +** Return the number of bytes of text that StrAccum is able to accept +** after the attempted enlargement. The value returned might be zero. +*/ +static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ + char *zNew; + assert( p->nChar+N >= p->nAlloc ); /* Only called if really needed */ + if( p->accError ){ + testcase(p->accError==STRACCUM_TOOBIG); + testcase(p->accError==STRACCUM_NOMEM); + return 0; + } + if( !p->useMalloc ){ + N = p->nAlloc - p->nChar - 1; + setStrAccumError(p, STRACCUM_TOOBIG); + return N; + }else{ + char *zOld = (p->zText==p->zBase ? 0 : p->zText); + i64 szNew = p->nChar; + szNew += N + 1; + if( szNew > p->mxAlloc ){ + sqlite3StrAccumReset(p); + setStrAccumError(p, STRACCUM_TOOBIG); + return 0; + }else{ + p->nAlloc = (int)szNew; + } + if( p->useMalloc==1 ){ + zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); + }else{ + zNew = sqlite3_realloc(zOld, p->nAlloc); + } + if( zNew ){ + if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); + p->zText = zNew; + }else{ + sqlite3StrAccumReset(p); + setStrAccumError(p, STRACCUM_NOMEM); + return 0; + } + } + return N; +} + +/* +** Append N space characters to the given string buffer. +*/ +void sqlite3AppendSpace(StrAccum *p, int N){ + if( p->nChar+N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ) return; + while( (N--)>0 ) p->zText[p->nChar++] = ' '; +} + +/* +** The StrAccum "p" is not large enough to accept N new bytes of z[]. +** So enlarge if first, then do the append. +** +** This is a helper routine to sqlite3StrAccumAppend() that does special-case +** work (enlarging the buffer) using tail recursion, so that the +** sqlite3StrAccumAppend() routine can use fast calling semantics. +*/ +static void enlargeAndAppend(StrAccum *p, const char *z, int N){ + N = sqlite3StrAccumEnlarge(p, N); + if( N>0 ){ + memcpy(&p->zText[p->nChar], z, N); + p->nChar += N; + } +} + +/* +** Append N bytes of text from z to the StrAccum object. Increase the +** size of the memory allocation for StrAccum if necessary. */ void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ assert( z!=0 ); @@ -759,43 +803,8 @@ void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ assert( N>=0 ); assert( p->accError==0 || p->nAlloc==0 ); if( p->nChar+N >= p->nAlloc ){ - char *zNew; - if( p->accError ){ - testcase(p->accError==STRACCUM_TOOBIG); - testcase(p->accError==STRACCUM_NOMEM); - return; - } - if( !p->useMalloc ){ - N = p->nAlloc - p->nChar - 1; - setStrAccumError(p, STRACCUM_TOOBIG); - if( N<=0 ){ - return; - } - }else{ - char *zOld = (p->zText==p->zBase ? 0 : p->zText); - i64 szNew = p->nChar; - szNew += N + 1; - if( szNew > p->mxAlloc ){ - sqlite3StrAccumReset(p); - setStrAccumError(p, STRACCUM_TOOBIG); - return; - }else{ - p->nAlloc = (int)szNew; - } - if( p->useMalloc==1 ){ - zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); - }else{ - zNew = sqlite3_realloc(zOld, p->nAlloc); - } - if( zNew ){ - if( zOld==0 && p->nChar>0 ) memcpy(zNew, p->zText, p->nChar); - p->zText = zNew; - }else{ - sqlite3StrAccumReset(p); - setStrAccumError(p, STRACCUM_NOMEM); - return; - } - } + enlargeAndAppend(p,z,N); + return; } assert( p->zText ); memcpy(&p->zText[p->nChar], z, N); From 0e682099a1ee1bf0b63937834b952f77ec55f1b5 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 17 Mar 2014 15:06:57 +0000 Subject: [PATCH 06/49] Clean up some obsolete "register" declarations in printf.c. FossilOrigin-Name: ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/printf.c | 6 ++---- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 7a2c6867be..bb21639801 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Performance\senhancements\sand\ssize\sreduction\sfor\ssqlite3VXPrintf() -D 2014-03-17T14:24:27.523 +C Clean\sup\ssome\sobsolete\s"register"\sdeclarations\sin\sprintf.c. +D 2014-03-17T15:06:57.359 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -213,7 +213,7 @@ F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c F src/pragma.c e78b4bf2a267de2c17ee09f90b6807cf8d40e6a3 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 -F src/printf.c d95ea1b8a1e825a2f4756322f2ce35f91289fa1a +F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 1277932b7e8bb36c7070ffafbf110a8e128c267b -R 6d92934790c660d014aba176737edbd6 +P eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 +R f35820126ad6a1c360258186873a2e79 U drh -Z c25cfd9c876d8aae4bc2be28c38a9b40 +Z b6382160ccdf74a44bc9653763bb82ef diff --git a/manifest.uuid b/manifest.uuid index 5f5b8ba0b3..3e3e1d6ac7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 \ No newline at end of file +ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 \ No newline at end of file diff --git a/src/printf.c b/src/printf.c index a5a3fbbe0c..01209c042d 100644 --- a/src/printf.c +++ b/src/printf.c @@ -406,10 +406,8 @@ void sqlite3VXPrintf( *(--bufpt) = zOrd[x*2]; } { - register const char *cset; /* Use registers for speed */ - register int base; - cset = &aDigits[infop->charset]; - base = infop->base; + const char *cset = &aDigits[infop->charset]; + u8 base = infop->base; do{ /* Convert to ascii */ *(--bufpt) = cset[longvalue%base]; longvalue = longvalue/base; From 0401acec910236ec9a7d9758e00a62cdebf684b8 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Mar 2014 15:30:27 +0000 Subject: [PATCH 07/49] Experiments with the optimization of ORDER BY and GROUP BY clauses. FossilOrigin-Name: b150902579d708b454efd5f8750e26a816f7f1a6 --- manifest | 17 +++++++----- manifest.uuid | 2 +- src/where.c | 72 ++++++++++++++++++++++++-------------------------- src/whereInt.h | 5 ++-- 4 files changed, 47 insertions(+), 49 deletions(-) diff --git a/manifest b/manifest index bb21639801..6f347f2fef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\ssome\sobsolete\s"register"\sdeclarations\sin\sprintf.c. -D 2014-03-17T15:06:57.359 +C Experiments\swith\sthe\soptimization\sof\sORDER\sBY\sand\sGROUP\sBY\sclauses. +D 2014-03-18T15:30:27.660 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,8 +291,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c bb50b5aed4f9b2284eb92c944253e60df2fb8259 -F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 +F src/where.c 222e6ed521c854887850e65a131d41fd59c28cf2 +F src/whereInt.h daa3cdf9f56c1dd12b63e4eda3182e64e3bba14f F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1156,7 +1156,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 -R f35820126ad6a1c360258186873a2e79 +P ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 +R 8abedb680a805ab9b5e400a1ac18119b +T *branch * orderby-planning +T *sym-orderby-planning * +T -sym-trunk * U drh -Z b6382160ccdf74a44bc9653763bb82ef +Z 6c19349f1c38f37cb3b2bffb6095d2a0 diff --git a/manifest.uuid b/manifest.uuid index 3e3e1d6ac7..26699b8468 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 \ No newline at end of file +b150902579d708b454efd5f8750e26a816f7f1a6 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6cd9c167a1..bed58ac752 100644 --- a/src/where.c +++ b/src/where.c @@ -4506,8 +4506,8 @@ static int whereLoopAddVirtual( pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; pIdxInfo->needToFreeIdxStr = 0; pNew->u.vtab.idxStr = pIdxInfo->idxStr; - pNew->u.vtab.isOrdered = (u8)((pIdxInfo->nOrderBy!=0) - && pIdxInfo->orderByConsumed); + pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? + pIdxInfo->nOrderBy : 0); pNew->rSetup = 0; pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); @@ -4668,11 +4668,11 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ /* ** Examine a WherePath (with the addition of the extra WhereLoop of the 5th ** parameters) to see if it outputs rows in the requested ORDER BY -** (or GROUP BY) without requiring a separate sort operation. Return: +** (or GROUP BY) without requiring a separate sort operation. Return N: ** -** 0: ORDER BY is not satisfied. Sorting required -** 1: ORDER BY is satisfied. Omit sorting -** -1: Unknown at this time +** N>0: N terms of the ORDER BY clause are satisfied +** N==0: No terms of the ORDER BY clause are satisfied +** N<0: Unknown yet how many terms of ORDER BY might be satisfied. ** ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as ** strict. With GROUP BY and DISTINCT the only requirement is that @@ -4682,7 +4682,7 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ ** the pOrderBy terms can be matched in any order. With ORDER BY, the ** pOrderBy terms must be matched in strict left-to-right order. */ -static int wherePathSatisfiesOrderBy( +static i8 wherePathSatisfiesOrderBy( WhereInfo *pWInfo, /* The WHERE clause */ ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ WherePath *pPath, /* The WherePath to check */ @@ -4915,7 +4915,7 @@ static int wherePathSatisfiesOrderBy( } } } /* End the loop over all WhereLoops from outer-most down to inner-most */ - if( obSat==obDone ) return 1; + if( obSat==obDone ) return nOrderBy; if( !isOrderDistinct ) return 0; return -1; } @@ -5001,12 +5001,13 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** to sqlite3WhereBegin() was concerned about sorting */ rSortCost = 0; if( pWInfo->pOrderBy==0 || nRowEst==0 ){ - aFrom[0].isOrderedValid = 1; + aFrom[0].isOrdered = 0; }else{ /* TUNING: Estimated cost of sorting is 48*N*log2(N) where N is the ** number of output rows. The 48 is the expected size of a row to sort. ** FIXME: compute a better estimate of the 48 multiplier based on the ** result set expressions. */ + aFrom[0].isOrdered = -1; rSortCost = nRowEst + estLog(nRowEst); WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost)); } @@ -5020,8 +5021,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ Bitmask maskNew; Bitmask revMask = 0; - u8 isOrderedValid = pFrom->isOrderedValid; - u8 isOrdered = pFrom->isOrdered; + i8 isOrdered = pFrom->isOrdered; if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; /* At this point, pWLoop is a candidate to be the next loop. @@ -5030,21 +5030,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ rCost = sqlite3LogEstAdd(rCost, pFrom->rCost); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; - if( !isOrderedValid ){ - switch( wherePathSatisfiesOrderBy(pWInfo, + if( isOrdered<0 ){ + isOrdered = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, - iLoop, pWLoop, &revMask) ){ - case 1: /* Yes. pFrom+pWLoop does satisfy the ORDER BY clause */ - isOrdered = 1; - isOrderedValid = 1; - break; - case 0: /* No. pFrom+pWLoop will require a separate sort */ - isOrdered = 0; - isOrderedValid = 1; - rCost = sqlite3LogEstAdd(rCost, rSortCost); - break; - default: /* Cannot tell yet. Try again on the next iteration */ - break; + iLoop, pWLoop, &revMask); + if( isOrdered==0 ){ + rCost = sqlite3LogEstAdd(rCost, rSortCost); } }else{ revMask = pFrom->revLoop; @@ -5052,7 +5043,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Check to see if pWLoop should be added to the mxChoice best so far */ for(jj=0, pTo=aTo; jjmaskLoop==maskNew - && pTo->isOrderedValid==isOrderedValid + && ((pTo->isOrdered^isOrdered)&80)==0 && ((pTo->rCost<=rCost && pTo->nRow<=nOut) || (pTo->rCost>=rCost && pTo->nRow>=nOut)) ){ @@ -5066,7 +5057,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n", wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + isOrdered>=0 ? isOrdered+'0' : '?'); } #endif continue; @@ -5084,7 +5075,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf("New %s cost=%-3d,%3d order=%c\n", wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + isOrdered>=0 ? isOrdered+'0' : '?'); } #endif }else{ @@ -5094,10 +5085,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ sqlite3DebugPrintf( "Skip %s cost=%-3d,%3d order=%c", wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + isOrdered>=0 ? isOrdered+'0' : '?'); sqlite3DebugPrintf(" vs %s cost=%-3d,%d order=%c\n", wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); + pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); } #endif testcase( pTo->rCost==rCost ); @@ -5110,10 +5101,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ sqlite3DebugPrintf( "Update %s cost=%-3d,%3d order=%c", wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, - isOrderedValid ? (isOrdered ? 'Y' : 'N') : '?'); + isOrdered>=0 ? isOrdered+'0' : '?'); sqlite3DebugPrintf(" was %s cost=%-3d,%3d order=%c\n", wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); + pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); } #endif } @@ -5122,7 +5113,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pTo->revLoop = revMask; pTo->nRow = nOut; pTo->rCost = rCost; - pTo->isOrderedValid = isOrderedValid; pTo->isOrdered = isOrdered; memcpy(pTo->aLoop, pFrom->aLoop, sizeof(WhereLoop*)*iLoop); pTo->aLoop[iLoop] = pWLoop; @@ -5147,8 +5137,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ for(ii=0, pTo=aTo; iirCost, pTo->nRow, - pTo->isOrderedValid ? (pTo->isOrdered ? 'Y' : 'N') : '?'); - if( pTo->isOrderedValid && pTo->isOrdered ){ + pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); + if( pTo->isOrdered>0 ){ sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); }else{ sqlite3DebugPrintf("\n"); @@ -5191,9 +5181,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ Bitmask notUsed; int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); - if( rc==1 ) pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + if( rc==pWInfo->pResultSet->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } } - if( pFrom->isOrdered ){ + if( pWInfo->pOrderBy && pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; }else{ @@ -5386,7 +5378,7 @@ WhereInfo *sqlite3WhereBegin( Parse *pParse, /* The parser context */ SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ - ExprList *pOrderBy, /* An ORDER BY clause, or NULL */ + ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ ExprList *pResultSet, /* Result set of the query */ u16 wctrlFlags, /* One of the WHERE_* flags defined in sqliteInt.h */ int iIdxCur /* If WHERE_ONETABLE_ONLY is set, index cursor number */ @@ -5408,6 +5400,10 @@ WhereInfo *sqlite3WhereBegin( /* Variable initialization */ db = pParse->db; memset(&sWLB, 0, sizeof(sWLB)); + + /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ + testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); + if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; sWLB.pOrderBy = pOrderBy; /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via diff --git a/src/whereInt.h b/src/whereInt.h index 23f929c461..233b331d42 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -121,7 +121,7 @@ struct WhereLoop { struct { /* Information for virtual tables */ int idxNum; /* Index number */ u8 needFree; /* True if sqlite3_free(idxStr) is needed */ - u8 isOrdered; /* True if satisfies ORDER BY */ + i8 isOrdered; /* True if satisfies ORDER BY */ u16 omitMask; /* Terms that may be omitted */ char *idxStr; /* Index identifier string */ } vtab; @@ -183,8 +183,7 @@ struct WherePath { Bitmask revLoop; /* aLoop[]s that should be reversed for ORDER BY */ LogEst nRow; /* Estimated number of rows generated by this path */ LogEst rCost; /* Total cost of this path */ - u8 isOrdered; /* True if this path satisfies ORDER BY */ - u8 isOrderedValid; /* True if the isOrdered field is valid */ + i8 isOrdered; /* No. of ORDER BY terms satisfied. -1 for unknown */ WhereLoop **aLoop; /* Array of WhereLoop objects implementing this path */ }; From d2de861cbece884fe9eb9f033308f6ca21cd7bab Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 18 Mar 2014 18:59:07 +0000 Subject: [PATCH 08/49] Adjust the query planner to keep track of the number of ORDER BY terms satisfied. Still doesn't do anything with this information. Some tests fail after this check-in, but all failures are believed to be benign. The failures will be addressed at a later stage. FossilOrigin-Name: 59d49b7fc49fa290e04a02653e7268c85836b27e --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/where.c | 30 ++++++++++++++++++++---------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 6f347f2fef..cf41f09f0d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Experiments\swith\sthe\soptimization\sof\sORDER\sBY\sand\sGROUP\sBY\sclauses. -D 2014-03-18T15:30:27.660 +C Adjust\sthe\squery\splanner\sto\skeep\strack\sof\sthe\snumber\sof\sORDER\sBY\sterms\nsatisfied.\s\sStill\sdoesn't\sdo\sanything\swith\sthis\sinformation.\s\sSome\stests\nfail\safter\sthis\scheck-in,\sbut\sall\sfailures\sare\sbelieved\sto\sbe\sbenign.\s\sThe\nfailures\swill\sbe\saddressed\sat\sa\slater\sstage. +D 2014-03-18T18:59:07.798 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 222e6ed521c854887850e65a131d41fd59c28cf2 +F src/where.c 81eea5ced2a2586ccd7ebd1d5bed7d34f4a2e99c F src/whereInt.h daa3cdf9f56c1dd12b63e4eda3182e64e3bba14f F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1156,10 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 -R 8abedb680a805ab9b5e400a1ac18119b -T *branch * orderby-planning -T *sym-orderby-planning * -T -sym-trunk * +P b150902579d708b454efd5f8750e26a816f7f1a6 +R 179b4f0ee34acb2c0f356485d1e56bc3 U drh -Z 6c19349f1c38f37cb3b2bffb6095d2a0 +Z adae3618c374e493bb2483c0db46dcf7 diff --git a/manifest.uuid b/manifest.uuid index 26699b8468..6d8c042ba1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b150902579d708b454efd5f8750e26a816f7f1a6 \ No newline at end of file +59d49b7fc49fa290e04a02653e7268c85836b27e \ No newline at end of file diff --git a/src/where.c b/src/where.c index bed58ac752..a766def035 100644 --- a/src/where.c +++ b/src/where.c @@ -4916,7 +4916,13 @@ static i8 wherePathSatisfiesOrderBy( } } /* End the loop over all WhereLoops from outer-most down to inner-most */ if( obSat==obDone ) return nOrderBy; - if( !isOrderDistinct ) return 0; + if( !isOrderDistinct ){ + for(i=nOrderBy-1; i>0; i--){ + Bitmask m = MASKBIT(i) - 1; + if( (obSat&m)==m ) return i; + } + return 0; + } return -1; } @@ -4953,11 +4959,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int iLoop; /* Loop counter over the terms of the join */ int ii, jj; /* Loop counters */ int mxI = 0; /* Index of next entry to replace */ + int nOrderBy; /* Number of ORDER BY clause terms */ LogEst rCost; /* Cost of a path */ LogEst nOut; /* Number of outputs */ LogEst mxCost = 0; /* Maximum cost of a set of paths */ LogEst mxOut = 0; /* Maximum nOut value on the set of paths */ - LogEst rSortCost; /* Cost to do a sort */ int nTo, nFrom; /* Number of valid entries in aTo[] and aFrom[] */ WherePath *aFrom; /* All nFrom paths at the previous level */ WherePath *aTo; /* The nTo best paths at the current level */ @@ -4999,17 +5005,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Precompute the cost of sorting the final result set, if the caller ** to sqlite3WhereBegin() was concerned about sorting */ - rSortCost = 0; if( pWInfo->pOrderBy==0 || nRowEst==0 ){ aFrom[0].isOrdered = 0; + nOrderBy = 0; }else{ - /* TUNING: Estimated cost of sorting is 48*N*log2(N) where N is the - ** number of output rows. The 48 is the expected size of a row to sort. - ** FIXME: compute a better estimate of the 48 multiplier based on the - ** result set expressions. */ aFrom[0].isOrdered = -1; - rSortCost = nRowEst + estLog(nRowEst); - WHERETRACE(0x002,("---- sort cost=%-3d\n", rSortCost)); + nOrderBy = pWInfo->pOrderBy->nExpr; } /* Compute successively longer WherePaths using the previous generation @@ -5034,7 +5035,16 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ isOrdered = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); - if( isOrdered==0 ){ + if( isOrdered>=0 && isOrdered Date: Tue, 18 Mar 2014 20:33:42 +0000 Subject: [PATCH 09/49] Make the partial-ORDER-BY information in the query planner available to the SELECT code generator. Still doesn't make a difference in the generated code. FossilOrigin-Name: e258df236b7de70087c8227cb209080e55b9bf9c --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 25 ++++++++++++++----------- src/where.c | 17 ++++++++++------- src/whereInt.h | 2 +- 5 files changed, 34 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index cf41f09f0d..d76e572699 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sthe\squery\splanner\sto\skeep\strack\sof\sthe\snumber\sof\sORDER\sBY\sterms\nsatisfied.\s\sStill\sdoesn't\sdo\sanything\swith\sthis\sinformation.\s\sSome\stests\nfail\safter\sthis\scheck-in,\sbut\sall\sfailures\sare\sbelieved\sto\sbe\sbenign.\s\sThe\nfailures\swill\sbe\saddressed\sat\sa\slater\sstage. -D 2014-03-18T18:59:07.798 +C Make\sthe\spartial-ORDER-BY\sinformation\sin\sthe\squery\splanner\savailable\sto\sthe\nSELECT\scode\sgenerator.\s\sStill\sdoesn't\smake\sa\sdifference\sin\sthe\sgenerated\scode. +D 2014-03-18T20:33:42.067 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 0adf172d33cc610ff5ffe26edfd2ba67c3551655 +F src/select.c 41df0e4dd1d167dc8b39047c84e1b8b51516a237 F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -291,8 +291,8 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 81eea5ced2a2586ccd7ebd1d5bed7d34f4a2e99c -F src/whereInt.h daa3cdf9f56c1dd12b63e4eda3182e64e3bba14f +F src/where.c 3f5de18e74b0f6da3218c2c609616a2f9e2c404a +F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b150902579d708b454efd5f8750e26a816f7f1a6 -R 179b4f0ee34acb2c0f356485d1e56bc3 +P 59d49b7fc49fa290e04a02653e7268c85836b27e +R 2b69bb33bc67b11eec1f6a4a37941d3e U drh -Z adae3618c374e493bb2483c0db46dcf7 +Z 43f4f2d39f5cfe334420891461bd8dda diff --git a/manifest.uuid b/manifest.uuid index 6d8c042ba1..4ef359f854 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59d49b7fc49fa290e04a02653e7268c85836b27e \ No newline at end of file +e258df236b7de70087c8227cb209080e55b9bf9c \ No newline at end of file diff --git a/src/select.c b/src/select.c index 850bc6a902..613ea25e26 100644 --- a/src/select.c +++ b/src/select.c @@ -562,6 +562,7 @@ static void selectInnerLoop( ExprList *pEList, /* List of values being extracted */ int srcTab, /* Pull data from this table */ ExprList *pOrderBy, /* If not NULL, sort results using this key */ + int nOBSat, /* Terms of ORDER BY already satisfied */ DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ int iContinue, /* Jump here to continue with next row */ @@ -1054,10 +1055,10 @@ static void explainComposite( static void generateSortTail( Parse *pParse, /* Parsing context */ Select *p, /* The SELECT statement */ - Vdbe *v, /* Generate code into this VDBE */ int nColumn, /* Number of columns of data */ SelectDest *pDest /* Write the sorted results here */ ){ + Vdbe *v = pParse->pVdbe; /* The prepared statement */ int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; @@ -1918,7 +1919,7 @@ static void generateWithRecursiveQuery( addrCont = sqlite3VdbeMakeLabel(v); codeOffset(v, regOffset, addrCont); selectInnerLoop(pParse, p, p->pEList, iCurrent, - 0, 0, pDest, addrCont, addrBreak); + 0, 0, 0, pDest, addrCont, addrBreak); if( regLimit ){ sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); VdbeCoverage(v); @@ -2192,7 +2193,7 @@ static int multiSelect( sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); iStart = sqlite3VdbeCurrentAddr(v); selectInnerLoop(pParse, p, p->pEList, unionTab, - 0, 0, &dest, iCont, iBreak); + 0, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); @@ -2270,7 +2271,7 @@ static int multiSelect( sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, r1); selectInnerLoop(pParse, p, p->pEList, tab1, - 0, 0, &dest, iCont, iBreak); + 0, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); @@ -4730,6 +4731,7 @@ int sqlite3Select( if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); + int nOBSat; /* Begin the database scan. */ pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, p->pEList, @@ -4741,7 +4743,8 @@ int sqlite3Select( if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); } - if( pOrderBy && sqlite3WhereIsOrdered(pWInfo) ) pOrderBy = 0; + nOBSat = sqlite3WhereIsOrdered(pWInfo); + if( pOrderBy && nOBSat==pOrderBy->nExpr ){ pOrderBy = 0; nOBSat = 0; } /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral @@ -4753,7 +4756,7 @@ int sqlite3Select( } /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, -1, pOrderBy, &sDistinct, pDest, + selectInnerLoop(pParse, p, pEList, -1, pOrderBy, nOBSat, &sDistinct, pDest, sqlite3WhereContinueLabel(pWInfo), sqlite3WhereBreakLabel(pWInfo)); @@ -4875,7 +4878,7 @@ int sqlite3Select( pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, WHERE_GROUPBY, 0); if( pWInfo==0 ) goto select_end; - if( sqlite3WhereIsOrdered(pWInfo) ){ + if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be ** cancelled later because we still need to use the pKeyInfo @@ -5026,7 +5029,7 @@ int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, -1, pOrderBy, + selectInnerLoop(pParse, p, p->pEList, -1, pOrderBy, 0, &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); @@ -5158,7 +5161,7 @@ int sqlite3Select( } updateAccumulator(pParse, &sAggInfo); assert( pMinMax==0 || pMinMax->nExpr==1 ); - if( sqlite3WhereIsOrdered(pWInfo) ){ + if( sqlite3WhereIsOrdered(pWInfo)>0 ){ sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3WhereBreakLabel(pWInfo)); VdbeComment((v, "%s() by index", (flag==WHERE_ORDERBY_MIN?"min":"max"))); @@ -5169,7 +5172,7 @@ int sqlite3Select( pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, + selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, 0, pDest, addrEnd, addrEnd); sqlite3ExprListDelete(db, pDel); } @@ -5186,7 +5189,7 @@ int sqlite3Select( */ if( pOrderBy ){ explainTempTable(pParse, "ORDER BY"); - generateSortTail(pParse, p, v, pEList->nExpr, pDest); + generateSortTail(pParse, p, pEList->nExpr, pDest); } /* Jump here to skip this query diff --git a/src/where.c b/src/where.c index a766def035..9d83636f4b 100644 --- a/src/where.c +++ b/src/where.c @@ -39,7 +39,7 @@ int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ ** Return FALSE if the output needs to be sorted. */ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->bOBSat!=0; + return pWInfo->nOBSat; } /* @@ -3036,8 +3036,11 @@ static Bitmask codeOneLoopStart( ** the first one after the nEq equality constraints in the index, ** this requires some special handling. */ + assert( pWInfo->pOrderBy==0 + || pWInfo->pOrderBy->nExpr==1 + || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && (pWInfo->bOBSat!=0) + && pWInfo->nOBSat>0 && (pIdx->nKeyCol>nEq) ){ assert( pLoop->u.btree.nSkip==0 ); @@ -5199,7 +5202,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; }else{ - pWInfo->bOBSat = 1; + pWInfo->nOBSat = pFrom->isOrdered; pWInfo->revMask = pFrom->revLoop; } } @@ -5284,7 +5287,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->maskSelf = getMask(&pWInfo->sMaskSet, iCur); pWInfo->a[0].iTabCur = iCur; pWInfo->nRowOut = 1; - if( pWInfo->pOrderBy ) pWInfo->bOBSat = 1; + if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } @@ -5492,7 +5495,7 @@ WhereInfo *sqlite3WhereBegin( /* Special case: No FROM clause */ if( nTabList==0 ){ - if( pOrderBy ) pWInfo->bOBSat = 1; + if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; if( wctrlFlags & WHERE_WANT_DISTINCT ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } @@ -5603,8 +5606,8 @@ WhereInfo *sqlite3WhereBegin( if( sqlite3WhereTrace ){ int ii; sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); - if( pWInfo->bOBSat ){ - sqlite3DebugPrintf(" ORDERBY=0x%llx", pWInfo->revMask); + if( pWInfo->nOBSat>0 ){ + sqlite3DebugPrintf(" ORDERBY=%d,0x%llx", pWInfo->nOBSat, pWInfo->revMask); } switch( pWInfo->eDistinct ){ case WHERE_DISTINCT_UNIQUE: { diff --git a/src/whereInt.h b/src/whereInt.h index 233b331d42..419bb81c0e 100644 --- a/src/whereInt.h +++ b/src/whereInt.h @@ -397,7 +397,7 @@ struct WhereInfo { Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ LogEst nRowOut; /* Estimated number of output rows */ u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ - u8 bOBSat; /* ORDER BY satisfied by indices */ + i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ u8 okOnePass; /* Ok to use one-pass algorithm for UPDATE/DELETE */ u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ From 079a3072590186c50ad93c4b4dfd911c2f2e08f5 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Mar 2014 14:10:55 +0000 Subject: [PATCH 10/49] First attempt at getting block-sort to work. This is an incremental check-in. There are many problems still to be worked out. FossilOrigin-Name: 59742dd4c5259883850044d0938248b009ebd045 --- manifest | 30 +++--- manifest.uuid | 2 +- src/btree.c | 9 ++ src/btree.h | 1 + src/expr.c | 6 +- src/pragma.c | 2 +- src/select.c | 249 +++++++++++++++++++++++++++++++----------------- src/sqliteInt.h | 7 +- src/vdbe.c | 32 +++++-- src/vdbeInt.h | 1 + src/vdbeaux.c | 4 +- src/where.c | 6 +- 12 files changed, 225 insertions(+), 124 deletions(-) diff --git a/manifest b/manifest index d76e572699..a367d8b9b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sthe\spartial-ORDER-BY\sinformation\sin\sthe\squery\splanner\savailable\sto\sthe\nSELECT\scode\sgenerator.\s\sStill\sdoesn't\smake\sa\sdifference\sin\sthe\sgenerated\scode. -D 2014-03-18T20:33:42.067 +C First\sattempt\sat\sgetting\sblock-sort\sto\swork.\s\sThis\sis\san\sincremental\scheck-in.\nThere\sare\smany\sproblems\sstill\sto\sbe\sworked\sout. +D 2014-03-19T14:10:55.625 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,8 +164,8 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 2a0e73f26c329f49e426237e71a879ffd205d778 -F src/btree.h da1b69b441ecee21f8b34ba73f0ae339540c4025 +F src/btree.c 029cec7b98fe0a985922c03f101630391044c4ad +F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98 @@ -173,7 +173,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c cdd57149543bb28304d8f717c243f2a86b1fc280 -F src/expr.c 014b8087a15c4c314bdd798cb1cb0b32693f8b40 +F src/expr.c e33fa25d2279c692ad899695cfb93db840498b65 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 @@ -211,18 +211,18 @@ F src/parse.y 2613ca5d609c2f3d71dd297351f010bcec16e1e0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c -F src/pragma.c e78b4bf2a267de2c17ee09f90b6807cf8d40e6a3 +F src/pragma.c 10f169b9650f0930a7a6df67e1387a4c2c449f38 F src/prepare.c 677521ab7132615a8a26107a1d1c3132f44ae337 F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 41df0e4dd1d167dc8b39047c84e1b8b51516a237 +F src/select.c bd6ac5dc27046877f53065ea25f36ca90db4c594 F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h fa7161b3de18a9c355d4148233f3563c92311fcc +F src/sqliteInt.h 03f7d4deeaa1a9558df62f6df7bc907bb473fcea F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -278,11 +278,11 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 5f0fffa9bf49a90c05dc3d46d8217603fd0ee00e +F src/vdbe.c e0fe939465f101a6f30f1864813698edc57f610e F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae -F src/vdbeInt.h e54fc4f289fce48e81b3371128446033d097733b +F src/vdbeInt.h aa7ea20696609966c71a4b58c6a4fdb974c5e877 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c e45e3f9daf38c5be3fd39e9aacc1c9066af57a06 +F src/vdbeaux.c 5078ca7de4fd5ba4535bd17fe44d5b56c2d3294c F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 3f5de18e74b0f6da3218c2c609616a2f9e2c404a +F src/where.c d4739c3900e4cb99dfd7d19853e6b4fb63c9d342 F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 59d49b7fc49fa290e04a02653e7268c85836b27e -R 2b69bb33bc67b11eec1f6a4a37941d3e +P e258df236b7de70087c8227cb209080e55b9bf9c +R edda8149383dd364d2984f7033840ec9 U drh -Z 43f4f2d39f5cfe334420891461bd8dda +Z c306e7ec68619896c760c46adb63dbc2 diff --git a/manifest.uuid b/manifest.uuid index 4ef359f854..837615cb34 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e258df236b7de70087c8227cb209080e55b9bf9c \ No newline at end of file +59742dd4c5259883850044d0938248b009ebd045 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 9ca600e8cb..486cf8e6d5 100644 --- a/src/btree.c +++ b/src/btree.c @@ -7425,6 +7425,15 @@ int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ return rc; } +/* +** Delete all information from the single table that pCur is open on. +** +** This routine only work for pCur on an ephemeral table. +*/ +int sqlite3BtreeClearTableOfCursor(BtCursor *pCur){ + return sqlite3BtreeClearTable(pCur->pBtree, pCur->pgnoRoot, 0); +} + /* ** Erase all information in a table and add the root of the table to ** the freelist. Except, the root of the principle table (the one on diff --git a/src/btree.h b/src/btree.h index 0c3fdfad63..a62c3c60d4 100644 --- a/src/btree.h +++ b/src/btree.h @@ -115,6 +115,7 @@ int sqlite3BtreeIncrVacuum(Btree *); int sqlite3BtreeDropTable(Btree*, int, int*); int sqlite3BtreeClearTable(Btree*, int, int*); +int sqlite3BtreeClearTableOfCursor(BtCursor*); void sqlite3BtreeTripAllCursors(Btree*, int); void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); diff --git a/src/expr.c b/src/expr.c index 722a77db73..2d7c95a682 100644 --- a/src/expr.c +++ b/src/expr.c @@ -949,7 +949,6 @@ ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ if( p==0 ) return 0; pNew = sqlite3DbMallocRaw(db, sizeof(*pNew) ); if( pNew==0 ) return 0; - pNew->iECursor = 0; pNew->nExpr = i = p->nExpr; if( (flags & EXPRDUP_REDUCE)==0 ) for(i=1; inExpr; i+=i){} pNew->a = pItem = sqlite3DbMallocRaw(db, i*sizeof(p->a[0]) ); @@ -1062,7 +1061,6 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; - pNew->addrOpenEphm[2] = -1; pNew->nSelectRow = p->nSelectRow; pNew->pWith = withDup(db, p->pWith); return pNew; @@ -2335,7 +2333,7 @@ void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ int i; struct yColCache *p; assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); - sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg-1); + sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); for(i=0, p=pParse->aColCache; iiReg; if( x>=iFrom && xaDb[i].zName), P4_DYNAMIC); - sqlite3VdbeAddOp2(v, OP_Move, 2, 4); + sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); sqlite3VdbeJumpHere(v, addr); diff --git a/src/select.c b/src/select.c index 613ea25e26..b3f7f861bc 100644 --- a/src/select.c +++ b/src/select.c @@ -14,6 +14,34 @@ */ #include "sqliteInt.h" +/* +** An instance of the following object is used to record information about +** how to process the DISTINCT keyword, to simplify passing that information +** into the selectInnerLoop() routine. +*/ +typedef struct DistinctCtx DistinctCtx; +struct DistinctCtx { + u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ + int tabTnct; /* Ephemeral table used for DISTINCT processing */ + int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ +}; + +/* +** An instance of the following object is used to record information about +** the ORDER BY (or GROUP BY) clause of query is being coded. +*/ +typedef struct SortCtx SortCtx; +struct SortCtx { + ExprList *pOrderBy; /* The ORDER BY (or GROUP BY clause) */ + int nOBSat; /* Number of ORDER BY terms satisfied by indices */ + int iECursor; /* Cursor number for the sorter */ + int regReturn; /* Register holding block-output return address */ + int labelBkOut; /* Start label for the block-output subroutine */ + int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ + u8 sortFlags; /* Zero or more SORTFLAG_* bits */ +}; +#define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ /* ** Delete all the content of a Select structure but do not deallocate @@ -87,7 +115,6 @@ Select *sqlite3SelectNew( assert( pOffset==0 || pLimit!=0 ); pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; - pNew->addrOpenEphm[2] = -1; if( db->mallocFailed ) { clearSelect(db, pNew); if( pNew!=&standin ) sqlite3DbFree(db, pNew); @@ -419,32 +446,69 @@ static int sqliteProcessJoin(Parse *pParse, Select *p){ return 0; } +/* Forward reference */ +static KeyInfo *keyInfoFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Form the KeyInfo object from this ExprList */ + int iStart, /* Begin with this column of pList */ + int nExtra /* Add this many extra columns to the end */ +); + /* -** Insert code into "v" that will push the record on the top of the -** stack into the sorter. +** Insert code into "v" that will push the record in register regData +** into the sorter. */ static void pushOntoSorter( Parse *pParse, /* Parser context */ - ExprList *pOrderBy, /* The ORDER BY clause */ + SortCtx *pSort, /* Information about the ORDER BY clause */ Select *pSelect, /* The whole SELECT statement */ int regData /* Register holding data to be sorted */ ){ Vdbe *v = pParse->pVdbe; - int nExpr = pOrderBy->nExpr; + int nExpr = pSort->pOrderBy->nExpr; int regBase = sqlite3GetTempRange(pParse, nExpr+2); int regRecord = sqlite3GetTempReg(pParse); + int nOBSat = pSort->nOBSat; int op; sqlite3ExprCacheClear(pParse); - sqlite3ExprCodeExprList(pParse, pOrderBy, regBase, 0); - sqlite3VdbeAddOp2(v, OP_Sequence, pOrderBy->iECursor, regBase+nExpr); + sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, 0); + sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+1, 1); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nExpr + 2, regRecord); - if( pSelect->selFlags & SF_UseSorter ){ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nExpr+2-nOBSat, regRecord); + if( nOBSat>0 ){ + int regPrevKey; /* The first nOBSat columns of the previous row */ + int addrFirst; /* Address of the OP_IfNot opcode */ + int addrJmp; /* Address of the OP_Jump opcode */ + VdbeOp *pOp; /* Opcode that opens the sorter */ + int nKey; /* Number of sorting key columns, including OP_Sequence */ + + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); + pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + pOp->opcode = OP_OpenEphemeral; + pSort->sortFlags &= ~SORTFLAG_UseSorter; + nKey = nExpr - pSort->nOBSat + 1; + pOp->p2 = nKey + 1; + regPrevKey = pParse->nMem+1; + pParse->nMem += pSort->nOBSat; + sqlite3VdbeAddOp4(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat, + (char*)pOp->p4.pKeyInfo, P4_KEYINFO); + pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); + addrJmp = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); + pSort->labelBkOut = sqlite3VdbeMakeLabel(v); + pSort->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeAddOp1(v, OP_ClearEphem, pSort->iECursor); + sqlite3VdbeJumpHere(v, addrFirst); + sqlite3VdbeAddOp3(v, OP_Move, regBase, regPrevKey, pSort->nOBSat); + sqlite3VdbeJumpHere(v, addrJmp); + } + if( pSort->sortFlags & SORTFLAG_UseSorter ){ op = OP_SorterInsert; }else{ op = OP_IdxInsert; } - sqlite3VdbeAddOp2(v, op, pOrderBy->iECursor, regRecord); + sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); if( pSelect->iLimit ){ @@ -459,8 +523,8 @@ static void pushOntoSorter( sqlite3VdbeAddOp2(v, OP_AddImm, iLimit, -1); addr2 = sqlite3VdbeAddOp0(v, OP_Goto); sqlite3VdbeJumpHere(v, addr1); - sqlite3VdbeAddOp1(v, OP_Last, pOrderBy->iECursor); - sqlite3VdbeAddOp1(v, OP_Delete, pOrderBy->iECursor); + sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); + sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); sqlite3VdbeJumpHere(v, addr2); } } @@ -534,19 +598,6 @@ static int checkForMultiColumnSelectError( } #endif -/* -** An instance of the following object is used to record information about -** how to process the DISTINCT keyword, to simplify passing that information -** into the selectInnerLoop() routine. -*/ -typedef struct DistinctCtx DistinctCtx; -struct DistinctCtx { - u8 isTnct; /* True if the DISTINCT keyword is present */ - u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ - int tabTnct; /* Ephemeral table used for DISTINCT processing */ - int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ -}; - /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. @@ -561,8 +612,7 @@ static void selectInnerLoop( Select *p, /* The complete select statement being coded */ ExprList *pEList, /* List of values being extracted */ int srcTab, /* Pull data from this table */ - ExprList *pOrderBy, /* If not NULL, sort results using this key */ - int nOBSat, /* Terms of ORDER BY already satisfied */ + SortCtx *pSort, /* If not NULL, info on how to process ORDER BY */ DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ int iContinue, /* Jump here to continue with next row */ @@ -579,7 +629,8 @@ static void selectInnerLoop( assert( v ); assert( pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; - if( pOrderBy==0 && !hasDistinct ){ + if( pSort && pSort->pOrderBy==0 ) pSort = 0; + if( pSort==0 && !hasDistinct ){ codeOffset(v, p->iOffset, iContinue); } @@ -669,7 +720,7 @@ static void selectInnerLoop( break; } } - if( pOrderBy==0 ){ + if( pSort==0 ){ codeOffset(v, p->iOffset, iContinue); } } @@ -717,11 +768,11 @@ static void selectInnerLoop( int addr = sqlite3VdbeCurrentAddr(v) + 4; sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); - assert( pOrderBy==0 ); + assert( pSort==0 ); } #endif - if( pOrderBy ){ - pushOntoSorter(pParse, pOrderBy, p, r1); + if( pSort ){ + pushOntoSorter(pParse, pSort, p, r1); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -742,12 +793,12 @@ static void selectInnerLoop( assert( nResultCol==1 ); pDest->affSdst = sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); - if( pOrderBy ){ + if( pSort ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pOrderBy, p, regResult); + pushOntoSorter(pParse, pSort, p, regResult); }else{ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); @@ -772,8 +823,8 @@ static void selectInnerLoop( */ case SRT_Mem: { assert( nResultCol==1 ); - if( pOrderBy ){ - pushOntoSorter(pParse, pOrderBy, p, regResult); + if( pSort ){ + pushOntoSorter(pParse, pSort, p, regResult); }else{ sqlite3ExprCodeMove(pParse, regResult, iParm, 1); /* The LIMIT clause will jump out of the loop for us */ @@ -786,10 +837,10 @@ static void selectInnerLoop( case SRT_Output: { /* Return the results */ testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); - if( pOrderBy ){ + if( pSort ){ int r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - pushOntoSorter(pParse, pOrderBy, p, r1); + pushOntoSorter(pParse, pSort, p, r1); sqlite3ReleaseTempReg(pParse, r1); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); @@ -866,7 +917,7 @@ static void selectInnerLoop( ** there is a sorter, in which case the sorter has already limited ** the output for us. */ - if( pOrderBy==0 && p->iLimit ){ + if( pSort==0 && p->iLimit ){ sqlite3VdbeAddOp3(v, OP_IfZero, p->iLimit, iBreak, -1); VdbeCoverage(v); } } @@ -937,7 +988,12 @@ int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } ** function is responsible for seeing that this structure is eventually ** freed. */ -static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList, int nExtra){ +static KeyInfo *keyInfoFromExprList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Form the KeyInfo object from this ExprList */ + int iStart, /* Begin with this column of pList */ + int nExtra /* Add this many extra columns to the end */ +){ int nExpr; KeyInfo *pInfo; struct ExprList_item *pItem; @@ -945,10 +1001,10 @@ static KeyInfo *keyInfoFromExprList(Parse *pParse, ExprList *pList, int nExtra){ int i; nExpr = pList->nExpr; - pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra, 1); + pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra-iStart, 1); if( pInfo ){ assert( sqlite3KeyInfoIsWriteable(pInfo) ); - for(i=0, pItem=pList->a; ia; ipExpr); if( !pColl ) pColl = db->pDfltColl; @@ -1055,6 +1111,7 @@ static void explainComposite( static void generateSortTail( Parse *pParse, /* Parsing context */ Select *p, /* The SELECT statement */ + SortCtx *pSort, /* Information on the ORDER BY clause */ int nColumn, /* Number of columns of data */ SelectDest *pDest /* Write the sorted results here */ ){ @@ -1062,17 +1119,23 @@ static void generateSortTail( int addrBreak = sqlite3VdbeMakeLabel(v); /* Jump here to exit loop */ int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ int addr; + int addrOnce = 0; int iTab; int pseudoTab = 0; - ExprList *pOrderBy = p->pOrderBy; - + ExprList *pOrderBy = pSort->pOrderBy; int eDest = pDest->eDest; int iParm = pDest->iSDParm; - int regRow; int regRowid; + int nKey; - iTab = pOrderBy->iECursor; + if( pSort->labelBkOut ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrBreak); + sqlite3VdbeResolveLabel(v, pSort->labelBkOut); + addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + } + iTab = pSort->iECursor; regRow = sqlite3GetTempReg(pParse); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ pseudoTab = pParse->nTab++; @@ -1081,20 +1144,23 @@ static void generateSortTail( }else{ regRowid = sqlite3GetTempReg(pParse); } - if( p->selFlags & SF_UseSorter ){ + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; int ptab2 = pParse->nTab++; - sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, pOrderBy->nExpr+2); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, ptab2, regSortOut, nKey+2); + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); sqlite3VdbeAddOp2(v, OP_SorterData, iTab, regSortOut); - sqlite3VdbeAddOp3(v, OP_Column, ptab2, pOrderBy->nExpr+1, regRow); + sqlite3VdbeAddOp3(v, OP_Column, ptab2, nKey+1, regRow); sqlite3VdbeChangeP5(v, OPFLAG_CLEARCACHE); }else{ + if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_Sort, iTab, addrBreak); VdbeCoverage(v); codeOffset(v, p->iOffset, addrContinue); - sqlite3VdbeAddOp3(v, OP_Column, iTab, pOrderBy->nExpr+1, regRow); + sqlite3VdbeAddOp3(v, OP_Column, iTab, nKey+1, regRow); } switch( eDest ){ case SRT_Table: @@ -1149,11 +1215,12 @@ static void generateSortTail( /* The bottom of the loop */ sqlite3VdbeResolveLabel(v, addrContinue); - if( p->selFlags & SF_UseSorter ){ + if( pSort->sortFlags & SORTFLAG_UseSorter ){ sqlite3VdbeAddOp2(v, OP_SorterNext, iTab, addr); VdbeCoverage(v); }else{ sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v); } + if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); sqlite3VdbeResolveLabel(v, addrBreak); if( eDest==SRT_Output || eDest==SRT_Coroutine ){ sqlite3VdbeAddOp2(v, OP_Close, pseudoTab, 0); @@ -1919,7 +1986,7 @@ static void generateWithRecursiveQuery( addrCont = sqlite3VdbeMakeLabel(v); codeOffset(v, regOffset, addrCont); selectInnerLoop(pParse, p, p->pEList, iCurrent, - 0, 0, 0, pDest, addrCont, addrBreak); + 0, 0, pDest, addrCont, addrBreak); if( regLimit ){ sqlite3VdbeAddOp3(v, OP_IfZero, regLimit, addrBreak, -1); VdbeCoverage(v); @@ -2193,7 +2260,7 @@ static int multiSelect( sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); iStart = sqlite3VdbeCurrentAddr(v); selectInnerLoop(pParse, p, p->pEList, unionTab, - 0, 0, 0, &dest, iCont, iBreak); + 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); @@ -2271,7 +2338,7 @@ static int multiSelect( sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v); sqlite3ReleaseTempReg(pParse, r1); selectInnerLoop(pParse, p, p->pEList, tab1, - 0, 0, 0, &dest, iCont, iBreak); + 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); @@ -4310,7 +4377,7 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ "argument"); pFunc->iDistinct = -1; }else{ - KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0); + KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0, 0); sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); } @@ -4465,12 +4532,11 @@ int sqlite3Select( ExprList *pEList; /* List of columns to extract. */ SrcList *pTabList; /* List of tables to select from */ Expr *pWhere; /* The WHERE clause. May be NULL */ - ExprList *pOrderBy; /* The ORDER BY clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ Expr *pHaving; /* The HAVING clause. May be NULL */ int rc = 1; /* Value to return from this function */ - int addrSortIndex; /* Address of an OP_OpenEphemeral instruction */ DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */ + SortCtx sSort; /* Info on how to code the ORDER BY clause */ AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ @@ -4497,7 +4563,8 @@ int sqlite3Select( p->selFlags &= ~SF_Distinct; } sqlite3SelectPrep(pParse, p, 0); - pOrderBy = p->pOrderBy; + memset(&sSort, 0, sizeof(sSort)); + sSort.pOrderBy = p->pOrderBy; pTabList = p->pSrc; pEList = p->pEList; if( pParse->nErr || db->mallocFailed ){ @@ -4619,7 +4686,7 @@ int sqlite3Select( pParse->nHeight -= sqlite3SelectExprHeight(p); pTabList = p->pSrc; if( !IgnorableOrderby(pDest) ){ - pOrderBy = p->pOrderBy; + sSort.pOrderBy = p->pOrderBy; } } pEList = p->pEList; @@ -4646,9 +4713,9 @@ int sqlite3Select( ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER ** to disable this optimization for testing purposes. */ - if( sqlite3ExprListCompare(p->pGroupBy, pOrderBy, -1)==0 + if( sqlite3ExprListCompare(p->pGroupBy, sSort.pOrderBy, -1)==0 && OptimizationEnabled(db, SQLITE_GroupByOrder) ){ - pOrderBy = 0; + sSort.pOrderBy = 0; } /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and @@ -4667,12 +4734,12 @@ int sqlite3Select( ** BY and DISTINCT, and an index or separate temp-table for the other. */ if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct - && sqlite3ExprListCompare(pOrderBy, p->pEList, -1)==0 + && sqlite3ExprListCompare(sSort.pOrderBy, p->pEList, -1)==0 ){ p->selFlags &= ~SF_Distinct; p->pGroupBy = sqlite3ExprListDup(db, p->pEList, 0); pGroupBy = p->pGroupBy; - pOrderBy = 0; + sSort.pOrderBy = 0; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ @@ -4686,16 +4753,16 @@ int sqlite3Select( ** we figure out that the sorting index is not needed. The addrSortIndex ** variable is used to facilitate that change. */ - if( pOrderBy ){ + if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, pOrderBy, 0); - pOrderBy->iECursor = pParse->nTab++; - p->addrOpenEphm[2] = addrSortIndex = + pKeyInfo = keyInfoFromExprList(pParse, sSort.pOrderBy, 0, 0); + sSort.iECursor = pParse->nTab++; + sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - pOrderBy->iECursor, pOrderBy->nExpr+2, 0, + sSort.iECursor, sSort.pOrderBy->nExpr+2, 0, (char*)pKeyInfo, P4_KEYINFO); }else{ - addrSortIndex = -1; + sSort.addrSortIndex = -1; } /* If the output is destined for a temporary table, open that table. @@ -4709,9 +4776,9 @@ int sqlite3Select( iEnd = sqlite3VdbeMakeLabel(v); p->nSelectRow = LARGEST_INT64; computeLimitRegisters(pParse, p, iEnd); - if( p->iLimit==0 && addrSortIndex>=0 ){ - sqlite3VdbeGetOp(v, addrSortIndex)->opcode = OP_SorterOpen; - p->selFlags |= SF_UseSorter; + if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ + sqlite3VdbeGetOp(v, sSort.addrSortIndex)->opcode = OP_SorterOpen; + sSort.sortFlags |= SORTFLAG_UseSorter; } /* Open a virtual index to use for the distinct set. @@ -4720,7 +4787,7 @@ int sqlite3Select( sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList, 0), + (char*)keyInfoFromExprList(pParse, p->pEList,0,0), P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; @@ -4731,11 +4798,10 @@ int sqlite3Select( if( !isAgg && pGroupBy==0 ){ /* No aggregate functions and no GROUP BY clause */ u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); - int nOBSat; /* Begin the database scan. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pOrderBy, p->pEList, - wctrlFlags, 0); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, + p->pEList, wctrlFlags, 0); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); @@ -4743,20 +4809,23 @@ int sqlite3Select( if( sDistinct.isTnct && sqlite3WhereIsDistinct(pWInfo) ){ sDistinct.eTnctType = sqlite3WhereIsDistinct(pWInfo); } - nOBSat = sqlite3WhereIsOrdered(pWInfo); - if( pOrderBy && nOBSat==pOrderBy->nExpr ){ pOrderBy = 0; nOBSat = 0; } + if( sSort.pOrderBy ){ + sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); + if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ + sSort.pOrderBy = 0; + } + } /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral ** into an OP_Noop. */ - if( addrSortIndex>=0 && pOrderBy==0 ){ - sqlite3VdbeChangeToNoop(v, addrSortIndex); - p->addrOpenEphm[2] = -1; + if( sSort.addrSortIndex>=0 && sSort.pOrderBy==0 ){ + sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); } /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, -1, pOrderBy, nOBSat, &sDistinct, pDest, + selectInnerLoop(pParse, p, pEList, -1, &sSort, &sDistinct, pDest, sqlite3WhereContinueLabel(pWInfo), sqlite3WhereBreakLabel(pWInfo)); @@ -4812,7 +4881,7 @@ int sqlite3Select( sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr+1 : 0; sAggInfo.pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); - sqlite3ExprAnalyzeAggList(&sNC, pOrderBy); + sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); if( pHaving ){ sqlite3ExprAnalyzeAggregates(&sNC, pHaving); } @@ -4846,7 +4915,7 @@ int sqlite3Select( ** will be converted into a Noop. */ sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0); + pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0, 0); addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, sAggInfo.sortingIdx, sAggInfo.nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); @@ -4875,7 +4944,7 @@ int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, WHERE_GROUPBY, 0); if( pWInfo==0 ) goto select_end; if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ @@ -5029,7 +5098,7 @@ int sqlite3Select( sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); finalizeAggFunctions(pParse, &sAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, -1, pOrderBy, 0, + selectInnerLoop(pParse, p, p->pEList, -1, &sSort, &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); @@ -5170,9 +5239,9 @@ int sqlite3Select( finalizeAggFunctions(pParse, &sAggInfo); } - pOrderBy = 0; + sSort.pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, 0, + selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, pDest, addrEnd, addrEnd); sqlite3ExprListDelete(db, pDel); } @@ -5187,9 +5256,9 @@ int sqlite3Select( /* If there is an ORDER BY clause, then we need to sort the results ** and send them to the callback one by one. */ - if( pOrderBy ){ + if( sSort.pOrderBy ){ explainTempTable(pParse, "ORDER BY"); - generateSortTail(pParse, p, pEList->nExpr, pDest); + generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } /* Jump here to skip this query diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ee39e5278..54aec9ccaa 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1958,7 +1958,6 @@ struct Expr { */ struct ExprList { int nExpr; /* Number of expressions on the list */ - int iECursor; /* VDBE Cursor associated with this ExprList */ struct ExprList_item { /* For each expression in the list */ Expr *pExpr; /* The list of expressions */ char *zName; /* Token associated with this expression */ @@ -2182,7 +2181,7 @@ struct Select { u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ u16 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ - int addrOpenEphm[3]; /* OP_OpenEphem opcodes related to this select */ + int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ u64 nSelectRow; /* Estimated number of result rows */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ @@ -2206,9 +2205,9 @@ struct Select { #define SF_UsesEphemeral 0x0008 /* Uses the OpenEphemeral opcode */ #define SF_Expanded 0x0010 /* sqlite3SelectExpand() called on this */ #define SF_HasTypeInfo 0x0020 /* FROM subqueries have Table metadata */ -#define SF_UseSorter 0x0040 /* Sort using a sorter */ + /* 0x0040 NOT USED */ #define SF_Values 0x0080 /* Synthesized from VALUES clause */ -#define SF_Materialize 0x0100 /* NOT USED */ + /* 0x0100 NOT USED */ #define SF_NestedFrom 0x0200 /* Part of a parenthesized FROM clause */ #define SF_MaybeConvert 0x0400 /* Need convertCompoundSelectToSubquery() */ #define SF_Recursive 0x0800 /* The recursive part of a recursive CTE */ diff --git a/src/vdbe.c b/src/vdbe.c index c58b5d9fda..9f02fa796c 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -1080,10 +1080,11 @@ case OP_Variable: { /* out2-prerelease */ /* Opcode: Move P1 P2 P3 * * ** Synopsis: r[P2@P3]=r[P1@P3] ** -** Move the values in register P1..P1+P3 over into -** registers P2..P2+P3. Registers P1..P1+P3 are +** Move the P3 values in register P1..P1+P3-1 over into +** registers P2..P2+P3-1. Registers P1..P1+P3-1 are ** left holding a NULL. It is an error for register ranges -** P1..P1+P3 and P2..P2+P3 to overlap. +** P1..P1+P3-1 and P2..P2+P3-1 to overlap. It is an error +** for P3 to be less than 1. */ case OP_Move: { char *zMalloc; /* Holding variable for allocated memory */ @@ -1094,7 +1095,7 @@ case OP_Move: { n = pOp->p3; p1 = pOp->p1; p2 = pOp->p2; - assert( n>=0 && p1>0 && p2>0 ); + assert( n>0 && p1>0 && p2>0 ); assert( p1+n<=p2 || p2+n<=p1 ); pIn1 = &aMem[p1]; @@ -1118,7 +1119,7 @@ case OP_Move: { REGISTER_TRACE(p2++, pOut); pIn1++; pOut++; - }while( n-- ); + }while( --n ); break; } @@ -1995,6 +1996,7 @@ case OP_Permutation: { } /* Opcode: Compare P1 P2 P3 P4 P5 +** Synopsis: r[P1@P3] <-> r[P2@P3] ** ** Compare two vectors of registers in reg(P1)..reg(P1+P3-1) (call this ** vector "A") and in reg(P2)..reg(P2+P3-1) ("B"). Save the result of @@ -3330,6 +3332,7 @@ case OP_OpenEphemeral: { pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, 1); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; + pCx->isEphemeral = 1; rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBt, BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); if( rc==SQLITE_OK ){ @@ -3820,7 +3823,7 @@ case OP_NotExists: { /* jump, in3 */ } /* Opcode: Sequence P1 P2 * * * -** Synopsis: r[P2]=rowid +** Synopsis: r[P2]=cursor[P1].ctr++ ** ** Find the next available sequence number for cursor P1. ** Write the sequence number into register P2. @@ -4869,6 +4872,23 @@ case OP_Clear: { break; } +/* Opcode: ClearEphem P1 * * * * +** +** Delete all contents from the ephemeral table that is open on cursor P1. +** +** See also: Clear, Destroy +*/ +case OP_ClearEphem: { + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->isEphemeral ); + rc = sqlite3BtreeClearTableOfCursor(pC->pCursor); + break; +} + /* Opcode: CreateTable P1 P2 * * * ** Synopsis: r[P2]=root iDb=P1 ** diff --git a/src/vdbeInt.h b/src/vdbeInt.h index 0394782f73..dc40187833 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -72,6 +72,7 @@ struct VdbeCursor { u8 nullRow; /* True if pointing to a row with no data */ u8 rowidIsValid; /* True if lastRowid is valid */ u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ + Bool isEphemeral:1; /* True for an ephemeral table */ Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */ Bool isTable:1; /* True if a table requiring integer keys */ Bool isOrdered:1; /* True if the underlying table is BTREE_UNORDERED */ diff --git a/src/vdbeaux.c b/src/vdbeaux.c index a641a72ec8..6f1fa2670b 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -783,7 +783,9 @@ void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int n){ addr = p->nOp - 1; } pOp = &p->aOp[addr]; - assert( pOp->p4type==P4_NOTUSED || pOp->p4type==P4_INT32 ); + assert( pOp->p4type==P4_NOTUSED + || pOp->p4type==P4_INT32 + || pOp->p4type==P4_KEYINFO ); freeP4(db, pOp->p4type, pOp->p4.p); pOp->p4.p = 0; if( n==P4_INT32 ){ diff --git a/src/where.c b/src/where.c index 9d83636f4b..a0bc027d97 100644 --- a/src/where.c +++ b/src/where.c @@ -5198,9 +5198,11 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } } - if( pWInfo->pOrderBy && pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ + if( pWInfo->pOrderBy ){ if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ - pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ + pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; + } }else{ pWInfo->nOBSat = pFrom->isOrdered; pWInfo->revMask = pFrom->revLoop; From ea6c36e2041410d64292bdc3493124dd457eb20c Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Mar 2014 14:30:55 +0000 Subject: [PATCH 11/49] Make sure the where.c query planner never reports that the number of ORDER BY terms that are satisfied by indices is negative. FossilOrigin-Name: b186d8d15a10d5e8fcae9ae72e8624d71831f5bd --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index a367d8b9b9..cdb6bd8483 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C First\sattempt\sat\sgetting\sblock-sort\sto\swork.\s\sThis\sis\san\sincremental\scheck-in.\nThere\sare\smany\sproblems\sstill\sto\sbe\sworked\sout. -D 2014-03-19T14:10:55.625 +C Make\ssure\sthe\swhere.c\squery\splanner\snever\sreports\sthat\sthe\snumber\sof\sORDER\sBY\nterms\sthat\sare\ssatisfied\sby\sindices\sis\snegative. +D 2014-03-19T14:30:55.491 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c d4739c3900e4cb99dfd7d19853e6b4fb63c9d342 +F src/where.c 1757f31c7f1c59a35ac8ffe931d22d66b43f9f1f F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e258df236b7de70087c8227cb209080e55b9bf9c -R edda8149383dd364d2984f7033840ec9 +P 59742dd4c5259883850044d0938248b009ebd045 +R 48f9fc76ec5916ff8df0bfaab631c0e0 U drh -Z c306e7ec68619896c760c46adb63dbc2 +Z 42480fcdcdf9fedc12ae658e080be0a9 diff --git a/manifest.uuid b/manifest.uuid index 837615cb34..ee59a2b175 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -59742dd4c5259883850044d0938248b009ebd045 \ No newline at end of file +b186d8d15a10d5e8fcae9ae72e8624d71831f5bd \ No newline at end of file diff --git a/src/where.c b/src/where.c index a0bc027d97..740e1607e0 100644 --- a/src/where.c +++ b/src/where.c @@ -5205,6 +5205,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } }else{ pWInfo->nOBSat = pFrom->isOrdered; + if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0; pWInfo->revMask = pFrom->revLoop; } } From ba194369add5ddbbacda3edb43116960b59e301d Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Mar 2014 15:17:24 +0000 Subject: [PATCH 12/49] Avoid reusing temporary registers in co-routines when doing block-sorting. FossilOrigin-Name: 463d921b519b08648a715e987c430f06b06bae84 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 6 ++++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index cdb6bd8483..34c7b8395d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\swhere.c\squery\splanner\snever\sreports\sthat\sthe\snumber\sof\sORDER\sBY\nterms\sthat\sare\ssatisfied\sby\sindices\sis\snegative. -D 2014-03-19T14:30:55.491 +C Avoid\sreusing\stemporary\sregisters\sin\sco-routines\swhen\sdoing\nblock-sorting. +D 2014-03-19T15:17:24.171 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c bd6ac5dc27046877f53065ea25f36ca90db4c594 +F src/select.c 45d3ac1273b15e98f94c261bd40eb54a8eff0ac7 F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 59742dd4c5259883850044d0938248b009ebd045 -R 48f9fc76ec5916ff8df0bfaab631c0e0 +P b186d8d15a10d5e8fcae9ae72e8624d71831f5bd +R b0910f2ab36ae08486561cabd81d63b0 U drh -Z 42480fcdcdf9fedc12ae658e080be0a9 +Z 0471d95e83a3c5ea30582909f120da3f diff --git a/manifest.uuid b/manifest.uuid index ee59a2b175..6cd642341c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b186d8d15a10d5e8fcae9ae72e8624d71831f5bd \ No newline at end of file +463d921b519b08648a715e987c430f06b06bae84 \ No newline at end of file diff --git a/src/select.c b/src/select.c index b3f7f861bc..087cd401f0 100644 --- a/src/select.c +++ b/src/select.c @@ -509,8 +509,10 @@ static void pushOntoSorter( op = OP_IdxInsert; } sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); - sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); + if( nOBSat==0 ){ + sqlite3ReleaseTempReg(pParse, regRecord); + sqlite3ReleaseTempRange(pParse, regBase, nExpr+2); + } if( pSelect->iLimit ){ int addr1, addr2; int iLimit; From 26d7e7c6b7dd234b5ae4cf6dd4ce96ed98b078c7 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Mar 2014 16:56:58 +0000 Subject: [PATCH 13/49] Fix a problem in the block-sort logic that can arise if the VDBE opcode array is resized while the block-sort logic is being coded. "make test" now runs to completion, though there are still 17 failures. FossilOrigin-Name: 62f3a22050e0f11c789374f8a3785df0a84c24cb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 10 +++++----- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 34c7b8395d..2fa2da7098 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sreusing\stemporary\sregisters\sin\sco-routines\swhen\sdoing\nblock-sorting. -D 2014-03-19T15:17:24.171 +C Fix\sa\sproblem\sin\sthe\sblock-sort\slogic\sthat\scan\sarise\sif\sthe\sVDBE\sopcode\narray\sis\sresized\swhile\sthe\sblock-sort\slogic\sis\sbeing\scoded.\s\s"make\stest"\nnow\sruns\sto\scompletion,\sthough\sthere\sare\sstill\s17\sfailures. +D 2014-03-19T16:56:58.204 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 45d3ac1273b15e98f94c261bd40eb54a8eff0ac7 +F src/select.c 0c5df3eb627983064207c543db7d46fc960ca3a4 F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b186d8d15a10d5e8fcae9ae72e8624d71831f5bd -R b0910f2ab36ae08486561cabd81d63b0 +P 463d921b519b08648a715e987c430f06b06bae84 +R 0046fa449ec804e57106e3417f064519 U drh -Z 0471d95e83a3c5ea30582909f120da3f +Z 208b34a8fc9197aeefd7f546d5d68bb0 diff --git a/manifest.uuid b/manifest.uuid index 6cd642341c..9945baac14 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -463d921b519b08648a715e987c430f06b06bae84 \ No newline at end of file +62f3a22050e0f11c789374f8a3785df0a84c24cb \ No newline at end of file diff --git a/src/select.c b/src/select.c index 087cd401f0..ca54ae635f 100644 --- a/src/select.c +++ b/src/select.c @@ -482,16 +482,16 @@ static void pushOntoSorter( VdbeOp *pOp; /* Opcode that opens the sorter */ int nKey; /* Number of sorting key columns, including OP_Sequence */ + regPrevKey = pParse->nMem+1; + pParse->nMem += pSort->nOBSat; + nKey = nExpr - pSort->nOBSat + 1; addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); pOp->opcode = OP_OpenEphemeral; pSort->sortFlags &= ~SORTFLAG_UseSorter; - nKey = nExpr - pSort->nOBSat + 1; pOp->p2 = nKey + 1; - regPrevKey = pParse->nMem+1; - pParse->nMem += pSort->nOBSat; - sqlite3VdbeAddOp4(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat, - (char*)pOp->p4.pKeyInfo, P4_KEYINFO); + sqlite3VdbeChangeP4(v, -1, (char*)pOp->p4.pKeyInfo, P4_KEYINFO); pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); From 65ea12cb4aef6f33970e74d4de705c5a7628f622 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Mar 2014 17:41:36 +0000 Subject: [PATCH 14/49] Make it possible for block-sort to use the OP_SorterOpen sorter in addition to a generic OP_OpenEphemeral. FossilOrigin-Name: 7ce2daafd39a117041bfdd0a7132e2764fe7a74b --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/select.c | 4 +--- src/vdbe.c | 18 ++++++++++++------ src/vdbeInt.h | 1 + src/vdbesort.c | 39 ++++++++++++++++++++++++++++----------- 6 files changed, 52 insertions(+), 30 deletions(-) diff --git a/manifest b/manifest index 2fa2da7098..d2192d8683 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sproblem\sin\sthe\sblock-sort\slogic\sthat\scan\sarise\sif\sthe\sVDBE\sopcode\narray\sis\sresized\swhile\sthe\sblock-sort\slogic\sis\sbeing\scoded.\s\s"make\stest"\nnow\sruns\sto\scompletion,\sthough\sthere\sare\sstill\s17\sfailures. -D 2014-03-19T16:56:58.204 +C Make\sit\spossible\sfor\sblock-sort\sto\suse\sthe\sOP_SorterOpen\ssorter\sin\saddition\nto\sa\sgeneric\sOP_OpenEphemeral. +D 2014-03-19T17:41:36.148 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 0c5df3eb627983064207c543db7d46fc960ca3a4 +F src/select.c f481ab847cfb3ad2b2dce671a4dcbfdb50f8e666 F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -278,14 +278,14 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c e0fe939465f101a6f30f1864813698edc57f610e +F src/vdbe.c 37cfae03a0c40515304fb0a8a1a97ac3aa965e87 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae -F src/vdbeInt.h aa7ea20696609966c71a4b58c6a4fdb974c5e877 +F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c 5078ca7de4fd5ba4535bd17fe44d5b56c2d3294c F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 +F src/vdbesort.c 553537f632e8925bb8a2563692d0ee6e2175018c F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 463d921b519b08648a715e987c430f06b06bae84 -R 0046fa449ec804e57106e3417f064519 +P 62f3a22050e0f11c789374f8a3785df0a84c24cb +R 83d570cd19fdb25aa0d409b1a1586b4b U drh -Z 208b34a8fc9197aeefd7f546d5d68bb0 +Z 5d298c6da64eb9d6d4284253fdadfab0 diff --git a/manifest.uuid b/manifest.uuid index 9945baac14..88b98e78a6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -62f3a22050e0f11c789374f8a3785df0a84c24cb \ No newline at end of file +7ce2daafd39a117041bfdd0a7132e2764fe7a74b \ No newline at end of file diff --git a/src/select.c b/src/select.c index ca54ae635f..6ea206d3f0 100644 --- a/src/select.c +++ b/src/select.c @@ -488,8 +488,6 @@ static void pushOntoSorter( addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); - pOp->opcode = OP_OpenEphemeral; - pSort->sortFlags &= ~SORTFLAG_UseSorter; pOp->p2 = nKey + 1; sqlite3VdbeChangeP4(v, -1, (char*)pOp->p4.pKeyInfo, P4_KEYINFO); pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); @@ -498,7 +496,7 @@ static void pushOntoSorter( pSort->labelBkOut = sqlite3VdbeMakeLabel(v); pSort->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); - sqlite3VdbeAddOp1(v, OP_ClearEphem, pSort->iECursor); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); sqlite3VdbeJumpHere(v, addrFirst); sqlite3VdbeAddOp3(v, OP_Move, regBase, regPrevKey, pSort->nOBSat); sqlite3VdbeJumpHere(v, addrJmp); diff --git a/src/vdbe.c b/src/vdbe.c index 9f02fa796c..7104c6d017 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4872,20 +4872,26 @@ case OP_Clear: { break; } -/* Opcode: ClearEphem P1 * * * * +/* Opcode: ResetSorter P1 * * * * ** -** Delete all contents from the ephemeral table that is open on cursor P1. +** Delete all contents from the ephemeral table or sorter +** that is open on cursor P1. ** -** See also: Clear, Destroy +** This opcode only works for cursors used for sorting and +** opened with OP_OpenEphemeral or OP_SorterOpen. */ -case OP_ClearEphem: { +case OP_ResetSorter: { VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->isEphemeral ); - rc = sqlite3BtreeClearTableOfCursor(pC->pCursor); + if( pC->pSorter ){ + sqlite3VdbeSorterReset(db, pC->pSorter); + }else{ + assert( pC->isEphemeral ); + rc = sqlite3BtreeClearTableOfCursor(pC->pCursor); + } break; } diff --git a/src/vdbeInt.h b/src/vdbeInt.h index dc40187833..b752478035 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -438,6 +438,7 @@ int sqlite3VdbeFrameRestore(VdbeFrame *); int sqlite3VdbeTransferError(Vdbe *p); int sqlite3VdbeSorterInit(sqlite3 *, VdbeCursor *); +void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); diff --git a/src/vdbesort.c b/src/vdbesort.c index b9ed97e8b3..8b6ecc5e0c 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -504,23 +504,40 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ } } +/* +** Reset a sorting cursor back to its original empty state. +*/ +void sqlite3VdbeSorterReset(sqlite3 *db, VdbeSorter *pSorter){ + if( pSorter->aIter ){ + int i; + for(i=0; inTree; i++){ + vdbeSorterIterZero(db, &pSorter->aIter[i]); + } + sqlite3DbFree(db, pSorter->aIter); + pSorter->aIter = 0; + } + if( pSorter->pTemp1 ){ + sqlite3OsCloseFree(pSorter->pTemp1); + pSorter->pTemp1 = 0; + } + vdbeSorterRecordFree(db, pSorter->pRecord); + pSorter->pRecord = 0; + pSorter->iWriteOff = 0; + pSorter->iReadOff = 0; + pSorter->nInMemory = 0; + pSorter->nTree = 0; + pSorter->nPMA = 0; + pSorter->aTree = 0; +} + + /* ** Free any cursor components allocated by sqlite3VdbeSorterXXX routines. */ void sqlite3VdbeSorterClose(sqlite3 *db, VdbeCursor *pCsr){ VdbeSorter *pSorter = pCsr->pSorter; if( pSorter ){ - if( pSorter->aIter ){ - int i; - for(i=0; inTree; i++){ - vdbeSorterIterZero(db, &pSorter->aIter[i]); - } - sqlite3DbFree(db, pSorter->aIter); - } - if( pSorter->pTemp1 ){ - sqlite3OsCloseFree(pSorter->pTemp1); - } - vdbeSorterRecordFree(db, pSorter->pRecord); + sqlite3VdbeSorterReset(db, pSorter); sqlite3DbFree(db, pSorter->pUnpacked); sqlite3DbFree(db, pSorter); pCsr->pSorter = 0; From 344510e62b113a570f1b2bc6c1a59c5de8ccd51a Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 19 Mar 2014 20:01:25 +0000 Subject: [PATCH 15/49] Avoid some unnecessary calls to sqlite3VdbeRecordUnpack() that were being made when merging data from two or more temp files together in vdbesort.c FossilOrigin-Name: 707ea170b3e26965b7e3982f7554d122d130b9a6 --- manifest | 14 ++++++------- manifest.uuid | 2 +- src/vdbesort.c | 53 ++++++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 55 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index bb21639801..8ad063c938 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clean\sup\ssome\sobsolete\s"register"\sdeclarations\sin\sprintf.c. -D 2014-03-17T15:06:57.359 +C Avoid\ssome\sunnecessary\scalls\sto\ssqlite3VdbeRecordUnpack()\sthat\swere\sbeing\smade\swhen\smerging\sdata\sfrom\stwo\sor\smore\stemp\sfiles\stogether\sin\svdbesort.c +D 2014-03-19T20:01:25.712 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e45e3f9daf38c5be3fd39e9aacc1c9066af57a06 F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 46801acb342e5e4c07ba1777fe58880c143abb59 +F src/vdbesort.c 0a9e5e63d7ce196651610d1a264eeb0b2152d3ba F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P eb071eb58cf675bd6cd9dd46aed9d57fbb7a77e8 -R f35820126ad6a1c360258186873a2e79 -U drh -Z b6382160ccdf74a44bc9653763bb82ef +P ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 +R 597feeb3c9b32d5be1e5c696f1b077da +U dan +Z 03e3772c0627172d43f05f231b29ede3 diff --git a/manifest.uuid b/manifest.uuid index 3e3e1d6ac7..5d0bb22e62 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 \ No newline at end of file +707ea170b3e26965b7e3982f7554d122d130b9a6 \ No newline at end of file diff --git a/src/vdbesort.c b/src/vdbesort.c index b9ed97e8b3..f971c2d933 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -956,14 +956,55 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ if( pSorter->aTree ){ int iPrev = pSorter->aTree[1];/* Index of iterator to advance */ - int i; /* Index of aTree[] to recalculate */ - rc = vdbeSorterIterNext(db, &pSorter->aIter[iPrev]); - for(i=(pSorter->nTree+iPrev)/2; rc==SQLITE_OK && i>0; i=i/2){ - rc = vdbeSorterDoCompare(pCsr, i); - } + if( rc==SQLITE_OK ){ + int i; /* Index of aTree[] to recalculate */ + VdbeSorterIter *pIter1; /* First iterator to compare */ + VdbeSorterIter *pIter2; /* Second iterator to compare */ + u8 *pKey2; /* To pIter2->aKey, or 0 if record cached */ - *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + /* Find the first two iterators to compare. The one that was just + ** advanced (iPrev) and the one next to it in the array. */ + pIter1 = &pSorter->aIter[(iPrev & 0xFFFE)]; + pIter2 = &pSorter->aIter[(iPrev | 0x0001)]; + pKey2 = pIter2->aKey; + + for(i=(pSorter->nTree+iPrev)/2; i>0; i=i/2){ + /* Compare pIter1 and pIter2. Store the result in variable iRes. */ + int iRes; + if( pIter1->pFile==0 ){ + iRes = +1; + }else if( pIter2->pFile==0 ){ + iRes = -1; + }else{ + vdbeSorterCompare(pCsr, 0, + pIter1->aKey, pIter1->nKey, pKey2, pIter2->nKey, &iRes + ); + } + + /* If pIter1 contained the smaller value, set aTree[i] to its index. + ** Then set pIter2 to the next iterator to compare to pIter1. In this + ** case there is no cache of pIter2 in pSorter->pUnpacked, so set + ** pKey2 to point to the record belonging to pIter2. + ** + ** Alternatively, if pIter2 contains the smaller of the two values, + ** set aTree[i] to its index and update pIter1. If vdbeSorterCompare() + ** was actually called above, then pSorter->pUnpacked now contains + ** a value equivalent to pIter2. So set pKey2 to NULL to prevent + ** vdbeSorterCompare() from decoding pIter2 again. */ + if( iRes<=0 ){ + pSorter->aTree[i] = (pIter1 - pSorter->aIter); + pIter2 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; + pKey2 = pIter2->aKey; + }else{ + if( pIter1->pFile ) pKey2 = 0; + pSorter->aTree[i] = (pIter2 - pSorter->aIter); + pIter1 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; + } + + } + *pbEof = (pSorter->aIter[pSorter->aTree[1]].pFile==0); + } }else{ SorterRecord *pFree = pSorter->pRecord; pSorter->pRecord = pFree->pNext; From 6284db908a0249feef9fba346e2a6bab57603861 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 19 Mar 2014 23:24:49 +0000 Subject: [PATCH 16/49] Fix query planner weights associated with choosing block-sorting. Fix block sorting of tables with collating functions. Fix various test cases. All "veryquick" tests are now passing, though more tests need to be added. FossilOrigin-Name: 01afbf97c0ff29667806e9a7c4d74ca717819de5 --- manifest | 20 ++++++++++---------- manifest.uuid | 2 +- src/select.c | 8 ++++---- src/where.c | 12 +++++++++--- test/distinct.test | 2 +- test/orderby5.test | 22 ++++++++++++++++++++-- test/whereG.test | 4 ++-- 7 files changed, 47 insertions(+), 23 deletions(-) diff --git a/manifest b/manifest index d2192d8683..f766ef6eb9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\sit\spossible\sfor\sblock-sort\sto\suse\sthe\sOP_SorterOpen\ssorter\sin\saddition\nto\sa\sgeneric\sOP_OpenEphemeral. -D 2014-03-19T17:41:36.148 +C Fix\squery\splanner\sweights\sassociated\swith\schoosing\sblock-sorting.\s\sFix\nblock\ssorting\sof\stables\swith\scollating\sfunctions.\s\sFix\svarious\stest\scases.\nAll\s"veryquick"\stests\sare\snow\spassing,\sthough\smore\stests\sneed\sto\sbe\sadded. +D 2014-03-19T23:24:49.240 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c f481ab847cfb3ad2b2dce671a4dcbfdb50f8e666 +F src/select.c 12534449e77ff54fca5d3e9fcddc10031b13fbc1 F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 1757f31c7f1c59a35ac8ffe931d22d66b43f9f1f +F src/where.c 7c74debe2e76711082d6cb503aa89dd2afaf84cf F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -431,7 +431,7 @@ F test/descidx1.test 6d03b44c8538fe0eb4924e19fba10cdd8f3c9240 F test/descidx2.test 9f1a0c83fd57f8667c82310ca21b30a350888b5d F test/descidx3.test 09ddbe3f5295f482d2f8b687cf6db8bad7acd9a2 F test/diskfull.test 106391384780753ea6896b7b4f005d10e9866b6e -F test/distinct.test c7b194ef95dbddb32d77acbbab2e023c6eed0cb2 +F test/distinct.test 086e70c765f172e8974e9f83b9ac5ca03c154e77 F test/distinctagg.test 1a6ef9c87a58669438fc771450d7a72577417376 F test/e_createtable.test ee95d48664503d40f6cc9ef4a7d03216188e2ada F test/e_delete.test d5186e2f5478b659f16a2c8b66c09892823e542a @@ -722,7 +722,7 @@ F test/orderby1.test 9b524aff9147288da43a6d7ddfdcff47fa2303c6 F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 -F test/orderby5.test 0eb82d5890c3f3d0563966560cfdc984ea69e30c +F test/orderby5.test 2490183fef54417209d1df253633a605d46bd350 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 @@ -1089,7 +1089,7 @@ F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a F test/whereD.test 6c2feb79ef1f68381b07f39017fe5f9b96da8d62 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 5b2ba0dbe8074aa13e416b37c753991f0a2492d7 -F test/whereG.test 2a3d5181decc801b36600fa1c40b0dad2ccc267f +F test/whereG.test eb3a46b3eaf38e25e3013433b2db8a25a866c215 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 62f3a22050e0f11c789374f8a3785df0a84c24cb -R 83d570cd19fdb25aa0d409b1a1586b4b +P 7ce2daafd39a117041bfdd0a7132e2764fe7a74b +R f3bf694b0186fa15aaedfb1a5982c72f U drh -Z 5d298c6da64eb9d6d4284253fdadfab0 +Z 15ce545d4c6482a35d82cbe6b958fe2e diff --git a/manifest.uuid b/manifest.uuid index 88b98e78a6..a006d6c80d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7ce2daafd39a117041bfdd0a7132e2764fe7a74b \ No newline at end of file +01afbf97c0ff29667806e9a7c4d74ca717819de5 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 6ea206d3f0..bb72e1ae70 100644 --- a/src/select.c +++ b/src/select.c @@ -1004,12 +1004,12 @@ static KeyInfo *keyInfoFromExprList( pInfo = sqlite3KeyInfoAlloc(db, nExpr+nExtra-iStart, 1); if( pInfo ){ assert( sqlite3KeyInfoIsWriteable(pInfo) ); - for(i=iStart, pItem=pList->a; ia+iStart; ipExpr); if( !pColl ) pColl = db->pDfltColl; - pInfo->aColl[i] = pColl; - pInfo->aSortOrder[i] = pItem->sortOrder; + pInfo->aColl[i-iStart] = pColl; + pInfo->aSortOrder[i-iStart] = pItem->sortOrder; } } return pInfo; @@ -5257,7 +5257,7 @@ int sqlite3Select( ** and send them to the callback one by one. */ if( sSort.pOrderBy ){ - explainTempTable(pParse, "ORDER BY"); + explainTempTable(pParse, sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } diff --git a/src/where.c b/src/where.c index 740e1607e0..ee608dd83f 100644 --- a/src/where.c +++ b/src/where.c @@ -5039,11 +5039,17 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); if( isOrdered>=0 && isOrderedwctrlFlags & WHERE_WANT_DISTINCT ){ + rSortCost += 16; + } WHERETRACE(0x002, ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", rSortCost, (nOrderBy-isOrdered), nOrderBy, rCost, diff --git a/test/distinct.test b/test/distinct.test index 5198113920..78c2c1df36 100644 --- a/test/distinct.test +++ b/test/distinct.test @@ -162,7 +162,7 @@ do_execsql_test 2.0 { foreach {tn sql temptables res} { 1 "a, b FROM t1" {} {A B a b} 2 "b, a FROM t1" {} {B A b a} - 3 "a, b, c FROM t1" {hash} {a b c A B C} + 3 "a, b, c FROM t1" {hash} {A B C a b c} 4 "a, b, c FROM t1 ORDER BY a, b, c" {btree} {A B C a b c} 5 "b FROM t1 WHERE a = 'a'" {} {b} 6 "b FROM t1 ORDER BY +b COLLATE binary" {btree hash} {B b} diff --git a/test/orderby5.test b/test/orderby5.test index 6da0dd4e76..bccd469f25 100644 --- a/test/orderby5.test +++ b/test/orderby5.test @@ -64,10 +64,28 @@ do_execsql_test 1.7 { EXPLAIN QUERY PLAN SELECT DISTINCT c, b, a FROM t1 WHERE +a=0; } {/B-TREE/} -do_execsql_test 2.1 { + +# In some cases, it is faster to do repeated index lookups than it is to +# sort. But in other cases, it is faster to sort than to do repeated index +# lookups. +# +do_execsql_test 2.1a { + CREATE TABLE t2(a,b,c); + CREATE INDEX t2bc ON t2(b,c); + ANALYZE; + INSERT INTO sqlite_stat1 VALUES('t1','t1bc','1000000 10 9'); + INSERT INTO sqlite_stat1 VALUES('t2','t2bc','100 10 5'); + ANALYZE sqlite_master; + + EXPLAIN QUERY PLAN + SELECT * FROM t2 WHERE a=0 ORDER BY a, b, c; +} {~/B-TREE/} +do_execsql_test 2.1b { EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=0 ORDER BY a, b, c; -} {~/B-TREE/} +} {/B-TREE/} + + do_execsql_test 2.2 { EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE +a=0 ORDER BY a, b, c; diff --git a/test/whereG.test b/test/whereG.test index 1b6a861855..490fffe64e 100644 --- a/test/whereG.test +++ b/test/whereG.test @@ -95,7 +95,7 @@ do_eqp_test whereG-1.5 { WHERE cname LIKE '%bach%' AND composer.cid=track.cid AND album.aid=track.aid; -} {/.*track.*composer.*album.*/} +} {/.*track.*(composer.*album|album.*composer).*/} do_execsql_test whereG-1.6 { SELECT DISTINCT aname FROM album, composer, track @@ -110,7 +110,7 @@ do_eqp_test whereG-1.7 { WHERE cname LIKE '%bach%' AND unlikely(composer.cid=track.cid) AND unlikely(album.aid=track.aid); -} {/.*track.*composer.*album.*/} +} {/.*track.*(composer.*album|album.*composer).*/} do_execsql_test whereG-1.8 { SELECT DISTINCT aname FROM album, composer, track From 781e34cde3908108e2cadf2c727e8fe0e8743961 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 20 Mar 2014 08:59:47 +0000 Subject: [PATCH 17/49] Add an experimental fix to avoid attempting to mmap memory from an offset that is not a multiple of the system page size on systems with page sizes larger than 32KB. FossilOrigin-Name: 6f3a5c24d254fc6faf607b505bdef4a7aafc21af --- manifest | 15 ++++++----- manifest.uuid | 2 +- src/os_unix.c | 71 ++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 60 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index 8ad063c938..0276975e81 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\ssome\sunnecessary\scalls\sto\ssqlite3VdbeRecordUnpack()\sthat\swere\sbeing\smade\swhen\smerging\sdata\sfrom\stwo\sor\smore\stemp\sfiles\stogether\sin\svdbesort.c -D 2014-03-19T20:01:25.712 +C Add\san\sexperimental\sfix\sto\savoid\sattempting\sto\smmap\smemory\sfrom\san\soffset\sthat\sis\snot\sa\smultiple\sof\sthe\ssystem\spage\ssize\son\ssystems\swith\spage\ssizes\slarger\sthan\s32KB. +D 2014-03-20T08:59:47.455 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -203,7 +203,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 +F src/os_unix.c 7e2f6348e99bd215d36cb5d40161b06456089e21 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -1156,7 +1156,10 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 -R 597feeb3c9b32d5be1e5c696f1b077da +P 707ea170b3e26965b7e3982f7554d122d130b9a6 +R df2b4ee35d794d54be353b7886f4773f +T *branch * shm-mapping-fix +T *sym-shm-mapping-fix * +T -sym-trunk * U dan -Z 03e3772c0627172d43f05f231b29ede3 +Z 8f1167a615e2deebf3a41d21381f8bfc diff --git a/manifest.uuid b/manifest.uuid index 5d0bb22e62..b57a947167 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -707ea170b3e26965b7e3982f7554d122d130b9a6 \ No newline at end of file +6f3a5c24d254fc6faf607b505bdef4a7aafc21af \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index deb9e51d07..151ed890ae 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4105,6 +4105,33 @@ static int unixShmSystemLock( return rc; } +/* +** Return the system page size. +*/ +static int unixGetPagesize(void){ +#if defined(_BSD_SOURCE) + return getpagesize(); +#else + return (int)sysconf(_SC_PAGESIZE); +#endif +} + +/* +** Return the minimum number of 32KB shm regions that should be mapped at +** a time, assuming that each mapping must be an integer multiple of the +** current system page-size. +** +** Usually, this is 1. The exception seems to be systems that are configured +** to use 64KB pages - in this case each mapping must cover at least two +** shm regions. +*/ +static int unixShmRegionPerMap(void){ + int shmsz = 32*1024; /* SHM region size */ + int pgsz = unixGetPagesize(); /* System page size */ + assert( ((pgsz-1)&pgsz)==0 ); /* Page size must be a power of 2 */ + if( pgszpInode->pShmNode; assert( unixMutexHeld() ); if( p && p->nRef==0 ){ + int nShmPerMap = unixShmRegionPerMap(); int i; assert( p->pInode==pFd->pInode ); sqlite3_mutex_free(p->mutex); - for(i=0; inRegion; i++){ + for(i=0; inRegion; i+=nShmPerMap){ if( p->h>=0 ){ osMunmap(p->apRegion[i], p->szRegion); }else{ @@ -4326,6 +4354,8 @@ static int unixShmMap( unixShm *p; unixShmNode *pShmNode; int rc = SQLITE_OK; + int nShmPerMap = unixShmRegionPerMap(); + int nReqRegion; /* If the shared-memory file has not yet been opened, open it now. */ if( pDbFd->pShm==0 ){ @@ -4341,9 +4371,12 @@ static int unixShmMap( assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); - if( pShmNode->nRegion<=iRegion ){ + /* Minimum number of regions required to be mapped. */ + nReqRegion = ((iRegion+nShmPerMap) / nShmPerMap) * nShmPerMap; + + if( pShmNode->nRegionszRegion = szRegion; @@ -4392,17 +4425,19 @@ static int unixShmMap( /* Map the requested memory region into this processes address space. */ apNew = (char **)sqlite3_realloc( - pShmNode->apRegion, (iRegion+1)*sizeof(char *) + pShmNode->apRegion, nReqRegion*sizeof(char *) ); if( !apNew ){ rc = SQLITE_IOERR_NOMEM; goto shmpage_out; } pShmNode->apRegion = apNew; - while(pShmNode->nRegion<=iRegion){ + while( pShmNode->nRegionh>=0 ){ - pMem = osMmap(0, szRegion, + pMem = osMmap(0, nMap, pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion ); @@ -4418,8 +4453,11 @@ static int unixShmMap( } memset(pMem, 0, szRegion); } - pShmNode->apRegion[pShmNode->nRegion] = pMem; - pShmNode->nRegion++; + + for(i=0; iapRegion[pShmNode->nRegion+i] = &((char*)pMem)[szRegion*i]; + } + pShmNode->nRegion += nShmPerMap; } } @@ -4633,19 +4671,6 @@ static void unixUnmapfile(unixFile *pFd){ } } -/* -** Return the system page size. -*/ -static int unixGetPagesize(void){ -#if HAVE_MREMAP - return 512; -#elif defined(_BSD_SOURCE) - return getpagesize(); -#else - return (int)sysconf(_SC_PAGESIZE); -#endif -} - /* ** Attempt to set the size of the memory mapping maintained by file ** descriptor pFd to nNew bytes. Any existing mapping is discarded. @@ -4682,8 +4707,12 @@ static void unixRemapfile( if( (pFd->ctrlFlags & UNIXFILE_RDONLY)==0 ) flags |= PROT_WRITE; if( pOrig ){ +#if HAVE_MREMAP + i64 nReuse = pFd->mmapSize; +#else const int szSyspage = unixGetPagesize(); i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); +#endif u8 *pReq = &pOrig[nReuse]; /* Unmap any pages of the existing mapping that cannot be reused. */ From bc76063cd2d0c7bdf59965af73a67a2ed1181641 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 20 Mar 2014 09:42:09 +0000 Subject: [PATCH 18/49] Add a test to ensure os_unix.c works with 64KiB OS pages. FossilOrigin-Name: e3d2be3ba47cdaafd26347620ae3bc2813203f16 --- manifest | 20 +++++++++---------- manifest.uuid | 2 +- src/os_unix.c | 15 ++++++++++---- src/test_syscall.c | 49 +++++++++++++++++++++++++++++++++++++++++++++- test/syscall.test | 1 + test/wal64k.test | 47 ++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 117 insertions(+), 17 deletions(-) create mode 100644 test/wal64k.test diff --git a/manifest b/manifest index 0276975e81..5dfca569cc 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sexperimental\sfix\sto\savoid\sattempting\sto\smmap\smemory\sfrom\san\soffset\sthat\sis\snot\sa\smultiple\sof\sthe\ssystem\spage\ssize\son\ssystems\swith\spage\ssizes\slarger\sthan\s32KB. -D 2014-03-20T08:59:47.455 +C Add\sa\stest\sto\sensure\sos_unix.c\sworks\swith\s64KiB\sOS\spages. +D 2014-03-20T09:42:09.744 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -203,7 +203,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c 1b147e4cf7cc39e618115c14a086aed44bc91ace F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 7e2f6348e99bd215d36cb5d40161b06456089e21 +F src/os_unix.c ae4b5240af4619d711301d7992396e182585269f F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 @@ -266,7 +266,7 @@ F src/test_server.c a2615049954cbb9cfb4a62e18e2f0616e4dc38fe F src/test_sqllog.c c1c1bbedbcaf82b93d83e4f9dd990e62476a680e F src/test_stat.c 9898687a6c2beca733b0dd6fe19163d987826d31 F src/test_superlock.c 2b97936ca127d13962c3605dbc9a4ef269c424cd -F src/test_syscall.c 16dbe79fb320fadb5acd7a0a59f49e52ab2d2091 +F src/test_syscall.c 2e21ca7f7dc54a028f1967b63f1e76155c356f9b F src/test_tclvar.c f4dc67d5f780707210d6bb0eb6016a431c04c7fa F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9 @@ -835,7 +835,7 @@ F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4 F test/substr.test 18f57c4ca8a598805c4d64e304c418734d843c1a F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 -F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 +F test/syscall.test d2fdaad713f103ac611fe7ef9b724c7b69f8149c F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 F test/table.test 580d23530187026d4502fae74a490f0408cf2cc7 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 @@ -1054,6 +1054,7 @@ F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e +F test/wal64k.test 63828c2161ad76ddd4109dee0a096b6ef6895698 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test 75c42e1bc4545c277fed212f8fc9b7723cd02216 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750 @@ -1156,10 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 707ea170b3e26965b7e3982f7554d122d130b9a6 -R df2b4ee35d794d54be353b7886f4773f -T *branch * shm-mapping-fix -T *sym-shm-mapping-fix * -T -sym-trunk * +P 6f3a5c24d254fc6faf607b505bdef4a7aafc21af +R 20782a7f4b1e1e0590156d829fb7cb71 U dan -Z 8f1167a615e2deebf3a41d21381f8bfc +Z 523769a43e90bd1d8c3037ea221853f3 diff --git a/manifest.uuid b/manifest.uuid index b57a947167..40e6d382ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6f3a5c24d254fc6faf607b505bdef4a7aafc21af \ No newline at end of file +e3d2be3ba47cdaafd26347620ae3bc2813203f16 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 151ed890ae..fc320a4926 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -323,6 +323,7 @@ static int posixFchown(int fd, uid_t uid, gid_t gid){ /* Forward reference */ static int openDirectory(const char*, int*); +static int unixGetpagesize(void); /* ** Many system calls are accessed through pointer-to-functions so that @@ -446,6 +447,9 @@ static struct unix_syscall { #define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) #endif + { "getpagesize", (sqlite3_syscall_ptr)unixGetpagesize, 0 }, +#define osGetpagesize ((int(*)(void))aSyscall[24].pCurrent) + }; /* End of the overrideable system calls */ /* @@ -4107,8 +4111,11 @@ static int unixShmSystemLock( /* ** Return the system page size. +** +** This function should not be called directly by other code in this file. +** Instead, it should be called via macro osGetpagesize(). */ -static int unixGetPagesize(void){ +static int unixGetpagesize(void){ #if defined(_BSD_SOURCE) return getpagesize(); #else @@ -4127,7 +4134,7 @@ static int unixGetPagesize(void){ */ static int unixShmRegionPerMap(void){ int shmsz = 32*1024; /* SHM region size */ - int pgsz = unixGetPagesize(); /* System page size */ + int pgsz = osGetpagesize(); /* System page size */ assert( ((pgsz-1)&pgsz)==0 ); /* Page size must be a power of 2 */ if( pgszmmapSize; #else - const int szSyspage = unixGetPagesize(); + const int szSyspage = osGetpagesize(); i64 nReuse = (pFd->mmapSize & ~(szSyspage-1)); #endif u8 *pReq = &pOrig[nReuse]; @@ -7458,7 +7465,7 @@ int sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==24 ); + assert( ArraySize(aSyscall)==25 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ diff --git a/src/test_syscall.c b/src/test_syscall.c index 7c0873c16d..0dac2e897e 100644 --- a/src/test_syscall.c +++ b/src/test_syscall.c @@ -67,6 +67,11 @@ ** test_syscall list ** Return a list of all system calls. The list is constructed using ** the xNextSystemCall() VFS method. +** +** test_syscall pagesize PGSZ +** If PGSZ is a power of two greater than 256, install a wrapper around +** OS function getpagesize() that reports the system page size as PGSZ. +** Or, if PGSZ is less than zero, remove any wrapper already installed. */ #include "sqliteInt.h" @@ -89,7 +94,9 @@ static struct TestSyscallGlobal { int bPersist; /* 1 for persistent errors, 0 for transient */ int nCount; /* Fail after this many more calls */ int nFail; /* Number of failures that have occurred */ -} gSyscall = { 0, 0 }; + int pgsz; + sqlite3_syscall_ptr orig_getpagesize; +} gSyscall = { 0, 0, 0, 0, 0 }; static int ts_open(const char *, int, int); static int ts_close(int fd); @@ -650,6 +657,45 @@ static int test_syscall_defaultvfs( return TCL_OK; } +static int ts_getpagesize(void){ + return gSyscall.pgsz; +} + +static int test_syscall_pagesize( + void * clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + int pgsz; + if( objc!=3 ){ + Tcl_WrongNumArgs(interp, 2, objv, "PGSZ"); + return TCL_ERROR; + } + if( Tcl_GetIntFromObj(interp, objv[2], &pgsz) ){ + return TCL_ERROR; + } + + if( pgsz<0 ){ + if( gSyscall.orig_getpagesize ){ + pVfs->xSetSystemCall(pVfs, "getpagesize", gSyscall.orig_getpagesize); + } + }else{ + if( pgsz<512 || (pgsz & (pgsz-1)) ){ + Tcl_AppendResult(interp, "pgsz out of range", 0); + return TCL_ERROR; + } + gSyscall.orig_getpagesize = pVfs->xGetSystemCall(pVfs, "getpagesize"); + gSyscall.pgsz = pgsz; + pVfs->xSetSystemCall( + pVfs, "getpagesize", (sqlite3_syscall_ptr)ts_getpagesize + ); + } + + return TCL_OK; +} + static int test_syscall( void * clientData, Tcl_Interp *interp, @@ -668,6 +714,7 @@ static int test_syscall( { "exists", test_syscall_exists }, { "list", test_syscall_list }, { "defaultvfs", test_syscall_defaultvfs }, + { "pagesize", test_syscall_pagesize }, { 0, 0 } }; int iCmd; diff --git a/test/syscall.test b/test/syscall.test index 5bf1225867..c2d9979031 100644 --- a/test/syscall.test +++ b/test/syscall.test @@ -61,6 +61,7 @@ foreach s { fcntl read pread write pwrite fchmod fallocate pread64 pwrite64 unlink openDirectory mkdir rmdir statvfs fchown umask mmap munmap mremap + getpagesize } { if {[test_syscall exists $s]} {lappend syscall_list $s} } diff --git a/test/wal64k.test b/test/wal64k.test new file mode 100644 index 0000000000..afd820778e --- /dev/null +++ b/test/wal64k.test @@ -0,0 +1,47 @@ +# 2010 April 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing the operation of the library in +# "PRAGMA journal_mode=WAL" mode. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix wal64k + +ifcapable !wal {finish_test ; return } + +db close +test_syscall pagesize 65536 +sqlite3 db test.db + +do_execsql_test 1.0 { + PRAGMA journal_mode = WAL; + CREATE TABLE t1(x); + CREATE INDEX i1 ON t1(x); +} {wal} +do_test 1.1 { file size test.db-shm } {65536} + +do_test 1.2 { + execsql BEGIN + while {[file size test.db-shm]==65536} { + execsql { INSERT INTO t1 VALUES( randstr(900,1100) ) } + } + execsql COMMIT + file size test.db-shm +} {131072} + +integrity_check 1.3 + +db close +test_syscall pagesize -1 +finish_test + From 0c8a934cb44f262ee60535db0a0167cb546b2aef Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 12:17:35 +0000 Subject: [PATCH 19/49] Fix an unnecessarily obtuse use of a bitmask flag. FossilOrigin-Name: ca3140813198ab7ce470cf86334e55207f3461f9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 8ad063c938..41cb481776 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\ssome\sunnecessary\scalls\sto\ssqlite3VdbeRecordUnpack()\sthat\swere\sbeing\smade\swhen\smerging\sdata\sfrom\stwo\sor\smore\stemp\sfiles\stogether\sin\svdbesort.c -D 2014-03-19T20:01:25.712 +C Fix\san\sunnecessarily\sobtuse\suse\sof\sa\sbitmask\sflag. +D 2014-03-20T12:17:35.051 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c bb50b5aed4f9b2284eb92c944253e60df2fb8259 +F src/where.c e433accd201ca482c761e679f4a2ce2f6a7348e6 F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ecd9d3f9453be0bb8e312d8027fd1a9e55882f36 -R 597feeb3c9b32d5be1e5c696f1b077da -U dan -Z 03e3772c0627172d43f05f231b29ede3 +P 707ea170b3e26965b7e3982f7554d122d130b9a6 +R 829b1d227d22721b3fefbad8e2ead1b7 +U drh +Z fcfbdaafabd38e7e728b08b0509fc847 diff --git a/manifest.uuid b/manifest.uuid index 5d0bb22e62..444963914a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -707ea170b3e26965b7e3982f7554d122d130b9a6 \ No newline at end of file +ca3140813198ab7ce470cf86334e55207f3461f9 \ No newline at end of file diff --git a/src/where.c b/src/where.c index 6cd9c167a1..d13258ad66 100644 --- a/src/where.c +++ b/src/where.c @@ -3208,8 +3208,7 @@ static Bitmask codeOneLoopStart( pLevel->op = OP_Next; } pLevel->p1 = iIdxCur; - assert( (WHERE_UNQ_WANTED>>16)==1 ); - pLevel->p3 = (pLoop->wsFlags>>16)&1; + pLevel->p3 = (pLoop->wsFlags&WHERE_UNQ_WANTED)!=0 ? 1:0; if( (pLoop->wsFlags & WHERE_CONSTRAINT)==0 ){ pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; }else{ From 2b59b3a4c80207e0c4692a8bbbcac5a0dce0c9d5 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 13:26:47 +0000 Subject: [PATCH 20/49] =?UTF-8?q?Convert=20expressions=20of=20the=20form?= =?UTF-8?q?=20"X=20IN=20(=3F)"=20with=20exactly=20one=20value=20on=20the?= =?UTF-8?q?=20RHS=20of=20the=20IN=20into=20equality=20tests:=20=20"X=3D=3F?= =?UTF-8?q?".=20=20Add=20test=20cases=20to=20verify=20that=20statements=20?= =?UTF-8?q?work=20correctly=20on=20this=20corner=20case.=20Fix=20for=20tic?= =?UTF-8?q?ket=20[e39d032577df6942].?= FossilOrigin-Name: e68b427afbc82e201c64474117851aa4c9eb0c92 --- manifest | 16 +++++----- manifest.uuid | 2 +- src/parse.y | 15 +++++++++ src/where.c | 2 ++ test/in4.test | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 114 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 41cb481776..16ca144d3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\san\sunnecessarily\sobtuse\suse\sof\sa\sbitmask\sflag. -D 2014-03-20T12:17:35.051 +C Convert\sexpressions\sof\sthe\sform\s"X\sIN\s(?)"\swith\sexactly\sone\svalue\son\sthe\nRHS\sof\sthe\sIN\sinto\sequality\stests:\s\s"X=?".\s\sAdd\stest\scases\sto\sverify\sthat\nstatements\swork\scorrectly\son\sthis\scorner\scase.\nFix\sfor\sticket\s[e39d032577df6942]. +D 2014-03-20T13:26:47.152 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -207,7 +207,7 @@ F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 2613ca5d609c2f3d71dd297351f010bcec16e1e0 +F src/parse.y 52d749e7fd1289d9a576fe88782d9637a2f7bb6d F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c e433accd201ca482c761e679f4a2ce2f6a7348e6 +F src/where.c a2f20bdc6565a3add38ac5e4d9e71523ed5b2f2b F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -593,7 +593,7 @@ F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 -F test/in4.test 64f3cc1acde1b9161ccdd8e5bde3daefdb5b2617 +F test/in4.test ed42587bed8c8e2219a09a6a6e3675edffc011da F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 707ea170b3e26965b7e3982f7554d122d130b9a6 -R 829b1d227d22721b3fefbad8e2ead1b7 +P ca3140813198ab7ce470cf86334e55207f3461f9 +R abb091fc1f04a82c7540caf08b56e8dd U drh -Z fcfbdaafabd38e7e728b08b0509fc847 +Z 8ed34cb3369a2d9b188815f2f4b333aa diff --git a/manifest.uuid b/manifest.uuid index 444963914a..6fc3864b4f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ca3140813198ab7ce470cf86334e55207f3461f9 \ No newline at end of file +e68b427afbc82e201c64474117851aa4c9eb0c92 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index ad9bedf0b4..1e6d859881 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1020,6 +1020,21 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { */ A.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[N]); sqlite3ExprDelete(pParse->db, X.pExpr); + }else if( Y->nExpr==1 ){ + /* Expressions of the form: + ** + ** expr1 IN (?1) + ** expr1 NOT IN (?2) + ** + ** with exactly one value on the RHS can be simplified to: + ** + ** expr1 == ?1 + ** expr1 <> ?2 + */ + Expr *pRHS = Y->a[0].pExpr; + Y->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, Y); + A.pExpr = sqlite3PExpr(pParse, N ? TK_NE : TK_EQ, X.pExpr, pRHS, 0); }else{ A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); if( A.pExpr ){ diff --git a/src/where.c b/src/where.c index d13258ad66..e3ce8b8697 100644 --- a/src/where.c +++ b/src/where.c @@ -4009,6 +4009,8 @@ static int whereLoopAddBtreeIndex( /* "x IN (value, value, ...)" */ nIn = sqlite3LogEst(pExpr->x.pList->nExpr); } + assert( nIn>0 ); /* RHS always has 2 or more terms... The parser + ** changes "x IN (?)" into "x=?". */ pNew->rRun += nIn; pNew->u.btree.nEq++; pNew->nOut = nRowEst + nInMul + nIn; diff --git a/test/in4.test b/test/in4.test index 470f4f0e34..b8ae4d9825 100644 --- a/test/in4.test +++ b/test/in4.test @@ -159,4 +159,92 @@ do_test in4-3.12 { execsql { SELECT * FROM t3 WHERE x IN (1, 2) AND y IN ()} } {} +# Tests for "... IN (?)" and "... NOT IN (?)". In other words, tests +# for when the RHS of IN is a single expression. This should work the +# same as the == and <> operators. +# +do_execsql_test in4-3.21 { + SELECT * FROM t3 WHERE x=10 AND y IN (10); +} {10 10 10} +do_execsql_test in4-3.22 { + SELECT * FROM t3 WHERE x IN (10) AND y=10; +} {10 10 10} +do_execsql_test in4-3.23 { + SELECT * FROM t3 WHERE x IN (10) AND y IN (10); +} {10 10 10} +do_execsql_test in4-3.24 { + SELECT * FROM t3 WHERE x=1 AND y NOT IN (10); +} {1 1 1} +do_execsql_test in4-3.25 { + SELECT * FROM t3 WHERE x NOT IN (10) AND y=1; +} {1 1 1} +do_execsql_test in4-3.26 { + SELECT * FROM t3 WHERE x NOT IN (10) AND y NOT IN (10); +} {1 1 1} + +# The query planner recognizes that "x IN (?)" only generates a +# single match and can use this information to optimize-out ORDER BY +# clauses. +# +do_execsql_test in4-3.31 { + DROP INDEX t3i1; + CREATE UNIQUE INDEX t3xy ON t3(x,y); + + SELECT *, '|' FROM t3 A, t3 B + WHERE A.x=10 AND A.y IN (10) + AND B.x=1 AND B.y IN (1); +} {10 10 10 1 1 1 |} +do_execsql_test in4-3.32 { + EXPLAIN QUERY PLAN + SELECT *, '|' FROM t3 A, t3 B + WHERE A.x=10 AND A.y IN (10) + AND B.x=1 AND B.y IN (1); +} {~/B-TREE/} ;# No separate sorting pass +do_execsql_test in4-3.33 { + SELECT *, '|' FROM t3 A, t3 B + WHERE A.x IN (10) AND A.y=10 + AND B.x IN (1) AND B.y=1; +} {10 10 10 1 1 1 |} +do_execsql_test in4-3.34 { + EXPLAIN QUERY PLAN + SELECT *, '|' FROM t3 A, t3 B + WHERE A.x IN (10) AND A.y=10 + AND B.x IN (1) AND B.y=1; +} {~/B-TREE/} ;# No separate sorting pass + +# An expression of the form "x IN (?,?)" creates an ephemeral table to +# hold the list of values on the RHS. But "x IN (?)" does not create +# an ephemeral table. +# +do_execsql_test in4-3.41 { + SELECT * FROM t3 WHERE x IN (10,11); +} {10 10 10} +do_execsql_test in4-3.42 { + EXPLAIN + SELECT * FROM t3 WHERE x IN (10,11); +} {/OpenEphemeral/} +do_execsql_test in4-3.43 { + SELECT * FROM t3 WHERE x IN (10); +} {10 10 10} +do_execsql_test in4-3.44 { + EXPLAIN + SELECT * FROM t3 WHERE x IN (10); +} {~/OpenEphemeral/} +do_execsql_test in4-3.45 { + SELECT * FROM t3 WHERE x NOT IN (10,11); +} {1 1 1} +do_execsql_test in4-3.46 { + EXPLAIN + SELECT * FROM t3 WHERE x NOT IN (10,11); +} {/OpenEphemeral/} +do_execsql_test in4-3.47 { + SELECT * FROM t3 WHERE x NOT IN (10); +} {1 1 1} +do_execsql_test in4-3.48 { + EXPLAIN + SELECT * FROM t3 WHERE x NOT IN (10); +} {~/OpenEphemeral/} + + + finish_test From 0a8d14261c7b47208176f750f95b2472fecd63bf Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 14:56:47 +0000 Subject: [PATCH 21/49] =?UTF-8?q?Previous=20check-in=20is=20not=20quite=20?= =?UTF-8?q?correct.=20=20"x=20IN=20(=3F)"=20is=20not=20exactly=20the=20sam?= =?UTF-8?q?e=20as=20"x=3D=3D=3F"=20do=20to=20collation=20and=20affinity=20?= =?UTF-8?q?issues.=20=20The=20correct=20converstion=20should=20be=20to=20"?= =?UTF-8?q?x=3D=3D(+=3F=20COLLATE=20binary)".=20=20The=20current=20check-i?= =?UTF-8?q?n=20fixes=20this=20problem=20and=20provides=20test=20cases.=20?= =?UTF-8?q?=20Ticket=20[e39d032577df69]?= FossilOrigin-Name: 2ff3b25f40fd117c8a2da1d1a3625f6b167b7b16 --- manifest | 14 ++++++------ manifest.uuid | 2 +- src/parse.y | 8 ++++--- test/in4.test | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 16ca144d3e..5d2b629021 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Convert\sexpressions\sof\sthe\sform\s"X\sIN\s(?)"\swith\sexactly\sone\svalue\son\sthe\nRHS\sof\sthe\sIN\sinto\sequality\stests:\s\s"X=?".\s\sAdd\stest\scases\sto\sverify\sthat\nstatements\swork\scorrectly\son\sthis\scorner\scase.\nFix\sfor\sticket\s[e39d032577df6942]. -D 2014-03-20T13:26:47.152 +C Previous\scheck-in\sis\snot\squite\scorrect.\s\s"x\sIN\s(?)"\sis\snot\sexactly\sthe\ssame\nas\s"x==?"\sdo\sto\scollation\sand\saffinity\sissues.\s\sThe\scorrect\sconverstion\sshould\nbe\sto\s"x==(+?\sCOLLATE\sbinary)".\s\sThe\scurrent\scheck-in\sfixes\sthis\sproblem\sand\nprovides\stest\scases.\s\sTicket\s[e39d032577df69] +D 2014-03-20T14:56:47.071 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -207,7 +207,7 @@ F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y 52d749e7fd1289d9a576fe88782d9637a2f7bb6d +F src/parse.y c9189eca0888ecd2711c77a199ae7a7c86f57546 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c @@ -593,7 +593,7 @@ F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 -F test/in4.test ed42587bed8c8e2219a09a6a6e3675edffc011da +F test/in4.test 18202389003284e8e019750c04e4bc6333df9c99 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P ca3140813198ab7ce470cf86334e55207f3461f9 -R abb091fc1f04a82c7540caf08b56e8dd +P e68b427afbc82e201c64474117851aa4c9eb0c92 +R 8739dc742c6ff38e89fff08724b87c5b U drh -Z 8ed34cb3369a2d9b188815f2f4b333aa +Z c6097cc93dfeb105af183d7216a869ad diff --git a/manifest.uuid b/manifest.uuid index 6fc3864b4f..7bb62d8aab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e68b427afbc82e201c64474117851aa4c9eb0c92 \ No newline at end of file +2ff3b25f40fd117c8a2da1d1a3625f6b167b7b16 \ No newline at end of file diff --git a/src/parse.y b/src/parse.y index 1e6d859881..24b8b98afc 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1028,12 +1028,14 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ** ** with exactly one value on the RHS can be simplified to: ** - ** expr1 == ?1 - ** expr1 <> ?2 + ** expr1 == (+?1 COLLATE binary) + ** expr1 <> (+?2 COLLATE binary) */ - Expr *pRHS = Y->a[0].pExpr; + static const Token collBin = { "binary", 6 }; + Expr *pRHS = sqlite3ExprAddCollateToken(pParse, Y->a[0].pExpr, &collBin); Y->a[0].pExpr = 0; sqlite3ExprListDelete(pParse->db, Y); + pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0, 0); A.pExpr = sqlite3PExpr(pParse, N ? TK_NE : TK_EQ, X.pExpr, pRHS, 0); }else{ A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); diff --git a/test/in4.test b/test/in4.test index b8ae4d9825..5828a83cc5 100644 --- a/test/in4.test +++ b/test/in4.test @@ -245,6 +245,65 @@ do_execsql_test in4-3.48 { SELECT * FROM t3 WHERE x NOT IN (10); } {~/OpenEphemeral/} +# Make sure that when "x IN (?)" is converted into "x==?" that collating +# sequence and affinity computations do not get messed up. +# +do_execsql_test in4-4.1 { + CREATE TABLE t4a(a TEXT, b TEXT COLLATE nocase, c); + INSERT INTO t4a VALUES('ABC','abc',1); + INSERT INTO t4a VALUES('def','xyz',2); + INSERT INTO t4a VALUES('ghi','ghi',3); + SELECT c FROM t4a WHERE a=b ORDER BY c; +} {3} +do_execsql_test in4-4.2 { + SELECT c FROM t4a WHERE b=a ORDER BY c; +} {1 3} +do_execsql_test in4-4.3 { + SELECT c FROM t4a WHERE (a||'')=b ORDER BY c; +} {1 3} +do_execsql_test in4-4.4 { + SELECT c FROM t4a WHERE (a||'')=(b||'') ORDER BY c; +} {3} +do_execsql_test in4-4.5 { + SELECT c FROM t4a WHERE a IN (b) ORDER BY c; +} {3} +do_execsql_test in4-4.6 { + SELECT c FROM t4a WHERE (a||'') IN (b) ORDER BY c; +} {3} + + +do_execsql_test in4-4.11 { + CREATE TABLE t4b(a TEXT, b NUMERIC, c); + INSERT INTO t4b VALUES('1.0',1,4); + SELECT c FROM t4b WHERE a=b; +} {4} +do_execsql_test in4-4.12 { + SELECT c FROM t4b WHERE b=a; +} {4} +do_execsql_test in4-4.13 { + SELECT c FROM t4b WHERE +a=b; +} {4} +do_execsql_test in4-4.14 { + SELECT c FROM t4b WHERE a=+b; +} {} +do_execsql_test in4-4.15 { + SELECT c FROM t4b WHERE +b=a; +} {} +do_execsql_test in4-4.16 { + SELECT c FROM t4b WHERE b=+a; +} {4} +do_execsql_test in4-4.17 { + SELECT c FROM t4b WHERE a IN (b); +} {} +do_execsql_test in4-4.18 { + SELECT c FROM t4b WHERE b IN (a); +} {4} +do_execsql_test in4-4.19 { + SELECT c FROM t4b WHERE +b IN (a); +} {} + + + finish_test From 4ef7efad5eccd9acc32c3ecce05026914beaa903 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 15:14:08 +0000 Subject: [PATCH 22/49] Fix harmless compiler warnings. FossilOrigin-Name: b1435f26b07b2208cfcca557f96342a5bd0d5328 --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 6 +++++- src/parse.y | 3 +-- src/sqliteInt.h | 2 +- src/vdbesort.c | 4 ++-- 6 files changed, 19 insertions(+), 16 deletions(-) diff --git a/manifest b/manifest index 5d2b629021..9590aa3ca2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Previous\scheck-in\sis\snot\squite\scorrect.\s\s"x\sIN\s(?)"\sis\snot\sexactly\sthe\ssame\nas\s"x==?"\sdo\sto\scollation\sand\saffinity\sissues.\s\sThe\scorrect\sconverstion\sshould\nbe\sto\s"x==(+?\sCOLLATE\sbinary)".\s\sThe\scurrent\scheck-in\sfixes\sthis\sproblem\sand\nprovides\stest\scases.\s\sTicket\s[e39d032577df69] -D 2014-03-20T14:56:47.071 +C Fix\sharmless\scompiler\swarnings. +D 2014-03-20T15:14:08.664 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c cdd57149543bb28304d8f717c243f2a86b1fc280 -F src/expr.c 014b8087a15c4c314bdd798cb1cb0b32693f8b40 +F src/expr.c b74939e7935c4ad9e7f87b31ce05713fd5dafc3a F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 @@ -207,7 +207,7 @@ F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y c9189eca0888ecd2711c77a199ae7a7c86f57546 +F src/parse.y d21075457487f84a72f848c2545e3319d6452e6f F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c @@ -222,7 +222,7 @@ F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h fa7161b3de18a9c355d4148233f3563c92311fcc +F src/sqliteInt.h db6d7cf6e44d1c862f4f5290716098a0b90f1310 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -285,7 +285,7 @@ F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 F src/vdbeaux.c e45e3f9daf38c5be3fd39e9aacc1c9066af57a06 F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 -F src/vdbesort.c 0a9e5e63d7ce196651610d1a264eeb0b2152d3ba +F src/vdbesort.c 0fef5fa890fda760f92f293a5b0f2ccfe5d2dc55 F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e68b427afbc82e201c64474117851aa4c9eb0c92 -R 8739dc742c6ff38e89fff08724b87c5b +P 2ff3b25f40fd117c8a2da1d1a3625f6b167b7b16 +R b4897cbb8b99db805aa2642c2b5c6a8f U drh -Z c6097cc93dfeb105af183d7216a869ad +Z 8f62a0caca3ff37dd6879b77616b6faf diff --git a/manifest.uuid b/manifest.uuid index 7bb62d8aab..82d7da5708 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2ff3b25f40fd117c8a2da1d1a3625f6b167b7b16 \ No newline at end of file +b1435f26b07b2208cfcca557f96342a5bd0d5328 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 722a77db73..83b948e8e1 100644 --- a/src/expr.c +++ b/src/expr.c @@ -65,7 +65,11 @@ char sqlite3ExprAffinity(Expr *pExpr){ ** If a memory allocation error occurs, that fact is recorded in pParse->db ** and the pExpr parameter is returned unchanged. */ -Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr *pExpr, Token *pCollName){ +Expr *sqlite3ExprAddCollateToken( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* Add the "COLLATE" clause to this expression */ + const Token *pCollName /* Name of collating sequence */ +){ if( pCollName->n>0 ){ Expr *pNew = sqlite3ExprAlloc(pParse->db, TK_COLLATE, pCollName, 1); if( pNew ){ diff --git a/src/parse.y b/src/parse.y index 24b8b98afc..aae30f5ee2 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1031,8 +1031,7 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ** expr1 == (+?1 COLLATE binary) ** expr1 <> (+?2 COLLATE binary) */ - static const Token collBin = { "binary", 6 }; - Expr *pRHS = sqlite3ExprAddCollateToken(pParse, Y->a[0].pExpr, &collBin); + Expr *pRHS = sqlite3ExprAddCollateString(pParse, Y->a[0].pExpr, "binary"); Y->a[0].pExpr = 0; sqlite3ExprListDelete(pParse->db, Y); pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0, 0); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 3ee39e5278..0974db7c91 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3291,7 +3291,7 @@ int sqlite3ReadSchema(Parse *pParse); CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); -Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, Token*); +Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*); Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); Expr *sqlite3ExprSkipCollate(Expr*); int sqlite3CheckCollSeq(Parse *, CollSeq *); diff --git a/src/vdbesort.c b/src/vdbesort.c index f971c2d933..839782b9c7 100644 --- a/src/vdbesort.c +++ b/src/vdbesort.c @@ -993,12 +993,12 @@ int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ ** a value equivalent to pIter2. So set pKey2 to NULL to prevent ** vdbeSorterCompare() from decoding pIter2 again. */ if( iRes<=0 ){ - pSorter->aTree[i] = (pIter1 - pSorter->aIter); + pSorter->aTree[i] = (int)(pIter1 - pSorter->aIter); pIter2 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; pKey2 = pIter2->aKey; }else{ if( pIter1->pFile ) pKey2 = 0; - pSorter->aTree[i] = (pIter2 - pSorter->aIter); + pSorter->aTree[i] = (int)(pIter2 - pSorter->aIter); pIter1 = &pSorter->aIter[ pSorter->aTree[i ^ 0x0001] ]; } From fbb24d1092c1c09a9493d382589176c1ba54bd52 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 17:03:30 +0000 Subject: [PATCH 23/49] The "x IN (?)" optimization in check-ins [2ff3b25f40] and [e68b427afb] is incorrect, as demonstrated by the in4-5.1 test case in this check-in. The "COLLATE binary" that was being added to the RHS of IN was overriding the implicit collating sequence of the LHS. This change defines the EP_Generic expression node property that blocks all affinity or collating sequence information in the expression subtree and adds that property to the expression taken from RHS of the IN operator. FossilOrigin-Name: 2ea4a9f75f46eaa928ba17e9e91bc0432750d46d --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/expr.c | 2 ++ src/parse.y | 19 ++++++++++++++----- src/sqliteInt.h | 4 ++-- test/in4.test | 30 ++++++++++++++++++++++++++++++ 6 files changed, 58 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 9590aa3ca2..8eaa4bfe54 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sharmless\scompiler\swarnings. -D 2014-03-20T15:14:08.664 +C The\s"x\sIN\s(?)"\soptimization\sin\scheck-ins\s[2ff3b25f40]\sand\s[e68b427afb]\sis\nincorrect,\sas\sdemonstrated\sby\sthe\sin4-5.1\stest\scase\sin\sthis\scheck-in.\nThe\s"COLLATE\sbinary"\sthat\swas\sbeing\sadded\sto\sthe\sRHS\sof\sIN\swas\soverriding\nthe\simplicit\scollating\ssequence\sof\sthe\sLHS.\s\sThis\schange\sdefines\sthe\sEP_Generic\nexpression\snode\sproperty\sthat\sblocks\sall\saffinity\sor\scollating\ssequence\ninformation\sin\sthe\sexpression\ssubtree\sand\sadds\sthat\sproperty\sto\sthe\sexpression\ntaken\sfrom\sRHS\sof\sthe\sIN\soperator. +D 2014-03-20T17:03:30.667 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c cdd57149543bb28304d8f717c243f2a86b1fc280 -F src/expr.c b74939e7935c4ad9e7f87b31ce05713fd5dafc3a +F src/expr.c 16ea9cefe7c8f998816b4eb8b8e7a88f0d2d3797 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 @@ -207,7 +207,7 @@ F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y d21075457487f84a72f848c2545e3319d6452e6f +F src/parse.y fb3280d85a103f623e5cf551b5b96b9df33151ac F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c @@ -222,7 +222,7 @@ F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h db6d7cf6e44d1c862f4f5290716098a0b90f1310 +F src/sqliteInt.h 42acfa3d3b793822915ceb7e83c0cbc774d37d66 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -593,7 +593,7 @@ F test/icu.test 70df4faca133254c042d02ae342c0a141f2663f4 F test/in.test 047c4671328e9032ab95666a67021adbbd36e98e F test/in2.test 5d4c61d17493c832f7d2d32bef785119e87bde75 F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0 -F test/in4.test 18202389003284e8e019750c04e4bc6333df9c99 +F test/in4.test 41c1c031aa46b1eb4411df2687ed2ed498da23b5 F test/in5.test 99f9a40af01711b06d2d614ecfe96129f334fba3 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328 F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2ff3b25f40fd117c8a2da1d1a3625f6b167b7b16 -R b4897cbb8b99db805aa2642c2b5c6a8f +P b1435f26b07b2208cfcca557f96342a5bd0d5328 +R a33b75c3e29794aff199068a6fe53400 U drh -Z 8f62a0caca3ff37dd6879b77616b6faf +Z 9b4a07b9ddb6d1760a1f90c03adda56a diff --git a/manifest.uuid b/manifest.uuid index 82d7da5708..5b4998caed 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b1435f26b07b2208cfcca557f96342a5bd0d5328 \ No newline at end of file +2ea4a9f75f46eaa928ba17e9e91bc0432750d46d \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 83b948e8e1..204d696179 100644 --- a/src/expr.c +++ b/src/expr.c @@ -33,6 +33,7 @@ char sqlite3ExprAffinity(Expr *pExpr){ int op; pExpr = sqlite3ExprSkipCollate(pExpr); + if( pExpr->flags & EP_Generic ) return SQLITE_AFF_NONE; op = pExpr->op; if( op==TK_SELECT ){ assert( pExpr->flags&EP_xIsSelect ); @@ -122,6 +123,7 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ Expr *p = pExpr; while( p ){ int op = p->op; + if( p->flags & EP_Generic ) break; if( op==TK_CAST || op==TK_UPLUS ){ p = p->pLeft; continue; diff --git a/src/parse.y b/src/parse.y index aae30f5ee2..ef00925254 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1026,15 +1026,24 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { ** expr1 IN (?1) ** expr1 NOT IN (?2) ** - ** with exactly one value on the RHS can be simplified to: + ** with exactly one value on the RHS can be simplified to something + ** like this: ** - ** expr1 == (+?1 COLLATE binary) - ** expr1 <> (+?2 COLLATE binary) + ** expr1 == ?1 + ** expr1 <> ?2 + ** + ** But, the RHS of the == or <> is marked with the EP_Generic flag + ** so that it may not contribute to the computation of comparison + ** affinity or the collating sequence to use for comparison. Otherwise, + ** the semantics would be subtly different from IN or NOT IN. */ - Expr *pRHS = sqlite3ExprAddCollateString(pParse, Y->a[0].pExpr, "binary"); + Expr *pRHS = Y->a[0].pExpr; Y->a[0].pExpr = 0; sqlite3ExprListDelete(pParse->db, Y); - pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0, 0); + if( pRHS ){ + pRHS->flags &= ~EP_Collate; + pRHS->flags |= EP_Generic; + } A.pExpr = sqlite3PExpr(pParse, N ? TK_NE : TK_EQ, X.pExpr, pRHS, 0); }else{ A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 0974db7c91..55b7e080ae 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1893,8 +1893,8 @@ struct Expr { #define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ #define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ #define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ -#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE opeartor */ - /* unused 0x000200 */ +#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ +#define EP_Generic 0x000200 /* Ignore COLLATE or affinity on this tree */ #define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ #define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ #define EP_Skip 0x001000 /* COLLATE, AS, or UNLIKELY */ diff --git a/test/in4.test b/test/in4.test index 5828a83cc5..0a4a75008b 100644 --- a/test/in4.test +++ b/test/in4.test @@ -302,8 +302,38 @@ do_execsql_test in4-4.19 { SELECT c FROM t4b WHERE +b IN (a); } {} +do_execsql_test in4-5.1 { + CREATE TABLE t5(c INTEGER PRIMARY KEY, d TEXT COLLATE nocase); + INSERT INTO t5 VALUES(17, 'fuzz'); + SELECT 1 FROM t5 WHERE 'fuzz' IN (d); -- match + SELECT 2 FROM t5 WHERE 'FUZZ' IN (d); -- no match + SELECT 3 FROM t5 WHERE d IN ('fuzz'); -- match + SELECT 4 FROM t5 WHERE d IN ('FUZZ'); -- match +} {1 3 4} +# An expression of the form "x IN (y)" can be used as "x=y" by the +# query planner when computing transitive constraints or to run the +# query using an index on y. +# +do_execsql_test in4-6.1 { + CREATE TABLE t6a(a INTEGER PRIMARY KEY, b); + INSERT INTO t6a VALUES(1,2),(3,4),(5,6); + CREATE TABLE t6b(c INTEGER PRIMARY KEY, d); + INSERT INTO t6b VALUES(4,44),(5,55),(6,66); + SELECT * FROM t6a, t6b WHERE a=3 AND b IN (c); +} {3 4 4 44} +do_execsql_test in4-6.1-eqp { + EXPLAIN QUERY PLAN + SELECT * FROM t6a, t6b WHERE a=3 AND b IN (c); +} {~/SCAN/} +do_execsql_test in4-6.2 { + SELECT * FROM t6a, t6b WHERE a=3 AND c IN (b); +} {3 4 4 44} +do_execsql_test in4-6.2-eqp { + EXPLAIN QUERY PLAN + SELECT * FROM t6a, t6b WHERE a=3 AND c IN (b); +} {~/SCAN/} finish_test From 8ad0de3f725e9d32d624d3102f4e1b75780e7ee6 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 18:45:27 +0000 Subject: [PATCH 24/49] Fix the EXPLAIN indenter in the command-line shell to correctly handle NextIfOpen and PrevIfOpen opcodes. FossilOrigin-Name: 01944c53f5c129f48ce8842faaedcf5607dd381e --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 3 ++- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 8eaa4bfe54..b902e242a6 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C The\s"x\sIN\s(?)"\soptimization\sin\scheck-ins\s[2ff3b25f40]\sand\s[e68b427afb]\sis\nincorrect,\sas\sdemonstrated\sby\sthe\sin4-5.1\stest\scase\sin\sthis\scheck-in.\nThe\s"COLLATE\sbinary"\sthat\swas\sbeing\sadded\sto\sthe\sRHS\sof\sIN\swas\soverriding\nthe\simplicit\scollating\ssequence\sof\sthe\sLHS.\s\sThis\schange\sdefines\sthe\sEP_Generic\nexpression\snode\sproperty\sthat\sblocks\sall\saffinity\sor\scollating\ssequence\ninformation\sin\sthe\sexpression\ssubtree\sand\sadds\sthat\sproperty\sto\sthe\sexpression\ntaken\sfrom\sRHS\sof\sthe\sIN\soperator. -D 2014-03-20T17:03:30.667 +C Fix\sthe\sEXPLAIN\sindenter\sin\sthe\scommand-line\sshell\sto\scorrectly\nhandle\sNextIfOpen\sand\sPrevIfOpen\sopcodes. +D 2014-03-20T18:45:27.340 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -218,7 +218,7 @@ F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 0adf172d33cc610ff5ffe26edfd2ba67c3551655 -F src/shell.c bab4de12b441369491812ecc93212ff4deda68fa +F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b1435f26b07b2208cfcca557f96342a5bd0d5328 -R a33b75c3e29794aff199068a6fe53400 +P 2ea4a9f75f46eaa928ba17e9e91bc0432750d46d +R 4c494d6d400709087b8c24162b58f63e U drh -Z 9b4a07b9ddb6d1760a1f90c03adda56a +Z f93731af177c8fb155193dbd5a919278 diff --git a/manifest.uuid b/manifest.uuid index 5b4998caed..80f6ef7e37 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2ea4a9f75f46eaa928ba17e9e91bc0432750d46d \ No newline at end of file +01944c53f5c129f48ce8842faaedcf5607dd381e \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 2cff102e91..1313112709 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1195,7 +1195,8 @@ static void explain_data_prepare(struct callback_data *p, sqlite3_stmt *pSql){ int nAlloc = 0; /* Allocated size of p->aiIndent[], abYield */ int iOp; /* Index of operation in p->aiIndent[] */ - const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", 0 }; + const char *azNext[] = { "Next", "Prev", "VPrev", "VNext", "SorterNext", + "NextIfOpen", "PrevIfOpen", 0 }; const char *azYield[] = { "Yield", "SeekLt", "SeekGt", "RowSetRead", "Rewind", 0 }; const char *azGoto[] = { "Goto", 0 }; From 5b1420e046aaf2aa3043fa1725b0ae57abe1578a Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 20 Mar 2014 19:04:56 +0000 Subject: [PATCH 25/49] Remove a testcase() that is now always true due to the "x IN (?)" optimization. Add an ALWAYS() around a conditional in the parser that cannot be false. FossilOrigin-Name: d5a1530bdc7ace053d05d1a037551110021d3758 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/expr.c | 1 - src/parse.y | 4 +++- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index b902e242a6..9c280ab5f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sEXPLAIN\sindenter\sin\sthe\scommand-line\sshell\sto\scorrectly\nhandle\sNextIfOpen\sand\sPrevIfOpen\sopcodes. -D 2014-03-20T18:45:27.340 +C Remove\sa\stestcase()\sthat\sis\snow\salways\strue\sdue\sto\sthe\s"x\sIN\s(?)"\soptimization.\nAdd\san\sALWAYS()\saround\sa\sconditional\sin\sthe\sparser\sthat\scannot\sbe\sfalse. +D 2014-03-20T19:04:56.106 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -173,7 +173,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 0231df905e2c4abba4483ee18ffc05adc321df2a F src/date.c 593c744b2623971e45affd0bde347631bdfa4625 F src/delete.c cdd57149543bb28304d8f717c243f2a86b1fc280 -F src/expr.c 16ea9cefe7c8f998816b4eb8b8e7a88f0d2d3797 +F src/expr.c 6e1e8bca468753f06758db7835a85e5408351e7a F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 5269ef07b100763134f71b889327c333bd0989cf F src/func.c 2945bb2c4cdc0ac43733046285a4434310be1811 @@ -207,7 +207,7 @@ F src/os_unix.c 18f7f95dc6bcb9cf4d4a238d8e2de96611bc2ae5 F src/os_win.c e71678ac927d0a0fb11d993db20a9748eabf808e F src/pager.c 97a8908bf4e6e7c3adea09d3597cfa48ae33ab4e F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428 -F src/parse.y fb3280d85a103f623e5cf551b5b96b9df33151ac +F src/parse.y 22d6a074e5f5a7258947a1dc55a9bf946b765dd0 F src/pcache.c d8eafac28290d4bb80332005435db44991d07fc2 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c 102e6f5a2fbc646154463eb856d1fd716867b64c @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2ea4a9f75f46eaa928ba17e9e91bc0432750d46d -R 4c494d6d400709087b8c24162b58f63e +P 01944c53f5c129f48ce8842faaedcf5607dd381e +R 608adcc59408221c1aef74de280cb552 U drh -Z f93731af177c8fb155193dbd5a919278 +Z 43de690b7d9666e906803652c8c9527b diff --git a/manifest.uuid b/manifest.uuid index 80f6ef7e37..416a2c393f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -01944c53f5c129f48ce8842faaedcf5607dd381e \ No newline at end of file +d5a1530bdc7ace053d05d1a037551110021d3758 \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index 204d696179..e3997f00c3 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1636,7 +1636,6 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){ *prNotFound = rMayHaveNull = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, *prNotFound); }else{ - testcase( pParse->nQueryLoop>0 ); pParse->nQueryLoop = 0; if( pX->pLeft->iColumn<0 && !ExprHasProperty(pX, EP_xIsSelect) ){ eType = IN_INDEX_ROWID; diff --git a/src/parse.y b/src/parse.y index ef00925254..dbc129ce63 100644 --- a/src/parse.y +++ b/src/parse.y @@ -1040,7 +1040,9 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] { Expr *pRHS = Y->a[0].pExpr; Y->a[0].pExpr = 0; sqlite3ExprListDelete(pParse->db, Y); - if( pRHS ){ + /* pRHS cannot be NULL because a malloc error would have been detected + ** before now and control would have never reached this point */ + if( ALWAYS(pRHS) ){ pRHS->flags &= ~EP_Collate; pRHS->flags |= EP_Generic; } From e99b9cdc70213d8e7d59a4b4447b7c59b56a4869 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Mar 2014 15:24:07 +0000 Subject: [PATCH 26/49] New test case for block-sorting. FossilOrigin-Name: e70cfa28aa393661ccc742ecd5e672d807bdd0a9 --- manifest | 11 +-- manifest.uuid | 2 +- test/orderby6.test | 183 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 6 deletions(-) create mode 100644 test/orderby6.test diff --git a/manifest b/manifest index 6bb5c6bb73..e0157af624 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\strunk\sfixes\sfor\s"x\sIN\s(?)"\shandling. -D 2014-03-20T20:56:49.315 +C New\stest\scase\sfor\sblock-sorting. +D 2014-03-21T15:24:07.126 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -723,6 +723,7 @@ F test/orderby2.test bc11009f7cd99d96b1b11e57b199b00633eb5b04 F test/orderby3.test 8619d06a3debdcd80a27c0fdea5c40b468854b99 F test/orderby4.test 4d39bfbaaa3ae64d026ca2ff166353d2edca4ba4 F test/orderby5.test 2490183fef54417209d1df253633a605d46bd350 +F test/orderby6.test 8b38138ab0972588240b3fca0985d2e400432859 F test/oserror.test 50417780d0e0d7cd23cf12a8277bb44024765df3 F test/pager1.test 1acbdb14c5952a72dd43129cabdbf69aaa3ed1fa F test/pager2.test 67b8f40ae98112bcdba1f2b2d03ea83266418c71 @@ -1156,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e4bfffb988283c077778c60696be0d285ad66c3c d5a1530bdc7ace053d05d1a037551110021d3758 -R a45d230e7f98fec92659279ce8b3552c +P eca35871c34374ca9189c7c9b6d490ac3c30357f +R f6c9f43506e02e19d33da5c08cba37a5 U drh -Z 027a45adb600d40b2564e06474072c41 +Z 983388db59b02f32fb79fe85b7106067 diff --git a/manifest.uuid b/manifest.uuid index 8a32b61e54..fe6f27d9c8 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -eca35871c34374ca9189c7c9b6d490ac3c30357f \ No newline at end of file +e70cfa28aa393661ccc742ecd5e672d807bdd0a9 \ No newline at end of file diff --git a/test/orderby6.test b/test/orderby6.test new file mode 100644 index 0000000000..403250ddb5 --- /dev/null +++ b/test/orderby6.test @@ -0,0 +1,183 @@ +# 2014-03-21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this file is testing that the block-sort optimization. +# + + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set ::testprefix orderby6 + +# Run all tests twice. Once with a normal table and a second time +# with a WITHOUT ROWID table +# +foreach {tn rowidclause} {1 {} 2 {WITHOUT ROWID}} { + + # Construct a table with 1000 rows and a split primary key + # + reset_db + do_test $tn.1 { + db eval "CREATE TABLE t1(a,b,c,PRIMARY KEY(b,c)) $rowidclause;" + db eval { + WITH RECURSIVE + cnt(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM cnt WHERE x<1000) + INSERT INTO t1 SELECT x, x%40, x/40 FROM cnt; + } + } {} + + # Run various ORDER BY queries that can benefit from block-sort. + # Compare the output to the same output using a full-sort enforced + # by adding + to each term of the ORDER BY clause. + # + do_execsql_test $tn.2 { + SELECT b,a,c FROM t1 ORDER BY b,a,c; + } [db eval {SELECT b,a,c FROM t1 ORDER BY +b,+a,+c}] + do_execsql_test $tn.3 { + SELECT b,a,c FROM t1 ORDER BY b,c DESC,a; + } [db eval {SELECT b,a,c FROM t1 ORDER BY +b,+c DESC,+a}] + do_execsql_test $tn.4 { + SELECT b,a,c FROM t1 ORDER BY b DESC,c,a; + } [db eval {SELECT b,a,c FROM t1 ORDER BY +b DESC,+c,+a}] + do_execsql_test $tn.5 { + SELECT b,a,c FROM t1 ORDER BY b DESC,a,c; + } [db eval {SELECT b,a,c FROM t1 ORDER BY +b DESC,+a,+c}] + + # LIMIT and OFFSET clauses on block-sort queries. + # + do_execsql_test $tn.11 { + SELECT a FROM t1 ORDER BY b, a LIMIT 10 OFFSET 20; + } {840 880 920 960 1000 1 41 81 121 161} + do_execsql_test $tn.11x { + SELECT a FROM t1 ORDER BY +b, a LIMIT 10 OFFSET 20; + } {840 880 920 960 1000 1 41 81 121 161} + + do_execsql_test $tn.12 { + SELECT a FROM t1 ORDER BY b DESC, a LIMIT 10 OFFSET 20; + } {839 879 919 959 999 38 78 118 158 198} + do_execsql_test $tn.12 { + SELECT a FROM t1 ORDER BY +b DESC, a LIMIT 10 OFFSET 20; + } {839 879 919 959 999 38 78 118 158 198} + + do_execsql_test $tn.13 { + SELECT a FROM t1 ORDER BY b, a DESC LIMIT 10 OFFSET 45; + } {161 121 81 41 1 962 922 882 842 802} + do_execsql_test $tn.13x { + SELECT a FROM t1 ORDER BY +b, a DESC LIMIT 10 OFFSET 45; + } {161 121 81 41 1 962 922 882 842 802} + + do_execsql_test $tn.14 { + SELECT a FROM t1 ORDER BY b DESC, a LIMIT 10 OFFSET 45; + } {838 878 918 958 998 37 77 117 157 197} + do_execsql_test $tn.14x { + SELECT a FROM t1 ORDER BY +b DESC, a LIMIT 10 OFFSET 45; + } {838 878 918 958 998 37 77 117 157 197} + + # Many test cases where the LIMIT+OFFSET window is in various + # alignments with block-sort boundaries. + # + foreach {tx limit offset orderby} { + 1 10 24 {+b,+a} + 2 10 25 {+b,+a} + 3 10 26 {+b,+a} + 4 10 39 {+b,+a} + 5 10 40 {+b,+a} + 6 10 41 {+b,+a} + 7 27 24 {+b,+a} + 8 27 49 {+b,+a} + 11 10 24 {+b DESC,+a} + 12 10 25 {+b DESC,+a} + 13 10 26 {+b DESC,+a} + 14 10 39 {+b DESC,+a} + 15 10 40 {+b DESC,+a} + 16 10 41 {+b DESC,+a} + 17 27 24 {+b DESC,+a} + 18 27 49 {+b DESC,+a} + 21 10 24 {+b,+a DESC} + 22 10 25 {+b,+a DESC} + 23 10 26 {+b,+a DESC} + 24 10 39 {+b,+a DESC} + 25 10 40 {+b,+a DESC} + 26 10 41 {+b,+a DESC} + 27 27 24 {+b,+a DESC} + 28 27 49 {+b,+a DESC} + 31 10 24 {+b DESC,+a DESC} + 32 10 25 {+b DESC,+a DESC} + 33 10 26 {+b DESC,+a DESC} + 34 10 39 {+b DESC,+a DESC} + 35 10 40 {+b DESC,+a DESC} + 36 10 41 {+b DESC,+a DESC} + 37 27 24 {+b DESC,+a DESC} + 38 27 49 {+b DESC,+a DESC} + } { + set sql1 "SELECT a FROM t1 ORDER BY $orderby LIMIT $limit OFFSET $offset;" + set sql2 [string map {+ {}} $sql1] + # puts $sql2\n$sql1\n[db eval $sql2] + do_test $tn.21.$tx {db eval $::sql2} [db eval $sql1] + } + + ######################################################################## + # A second test table, t2, has many columns open to sorting. + do_test $tn.31 { + db eval "CREATE TABLE t2(a,b,c,d,e,f,PRIMARY KEY(b,c,d,e,f)) $rowidclause;" + db eval { + WITH RECURSIVE + cnt(x) AS (VALUES(0) UNION ALL SELECT x+1 FROM cnt WHERE x<242) + INSERT INTO t2 SELECT x, x%3, (x/3)%3, (x/9)%3, (x/27)%3, (x/81)%3 + FROM cnt; + } + } {} + + do_execsql_test $tn.32 { + SELECT a FROM t2 ORDER BY b,c,d,e,f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f;}] + do_execsql_test $tn.33 { + SELECT a FROM t2 ORDER BY b,c,d,e,+f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f;}] + do_execsql_test $tn.34 { + SELECT a FROM t2 ORDER BY b,c,d,+e,+f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f;}] + do_execsql_test $tn.35 { + SELECT a FROM t2 ORDER BY b,c,+d,+e,+f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f;}] + do_execsql_test $tn.36 { + SELECT a FROM t2 ORDER BY b,+c,+d,+e,+f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f;}] + + do_execsql_test $tn.37 { + SELECT a FROM t2 ORDER BY b,c,d,e,f DESC; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f DESC;}] + do_execsql_test $tn.38 { + SELECT a FROM t2 ORDER BY b,c,d,e DESC,f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e DESC,+f;}] + do_execsql_test $tn.39 { + SELECT a FROM t2 ORDER BY b,c,d DESC,e,f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d DESC,+e,+f;}] + do_execsql_test $tn.40 { + SELECT a FROM t2 ORDER BY b,c DESC,d,e,f; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c DESC,+d,+e,+f;}] + do_execsql_test $tn.41 { + SELECT a FROM t2 ORDER BY b DESC,c,d,e,f; + } [db eval {SELECT a FROM t2 ORDER BY +b DESC,+c,+d,+e,+f;}] + + do_execsql_test $tn.42 { + SELECT a FROM t2 ORDER BY b DESC,c DESC,d,e,f LIMIT 31; + } [db eval {SELECT a FROM t2 ORDER BY +b DESC,+c DESC,+d,+e,+f LIMIT 31}] + do_execsql_test $tn.43 { + SELECT a FROM t2 ORDER BY b,c,d,e,f DESC LIMIT 8 OFFSET 7; + } [db eval {SELECT a FROM t2 ORDER BY +b,+c,+d,+e,+f DESC LIMIT 8 OFFSET 7}] + + +} + + + +finish_test From a22a75e5ca62599b22744fd089b3a4f504bc83d5 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Mar 2014 18:16:23 +0000 Subject: [PATCH 27/49] Fix the OFFSET clause so that it works correctly on queries that lack a FROM clause. Ticket [07d6a0453d4ed8]. FossilOrigin-Name: 179ef81648b0ad557df78b7712f216b876b6fb65 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 3 ++- src/where.c | 3 ++- test/limit.test | 20 ++++++++++++++++++++ 5 files changed, 33 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 9c280ab5f2..68306cb2c1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sa\stestcase()\sthat\sis\snow\salways\strue\sdue\sto\sthe\s"x\sIN\s(?)"\soptimization.\nAdd\san\sALWAYS()\saround\sa\sconditional\sin\sthe\sparser\sthat\scannot\sbe\sfalse. -D 2014-03-20T19:04:56.106 +C Fix\sthe\sOFFSET\sclause\sso\sthat\sit\sworks\scorrectly\son\squeries\sthat\slack\na\sFROM\sclause.\s\sTicket\s[07d6a0453d4ed8]. +D 2014-03-21T18:16:23.398 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 0adf172d33cc610ff5ffe26edfd2ba67c3551655 +F src/select.c bea5181682a75940f4f555bfd1020169cc8f1657 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c a2f20bdc6565a3add38ac5e4d9e71523ed5b2f2b +F src/where.c eed0b83bd7ffef4ac2f7a2fcc9dcfa66b18b8b9e F src/whereInt.h 921f935af8b684ffb49705610bda7284db1db138 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -648,7 +648,7 @@ F test/lastinsert.test 42e948fd6442f07d60acbd15d33fb86473e0ef63 F test/laststmtchanges.test ae613f53819206b3222771828d024154d51db200 F test/like.test e191e536d0fcd722a6b965e7cd1ee0bfd12a5991 F test/like2.test 3b2ee13149ba4a8a60b59756f4e5d345573852da -F test/limit.test cc0ab63385239b63c72452b0e93700bf5e8f0b99 +F test/limit.test 3d7df19c35ac672a11f7de406cd3205d592babbb F test/loadext.test 648cb95f324d1775c54a55c12271b2d1156b633b F test/loadext2.test 0408380b57adca04004247179837a18e866a74f7 F test/lock.test 87af515b0c4cf928576d0f89946d67d7c265dfb4 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 01944c53f5c129f48ce8842faaedcf5607dd381e -R 608adcc59408221c1aef74de280cb552 +P d5a1530bdc7ace053d05d1a037551110021d3758 +R e96408bb9b93596f5e755db9dbf77630 U drh -Z 43de690b7d9666e906803652c8c9527b +Z f1228f80df0ab7ad7b6a23acd634c66b diff --git a/manifest.uuid b/manifest.uuid index 416a2c393f..0c9a994b80 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -d5a1530bdc7ace053d05d1a037551110021d3758 \ No newline at end of file +179ef81648b0ad557df78b7712f216b876b6fb65 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 850bc6a902..65e4ae7d67 100644 --- a/src/select.c +++ b/src/select.c @@ -473,7 +473,7 @@ static void codeOffset( int iOffset, /* Register holding the offset counter */ int iContinue /* Jump here to skip the current record */ ){ - if( iOffset>0 && iContinue!=0 ){ + if( iOffset>0 ){ int addr; sqlite3VdbeAddOp2(v, OP_AddImm, iOffset, -1); addr = sqlite3VdbeAddOp1(v, OP_IfNeg, iOffset); VdbeCoverage(v); @@ -579,6 +579,7 @@ static void selectInnerLoop( assert( pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; if( pOrderBy==0 && !hasDistinct ){ + assert( iContinue!=0 ); codeOffset(v, p->iOffset, iContinue); } diff --git a/src/where.c b/src/where.c index e3ce8b8697..bc5dc04d14 100644 --- a/src/where.c +++ b/src/where.c @@ -47,6 +47,7 @@ int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ ** immediately with the next row of a WHERE clause. */ int sqlite3WhereContinueLabel(WhereInfo *pWInfo){ + assert( pWInfo->iContinue!=0 ); return pWInfo->iContinue; } @@ -5453,7 +5454,7 @@ WhereInfo *sqlite3WhereBegin( pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; pWInfo->pResultSet = pResultSet; - pWInfo->iBreak = sqlite3VdbeMakeLabel(v); + pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); pWInfo->wctrlFlags = wctrlFlags; pWInfo->savedNQueryLoop = pParse->nQueryLoop; pMaskSet = &pWInfo->sMaskSet; diff --git a/test/limit.test b/test/limit.test index c5b75c2b69..5d6ef336b2 100644 --- a/test/limit.test +++ b/test/limit.test @@ -616,4 +616,24 @@ do_test limit-13.81 { db eval {SELECT z FROM v13c LIMIT 1 OFFSET 8} } {} +do_execsql_test limit-14.1 { + SELECT 123 LIMIT 1 OFFSET 0 +} {123} +do_execsql_test limit-14.2 { + SELECT 123 LIMIT 1 OFFSET 1 +} {} +do_execsql_test limit-14.3 { + SELECT 123 LIMIT 0 OFFSET 0 +} {} +do_execsql_test limit-14.4 { + SELECT 123 LIMIT 0 OFFSET 1 +} {} +do_execsql_test limit-14.6 { + SELECT 123 LIMIT -1 OFFSET 0 +} {123} +do_execsql_test limit-14.7 { + SELECT 123 LIMIT -1 OFFSET 1 +} {} + + finish_test From 9afccba269eb1910404fc003fc8e96ca6fbd742e Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 21 Mar 2014 19:27:54 +0000 Subject: [PATCH 28/49] Avoid leaking memory in an obscure case where the flattener adds an ORDER BY clause to the recursive part of a recursive query. FossilOrigin-Name: 1f413aca00015100224273480e1ce39a76bf93ab --- manifest | 18 +++++++++--------- manifest.uuid | 2 +- src/select.c | 8 +++++++- src/sqliteInt.h | 2 +- test/with2.test | 27 +++++++++++++++++++++++++++ 5 files changed, 45 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 68306cb2c1..9a15563c3f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\sOFFSET\sclause\sso\sthat\sit\sworks\scorrectly\son\squeries\sthat\slack\na\sFROM\sclause.\s\sTicket\s[07d6a0453d4ed8]. -D 2014-03-21T18:16:23.398 +C Avoid\sleaking\smemory\sin\san\sobscure\scase\swhere\sthe\sflattener\sadds\san\sORDER\sBY\sclause\sto\sthe\srecursive\spart\sof\sa\srecursive\squery. +D 2014-03-21T19:27:54.828 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,12 +217,12 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c bea5181682a75940f4f555bfd1020169cc8f1657 +F src/select.c 0f7542b85f92d8b0c59a21d7ddcf5c0228e60fdc F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 42acfa3d3b793822915ceb7e83c0cbc774d37d66 +F src/sqliteInt.h 828f61d3c58608ad2e871369950f3cf70428e917 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1096,7 +1096,7 @@ F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d F test/with1.test 268081a6b14817a262ced4d0ee34d4d2a1dd2068 -F test/with2.test 2fe78fcd8deef2a0f9cfc49bfc755911d0b3fd64 +F test/with2.test ee227a663586aa09771cafd4fa269c5217eaf775 F test/withM.test e97f2a8c506ab3ea9eab94e6f6072f6cc924c991 F test/without_rowid1.test e00a0a9dc9f0be651f011d61e8a32b7add5afb30 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99 @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P d5a1530bdc7ace053d05d1a037551110021d3758 -R e96408bb9b93596f5e755db9dbf77630 -U drh -Z f1228f80df0ab7ad7b6a23acd634c66b +P 179ef81648b0ad557df78b7712f216b876b6fb65 +R d6ea5c2b32ee3608a1c16a03630c7806 +U dan +Z 37d6162a1c996286e576ee6c912ca01b diff --git a/manifest.uuid b/manifest.uuid index 0c9a994b80..e3acb3d652 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -179ef81648b0ad557df78b7712f216b876b6fb65 \ No newline at end of file +1f413aca00015100224273480e1ce39a76bf93ab \ No newline at end of file diff --git a/src/select.c b/src/select.c index 65e4ae7d67..93cf15f79b 100644 --- a/src/select.c +++ b/src/select.c @@ -1939,6 +1939,7 @@ static void generateWithRecursiveQuery( sqlite3VdbeResolveLabel(v, addrBreak); end_of_recursive_query: + sqlite3ExprListDelete(pParse->db, p->pOrderBy); p->pOrderBy = pOrderBy; p->pLimit = pLimit; p->pOffset = pOffset; @@ -4487,9 +4488,14 @@ int sqlite3Select( if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; memset(&sAggInfo, 0, sizeof(sAggInfo)); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistTable ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); if( IgnorableOrderby(pDest) ){ assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || - pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard); + pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || + pDest->eDest==SRT_Queue || pDest->eDest==SRT_DistTable || + pDest->eDest==SRT_DistQueue); /* If ORDER BY makes no difference in the output then neither does ** DISTINCT so it can be removed too. */ sqlite3ExprListDelete(db, p->pOrderBy); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 55b7e080ae..b31686c3ff 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2283,7 +2283,7 @@ struct Select { #define SRT_Discard 4 /* Do not save the results anywhere */ /* The ORDER BY clause is ignored for all of the above */ -#define IgnorableOrderby(X) ((X->eDest)<=SRT_Discard) +#define IgnorableOrderby(X) ((X->eDest)<=SRT_Discard || (X->eDest)>SRT_Table) #define SRT_Output 5 /* Output each row of result */ #define SRT_Mem 6 /* Store result in a memory cell */ diff --git a/test/with2.test b/test/with2.test index d702f8c962..eb0614729b 100644 --- a/test/with2.test +++ b/test/with2.test @@ -385,6 +385,33 @@ do_execsql_test 7.5 { ) } {14 28 42} +#------------------------------------------------------------------------- +# At one point the following was causing an assertion failure and a +# memory leak. +# +do_execsql_test 8.1 { + CREATE TABLE t7(y); + INSERT INTO t7 VALUES(NULL); + CREATE VIEW v AS SELECT * FROM t7 ORDER BY y; +} + +do_execsql_test 8.2 { + WITH q(a) AS ( + SELECT 1 + UNION + SELECT a+1 FROM q, v WHERE a<5 + ) + SELECT * FROM q; +} {1 2 3 4 5} + +do_execsql_test 8.3 { + WITH q(a) AS ( + SELECT 1 + UNION ALL + SELECT a+1 FROM q, v WHERE a<5 + ) + SELECT * FROM q; +} {1 2 3 4 5} finish_test From 8e1ee88c8811f9653882c4c1a894951d65d36bbf Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 21 Mar 2014 19:56:09 +0000 Subject: [PATCH 29/49] Change the names of SRT_DistTable and SRT_Table used by CTE to more meaningful SRT_DistFifo and SRT_Fifo, respectively. Simplify the IgnorableOrderby() macro in the process. FossilOrigin-Name: 45d8cc678d128f1dda6469864215a8ed9de4366a --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/select.c | 22 ++++++++++++---------- src/sqliteInt.h | 31 +++++++++++++++++-------------- 4 files changed, 38 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 9a15563c3f..88f61607f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sleaking\smemory\sin\san\sobscure\scase\swhere\sthe\sflattener\sadds\san\sORDER\sBY\sclause\sto\sthe\srecursive\spart\sof\sa\srecursive\squery. -D 2014-03-21T19:27:54.828 +C Change\sthe\snames\sof\sSRT_DistTable\sand\sSRT_Table\sused\sby\sCTE\sto\smore\nmeaningful\sSRT_DistFifo\sand\sSRT_Fifo,\srespectively.\s\sSimplify\sthe\nIgnorableOrderby()\smacro\sin\sthe\sprocess. +D 2014-03-21T19:56:09.443 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,12 +217,12 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 0f7542b85f92d8b0c59a21d7ddcf5c0228e60fdc +F src/select.c cbee644f795a5fa14fa79bd760c3c9c883a6b4d8 F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 828f61d3c58608ad2e871369950f3cf70428e917 +F src/sqliteInt.h 76b1cf0836f0d022aec5045474f5c6c90f7b9730 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1156,7 +1156,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 179ef81648b0ad557df78b7712f216b876b6fb65 -R d6ea5c2b32ee3608a1c16a03630c7806 -U dan -Z 37d6162a1c996286e576ee6c912ca01b +P 1f413aca00015100224273480e1ce39a76bf93ab +R 26118dcbaf585402aab527afe8f97345 +U drh +Z d7e9ffdc5a42e976a5961e8287a0561b diff --git a/manifest.uuid b/manifest.uuid index e3acb3d652..253fab48e6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1f413aca00015100224273480e1ce39a76bf93ab \ No newline at end of file +45d8cc678d128f1dda6469864215a8ed9de4366a \ No newline at end of file diff --git a/src/select.c b/src/select.c index 93cf15f79b..973b444407 100644 --- a/src/select.c +++ b/src/select.c @@ -700,7 +700,8 @@ static void selectInnerLoop( /* Store the result as data using a unique key. */ - case SRT_DistTable: + case SRT_Fifo: + case SRT_DistFifo: case SRT_Table: case SRT_EphemTab: { int r1 = sqlite3GetTempReg(pParse); @@ -708,8 +709,8 @@ static void selectInnerLoop( testcase( eDest==SRT_EphemTab ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); #ifndef SQLITE_OMIT_CTE - if( eDest==SRT_DistTable ){ - /* If the destination is DistTable, then cursor (iParm+1) is open + if( eDest==SRT_DistFifo ){ + /* If the destination is DistFifo, then cursor (iParm+1) is open ** on an ephemeral index. If the current row is already present ** in the index, do not write it to the output. If not, add the ** current row to the index and proceed with writing it to the @@ -1835,7 +1836,7 @@ static void generateWithRecursiveQuery( int regCurrent; /* Register holding Current table */ int iQueue; /* The Queue table */ int iDistinct = 0; /* To ensure unique results if UNION */ - int eDest = SRT_Table; /* How to write to Queue */ + int eDest = SRT_Fifo; /* How to write to Queue */ SelectDest destQueue; /* SelectDest targetting the Queue table */ int i; /* Loop counter */ int rc; /* Result code */ @@ -1867,13 +1868,13 @@ static void generateWithRecursiveQuery( /* Allocate cursors numbers for Queue and Distinct. The cursor number for ** the Distinct table must be exactly one greater than Queue in order - ** for the SRT_DistTable and SRT_DistQueue destinations to work. */ + ** for the SRT_DistFifo and SRT_DistQueue destinations to work. */ iQueue = pParse->nTab++; if( p->op==TK_UNION ){ - eDest = pOrderBy ? SRT_DistQueue : SRT_DistTable; + eDest = pOrderBy ? SRT_DistQueue : SRT_DistFifo; iDistinct = pParse->nTab++; }else{ - eDest = pOrderBy ? SRT_Queue : SRT_Table; + eDest = pOrderBy ? SRT_Queue : SRT_Fifo; } sqlite3SelectDestInit(&destQueue, eDest, iQueue); @@ -4488,14 +4489,15 @@ int sqlite3Select( if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; memset(&sAggInfo, 0, sizeof(sAggInfo)); - assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistTable ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); + assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); if( IgnorableOrderby(pDest) ){ assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || - pDest->eDest==SRT_Queue || pDest->eDest==SRT_DistTable || - pDest->eDest==SRT_DistQueue); + pDest->eDest==SRT_Queue || pDest->eDest==SRT_DistFifo || + pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_Fifo); /* If ORDER BY makes no difference in the output then neither does ** DISTINCT so it can be removed too. */ sqlite3ExprListDelete(db, p->pOrderBy); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index b31686c3ff..724b22fec1 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2261,13 +2261,15 @@ struct Select { ** starting with pDest->iSdst. ** ** SRT_Table Store results in temporary table pDest->iSDParm. -** This is like SRT_EphemTab except that the table -** is assumed to already be open. +** SRT_Fifo This is like SRT_EphemTab except that the table +** is assumed to already be open. SRT_Fifo has +** the additional property of being able to ignore +** the ORDER BY clause. ** -** SRT_DistTable Store results in a temporary table pDest->iSDParm. +** SRT_DistFifo Store results in a temporary table pDest->iSDParm. ** But also use temporary table pDest->iSDParm+1 as ** a record of all prior results and ignore any duplicate -** rows. Name means: "Distinct Table". +** rows. Name means: "Distinct Fifo". ** ** SRT_Queue Store results in priority queue pDest->iSDParm (really ** an index). Append a sequence number so that all entries @@ -2281,19 +2283,20 @@ struct Select { #define SRT_Except 2 /* Remove result from a UNION index */ #define SRT_Exists 3 /* Store 1 if the result is not empty */ #define SRT_Discard 4 /* Do not save the results anywhere */ +#define SRT_Fifo 5 /* Store result as data with an automatic rowid */ +#define SRT_DistFifo 6 /* Like SRT_Fifo, but unique results only */ +#define SRT_Queue 7 /* Store result in an queue */ +#define SRT_DistQueue 8 /* Like SRT_Queue, but unique results only */ /* The ORDER BY clause is ignored for all of the above */ -#define IgnorableOrderby(X) ((X->eDest)<=SRT_Discard || (X->eDest)>SRT_Table) +#define IgnorableOrderby(X) ((X->eDest)<=SRT_DistQueue) -#define SRT_Output 5 /* Output each row of result */ -#define SRT_Mem 6 /* Store result in a memory cell */ -#define SRT_Set 7 /* Store results as keys in an index */ -#define SRT_EphemTab 8 /* Create transient tab and store like SRT_Table */ -#define SRT_Coroutine 9 /* Generate a single row of result */ -#define SRT_Table 10 /* Store result as data with an automatic rowid */ -#define SRT_DistTable 11 /* Like SRT_Table, but unique results only */ -#define SRT_Queue 12 /* Store result in an queue */ -#define SRT_DistQueue 13 /* Like SRT_Queue, but unique results only */ +#define SRT_Output 9 /* Output each row of result */ +#define SRT_Mem 10 /* Store result in a memory cell */ +#define SRT_Set 11 /* Store results as keys in an index */ +#define SRT_EphemTab 12 /* Create transient tab and store like SRT_Table */ +#define SRT_Coroutine 13 /* Generate a single row of result */ +#define SRT_Table 14 /* Store result as data with an automatic rowid */ /* ** An instance of this object describes where to put of the results of From 59b8f2e108b3de087b31a2bd75f38a9114fb71d7 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Mar 2014 00:27:14 +0000 Subject: [PATCH 30/49] Fix the ORDER BY optimization logic so that it will do a block-sort on a partial DESC ORDER BY. This enhancement uncovered a memory leak in pushUntoSorter() which is also fixed. FossilOrigin-Name: c36f74611cf17ad2ff198a2ac2054d7ab451a72c --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/select.c | 1 + src/where.c | 22 +++++++++++----------- 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/manifest b/manifest index 5abb89ba80..eca837d211 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\sability\sto\suse\sindices\sfor\sthe\sfirst\sfew\sterms\sof\san\sORDER\sBY\sclause,\nthen\ssort\sin\sbatches\sto\shandle\sthe\slater\sterms. -D 2014-03-21T20:58:42.176 +C Fix\sthe\sORDER\sBY\soptimization\slogic\sso\sthat\sit\swill\sdo\sa\sblock-sort\son\na\spartial\sDESC\sORDER\sBY.\s\sThis\senhancement\suncovered\sa\smemory\sleak\sin\npushUntoSorter()\swhich\sis\salso\sfixed. +D 2014-03-22T00:27:14.500 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 633c56adeb382173d01016bea9e04e4dac54dfbc +F src/select.c 89bfc2228c6dda5200040d3c61a67a7fc815a24e F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c f394453e4960462e62671158199f060ac956f8bc +F src/where.c da8ec216f14af617505799b0b4e52c73dda7a5ca F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 45d8cc678d128f1dda6469864215a8ed9de4366a 71e9ae72c272dc86720b2bfe719f57de437c400b -R 3371108f475a5908c384cc490e83cfb1 +P fa06a6fed9f48322d9b89721799ba12c46efa898 +R c7b944946040f3192afd04e740c2e57c U drh -Z 76333cbbd5592a854905d3a7e7bd7626 +Z 5cc077b206326620cdb0818e6e6de660 diff --git a/manifest.uuid b/manifest.uuid index 4099c22d66..a151346c9a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa06a6fed9f48322d9b89721799ba12c46efa898 \ No newline at end of file +c36f74611cf17ad2ff198a2ac2054d7ab451a72c \ No newline at end of file diff --git a/src/select.c b/src/select.c index fe65e2909a..77ca0de8cc 100644 --- a/src/select.c +++ b/src/select.c @@ -488,6 +488,7 @@ static void pushOntoSorter( addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_Compare, regPrevKey, regBase, pSort->nOBSat); pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + if( pParse->db->mallocFailed ) return; pOp->p2 = nKey + 1; sqlite3VdbeChangeP4(v, -1, (char*)pOp->p4.pKeyInfo, P4_KEYINFO); pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); diff --git a/src/where.c b/src/where.c index 03066043d4..15084f099a 100644 --- a/src/where.c +++ b/src/where.c @@ -4872,23 +4872,23 @@ static i8 wherePathSatisfiesOrderBy( isMatch = 1; break; } + if( isMatch && (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ + /* Make sure the sort order is compatible in an ORDER BY clause. + ** Sort order is irrelevant for a GROUP BY clause. */ + if( revSet ){ + if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) isMatch = 0; + }else{ + rev = revIdx ^ pOrderBy->a[i].sortOrder; + if( rev ) *pRevMask |= MASKBIT(iLoop); + revSet = 1; + } + } if( isMatch ){ if( iColumn<0 ){ testcase( distinctColumns==0 ); distinctColumns = 1; } obSat |= MASKBIT(i); - if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ - /* Make sure the sort order is compatible in an ORDER BY clause. - ** Sort order is irrelevant for a GROUP BY clause. */ - if( revSet ){ - if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) return 0; - }else{ - rev = revIdx ^ pOrderBy->a[i].sortOrder; - if( rev ) *pRevMask |= MASKBIT(iLoop); - revSet = 1; - } - } }else{ /* No match found */ if( j==0 || j Date: Sat, 22 Mar 2014 01:52:17 +0000 Subject: [PATCH 31/49] Increase the version number to 3.8.5. FossilOrigin-Name: 9760879d086b61476890927f01806a93798d4eee --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index a93056bbe2..0cbfaed0d9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.4.1 +3.8.5 diff --git a/configure b/configure index 56d58c7b73..8be06033f0 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.4.1. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.5. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.4.1' -PACKAGE_STRING='sqlite 3.8.4.1' +PACKAGE_VERSION='3.8.5' +PACKAGE_STRING='sqlite 3.8.5' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1483,7 +1483,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.4.1 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.5 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1548,7 +1548,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.4.1:";; + short | recursive ) echo "Configuration of sqlite 3.8.5:";; esac cat <<\_ACEOF @@ -1664,7 +1664,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.4.1 +sqlite configure 3.8.5 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1678,7 +1678,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.4.1, which was +It was created by sqlite $as_me 3.8.5, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14021,7 +14021,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.4.1, which was +This file was extended by sqlite $as_me 3.8.5, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14074,7 +14074,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.4.1 +sqlite config.status 3.8.5 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index eca837d211..4300f5a3d4 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Fix\sthe\sORDER\sBY\soptimization\slogic\sso\sthat\sit\swill\sdo\sa\sblock-sort\son\na\spartial\sDESC\sORDER\sBY.\s\sThis\senhancement\suncovered\sa\smemory\sleak\sin\npushUntoSorter()\swhich\sis\salso\sfixed. -D 2014-03-22T00:27:14.500 +C Increase\sthe\sversion\snumber\sto\s3.8.5. +D 2014-03-22T01:52:17.580 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc fdacba6fb574868c71fde6db6b77789a383a4c19 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README.md 64f270c43c38c46de749e419c22f0ae2f4499fe8 -F VERSION c3b0d47c3c5cf25c5bd4ff9e6f3af2f9d7934ea6 +F VERSION 9f823c026c6a32fc5f84d212a8aae0a221dba45c F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 9eb448a552d5c0185cf62c463f9c173cedae3811 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 6a7a482649dd20b7ee31424bf6553c6f20ca7936 x +F configure 5de98fac90452f876511fee8a35565a55dfb191f x F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P fa06a6fed9f48322d9b89721799ba12c46efa898 -R c7b944946040f3192afd04e740c2e57c +P c36f74611cf17ad2ff198a2ac2054d7ab451a72c +R c87966e812e5349386ca47468f11243f U drh -Z 5cc077b206326620cdb0818e6e6de660 +Z 4135b8e90212e093f926ac90192216e8 diff --git a/manifest.uuid b/manifest.uuid index a151346c9a..4af93967f6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c36f74611cf17ad2ff198a2ac2054d7ab451a72c \ No newline at end of file +9760879d086b61476890927f01806a93798d4eee \ No newline at end of file From dbfca2b7f89d761d330e7c8da551e1ca11b4f185 Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 22 Mar 2014 02:19:53 +0000 Subject: [PATCH 32/49] Clear the sort-order flags on the KeyInfo object associated with the OP_Compare/OP_Jump of the block-sort logic, in order to make the OP_Jump fully testable. FossilOrigin-Name: 9b2838f23e5f52369eeb078f60bd231cc54362f8 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/select.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 4300f5a3d4..8802b6c3e7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.8.5. -D 2014-03-22T01:52:17.580 +C Clear\sthe\ssort-order\sflags\son\sthe\sKeyInfo\sobject\sassociated\swith\sthe\nOP_Compare/OP_Jump\sof\sthe\sblock-sort\slogic,\sin\sorder\sto\smake\sthe\sOP_Jump\nfully\stestable. +D 2014-03-22T02:19:53.523 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/printf.c e5a0005f8b3de21f85da6a709d2fbee76775bf4b F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece F src/resolve.c 273d5f47c4e2c05b2d3d2bffeda939551ab59e66 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 -F src/select.c 89bfc2228c6dda5200040d3c61a67a7fc815a24e +F src/select.c 269c3e31a450fce642a10569221a49180348c88e F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P c36f74611cf17ad2ff198a2ac2054d7ab451a72c -R c87966e812e5349386ca47468f11243f +P 9760879d086b61476890927f01806a93798d4eee +R 9c4612b02334319090d26af6339ff92f U drh -Z 4135b8e90212e093f926ac90192216e8 +Z c09bf15737bf88995e1679f9f666a1a6 diff --git a/manifest.uuid b/manifest.uuid index 4af93967f6..f209cb2ae6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9760879d086b61476890927f01806a93798d4eee \ No newline at end of file +9b2838f23e5f52369eeb078f60bd231cc54362f8 \ No newline at end of file diff --git a/src/select.c b/src/select.c index 77ca0de8cc..c7f0b24a4a 100644 --- a/src/select.c +++ b/src/select.c @@ -481,6 +481,7 @@ static void pushOntoSorter( int addrJmp; /* Address of the OP_Jump opcode */ VdbeOp *pOp; /* Opcode that opens the sorter */ int nKey; /* Number of sorting key columns, including OP_Sequence */ + KeyInfo *pKI; /* Original KeyInfo on the sorter table */ regPrevKey = pParse->nMem+1; pParse->nMem += pSort->nOBSat; @@ -490,7 +491,9 @@ static void pushOntoSorter( pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); if( pParse->db->mallocFailed ) return; pOp->p2 = nKey + 1; - sqlite3VdbeChangeP4(v, -1, (char*)pOp->p4.pKeyInfo, P4_KEYINFO); + pKI = pOp->p4.pKeyInfo; + memset(pKI->aSortOrder, 0, pKI->nField); /* Makes OP_Jump below testable */ + sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, 1); addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); From 323913c88080d5013c404cd90ef689467e29406a Mon Sep 17 00:00:00 2001 From: drh Date: Sun, 23 Mar 2014 16:29:23 +0000 Subject: [PATCH 33/49] Avoid a possible use of an uninitialized variable following an I/O or OOM error. FossilOrigin-Name: 641408a1395bfc911ca619ef9e5f073b913d856b --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbe.c | 1 + 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 8802b6c3e7..b22c63dd88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Clear\sthe\ssort-order\sflags\son\sthe\sKeyInfo\sobject\sassociated\swith\sthe\nOP_Compare/OP_Jump\sof\sthe\sblock-sort\slogic,\sin\sorder\sto\smake\sthe\sOP_Jump\nfully\stestable. -D 2014-03-22T02:19:53.523 +C Avoid\sa\spossible\suse\sof\san\suninitialized\svariable\sfollowing\san\sI/O\sor\sOOM\nerror. +D 2014-03-23T16:29:23.966 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -278,7 +278,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 37cfae03a0c40515304fb0a8a1a97ac3aa965e87 +F src/vdbe.c 5c0feeb6c9e6a0e0cc2a9715aa6045830643809d F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -1157,7 +1157,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9760879d086b61476890927f01806a93798d4eee -R 9c4612b02334319090d26af6339ff92f +P 9b2838f23e5f52369eeb078f60bd231cc54362f8 +R 5e47a0912e77eb3fe11c659549f1088a U drh -Z c09bf15737bf88995e1679f9f666a1a6 +Z 7a01e94679f2d88e0c9f56d7135921af diff --git a/manifest.uuid b/manifest.uuid index f209cb2ae6..7e88ad746f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b2838f23e5f52369eeb078f60bd231cc54362f8 \ No newline at end of file +641408a1395bfc911ca619ef9e5f073b913d856b \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index 7104c6d017..b70c5e1c9d 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4514,6 +4514,7 @@ case OP_SorterNext: { /* jump */ pC = p->apCsr[pOp->p1]; assert( isSorter(pC) ); + res = 0; rc = sqlite3VdbeSorterNext(db, pC, &res); goto next_tail; case OP_PrevIfOpen: /* jump */ From 3d1d90a13da2d8a617f86756b79d9289b03ed566 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 24 Mar 2014 15:00:15 +0000 Subject: [PATCH 34/49] Fix arithmetic operators so that they do not change the affinity of their input operands. Ticket [a8a0d2996a]. FossilOrigin-Name: 221f8f944703108e47d789fa8ce6c00fe2abcbb6 --- manifest | 15 ++++--- manifest.uuid | 2 +- src/vdbe.c | 35 ++++++++++++--- test/tkt-a8a0d2996a.test | 93 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+), 13 deletions(-) create mode 100644 test/tkt-a8a0d2996a.test diff --git a/manifest b/manifest index 3c6309f95b..9e30ebbe2c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sattempting\sto\smmap\smemory\sfrom\san\soffset\sthat\sis\snot\sa\smultiple\sof\sthe\ssystem\spage\ssize\son\ssystems\swith\spage\ssizes\slarger\sthan\s32KB. -D 2014-03-24T11:23:17.736 +C Fix\sarithmetic\soperators\sso\sthat\sthey\sdo\snot\schange\sthe\saffinity\sof\stheir\ninput\soperands.\s\sTicket\s[a8a0d2996a]. +D 2014-03-24T15:00:15.271 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -278,7 +278,7 @@ F src/update.c 5b3e74a03b3811e586b4f2b4cbd7c49f01c93115 F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 -F src/vdbe.c 5c0feeb6c9e6a0e0cc2a9715aa6045830643809d +F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 @@ -892,6 +892,7 @@ F test/tkt-94c04eaadb.test f738c57c7f68ab8be1c054415af7774617cb6223 F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 F test/tkt-9f2eb3abac.test 85bc63e749f050e6a61c8f9207f1eee65c9d3395 F test/tkt-a7b7803e.test 159ef554234fa1f9fb318c751b284bd1cf858da4 +F test/tkt-a8a0d2996a.test eb597379dbcefa24765763d7f682c00cb5924fa9 F test/tkt-b1d3a2e531.test 8f7576e41ca179289ee1a8fee28386fd8e4b0550 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 @@ -1158,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 641408a1395bfc911ca619ef9e5f073b913d856b e3d2be3ba47cdaafd26347620ae3bc2813203f16 -R aadeef09c98319ee1e1acbdf39605080 -U dan -Z a7c50b19ab534dd04ee960894cca4791 +P db7d62c8d58eb1e8654a762c9b199ae4e2759038 +R c2ec5fb627231252d937ea015ddf100a +U drh +Z 7360db1b973f463e9c0480db05633068 diff --git a/manifest.uuid b/manifest.uuid index 3a41778163..9344d79651 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -db7d62c8d58eb1e8654a762c9b199ae4e2759038 \ No newline at end of file +221f8f944703108e47d789fa8ce6c00fe2abcbb6 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index b70c5e1c9d..84f720b526 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -310,6 +310,29 @@ void sqlite3ValueApplyAffinity( applyAffinity((Mem *)pVal, affinity, enc); } +/* +** Return the numeric type for pMem, either MEM_Int or MEM_Real or both or +** none. +** +** Unlike applyNumericAffinity(), this routine does not modify pMem->flags. +** But it does set pMem->r and pMem->u.i appropriately. +*/ +static u16 numericType(Mem *pMem){ + if( pMem->flags & (MEM_Int|MEM_Real) ){ + return pMem->flags & (MEM_Int|MEM_Real); + } + if( pMem->flags & (MEM_Str|MEM_Blob) ){ + if( sqlite3AtoF(pMem->z, &pMem->r, pMem->n, pMem->enc)==0 ){ + return 0; + } + if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){ + return MEM_Int; + } + return MEM_Real; + } + return 0; +} + #ifdef SQLITE_DEBUG /* ** Write a nice string representation of the contents of cell pMem @@ -1351,20 +1374,22 @@ case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ char bIntint; /* Started out as two integer operands */ - int flags; /* Combined MEM_* flags from both inputs */ + u16 flags; /* Combined MEM_* flags from both inputs */ + u16 type1; /* Numeric type of left operand */ + u16 type2; /* Numeric type of right operand */ i64 iA; /* Integer value of left operand */ i64 iB; /* Integer value of right operand */ double rA; /* Real value of left operand */ double rB; /* Real value of right operand */ pIn1 = &aMem[pOp->p1]; - applyNumericAffinity(pIn1); + type1 = numericType(pIn1); pIn2 = &aMem[pOp->p2]; - applyNumericAffinity(pIn2); + type2 = numericType(pIn2); pOut = &aMem[pOp->p3]; flags = pIn1->flags | pIn2->flags; if( (flags & MEM_Null)!=0 ) goto arithmetic_result_is_null; - if( (pIn1->flags & pIn2->flags & MEM_Int)==MEM_Int ){ + if( (type1 & type2 & MEM_Int)!=0 ){ iA = pIn1->u.i; iB = pIn2->u.i; bIntint = 1; @@ -1420,7 +1445,7 @@ fp_math: } pOut->r = rB; MemSetTypeFlag(pOut, MEM_Real); - if( (flags & MEM_Real)==0 && !bIntint ){ + if( ((type1|type2)&MEM_Real)==0 && !bIntint ){ sqlite3VdbeIntegerAffinity(pOut); } #endif diff --git a/test/tkt-a8a0d2996a.test b/test/tkt-a8a0d2996a.test new file mode 100644 index 0000000000..6b15e410e7 --- /dev/null +++ b/test/tkt-a8a0d2996a.test @@ -0,0 +1,93 @@ +# 2014-03-24 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests to verify that arithmetic operators do not change the type of +# input operands. Ticket [a8a0d2996a] +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix tkt-a8a0d2996a + +do_execsql_test 1.0 { + CREATE TABLE t(x,y); + INSERT INTO t VALUES('1','1'); + SELECT typeof(x), typeof(y) FROM t WHERE 1=x+0 AND y=='1'; +} {text text} +do_execsql_test 1.1 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x-0 AND y=='1'; +} {text text} +do_execsql_test 1.2 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x*1 AND y=='1'; +} {text text} +do_execsql_test 1.3 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x/1 AND y=='1'; +} {text text} +do_execsql_test 1.4 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x%4 AND y=='1'; +} {text text} + +do_execsql_test 2.0 { + UPDATE t SET x='1xyzzy'; + SELECT typeof(x), typeof(y) FROM t WHERE 1=x+0 AND y=='1'; +} {text text} +do_execsql_test 2.1 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x-0 AND y=='1'; +} {text text} +do_execsql_test 2.2 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x*1 AND y=='1'; +} {text text} +do_execsql_test 2.3 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x/1 AND y=='1'; +} {text text} +do_execsql_test 2.4 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x%4 AND y=='1'; +} {text text} + + +do_execsql_test 3.0 { + UPDATE t SET x='1.0'; + SELECT typeof(x), typeof(y) FROM t WHERE 1=x+0 AND y=='1'; +} {text text} +do_execsql_test 3.1 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x-0 AND y=='1'; +} {text text} +do_execsql_test 3.2 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x*1 AND y=='1'; +} {text text} +do_execsql_test 3.3 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x/1 AND y=='1'; +} {text text} +do_execsql_test 3.4 { + SELECT typeof(x), typeof(y) FROM t WHERE 1=x%4 AND y=='1'; +} {text text} + +do_execsql_test 4.0 { + SELECT 1+1.; +} {2.0} +do_execsql_test 4.1 { + SELECT '1.23e64'/'1.0000e+62'; +} {123.0} +do_execsql_test 4.2 { + SELECT '100x'+'-2y'; +} {98} +do_execsql_test 4.3 { + SELECT '100x'+'4.5y'; +} {104.5} +do_execsql_test 4.4 { + SELECT '-9223372036854775807x'-'1x'; +} {-9.22337203685478e+18} +do_execsql_test 4.5 { + SELECT '9223372036854775806x'+'1x'; +} {9.22337203685478e+18} +do_execsql_test 4.6 { + SELECT '1234x'/'10y'; +} {123.4} From b22f38a7791fed647b172dff6b2b1cf308b6e66c Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 24 Mar 2014 16:30:06 +0000 Subject: [PATCH 35/49] Remove unused variables Parse.nColCache and Parse.iColCache. FossilOrigin-Name: 4d7551ce464c8038147e81667368924f2a7485a6 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/sqliteInt.h | 2 -- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 9e30ebbe2c..58b47e5f6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sarithmetic\soperators\sso\sthat\sthey\sdo\snot\schange\sthe\saffinity\sof\stheir\ninput\soperands.\s\sTicket\s[a8a0d2996a]. -D 2014-03-24T15:00:15.271 +C Remove\sunused\svariables\sParse.nColCache\sand\sParse.iColCache. +D 2014-03-24T16:30:06.093 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -222,7 +222,7 @@ F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 1e16bac177ce6396e59af867afb90c92949895de +F src/sqliteInt.h afbf39e96736ceb85e1d896b281ba2406dd70aa0 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P db7d62c8d58eb1e8654a762c9b199ae4e2759038 -R c2ec5fb627231252d937ea015ddf100a -U drh -Z 7360db1b973f463e9c0480db05633068 +P 221f8f944703108e47d789fa8ce6c00fe2abcbb6 +R dc20bd716d0015e30edc93a05e72ad4b +U dan +Z 0a61715f43b39f80ef65df2d7add7843 diff --git a/manifest.uuid b/manifest.uuid index 9344d79651..de90fd9eab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -221f8f944703108e47d789fa8ce6c00fe2abcbb6 \ No newline at end of file +4d7551ce464c8038147e81667368924f2a7485a6 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 27dfc08e0f..5e0dd91ec9 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -2393,8 +2393,6 @@ struct Parse { u8 checkSchema; /* Causes schema cookie check after an error */ u8 nested; /* Number of nested calls to the parser/code generator */ u8 nTempReg; /* Number of temporary registers in aTempReg[] */ - u8 nColCache; /* Number of entries in aColCache[] */ - u8 iColCache; /* Next entry in aColCache[] to replace */ u8 isMultiWrite; /* True if statement may modify/insert multiple rows */ u8 mayAbort; /* True if statement may throw an ABORT exception */ u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ From 86dd3716b998bbc2ca2f1aad2a5c01eeb0d7ece8 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Mar 2014 11:00:21 +0000 Subject: [PATCH 36/49] Detect when a VdbeCursor is still pointing at a valid row but that row has moved, and invalidated the return from prior sqlite3BtreeDataFetch() or sqlite3BtreeKeyFetch() calls. FossilOrigin-Name: e6798871ce94961135762669af418cd78540c121 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/btree.c | 23 +++++++++++++++++++---- src/vdbeaux.c | 2 +- 4 files changed, 29 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 58b47e5f6f..e6372d6699 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sunused\svariables\sParse.nColCache\sand\sParse.iColCache. -D 2014-03-24T16:30:06.093 +C Detect\swhen\sa\sVdbeCursor\sis\sstill\spointing\sat\sa\svalid\srow\sbut\sthat\srow\shas\nmoved,\sand\sinvalidated\sthe\sreturn\sfrom\sprior\ssqlite3BtreeDataFetch()\sor\nsqlite3BtreeKeyFetch()\scalls. +D 2014-03-25T11:00:21.320 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 029cec7b98fe0a985922c03f101630391044c4ad +F src/btree.c 8d7e432bdd27d63182865c708ea0e7606489b6d1 F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 @@ -282,7 +282,7 @@ F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 5078ca7de4fd5ba4535bd17fe44d5b56c2d3294c +F src/vdbeaux.c 68dbdc77cdc008eeabc088b8b8a60aa743ba8d2a F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 221f8f944703108e47d789fa8ce6c00fe2abcbb6 -R dc20bd716d0015e30edc93a05e72ad4b -U dan -Z 0a61715f43b39f80ef65df2d7add7843 +P 4d7551ce464c8038147e81667368924f2a7485a6 +R 8613a90c52863cb0b93f254b2d121d4b +U drh +Z f61d5d3cf5132dbd8d9ed764bc0739f9 diff --git a/manifest.uuid b/manifest.uuid index de90fd9eab..b106017e83 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d7551ce464c8038147e81667368924f2a7485a6 \ No newline at end of file +e6798871ce94961135762669af418cd78540c121 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 486cf8e6d5..43d41d67e9 100644 --- a/src/btree.c +++ b/src/btree.c @@ -746,20 +746,32 @@ static int btreeRestoreCursorPosition(BtCursor *pCur){ ** at is deleted out from under them. ** ** This routine returns an error code if something goes wrong. The -** integer *pHasMoved is set to one if the cursor has moved and 0 if not. +** integer *pHasMoved is set as follows: +** +** 0: The cursor is unchanged +** 1: The cursor is still pointing at the same row, but the pointers +** returned by sqlite3BtreeKeyFetch() or sqlite3BtreeDataFetch() +** might now be invalid because of a balance() or other change to the +** b-tree. +** 2: The cursor is no longer pointing to the row. The row might have +** been deleted out from under the cursor. */ int sqlite3BtreeCursorHasMoved(BtCursor *pCur, int *pHasMoved){ int rc; + if( pCur->eState==CURSOR_VALID ){ + *pHasMoved = 0; + return SQLITE_OK; + } rc = restoreCursorPosition(pCur); if( rc ){ - *pHasMoved = 1; + *pHasMoved = 2; return rc; } if( pCur->eState!=CURSOR_VALID || NEVER(pCur->skipNext!=0) ){ - *pHasMoved = 1; + *pHasMoved = 2; }else{ - *pHasMoved = 0; + *pHasMoved = 1; } return SQLITE_OK; } @@ -4188,10 +4200,13 @@ static const void *fetchPayload( assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert( cursorHoldsMutex(pCur) ); assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + assert( pCur->info.nSize>0 ); +#if 0 if( pCur->info.nSize==0 ){ btreeParseCell(pCur->apPage[pCur->iPage], pCur->aiIdx[pCur->iPage], &pCur->info); } +#endif *pAmt = pCur->info.nLocal; return (void*)(pCur->info.pCell + pCur->info.nHeader); } diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 6f1fa2670b..0d2299bab1 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -2735,7 +2735,7 @@ int sqlite3VdbeCursorMoveto(VdbeCursor *p){ if( rc ) return rc; if( hasMoved ){ p->cacheStatus = CACHE_STALE; - p->nullRow = 1; + if( hasMoved==2 ) p->nullRow = 1; } } return SQLITE_OK; From 0c60c1fe6674d50ea394ffd9ec08201414c5251b Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Mar 2014 14:54:36 +0000 Subject: [PATCH 37/49] Add an ORDER BY test case to speedtest1.c FossilOrigin-Name: 588122641e57e957813d329ea071e13ccbde5acd --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/speedtest1.c | 20 ++++++++++++++++++++ 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index e6372d6699..5a5497d41f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Detect\swhen\sa\sVdbeCursor\sis\sstill\spointing\sat\sa\svalid\srow\sbut\sthat\srow\shas\nmoved,\sand\sinvalidated\sthe\sreturn\sfrom\sprior\ssqlite3BtreeDataFetch()\sor\nsqlite3BtreeKeyFetch()\scalls. -D 2014-03-25T11:00:21.320 +C Add\san\sORDER\sBY\stest\scase\sto\sspeedtest1.c +D 2014-03-25T14:54:36.731 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -825,7 +825,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 1603da7b4897716f9df15bd71b0310f56ec3181e +F test/speedtest1.c bbf90952e0464e464467c72785602cca82a4d842 F test/spellfix.test 61309f5efbec53603b3f86457d34a504f80abafe F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 4d7551ce464c8038147e81667368924f2a7485a6 -R 8613a90c52863cb0b93f254b2d121d4b +P e6798871ce94961135762669af418cd78540c121 +R 63e64c2147fb938edf250ea589ed8a9b U drh -Z f61d5d3cf5132dbd8d9ed764bc0739f9 +Z 815d7c70c40cd78a37bdf42f1d8f6b2e diff --git a/manifest.uuid b/manifest.uuid index b106017e83..f5a99b9470 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e6798871ce94961135762669af418cd78540c121 \ No newline at end of file +588122641e57e957813d329ea071e13ccbde5acd \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index 05160e0eeb..e525f9cb18 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -512,6 +512,26 @@ void testset_main(void){ speedtest1_end_test(); + n = g.szTest/5; + speedtest1_begin_test(145, "%d SELECTS w/ORDER BY, unindexed", n); + speedtest1_exec("BEGIN"); + speedtest1_prepare( + "SELECT a, b, c FROM t1 WHERE c LIKE ?1\n" + " ORDER BY a LIMIT 10; -- %d times", n + ); + for(i=1; i<=n; i++){ + x1 = speedtest1_random()%maxb; + zNum[0] = '%'; + len = speedtest1_numbername(i, zNum+1, sizeof(zNum)-2); + zNum[len] = '%'; + zNum[len+1] = 0; + sqlite3_bind_text(g.pStmt, 1, zNum, len, SQLITE_STATIC); + speedtest1_run(); + } + speedtest1_exec("COMMIT"); + speedtest1_end_test(); + + speedtest1_begin_test(150, "CREATE INDEX five times"); speedtest1_exec("BEGIN;"); speedtest1_exec("CREATE UNIQUE INDEX t1b ON t1(b);"); From c872966ef483aee010e327af97e7e66b065236bb Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Mar 2014 17:45:49 +0000 Subject: [PATCH 38/49] Add another performance test case to speedtest1. This case is another ORDER BY test but this time without LIMIT. FossilOrigin-Name: 9ab7ffd59209aef0ffbf384b3902a93fd3b86a6d --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/speedtest1.c | 21 ++++++++++++++++++++- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 5a5497d41f..d9f3760dc0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sORDER\sBY\stest\scase\sto\sspeedtest1.c -D 2014-03-25T14:54:36.731 +C Add\sanother\sperformance\stest\scase\sto\sspeedtest1.\s\sThis\scase\sis\sanother\nORDER\sBY\stest\sbut\sthis\stime\swithout\sLIMIT. +D 2014-03-25T17:45:49.623 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -825,7 +825,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c bbf90952e0464e464467c72785602cca82a4d842 +F test/speedtest1.c 2bec93920c6d26adfa781b509b1c9ca0a4d8f39d F test/spellfix.test 61309f5efbec53603b3f86457d34a504f80abafe F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P e6798871ce94961135762669af418cd78540c121 -R 63e64c2147fb938edf250ea589ed8a9b +P 588122641e57e957813d329ea071e13ccbde5acd +R 6b6fe86503bb240eaed31036f5ef1e04 U drh -Z 815d7c70c40cd78a37bdf42f1d8f6b2e +Z 513f3088d16b4a60fe04e39489233db6 diff --git a/manifest.uuid b/manifest.uuid index f5a99b9470..0289c1d2a0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -588122641e57e957813d329ea071e13ccbde5acd \ No newline at end of file +9ab7ffd59209aef0ffbf384b3902a93fd3b86a6d \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index e525f9cb18..b39493354d 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -513,7 +513,26 @@ void testset_main(void){ n = g.szTest/5; - speedtest1_begin_test(145, "%d SELECTS w/ORDER BY, unindexed", n); + speedtest1_begin_test(142, "%d SELECTS w/ORDER BY, unindexed", n); + speedtest1_exec("BEGIN"); + speedtest1_prepare( + "SELECT a, b, c FROM t1 WHERE c LIKE ?1\n" + " ORDER BY a; -- %d times", n + ); + for(i=1; i<=n; i++){ + x1 = speedtest1_random()%maxb; + zNum[0] = '%'; + len = speedtest1_numbername(i, zNum+1, sizeof(zNum)-2); + zNum[len] = '%'; + zNum[len+1] = 0; + sqlite3_bind_text(g.pStmt, 1, zNum, len, SQLITE_STATIC); + speedtest1_run(); + } + speedtest1_exec("COMMIT"); + speedtest1_end_test(); + + n = g.szTest/5; + speedtest1_begin_test(145, "%d SELECTS w/ORDER BY and LIMIT, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( "SELECT a, b, c FROM t1 WHERE c LIKE ?1\n" From 5e8980ddb1394cf09d39ced95e8ce69680289daa Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 25 Mar 2014 20:28:38 +0000 Subject: [PATCH 39/49] Adjust the repeat counts on several tests in speedtest1 so that the relative time spent on the test is roughly the same regardless of --size. FossilOrigin-Name: 7922809ee0d3978ce9221fbb7df63aa0684d0c4d --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/speedtest1.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index d9f3760dc0..c0390453c4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sanother\sperformance\stest\scase\sto\sspeedtest1.\s\sThis\scase\sis\sanother\nORDER\sBY\stest\sbut\sthis\stime\swithout\sLIMIT. -D 2014-03-25T17:45:49.623 +C Adjust\sthe\srepeat\scounts\son\sseveral\stests\sin\sspeedtest1\sso\sthat\sthe\srelative\ntime\sspent\son\sthe\stest\sis\sroughly\sthe\ssame\sregardless\sof\s--size. +D 2014-03-25T20:28:38.512 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -825,7 +825,7 @@ F test/speed3.test d32043614c08c53eafdc80f33191d5bd9b920523 F test/speed4.test abc0ad3399dcf9703abed2fff8705e4f8e416715 F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b -F test/speedtest1.c 2bec93920c6d26adfa781b509b1c9ca0a4d8f39d +F test/speedtest1.c 90446861e566a9965a8d005381a3c964ff333646 F test/spellfix.test 61309f5efbec53603b3f86457d34a504f80abafe F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298 F test/stat.test 76fd746b85459e812a0193410fb599f0531f22de @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 588122641e57e957813d329ea071e13ccbde5acd -R 6b6fe86503bb240eaed31036f5ef1e04 +P 9ab7ffd59209aef0ffbf384b3902a93fd3b86a6d +R 8ec1b9b9d2439aad92a9a0c48f5fc5b8 U drh -Z 513f3088d16b4a60fe04e39489233db6 +Z 44fa9dae5e4277ca9cc29c89c29e96c1 diff --git a/manifest.uuid b/manifest.uuid index 0289c1d2a0..52ad9d7d36 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9ab7ffd59209aef0ffbf384b3902a93fd3b86a6d \ No newline at end of file +7922809ee0d3978ce9221fbb7df63aa0684d0c4d \ No newline at end of file diff --git a/test/speedtest1.c b/test/speedtest1.c index b39493354d..28c24e0885 100644 --- a/test/speedtest1.c +++ b/test/speedtest1.c @@ -474,7 +474,7 @@ void testset_main(void){ speedtest1_end_test(); - n = g.szTest/2; + n = 25; speedtest1_begin_test(130, "%d SELECTS, numeric BETWEEN, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( @@ -492,7 +492,7 @@ void testset_main(void){ speedtest1_end_test(); - n = g.szTest/5; + n = 10; speedtest1_begin_test(140, "%d SELECTS, LIKE, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( @@ -512,7 +512,7 @@ void testset_main(void){ speedtest1_end_test(); - n = g.szTest/5; + n = 10; speedtest1_begin_test(142, "%d SELECTS w/ORDER BY, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( @@ -531,7 +531,7 @@ void testset_main(void){ speedtest1_exec("COMMIT"); speedtest1_end_test(); - n = g.szTest/5; + n = 10; //g.szTest/5; speedtest1_begin_test(145, "%d SELECTS w/ORDER BY and LIMIT, unindexed", n); speedtest1_exec("BEGIN"); speedtest1_prepare( From e2ac506704e76a26352626788910d62f00dd1f63 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Mar 2014 12:02:38 +0000 Subject: [PATCH 40/49] Add an assert() and a comment to clarify the operation of the vdbeRecordCompareInt() routine. FossilOrigin-Name: 851abdb8fd9b5a8a6ce21db53d30dbac3c430cc7 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbeaux.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index c0390453c4..ce3e7af4be 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sthe\srepeat\scounts\son\sseveral\stests\sin\sspeedtest1\sso\sthat\sthe\srelative\ntime\sspent\son\sthe\stest\sis\sroughly\sthe\ssame\sregardless\sof\s--size. -D 2014-03-25T20:28:38.512 +C Add\san\sassert()\sand\sa\scomment\sto\sclarify\sthe\soperation\sof\sthe\nvdbeRecordCompareInt()\sroutine. +D 2014-03-26T12:02:38.907 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,7 +282,7 @@ F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 68dbdc77cdc008eeabc088b8b8a60aa743ba8d2a +F src/vdbeaux.c 9898720db88a8e733acce7d72fbd56f4565d6d82 F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9ab7ffd59209aef0ffbf384b3902a93fd3b86a6d -R 8ec1b9b9d2439aad92a9a0c48f5fc5b8 +P 7922809ee0d3978ce9221fbb7df63aa0684d0c4d +R 0dc0a56d9b4e58b2ad9ea1b209a9c778 U drh -Z 44fa9dae5e4277ca9cc29c89c29e96c1 +Z d2e0055a9ca933f7d0981ccf44961bc7 diff --git a/manifest.uuid b/manifest.uuid index 52ad9d7d36..7ab08ac131 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7922809ee0d3978ce9221fbb7df63aa0684d0c4d \ No newline at end of file +851abdb8fd9b5a8a6ce21db53d30dbac3c430cc7 \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 0d2299bab1..18862028e4 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3589,6 +3589,9 @@ int sqlite3VdbeRecordCompare( ** that (a) the first field of pPKey2 is an integer, and (b) the ** size-of-header varint at the start of (pKey1/nKey1) fits in a single ** byte (i.e. is less than 128). +** +** To avoid concerns about buffer overreads, this routine is only used +** on schemas where the maximum valid header size is 63 bytes or less. */ static int vdbeRecordCompareInt( int nKey1, const void *pKey1, /* Left key */ @@ -3605,6 +3608,7 @@ static int vdbeRecordCompareInt( UNUSED_PARAMETER(bSkip); assert( bSkip==0 ); + assert( (*(u8*)pKey1)<=0x3F || CORRUPT_DB ); switch( serial_type ){ case 1: { /* 1-byte signed integer */ lhs = ONE_BYTE_INT(aKey); From 182ea4e3246fce6a468625812f1bf603b9690a30 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Mar 2014 14:51:07 +0000 Subject: [PATCH 41/49] Earlier detection of corruption in sqlite3VdbeRecordCompare() in order to avoid a potential buffer overread. FossilOrigin-Name: 28ddecff044dbc2dff50a7d8406ab67dfe06587f --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/vdbeaux.c | 1 + test/corruptI.test | 6 +----- 4 files changed, 10 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index ce3e7af4be..b9c9cba60c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sassert()\sand\sa\scomment\sto\sclarify\sthe\soperation\sof\sthe\nvdbeRecordCompareInt()\sroutine. -D 2014-03-26T12:02:38.907 +C Earlier\sdetection\sof\scorruption\sin\ssqlite3VdbeRecordCompare()\sin\sorder\sto\navoid\sa\spotential\sbuffer\soverread. +D 2014-03-26T14:51:07.017 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -282,7 +282,7 @@ F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7 F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c 9898720db88a8e733acce7d72fbd56f4565d6d82 +F src/vdbeaux.c f81ef920dcf76aceaa1ce77081e9fc5d7a0993dd F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 @@ -405,7 +405,7 @@ F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb -F test/corruptI.test 88886ec9cd1bdba835263566bbf60ee009c6ea09 +F test/corruptI.test 645794bfc0bbcb962df34302a52118b0ed6a21fb F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7922809ee0d3978ce9221fbb7df63aa0684d0c4d -R 0dc0a56d9b4e58b2ad9ea1b209a9c778 +P 851abdb8fd9b5a8a6ce21db53d30dbac3c430cc7 +R 08734d76a115f4a142a4825f21f3cdab U drh -Z d2e0055a9ca933f7d0981ccf44961bc7 +Z 2973842cba6cb939f8ad2063fb07de64 diff --git a/manifest.uuid b/manifest.uuid index 7ab08ac131..8e8ee672b5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -851abdb8fd9b5a8a6ce21db53d30dbac3c430cc7 \ No newline at end of file +28ddecff044dbc2dff50a7d8406ab67dfe06587f \ No newline at end of file diff --git a/src/vdbeaux.c b/src/vdbeaux.c index 18862028e4..f5e4b0a9fc 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3434,6 +3434,7 @@ int sqlite3VdbeRecordCompare( }else{ idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; + if( d1>(unsigned)nKey1 ) return 1; /* Corruption */ i = 0; } diff --git a/test/corruptI.test b/test/corruptI.test index 51cf64c9be..c2eec8ba70 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -40,13 +40,9 @@ do_test 1.2 { set offset [hexio_get_int [hexio_read test.db [expr 2*1024 + 8] 2]] set off [expr 2*1024 + $offset + 1] hexio_write test.db $off FF06 - - breakpoint - sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE a = 10 } -} {1 {database disk image is malformed}} +} {0 {}} finish_test - From d9c131c3664b224dc7be2f30149bc73b6c2b47c5 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 26 Mar 2014 15:05:56 +0000 Subject: [PATCH 42/49] Add a test case to verify that the previous change avoids excess buffer overread in sqlite3VdbeRecordCompare(). FossilOrigin-Name: 2b28e8d582cf10936fa1faca04a16ca2eeead66f --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/corruptI.test | 13 +++++++++++-- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index b9c9cba60c..be675fdb5a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Earlier\sdetection\sof\scorruption\sin\ssqlite3VdbeRecordCompare()\sin\sorder\sto\navoid\sa\spotential\sbuffer\soverread. -D 2014-03-26T14:51:07.017 +C Add\sa\stest\scase\sto\sverify\sthat\sthe\sprevious\schange\savoids\sexcess\sbuffer\noverread\sin\ssqlite3VdbeRecordCompare(). +D 2014-03-26T15:05:56.169 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -405,7 +405,7 @@ F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb -F test/corruptI.test 645794bfc0bbcb962df34302a52118b0ed6a21fb +F test/corruptI.test 487e9e5eac227fca974eed0a0bd5791cccc9eb86 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 851abdb8fd9b5a8a6ce21db53d30dbac3c430cc7 -R 08734d76a115f4a142a4825f21f3cdab +P 28ddecff044dbc2dff50a7d8406ab67dfe06587f +R 6b54752b8296f7dffb679821462bd1b5 U drh -Z 2973842cba6cb939f8ad2063fb07de64 +Z 7c53e6dad71c7413b3950e5c5e55a2eb diff --git a/manifest.uuid b/manifest.uuid index 8e8ee672b5..bdd003989c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -28ddecff044dbc2dff50a7d8406ab67dfe06587f \ No newline at end of file +2b28e8d582cf10936fa1faca04a16ca2eeead66f \ No newline at end of file diff --git a/test/corruptI.test b/test/corruptI.test index c2eec8ba70..bb8d984938 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -32,14 +32,23 @@ do_execsql_test 1.1 { PRAGMA auto_vacuum=0; CREATE TABLE t1(a); CREATE INDEX i1 ON t1(a); - INSERT INTO t1 VALUES('a'); + INSERT INTO t1 VALUES('abcdefghijklmnop'); } {} db close do_test 1.2 { set offset [hexio_get_int [hexio_read test.db [expr 2*1024 + 8] 2]] set off [expr 2*1024 + $offset + 1] - hexio_write test.db $off FF06 + hexio_write test.db $off 7f06 + sqlite3 db test.db + catchsql { SELECT * FROM t1 WHERE a = 10 } +} {0 {}} + +do_test 1.3 { + db close + set offset [hexio_get_int [hexio_read test.db [expr 2*1024 + 8] 2]] + set off [expr 2*1024 + $offset + 1] + hexio_write test.db $off FFFF7f02 sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE a = 10 } } {0 {}} From 68916c9fd3d9829a462a993ded1e58fca82637d2 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 26 Mar 2014 15:14:59 +0000 Subject: [PATCH 43/49] Add an extra test case for the potential buffer overread patched by [28ddecff04]. FossilOrigin-Name: f585f5d7a0f9bf8c590388654a3638231eba8892 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/corruptI.test | 22 ++++++++++++++++++++++ 3 files changed, 30 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index be675fdb5a..fed419c7cf 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sto\sverify\sthat\sthe\sprevious\schange\savoids\sexcess\sbuffer\noverread\sin\ssqlite3VdbeRecordCompare(). -D 2014-03-26T15:05:56.169 +C Add\san\sextra\stest\scase\sfor\sthe\spotential\sbuffer\soverread\spatched\sby\s[28ddecff04]. +D 2014-03-26T15:14:59.762 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -405,7 +405,7 @@ F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb -F test/corruptI.test 487e9e5eac227fca974eed0a0bd5791cccc9eb86 +F test/corruptI.test 1b796461e5b635e0a74e3c4ecb1121c82d319dff F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 28ddecff044dbc2dff50a7d8406ab67dfe06587f -R 6b54752b8296f7dffb679821462bd1b5 -U drh -Z 7c53e6dad71c7413b3950e5c5e55a2eb +P 2b28e8d582cf10936fa1faca04a16ca2eeead66f +R 76c12b0a4284d7654cb2771b2fc9c8eb +U dan +Z 22c3e1f8cdae5a20dae420dc313d8edd diff --git a/manifest.uuid b/manifest.uuid index bdd003989c..bdc4496d4e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2b28e8d582cf10936fa1faca04a16ca2eeead66f \ No newline at end of file +f585f5d7a0f9bf8c590388654a3638231eba8892 \ No newline at end of file diff --git a/test/corruptI.test b/test/corruptI.test index bb8d984938..087a0f3b01 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -53,5 +53,27 @@ do_test 1.3 { catchsql { SELECT * FROM t1 WHERE a = 10 } } {0 {}} +do_test 2.0 { + execsql { + CREATE TABLE r(x); + INSERT INTO r VALUES('ABCDEFGHIJK'); + CREATE INDEX r1 ON r(x); + } + set pg [db one {SELECT rootpage FROM sqlite_master WHERE name = 'r1'}] +} {5} + +do_test 2.1 { + db close + set offset [hexio_get_int [hexio_read test.db [expr (5-1)*1024 + 8] 2]] + set off [expr (5-1)*1024 + $offset + 1] + hexio_write test.db $off FFFF0004 + sqlite3 db test.db + catchsql { SELECT * FROM r WHERE x >= 10.0 } +} {1 {database disk image is malformed}} + +do_test 2.2 { + catchsql { SELECT * FROM r WHERE x >= 10 } +} {1 {database disk image is malformed}} + finish_test From 382bdeabefc740398f8fd6e55d12a8d952965d85 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Mar 2014 14:05:38 +0000 Subject: [PATCH 44/49] Enhance the logest.c utility with new operators: "dup", "inv", "log", and "nlogn". Provide help on an invalid input. FossilOrigin-Name: b4bd2a062c4baf5f622d61b7411f00de5904ef56 --- manifest | 14 ++++++------ manifest.uuid | 2 +- tool/logest.c | 62 ++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 52 insertions(+), 26 deletions(-) diff --git a/manifest b/manifest index fed419c7cf..fc133ce7b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\san\sextra\stest\scase\sfor\sthe\spotential\sbuffer\soverread\spatched\sby\s[28ddecff04]. -D 2014-03-26T15:14:59.762 +C Enhance\sthe\slogest.c\sutility\swith\snew\soperators:\s"dup",\s"inv",\s"log",\sand\n"nlogn".\s\sProvide\shelp\son\san\sinvalid\sinput. +D 2014-03-27T14:05:38.733 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1121,7 +1121,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce F tool/lemon.c 07aba6270d5a5016ba8107b09e431eea4ecdc123 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc -F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75 +F tool/logest.c 388c318c7ac8b52b7c08ca1e2de0f4ca9a8f7e81 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383 F tool/mkkeywordhash.c c9e05e4a7bcab8fab9f583d5b321fb72f565ad97 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 2b28e8d582cf10936fa1faca04a16ca2eeead66f -R 76c12b0a4284d7654cb2771b2fc9c8eb -U dan -Z 22c3e1f8cdae5a20dae420dc313d8edd +P f585f5d7a0f9bf8c590388654a3638231eba8892 +R fc449dde2f7cf3aac00c9e1a5da52aee +U drh +Z ebedee01c152a936b8dfbac0e8a85bc4 diff --git a/manifest.uuid b/manifest.uuid index bdc4496d4e..bf7192882e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f585f5d7a0f9bf8c590388654a3638231eba8892 \ No newline at end of file +b4bd2a062c4baf5f622d61b7411f00de5904ef56 \ No newline at end of file diff --git a/tool/logest.c b/tool/logest.c index 8dad6cc9b6..1ac337d36c 100644 --- a/tool/logest.c +++ b/tool/logest.c @@ -17,13 +17,7 @@ ** ** ./LogEst ARGS ** -** Arguments: -** -** 'x' Multiple the top two elements of the stack -** '+' Add the top two elements of the stack -** NUM Convert NUM from integer to LogEst and push onto the stack -** ^NUM Interpret NUM as a LogEst and push onto stack. -** +** See the showHelp() routine for a description of valid arguments. ** Examples: ** ** To convert 123 from LogEst to integer: @@ -97,12 +91,31 @@ static LogEst logEstFromDouble(double x){ return e*10; } +int isInteger(const char *z){ + while( z[0]>='0' && z[0]<='9' ) z++; + return z[0]==0; +} + int isFloat(const char *z){ - while( z[0] ){ - if( z[0]=='.' || z[0]=='E' || z[0]=='e' ) return 1; - z++; - } - return 0; + char c; + while( ((c=z[0])>='0' && c<='9') || c=='.' || c=='E' || c=='e' + || c=='+' || c=='-' ) z++; + return z[0]==0; +} + +static void showHelp(const char *zArgv0){ + printf("Usage: %s ARGS...\n", zArgv0); + printf("Arguments:\n" + " NUM Convert NUM from integer to LogEst and push onto the stack\n" + " ^NUM Interpret NUM as a LogEst and push onto stack\n" + " x Multiple the top two elements of the stack\n" + " + Add the top two elements of the stack\n" + " dup Dupliate the top element on the stack\n" + " inv Take the reciprocal of the top of stack. N = 1/N.\n" + " log Find the LogEst of the number on top of stack\n" + " nlogn Compute NlogN where N is the top of stack\n" + ); + exit(1); } int main(int argc, char **argv){ @@ -111,30 +124,43 @@ int main(int argc, char **argv){ LogEst a[100]; for(i=1; i=2 ){ a[n-2] = logEstAdd(a[n-2],a[n-1]); n--; } - }else if( z[0]=='x' ){ + }else if( strcmp(z,"x")==0 ){ if( n>=2 ){ a[n-2] = logEstMultiply(a[n-2],a[n-1]); n--; } + }else if( strcmp(z,"dup")==0 ){ + if( n>0 ){ + a[n] = a[n-1]; + n++; + } + }else if( strcmp(z,"log")==0 ){ + if( n>0 ) a[n-1] = logEstFromInteger(a[n-1]) - 33; + }else if( strcmp(z,"nlogn")==0 ){ + if( n>0 ) a[n-1] += logEstFromInteger(a[n-1]) - 33; + }else if( strcmp(z,"inv")==0 ){ + if( n>0 ) a[n-1] = -a[n-1]; }else if( z[0]=='^' ){ a[n++] = atoi(z+1); - }else if( isFloat(z) ){ + }else if( isInteger(z) ){ + a[n++] = logEstFromInteger(atoi(z)); + }else if( isFloat(z) && z[0]!='-' ){ a[n++] = logEstFromDouble(atof(z)); }else{ - a[n++] = logEstFromInteger(atoi(z)); + showHelp(argv[0]); } } for(i=n-1; i>=0; i--){ if( a[i]<0 ){ - printf("%d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); + printf("%5d (%f)\n", a[i], 1.0/(double)logEstToInt(-a[i])); }else{ sqlite3_uint64 x = logEstToInt(a[i]+100)*100/1024; - printf("%d (%lld.%02lld)\n", a[i], x/100, x%100); + printf("%5d (%lld.%02lld)\n", a[i], x/100, x%100); } } return 0; From 27de5c5cb623fcdbfe4aa128a30a08a5efa268f4 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 27 Mar 2014 18:36:34 +0000 Subject: [PATCH 45/49] Minor cleanup of the code in the query planner that computes the costs estimates for the various plans. There are no changes to the costs at this time. But the code is slightly more readable now and that might facilitate future enhancements. FossilOrigin-Name: 9b4d7226bcee38be5ac68a54bee03b4179cb69fc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 44 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index fc133ce7b9..c8233bfa12 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\slogest.c\sutility\swith\snew\soperators:\s"dup",\s"inv",\s"log",\sand\n"nlogn".\s\sProvide\shelp\son\san\sinvalid\sinput. -D 2014-03-27T14:05:38.733 +C Minor\scleanup\sof\sthe\scode\sin\sthe\squery\splanner\sthat\scomputes\sthe\scosts\s\nestimates\sfor\sthe\svarious\splans.\s\sThere\sare\sno\schanges\sto\sthe\scosts\sat\sthis\ntime.\s\sBut\sthe\scode\sis\sslightly\smore\sreadable\snow\sand\sthat\smight\sfacilitate\nfuture\senhancements. +D 2014-03-27T18:36:34.321 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c da8ec216f14af617505799b0b4e52c73dda7a5ca +F src/where.c 7c53de68bd6762848b746510cf4eb077ffd7d70d F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P f585f5d7a0f9bf8c590388654a3638231eba8892 -R fc449dde2f7cf3aac00c9e1a5da52aee +P b4bd2a062c4baf5f622d61b7411f00de5904ef56 +R 96966a646967b8a2388d32a924ac0ae2 U drh -Z ebedee01c152a936b8dfbac0e8a85bc4 +Z 308d20d60c0618b1b3ae79dc397c2638 diff --git a/manifest.uuid b/manifest.uuid index bf7192882e..65bd2a013d 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b4bd2a062c4baf5f622d61b7411f00de5904ef56 \ No newline at end of file +9b4d7226bcee38be5ac68a54bee03b4179cb69fc \ No newline at end of file diff --git a/src/where.c b/src/where.c index 15084f099a..16e7e40bc3 100644 --- a/src/where.c +++ b/src/where.c @@ -4328,18 +4328,34 @@ static int whereLoopAddBtree( ) ){ pNew->iSortIdx = b ? iSortIdx : 0; + /* TUNING: The base cost of an index scan is N + log2(N). + ** The log2(N) is for the initial seek to the beginning and the N + ** is for the scan itself. */ + pNew->rRun = sqlite3LogEstAdd(rSize, rLogSize); if( m==0 ){ /* TUNING: Cost of a covering index scan is K*(N + log2(N)). ** + The extra factor K of between 1.1 and 3.0 that depends ** on the relative sizes of the table and the index. K ** is smaller for smaller indices, thus favoring them. + ** The upper bound on K (3.0) matches the penalty factor + ** on a full table scan that tries to encourage the use of + ** indexed lookups over full scans. */ - pNew->rRun = sqlite3LogEstAdd(rSize,rLogSize) + 1 + - (15*pProbe->szIdxRow)/pTab->szTabRow; + pNew->rRun += 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; }else{ - /* TUNING: Cost of scanning a non-covering index is (N+1)*log2(N) - ** which we will simplify to just N*log2(N) */ - pNew->rRun = rSize + rLogSize; + /* TUNING: The cost of scanning a non-covering index is multiplied + ** by log2(N) to account for the binary search of the main table + ** that must happen for each row of the index. + ** TODO: Should there be a multiplier here, analogous to the 3x + ** multiplier for a fulltable scan or covering index scan, to + ** further discourage the use of an index scan? Or is the log2(N) + ** term sufficient discouragement? + ** TODO: What if some or all of the WHERE clause terms can be + ** computed without reference to the original table. Then the + ** penality should reduce to logK where K is the number of output + ** rows. + */ + pNew->rRun += rLogSize; } whereLoopOutputAdjust(pWC, pNew); rc = whereLoopInsert(pBuilder, pNew); @@ -5041,11 +5057,19 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); if( isOrdered>=0 && isOrdered0 ); + rScale = sqlite3LogEst((nOrderBy-isOrdered)*100/nOrderBy) - 66; + rSortCost = nRowEst + estLog(nRowEst) + rScale; /* TUNING: The cost of implementing DISTINCT using a B-TREE is ** also N*log(N) but it has a larger constant of proportionality. ** Multiply by 3.0. */ From a1f7c0a21c65431a3fbfeba25ab7f1e9ad827273 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Mar 2014 03:12:48 +0000 Subject: [PATCH 46/49] Enhance the sqlite3VdbeRecordCompare() routines so that if they encounter database corruption, they will set the UnpackedRecord.isCorrupt field and return 0. The sqlite3BtreeMovetoUnpacked() routine detects this and returns SQLITE_CORRUPT, causing the corruption to be reported back to the top-level. FossilOrigin-Name: 7fa85eaaaf6d211378620d728a759fdfe30a15b0 --- manifest | 24 ++++++++++++------------ manifest.uuid | 2 +- src/btree.c | 3 +++ src/sqliteInt.h | 1 + src/vdbe.h | 4 ++-- src/vdbeInt.h | 2 +- src/vdbeaux.c | 27 +++++++++++++++++++-------- test/corruptG.test | 11 +++-------- test/corruptI.test | 2 +- 9 files changed, 43 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index c8233bfa12..48ca1421b2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Minor\scleanup\sof\sthe\scode\sin\sthe\squery\splanner\sthat\scomputes\sthe\scosts\s\nestimates\sfor\sthe\svarious\splans.\s\sThere\sare\sno\schanges\sto\sthe\scosts\sat\sthis\ntime.\s\sBut\sthe\scode\sis\sslightly\smore\sreadable\snow\sand\sthat\smight\sfacilitate\nfuture\senhancements. -D 2014-03-27T18:36:34.321 +C Enhance\sthe\ssqlite3VdbeRecordCompare()\sroutines\sso\sthat\sif\sthey\sencounter\ndatabase\scorruption,\sthey\swill\sset\sthe\sUnpackedRecord.isCorrupt\sfield\sand\nreturn\s0.\s\sThe\ssqlite3BtreeMovetoUnpacked()\sroutine\sdetects\sthis\sand\sreturns\nSQLITE_CORRUPT,\scausing\sthe\scorruption\sto\sbe\sreported\sback\sto\sthe\stop-level. +D 2014-03-28T03:12:48.763 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -164,7 +164,7 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 8d7e432bdd27d63182865c708ea0e7606489b6d1 +F src/btree.c a59a199f21338ae1847d69f5db87c3e8ef1b1578 F src/btree.h 232836cb51753f2e96aa8ce0f052c6df850f76ba F src/btreeInt.h 0be66063468a520e4d66b80c7a1dc26d04ee6ea4 F src/build.c 0d50ef95aad63f4c4fc47f3fa2670d4557c45db0 @@ -222,7 +222,7 @@ F src/shell.c cee9f46f2688a261601b1fd3d7f4b3cddf9b5cdf F src/sqlite.h.in a2ef671f92747a5a1c8a47bad5c585a8dd9eca80 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h afbf39e96736ceb85e1d896b281ba2406dd70aa0 +F src/sqliteInt.h cc9582a91b2910404ccda7b7e198815ea0f75948 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -279,10 +279,10 @@ F src/utf.c 6dc9ec9f1b3db43ae8ba0365377f11df1ee4c01c F src/util.c c46c90459ef9bdc0c6c73803cf4c55425b4771cf F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179 F src/vdbe.c 74c7386e83eee56f921a17bb4a0396c9551f5bc7 -F src/vdbe.h fb2c48c198300a7c632f09fc940011d2ad2fc2ae -F src/vdbeInt.h 2b9a6849166d0014c843ae3fd83a062be4efa325 +F src/vdbe.h 394464909ed682334aa3d5831aae0c2fe2abef94 +F src/vdbeInt.h e6d83e5bfd62fc6685ba1ed6153f7099f82de9f7 F src/vdbeapi.c 0ed6053f947edd0b30f64ce5aeb811872a3450a4 -F src/vdbeaux.c f81ef920dcf76aceaa1ce77081e9fc5d7a0993dd +F src/vdbeaux.c 1153175fb57a8454e1c8cf79b59b7bf92b26779d F src/vdbeblob.c 15377abfb59251bccedd5a9c7d014a895f0c04aa F src/vdbemem.c 6fc77594c60f6155404f3f8d71bf36d1fdeb4447 F src/vdbesort.c 4abb7c0f8f19b7d7d82f4558d5da1a30fdf9ea38 @@ -403,9 +403,9 @@ F test/corruptC.test 02405cf7ed0c1e989060e1aab6d02ffbc3906fbb F test/corruptD.test b3c205fac7952b1de645ce44bb02335cd9e3e040 F test/corruptE.test 193b4ca4e927e77c1d5f4f56203ddc998432a7ee F test/corruptF.test be9fde98e4c93648f1ba52b74e5318edc8f59fe4 -F test/corruptG.test 58ec333a01997fe655e34e5bea52b7a2a6b9704d +F test/corruptG.test 1ab3bf97ee7bdba70e0ff3ba2320657df55d1804 F test/corruptH.test 88ed71a086e13591c917aac6de32750e7c7281cb -F test/corruptI.test 1b796461e5b635e0a74e3c4ecb1121c82d319dff +F test/corruptI.test b3e4203d420490fc3d3062711597bc1dea06a789 F test/count.test 42a251178e32f617eda33f76236a7f79825a50b5 F test/coveridxscan.test cdb47d01acc4a634a34fd25abe85189e0d0f1e62 F test/crash.test fb9dc4a02dcba30d4aa5c2c226f98b220b2b959f @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P b4bd2a062c4baf5f622d61b7411f00de5904ef56 -R 96966a646967b8a2388d32a924ac0ae2 +P 9b4d7226bcee38be5ac68a54bee03b4179cb69fc +R e6ff5922142fb694f528da1c1b2a1baf U drh -Z 308d20d60c0618b1b3ae79dc397c2638 +Z 2ba581858b0e7a94365bc5b4fc44e179 diff --git a/manifest.uuid b/manifest.uuid index 65bd2a013d..ef58fd2f6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9b4d7226bcee38be5ac68a54bee03b4179cb69fc \ No newline at end of file +7fa85eaaaf6d211378620d728a759fdfe30a15b0 \ No newline at end of file diff --git a/src/btree.c b/src/btree.c index 43d41d67e9..c3055836c9 100644 --- a/src/btree.c +++ b/src/btree.c @@ -4588,6 +4588,7 @@ int sqlite3BtreeMovetoUnpacked( if( pIdxKey ){ xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); + pIdxKey->isCorrupt = 0; assert( pIdxKey->default_rc==1 || pIdxKey->default_rc==0 || pIdxKey->default_rc==-1 @@ -4711,6 +4712,7 @@ int sqlite3BtreeMovetoUnpacked( c = xRecordCompare(nCell, pCellKey, pIdxKey, 0); sqlite3_free(pCellKey); } + assert( pIdxKey->isCorrupt==0 || c==0 ); if( c<0 ){ lwr = idx+1; }else if( c>0 ){ @@ -4720,6 +4722,7 @@ int sqlite3BtreeMovetoUnpacked( *pRes = 0; rc = SQLITE_OK; pCur->aiIdx[pCur->iPage] = (u16)idx; + if( pIdxKey->isCorrupt ) rc = SQLITE_CORRUPT; goto moveto_finish; } if( lwr>upr ) break; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 5e0dd91ec9..0f67748fc8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1627,6 +1627,7 @@ struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ + u8 isCorrupt; /* Corruption detected by xRecordCompare() */ Mem *aMem; /* Values */ int r1; /* Value to return if (lhs > rhs) */ int r2; /* Value to return if (rhs < lhs) */ diff --git a/src/vdbe.h b/src/vdbe.h index 8e300b88a6..10a4140ee1 100644 --- a/src/vdbe.h +++ b/src/vdbe.h @@ -211,10 +211,10 @@ void sqlite3VdbeSetVarmask(Vdbe*, int); #endif void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); -int sqlite3VdbeRecordCompare(int,const void*,const UnpackedRecord*,int); +int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*,int); UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); -typedef int (*RecordCompare)(int,const void*,const UnpackedRecord*,int); +typedef int (*RecordCompare)(int,const void*,UnpackedRecord*,int); RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); #ifndef SQLITE_OMIT_TRIGGER diff --git a/src/vdbeInt.h b/src/vdbeInt.h index b752478035..043b56da59 100644 --- a/src/vdbeInt.h +++ b/src/vdbeInt.h @@ -392,7 +392,7 @@ u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); void sqlite3VdbeDeleteAuxData(Vdbe*, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); -int sqlite3VdbeIdxKeyCompare(VdbeCursor*,const UnpackedRecord*,int*); +int sqlite3VdbeIdxKeyCompare(VdbeCursor*,UnpackedRecord*,int*); int sqlite3VdbeIdxRowid(sqlite3*, BtCursor *, i64 *); int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); int sqlite3VdbeExec(Vdbe*); diff --git a/src/vdbeaux.c b/src/vdbeaux.c index f5e4b0a9fc..0ce21378d5 100644 --- a/src/vdbeaux.c +++ b/src/vdbeaux.c @@ -3405,10 +3405,13 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** Key1 and Key2 do not have to contain the same number of fields. If all ** fields that appear in both keys are equal, then pPKey2->default_rc is ** returned. +** +** If database corruption is discovered, set pPKey2->isCorrupt to non-zero +** and return 0. */ int sqlite3VdbeRecordCompare( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2, /* Right key */ + UnpackedRecord *pPKey2, /* Right key */ int bSkip /* If true, skip the first field */ ){ u32 d1; /* Offset into aKey[] of next data element */ @@ -3434,7 +3437,10 @@ int sqlite3VdbeRecordCompare( }else{ idx1 = getVarint32(aKey1, szHdr1); d1 = szHdr1; - if( d1>(unsigned)nKey1 ) return 1; /* Corruption */ + if( d1>(unsigned)nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } i = 0; } @@ -3511,7 +3517,8 @@ int sqlite3VdbeRecordCompare( testcase( (d1+mem1.n)==(unsigned)nKey1 ); testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); if( (d1+mem1.n) > (unsigned)nKey1 ){ - rc = 1; /* Corruption */ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ }else if( pKeyInfo->aColl[i] ){ mem1.enc = pKeyInfo->enc; mem1.db = pKeyInfo->db; @@ -3537,7 +3544,8 @@ int sqlite3VdbeRecordCompare( testcase( (d1+nStr)==(unsigned)nKey1 ); testcase( (d1+nStr+1)==(unsigned)nKey1 ); if( (d1+nStr) > (unsigned)nKey1 ){ - rc = 1; /* Corruption */ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ }else{ int nCmp = MIN(nStr, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); @@ -3596,7 +3604,7 @@ int sqlite3VdbeRecordCompare( */ static int vdbeRecordCompareInt( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2, /* Right key */ + UnpackedRecord *pPKey2, /* Right key */ int bSkip /* Ignored */ ){ const u8 *aKey = &((const u8*)pKey1)[*(const u8*)pKey1 & 0x3F]; @@ -3694,7 +3702,7 @@ static int vdbeRecordCompareInt( */ static int vdbeRecordCompareString( int nKey1, const void *pKey1, /* Left key */ - const UnpackedRecord *pPKey2, /* Right key */ + UnpackedRecord *pPKey2, /* Right key */ int bSkip ){ const u8 *aKey1 = (const u8*)pKey1; @@ -3715,7 +3723,10 @@ static int vdbeRecordCompareString( int szHdr = aKey1[0]; nStr = (serial_type-12) / 2; - if( (szHdr + nStr) > nKey1 ) return 0; /* Corruption */ + if( (szHdr + nStr) > nKey1 ){ + pPKey2->isCorrupt = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } nCmp = MIN( pPKey2->aMem[0].n, nStr ); res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); @@ -3880,7 +3891,7 @@ idx_rowid_corruption: */ int sqlite3VdbeIdxKeyCompare( VdbeCursor *pC, /* The cursor to compare against */ - const UnpackedRecord *pUnpacked, /* Unpacked version of key */ + UnpackedRecord *pUnpacked, /* Unpacked version of key */ int *res /* Write the comparison result here */ ){ i64 nCellKey = 0; diff --git a/test/corruptG.test b/test/corruptG.test index 1ec5d6f6a9..af920edf41 100644 --- a/test/corruptG.test +++ b/test/corruptG.test @@ -47,12 +47,12 @@ do_test 1.2 { catchsql { SELECT c FROM t1 WHERE a>'abc'; } -} {0 {}} +} {1 {database disk image is malformed}} do_test 1.3 { catchsql { PRAGMA integrity_check } -} {0 ok} +} {1 {database disk image is malformed}} do_test 1.4 { catchsql { SELECT c FROM t1 ORDER BY a; @@ -71,11 +71,6 @@ do_test 2.1 { catchsql { SELECT rowid FROM t1 WHERE a='abc' and b='xyz123456789XYZ'; } - # The following test result is brittle. The point above is to try to - # force a buffer overread by a corrupt database file. If we get an - # incorrect answer from a corrupt database file, that is OK. If the - # result below changes, that just means that "undefined behavior" has - # changed. -} {/0 .*/} +} {1 {database disk image is malformed}} finish_test diff --git a/test/corruptI.test b/test/corruptI.test index 087a0f3b01..ed34c0f8c3 100644 --- a/test/corruptI.test +++ b/test/corruptI.test @@ -51,7 +51,7 @@ do_test 1.3 { hexio_write test.db $off FFFF7f02 sqlite3 db test.db catchsql { SELECT * FROM t1 WHERE a = 10 } -} {0 {}} +} {1 {database disk image is malformed}} do_test 2.0 { execsql { From 36ed03428d501b8de6b86aeca09f27c9ec5869ad Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Mar 2014 12:56:57 +0000 Subject: [PATCH 47/49] Fix a harmless compiler warning. FossilOrigin-Name: a4e47150f32b3a4120b1f89ccc66d633d829e3bb --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 48ca1421b2..3f74055a6f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enhance\sthe\ssqlite3VdbeRecordCompare()\sroutines\sso\sthat\sif\sthey\sencounter\ndatabase\scorruption,\sthey\swill\sset\sthe\sUnpackedRecord.isCorrupt\sfield\sand\nreturn\s0.\s\sThe\ssqlite3BtreeMovetoUnpacked()\sroutine\sdetects\sthis\sand\sreturns\nSQLITE_CORRUPT,\scausing\sthe\scorruption\sto\sbe\sreported\sback\sto\sthe\stop-level. -D 2014-03-28T03:12:48.763 +C Fix\sa\sharmless\scompiler\swarning. +D 2014-03-28T12:56:57.685 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -291,7 +291,7 @@ F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd F src/wal.c 76e7fc6de229bea8b30bb2539110f03a494dc3a8 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45 -F src/where.c 7c53de68bd6762848b746510cf4eb077ffd7d70d +F src/where.c 7d539cedb1c6a6d6b5d2075b8fea3a48db4838eb F src/whereInt.h 2564055b440e44ebec8b47f237bbccae6719b7af F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 9b4d7226bcee38be5ac68a54bee03b4179cb69fc -R e6ff5922142fb694f528da1c1b2a1baf +P 7fa85eaaaf6d211378620d728a759fdfe30a15b0 +R 84c408a5c4c5e7a44048ef6100ea9043 U drh -Z 2ba581858b0e7a94365bc5b4fc44e179 +Z 645b10a99877639c66e2a5f7ed2d5a66 diff --git a/manifest.uuid b/manifest.uuid index ef58fd2f6f..b73eec43f2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7fa85eaaaf6d211378620d728a759fdfe30a15b0 \ No newline at end of file +a4e47150f32b3a4120b1f89ccc66d633d829e3bb \ No newline at end of file diff --git a/src/where.c b/src/where.c index 16e7e40bc3..93ee8c59c3 100644 --- a/src/where.c +++ b/src/where.c @@ -4936,7 +4936,7 @@ static i8 wherePathSatisfiesOrderBy( } } } /* End the loop over all WhereLoops from outer-most down to inner-most */ - if( obSat==obDone ) return nOrderBy; + if( obSat==obDone ) return (i8)nOrderBy; if( !isOrderDistinct ){ for(i=nOrderBy-1; i>0; i--){ Bitmask m = MASKBIT(i) - 1; From 70c93204ea3e1b37750dd7f362dfe379101cba68 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 28 Mar 2014 14:41:35 +0000 Subject: [PATCH 48/49] Disable the wal64k.test script for non-unix systems since it depends on unix-only features. FossilOrigin-Name: 27deb6e49bcc76714dbdc61b34748603155ac770 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/wal64k.test | 6 +++++- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 3f74055a6f..52d212de1d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sharmless\scompiler\swarning. -D 2014-03-28T12:56:57.685 +C Disable\sthe\swal64k.test\sscript\sfor\snon-unix\ssystems\ssince\sit\sdepends\son\nunix-only\sfeatures. +D 2014-03-28T14:41:35.536 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1056,7 +1056,7 @@ F test/wal3.test b22eb662bcbc148c5f6d956eaf94b047f7afe9c0 F test/wal4.test 4744e155cd6299c6bd99d3eab1c82f77db9cdb3c F test/wal5.test 8f888b50f66b78821e61ed0e233ded5de378224b F test/wal6.test 527581f5527bf9c24394991e2be83000aace5f9e -F test/wal64k.test 63828c2161ad76ddd4109dee0a096b6ef6895698 +F test/wal64k.test 163655ecd2cb8afef4737cac2a40fdd2eeaf20b8 F test/wal7.test 2ae8f427d240099cc4b2dfef63cff44e2a68a1bd F test/wal8.test 75c42e1bc4545c277fed212f8fc9b7723cd02216 F test/wal9.test 378e76a9ad09cd9bee06c172ad3547b0129a6750 @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P 7fa85eaaaf6d211378620d728a759fdfe30a15b0 -R 84c408a5c4c5e7a44048ef6100ea9043 +P a4e47150f32b3a4120b1f89ccc66d633d829e3bb +R 578568415288cd31fe0adba6128329da U drh -Z 645b10a99877639c66e2a5f7ed2d5a66 +Z 045e2748905f8bd05ecf1197b97d7f20 diff --git a/manifest.uuid b/manifest.uuid index b73eec43f2..d9c988d4d5 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a4e47150f32b3a4120b1f89ccc66d633d829e3bb \ No newline at end of file +27deb6e49bcc76714dbdc61b34748603155ac770 \ No newline at end of file diff --git a/test/wal64k.test b/test/wal64k.test index afd820778e..e962da128e 100644 --- a/test/wal64k.test +++ b/test/wal64k.test @@ -19,6 +19,11 @@ set testprefix wal64k ifcapable !wal {finish_test ; return } +if {$tcl_platform(platform) != "unix"} { + finish_test + return +} + db close test_syscall pagesize 65536 sqlite3 db test.db @@ -44,4 +49,3 @@ integrity_check 1.3 db close test_syscall pagesize -1 finish_test - From b663587890f557c9926964710f6f1e8121040c88 Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 31 Mar 2014 13:42:42 +0000 Subject: [PATCH 49/49] Avoid a (harmless) buffer overread that is possible on an OOM when MEMSYS5 is engaged. FossilOrigin-Name: b3296267fb67b9f59719a37093253062edde3746 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/mem5.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index 52d212de1d..887bb92f5b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Disable\sthe\swal64k.test\sscript\sfor\snon-unix\ssystems\ssince\sit\sdepends\son\nunix-only\sfeatures. -D 2014-03-28T14:41:35.536 +C Avoid\sa\s(harmless)\sbuffer\soverread\sthat\sis\spossible\son\san\sOOM\swhen\nMEMSYS5\sis\sengaged. +D 2014-03-31T13:42:42.977 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -192,7 +192,7 @@ F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c c0c990fcaddff810ea277b4fb5d9138603dd5d4b F src/mem2.c dce31758da87ec2cfa52ba4c5df1aed6e07d8e8f F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 -F src/mem5.c aeb019f271ea53de83d651ec526877e6ba863450 +F src/mem5.c 74670012946c4adc8a6ad84d03acc80959c3e529 F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea @@ -1159,7 +1159,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff -P a4e47150f32b3a4120b1f89ccc66d633d829e3bb -R 578568415288cd31fe0adba6128329da +P 27deb6e49bcc76714dbdc61b34748603155ac770 +R 74e3f11f049d7792447568e00e413b7f U drh -Z 045e2748905f8bd05ecf1197b97d7f20 +Z 4989508149fff1b415b46911a02fce9e diff --git a/manifest.uuid b/manifest.uuid index d9c988d4d5..70f95ac337 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -27deb6e49bcc76714dbdc61b34748603155ac770 \ No newline at end of file +b3296267fb67b9f59719a37093253062edde3746 \ No newline at end of file diff --git a/src/mem5.c b/src/mem5.c index 5d75611a32..67615bb964 100644 --- a/src/mem5.c +++ b/src/mem5.c @@ -248,7 +248,7 @@ static void *memsys5MallocUnsafe(int nByte){ ** block. If not, then split a block of the next larger power of ** two in order to create a new free block of size iLogsize. */ - for(iBin=iLogsize; mem5.aiFreelist[iBin]<0 && iBin<=LOGMAX; iBin++){} + for(iBin=iLogsize; iBin<=LOGMAX && mem5.aiFreelist[iBin]<0; iBin++){} if( iBin>LOGMAX ){ testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes", nByte);