From f52bb8d3857ebd8359d22a4d4a802f681d4c55c4 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 3 Aug 2013 20:24:58 +0000 Subject: [PATCH 01/67] Begin adding experimental sqlite_stat4 table. This commit is buggy. FossilOrigin-Name: 2beea303a1d609cd2ff252412c50b966b9e5e8f1 --- manifest | 60 ++-- manifest.uuid | 2 +- src/analyze.c | 711 +++++++++++++++++++++++++-------------- src/btree.c | 1 + src/build.c | 2 +- src/ctime.c | 4 +- src/sqliteInt.h | 15 +- src/test_config.c | 6 +- src/utf.c | 2 +- src/vdbe.c | 1 + src/vdbemem.c | 6 +- src/where.c | 156 ++------- test/alter.test | 2 +- test/analyze.test | 21 +- test/analyze3.test | 2 +- test/analyze5.test | 23 +- test/analyze6.test | 2 +- test/analyze7.test | 8 +- test/analyze8.test | 2 +- test/analyze9.test | 55 +++ test/auth.test | 8 +- test/dbstatus.test | 4 +- test/tkt-cbd054fa6b.test | 29 +- test/where9.test | 6 +- test/wild001.test | 4 + 25 files changed, 663 insertions(+), 469 deletions(-) create mode 100644 test/analyze9.test diff --git a/manifest b/manifest index 478a6e4f1d..7dbbcbff26 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Updates\sto\srequirements\smarks.\s\sNo\scode\schanges. -D 2013-08-02T23:40:45.340 +C Begin\sadding\sexperimental\ssqlite_stat4\stable.\sThis\scommit\sis\sbuggy. +D 2013-08-03T20:24:58.082 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,19 +157,19 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 -F src/analyze.c a33fcb0b3a399d966951feb9f32115106b3ecc2e +F src/analyze.c 7c272ae217eb34439ba2ea560b506eec667bd1e0 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7 -F src/btree.c 3f7bbfd72efb1cbf6a49515c376a031767ec930a +F src/btree.c 15ea4e980ba5edeb9b495f001e86a0688f46ee2c F src/btree.h 6fa8a3ff2483d0bb64a9f0105a8cedeac9e00cca F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 -F src/build.c fc76e1cd014840781e175f57f6de38917986943b +F src/build.c b2771cc57484ee4225a9eb6e57e6933be3f96e3b F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 4262c227bc91cecc61ae37ed3a40f08069cfa267 +F src/ctime.c 177fa0cbf28b8deda3f216603beee0b883408a40 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c 2317c814866d9aa71fea16b3faf4fdd4d6a49b94 F src/expr.c 2068a7c17e45f8bee6e44205b059aa30acbc71c5 @@ -221,7 +221,7 @@ F src/shell.c 52f975eae87c8338c4dfbf4c2842d2a0971f01fd F src/sqlite.h.in 442c109e0c3447c34b1794971ecdb673ce08a843 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 7c6ad474ce49ed18393c027be65c9532b7c9168f +F src/sqliteInt.h 37e2952ce88a64d6572b2c40fcb78956e5aa2f7a F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -239,7 +239,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c 32cff3d01cdd3202486e623c3f8103ed04cb57fa F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c 95bb33e9dcaa340a296c0bf0e0ba3d1a1c8004c0 +F src/test_config.c 636ecd15a6ba18bf97a590b5a21f47573c8c2b65 F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c 8f786bfd0ad48030cf2a06fb1f050e9c60a150d7 @@ -274,40 +274,41 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 70061085a51f2f4fc15ece94f32c03bcb78e63b2 F src/trigger.c 5c0ea9b8755e7c5e1a700f3e27ac4f8d92dd221e F src/update.c 7f3fe64d8f3b44c44a1eac293f0f85f87c355b7a -F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f +F src/utf.c acd0b6f8beb8df4e0ed178c48c81c693bcc31967 F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 F src/vacuum.c d9c5759f4c5a438bb43c2086f72c5d2edabc36c8 -F src/vdbe.c d6048a720c197db2f0e7d618e918bd2e2eff0322 +F src/vdbe.c 91fa72a040cb1065045fce5e84196be093e29918 F src/vdbe.h 4f554b5627f26710c4c36d919110a3fc611ca5c4 F src/vdbeInt.h e9b7c6b165a31a4715c5aa97223d20d265515231 F src/vdbeapi.c 4d13580bd058b39623e8fcfc233b7df4b8191e8b F src/vdbeaux.c 4389b3692969b4415fcfd00de36818a02f84df28 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 -F src/vdbemem.c 833005f1cbbf447289f1973dba2a0c2228c7b8ab +F src/vdbemem.c 69c6d1c3ef07f4442e074def9a92d15d02f06eba F src/vdbesort.c 3937e06b2a0e354500e17dc206ef4c35770a5017 F src/vdbetrace.c e7ec40e1999ff3c6414424365d5941178966dcbc F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 67245bb73fd9cc04ee7796a0f358d4910c45cfe9 +F src/where.c 072abd8556bf1d05676cc14b561da79acb70abb9 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87 F test/all.test 6ff7b43c2b4b905c74dc4a813d201d0fa64c5783 -F test/alter.test 57d96ec9b320bd07af77567034488dcb6642c748 +F test/alter.test 7e771c3c3f401198557dbbcf4a2c21950ba934f3 F test/alter2.test 7ea05c7d92ac99349a802ef7ada17294dd647060 F test/alter3.test 49c9d9fba2b8fcdce2dedeca97bbf1f369cc548d F test/alter4.test b2debc14d8cbe4c1d12ccd6a41eef88a8c1f15d5 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc -F test/analyze.test f8ab7d15858b4093b06caf5e57e2a5ff7104bdae -F test/analyze3.test 53cfd07625d110e70b92b57a6ff656ea844dfbee +F test/analyze.test 4d08a739c5ec28db93e0465e3b5a468befdf145f +F test/analyze3.test 4532e5475d2aa68d752627548bdcaf70aff51010 F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 -F test/analyze5.test 3e57f192307be931f9ab2f6ff456f9063358ac77 -F test/analyze6.test cdbf9887d40ab41301f570fe85c6e1525dd3b1c9 -F test/analyze7.test 7de3ab66e1981303e783102a012d62cb876bf1d5 -F test/analyze8.test ea4972c76820ac8d6a0754e6f5b851292f0f5a61 +F test/analyze5.test e3eece09761c935ec0b85dc4ed70dbf6cac1ed77 +F test/analyze6.test 3c01e084309706a1033f850330ea24f6f7846297 +F test/analyze7.test c0af22c5e0140e2e4ac556a21c2b6fff58229c98 +F test/analyze8.test 092425439c12f62f9d5c3127e2b4f6e7b3e170cc +F test/analyze9.test 238e3d1fa17e53d80da1d84a151c0465ef8ba666 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -319,7 +320,7 @@ F test/attach2.test e54436ed956d3d88bdee61221da59bf3935a0966 F test/attach3.test d89ccfe4fe6e2b5e368d480fcdfe4b496c54cf4e F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 -F test/auth.test 4a4c3b034fff7750513520defa910f376c96ab49 +F test/auth.test cb43e31c14b8f5849d04edd6d3a1f844687cc855 F test/auth2.test a2a371aa6df15f8b0c8109b33d3d7f0f73e4c9aa F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test bd30d372d00045252f6c2e41b5f41455e1975acf @@ -411,7 +412,7 @@ F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8 F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test f3228180c87bbe5d39c9397bf001c0095c3821b9 -F test/dbstatus.test 207e5b63fcb7b9c3bb8e1fdf38ebd4654ad0e54b +F test/dbstatus.test 1e64356a8c0407d7aeead201852fc4de9418196a F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 @@ -864,7 +865,7 @@ F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 F test/tkt-bdc6bbbb38.test fc38bb09bdd440e3513a1f5f98fc60a075182d7d F test/tkt-c48d99d690.test ba61977d62ab612fc515b3c488a6fbd6464a2447 -F test/tkt-cbd054fa6b.test 9c27ed07b333eed458e5d4543f91ecdcf05aeb19 +F test/tkt-cbd054fa6b.test 2fd674fda943346a31cd020883f70bf6c037e98c F test/tkt-d11f09d36e.test d999b548fef885d1d1afa49a0e8544ecf436869d F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 @@ -1047,7 +1048,7 @@ F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5a4b0abc207d71da4deecd734ad8579e8dd40aa8 F test/where8.test 6f95896633cf2d307b5263145b942b7d33e837c6 F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 -F test/where9.test 9a7fda4a4512abc26a855e8b2b6572b200f6019b +F test/where9.test 71aa15cc17cb3343566a2de3aef47d4548610e4d F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a @@ -1055,7 +1056,7 @@ F test/whereD.test 6c2feb79ef1f68381b07f39017fe5f9b96da8d62 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 136a7301512d72a08a272806c8767066311b7bc1 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 -F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c +F test/wild001.test 384db4b30fbe82eaaefe8921f8a702c09e26db27 F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test 209d7ed441f44cc5299e4ebffbef06fd5aabfefd @@ -1105,7 +1106,10 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c5c0a8ab6c222185d5f9d4321e64d9f93cd36b7d -R 20b1dec2302f0bdaf9f8a82570231a5d -U drh -Z 33c56d2532f56c7cb339c035fb818e1a +P 213020769f310aec1591d97756b53891d0b64005 +R b9be00b2213e6b6e3de39cb20a1e763e +T *branch * sqlite_stat4 +T *sym-sqlite_stat4 * +T -sym-trunk * +U dan +Z 57387cc568421f9309aeb8be1def5190 diff --git a/manifest.uuid b/manifest.uuid index 0dc138d599..8f3d3831d4 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -213020769f310aec1591d97756b53891d0b64005 \ No newline at end of file +2beea303a1d609cd2ff252412c50b966b9e5e8f1 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index d25a9b196c..d2057ffc39 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -119,16 +119,16 @@ /* ** This routine generates code that opens the sqlite_stat1 table for ** writing with cursor iStatCur. If the library was built with the -** SQLITE_ENABLE_STAT3 macro defined, then the sqlite_stat3 table is +** SQLITE_ENABLE_STAT4 macro defined, then the sqlite_stat4 table is ** opened for writing using cursor (iStatCur+1) ** ** If the sqlite_stat1 tables does not previously exist, it is created. -** Similarly, if the sqlite_stat3 table does not exist and the library -** is compiled with SQLITE_ENABLE_STAT3 defined, it is created. +** Similarly, if the sqlite_stat4 table does not exist and the library +** is compiled with SQLITE_ENABLE_STAT4 defined, it is created. ** ** Argument zWhere may be a pointer to a buffer containing a table name, ** or it may be a NULL pointer. If it is not NULL, then all entries in -** the sqlite_stat1 and (if applicable) sqlite_stat3 tables associated +** the sqlite_stat1 and (if applicable) sqlite_stat4 tables associated ** with the named table are deleted. If zWhere==0, then code is generated ** to delete all stat table entries. */ @@ -144,8 +144,8 @@ static void openStatTable( const char *zCols; } aTable[] = { { "sqlite_stat1", "tbl,idx,stat" }, -#ifdef SQLITE_ENABLE_STAT3 - { "sqlite_stat3", "tbl,idx,neq,nlt,ndlt,sample" }, +#ifdef SQLITE_ENABLE_STAT4 + { "sqlite_stat4", "tbl,idx,neq,nlt,ndlt,sample" }, #endif }; @@ -194,7 +194,7 @@ static void openStatTable( } } - /* Open the sqlite_stat[13] tables for writing. */ + /* Open the sqlite_stat[14] tables for writing. */ for(i=0; iaSample[] */ + /* Decode the three function arguments */ UNUSED_PARAMETER(argc); nRow = (tRowcnt)sqlite3_value_int64(argv[0]); - mxSample = sqlite3_value_int(argv[1]); - n = sizeof(*p) + sizeof(p->a[0])*mxSample; + nCol = sqlite3_value_int(argv[1]); + mxSample = sqlite3_value_int(argv[2]); + assert( nCol>0 ); + + /* Allocate the space required for the Stat4Accum object */ + n = sizeof(*p) + (sizeof(p->a[0]) + 3*sizeof(tRowcnt)*nCol)*mxSample; p = sqlite3MallocZero( n ); if( p==0 ){ sqlite3_result_error_nomem(context); return; } - p->a = (struct Stat3Sample*)&p[1]; + + /* Populate the new Stat4Accum object */ p->nRow = nRow; + p->nCol = nCol; p->mxSample = mxSample; p->nPSample = p->nRow/(mxSample/3+1) + 1; sqlite3_randomness(sizeof(p->iPrn), &p->iPrn); + p->a = (struct Stat4Sample*)&p[1]; + pSpace = (u8*)(&p->a[mxSample]); + for(i=0; ia[i].anEq = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nCol); + p->a[i].anLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nCol); + p->a[i].anDLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nCol); + } + assert( (pSpace - (u8*)p)==n ); + + /* Return a pointer to the allocated object to the caller */ sqlite3_result_blob(context, p, sizeof(p), sqlite3_free); } -static const FuncDef stat3InitFuncdef = { - 2, /* nArg */ +static const FuncDef stat4InitFuncdef = { + 3, /* nArg */ SQLITE_UTF8, /* iPrefEnc */ 0, /* flags */ 0, /* pUserData */ 0, /* pNext */ - stat3Init, /* xFunc */ + stat4Init, /* xFunc */ 0, /* xStep */ 0, /* xFinalize */ - "stat3_init", /* zName */ + "stat4_init", /* zName */ 0, /* pHash */ 0 /* pDestructor */ }; /* -** Implementation of the stat3_push(nEq,nLt,nDLt,rowid,P) SQL function. The -** arguments describe a single key instance. This routine makes the -** decision about whether or not to retain this key for the sqlite_stat3 -** table. +** Implementation of the stat4_push SQL function. The arguments describe a +** single key instance. This routine makes the decision about whether or +** not to retain this key for the sqlite_stat4 table. +** +** The calling convention is: ** -** The return value is NULL. +** stat4_push(P, rowid, ...nEq args..., ...nLt args..., ...nDLt args...) +** +** where each instance of the "...nXX args..." is replaced by an array of +** nCol arguments, where nCol is the number of columns in the index being +** sampled (if the index being sampled is "CREATE INDEX i ON t(a, b)", a +** total of 8 arguments are passed when this function is invoked). +** +** The return value is always NULL. */ -static void stat3Push( +static void stat4Push( sqlite3_context *context, int argc, sqlite3_value **argv ){ - Stat3Accum *p = (Stat3Accum*)sqlite3_value_blob(argv[4]); - tRowcnt nEq = sqlite3_value_int64(argv[0]); - tRowcnt nLt = sqlite3_value_int64(argv[1]); - tRowcnt nDLt = sqlite3_value_int64(argv[2]); - i64 rowid = sqlite3_value_int64(argv[3]); + Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); + i64 rowid = sqlite3_value_int64(argv[1]); + i64 nSumEq = 0; /* Sum of all nEq parameters */ + struct Stat4Sample *pSample; + u32 h; + int iMin = p->iMin; + int i; u8 isPSample = 0; u8 doInsert = 0; - int iMin = p->iMin; - struct Stat3Sample *pSample; - int i; - u32 h; + + sqlite3_value **aEq = &argv[2]; + sqlite3_value **aLt = &argv[2+p->nCol]; + sqlite3_value **aDLt = &argv[2+p->nCol+p->nCol]; + + i64 nEq = sqlite3_value_int64(aEq[p->nCol-1]); + i64 nLt = sqlite3_value_int64(aLt[p->nCol-1]); UNUSED_PARAMETER(context); UNUSED_PARAMETER(argc); - if( nEq==0 ) return; + + assert( p->nCol>0 ); + assert( argc==(2 + 3*p->nCol) ); + + /* Set nSumEq to the sum of all nEq parameters. */ + for(i=0; inCol; i++){ + nSumEq += sqlite3_value_int64(aEq[i]); + } + if( nSumEq==0 ) return; + + /* Figure out if this sample will be used. Set isPSample to true if this + ** is a periodic sample, or false if it is being captured because of a + ** large nSumEq value. If the sample will not be used, return early. */ h = p->iPrn = p->iPrn*1103515245 + 12345; if( (nLt/p->nPSample)!=((nEq+nLt)/p->nPSample) ){ doInsert = isPSample = 1; - }else if( p->nSamplemxSample ){ + }else if( (p->nSamplemxSample) + || (nSumEq>p->a[iMin].nSumEq) + || (nSumEq==p->a[iMin].nSumEq && h>p->a[iMin].iHash) + ){ doInsert = 1; - }else{ - if( nEq>p->a[iMin].nEq || (nEq==p->a[iMin].nEq && h>p->a[iMin].iHash) ){ - doInsert = 1; - } } if( !doInsert ) return; + + /* Fill in the new Stat4Sample object. */ if( p->nSample==p->mxSample ){ assert( p->nSample - iMin - 1 >= 0 ); memmove(&p->a[iMin], &p->a[iMin+1], sizeof(p->a[0])*(p->nSample-iMin-1)); @@ -330,47 +378,45 @@ static void stat3Push( pSample = &p->a[p->nSample++]; } pSample->iRowid = rowid; - pSample->nEq = nEq; - pSample->nLt = nLt; - pSample->nDLt = nDLt; pSample->iHash = h; pSample->isPSample = isPSample; + pSample->nSumEq = nSumEq; + for(i=0; inCol; i++){ + pSample->anEq[i] = sqlite3_value_int64(aEq[i]); + pSample->anLt[i] = sqlite3_value_int64(aLt[i]); + pSample->anDLt[i] = sqlite3_value_int64(aDLt[i]); + } /* Find the new minimum */ if( p->nSample==p->mxSample ){ - pSample = p->a; - i = 0; - while( pSample->isPSample ){ - i++; - pSample++; - assert( inSample ); - } - nEq = pSample->nEq; - h = pSample->iHash; - iMin = i; - for(i++, pSample++; inSample; i++, pSample++){ - if( pSample->isPSample ) continue; - if( pSample->nEqnEq==nEq && pSample->iHashmxSample; i++){ + if( p->a[i].isPSample ) continue; + if( (p->a[i].nSumEqa[i].nSumEq==nMinEq && p->a[i].iHashnEq; - h = pSample->iHash; + nMinEq = p->a[i].nSumEq; + iHash = p->a[i].iHash; } } + assert( iMin>=0 ); p->iMin = iMin; } } -static const FuncDef stat3PushFuncdef = { - 5, /* nArg */ +static const FuncDef stat4PushFuncdef = { + -1, /* nArg */ SQLITE_UTF8, /* iPrefEnc */ 0, /* flags */ 0, /* pUserData */ 0, /* pNext */ - stat3Push, /* xFunc */ + stat4Push, /* xFunc */ 0, /* xStep */ 0, /* xFinalize */ - "stat3_push", /* zName */ + "stat4_push", /* zName */ 0, /* pHash */ 0 /* pDestructor */ }; @@ -386,37 +432,58 @@ static const FuncDef stat3PushFuncdef = { ** argc==4 result: nLt ** argc==5 result: nDLt */ -static void stat3Get( +static void stat4Get( sqlite3_context *context, int argc, sqlite3_value **argv ){ + Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); int n = sqlite3_value_int(argv[1]); - Stat3Accum *p = (Stat3Accum*)sqlite3_value_blob(argv[0]); assert( p!=0 ); - if( p->nSample<=n ) return; - switch( argc ){ - case 2: sqlite3_result_int64(context, p->a[n].iRowid); break; - case 3: sqlite3_result_int64(context, p->a[n].nEq); break; - case 4: sqlite3_result_int64(context, p->a[n].nLt); break; - default: sqlite3_result_int64(context, p->a[n].nDLt); break; + if( nnSample ){ + tRowcnt *aCnt = 0; + char *zRet; + + switch( argc ){ + case 2: + sqlite3_result_int64(context, p->a[n].iRowid); + return; + case 3: aCnt = p->a[n].anEq; break; + case 4: aCnt = p->a[n].anLt; break; + default: aCnt = p->a[n].anDLt; break; + } + + zRet = sqlite3MallocZero(p->nCol * 25); + if( zRet==0 ){ + sqlite3_result_error_nomem(context); + }else{ + int i; + char *z = zRet; + for(i=0; inCol; i++){ + sqlite3_snprintf(24, z, "%lld ", aCnt[i]); + z += sqlite3Strlen30(z); + } + assert( z[0]=='\0' && z>zRet ); + z[-1] = '\0'; + sqlite3_result_text(context, zRet, -1, sqlite3_free); + } } } -static const FuncDef stat3GetFuncdef = { +static const FuncDef stat4GetFuncdef = { -1, /* nArg */ SQLITE_UTF8, /* iPrefEnc */ 0, /* flags */ 0, /* pUserData */ 0, /* pNext */ - stat3Get, /* xFunc */ + stat4Get, /* xFunc */ 0, /* xStep */ 0, /* xFinalize */ - "stat3_get", /* zName */ + "stat4_get", /* zName */ 0, /* pHash */ 0 /* pDestructor */ }; -#endif /* SQLITE_ENABLE_STAT3 */ +#endif /* SQLITE_ENABLE_STAT4 */ @@ -445,13 +512,13 @@ static void analyzeOneTable( int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ int regStat1 = iMem++; /* The stat column of sqlite_stat1 */ -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 int regNumEq = regStat1; /* Number of instances. Same as regStat1 */ int regNumLt = iMem++; /* Number of keys less than regSample */ int regNumDLt = iMem++; /* Number of distinct keys less than regSample */ int regSample = iMem++; /* The next sample value */ int regRowid = regSample; /* Rowid of a sample */ - int regAccum = iMem++; /* Register to hold Stat3Accum object */ + int regAccum = iMem++; /* Register to hold Stat4Accum object */ int regLoop = iMem++; /* Loop counter */ int regCount = iMem++; /* Number of rows in the table or index */ int regTemp1 = iMem++; /* Intermediate register */ @@ -465,6 +532,7 @@ static void analyzeOneTable( int regTemp = iMem++; /* Temporary use register */ int regNewRowid = iMem++; /* Rowid for the inserted record */ + int regStat4 = iMem++; /* Register to hold Stat4Accum object */ v = sqlite3GetVdbe(pParse); if( v==0 || NEVER(pTab==0) ){ @@ -495,10 +563,19 @@ static void analyzeOneTable( iIdxCur = pParse->nTab++; sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - int nCol; - KeyInfo *pKey; - int addrIfNot = 0; /* address of OP_IfNot */ - int *aChngAddr; /* Array of jump instruction addresses */ + int nCol; /* Number of columns indexed by pIdx */ + KeyInfo *pKey; /* KeyInfo structure for pIdx */ + int addrIfNot = 0; /* address of OP_IfNot */ + int *aChngAddr; /* Array of jump instruction addresses */ + + int regRowid; /* Register for rowid of current row */ + int regPrev; /* First in array of previous values */ + int regDLt; /* First in array of nDlt registers */ + int regLt; /* First in array of nLt registers */ + int regEq; /* First in array of nEq registers */ + int regCnt; /* Number of index entries */ + + int addrGoto; if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0; @@ -507,9 +584,6 @@ static void analyzeOneTable( aChngAddr = sqlite3DbMallocRaw(db, sizeof(int)*nCol); if( aChngAddr==0 ) continue; pKey = sqlite3IndexKeyinfo(pParse, pIdx); - if( iMem+1+(nCol*2)>pParse->nMem ){ - pParse->nMem = iMem+1+(nCol*2); - } /* Open a cursor to the index to be analyzed. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); @@ -520,30 +594,39 @@ static void analyzeOneTable( /* Populate the register containing the index name. */ sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, pIdx->zName, 0); -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 if( once ){ once = 0; sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); } - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regCount); - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_STAT3_SAMPLES, regTemp1); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regNumEq); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regNumLt); - sqlite3VdbeAddOp2(v, OP_Integer, -1, regNumDLt); - sqlite3VdbeAddOp3(v, OP_Null, 0, regSample, regAccum); - sqlite3VdbeAddOp4(v, OP_Function, 1, regCount, regAccum, - (char*)&stat3InitFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 2); -#endif /* SQLITE_ENABLE_STAT3 */ - /* The block of memory cells initialized here is used as follows. + /* Invoke the stat4_init() function. The arguments are: + ** + ** * the number of rows in the index, + ** * the number of columns in the index, + ** * the recommended number of samples for the stat4 table. + */ + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+1); + sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+2); + sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_STAT4_SAMPLES, regStat4+3); + sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); + sqlite3VdbeChangeP4(v, -1, (char*)&stat4InitFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 3); +#endif /* SQLITE_ENABLE_STAT4 */ + + /* The block of (1 + 4*nCol) memory cells initialized here is used + ** as follows: + ** + ** TODO: Update this comment: ** ** iMem: - ** The total number of rows in the table. + ** Loop counter. The number of rows visited so far, including + ** the current row (i.e. this register is set to 1 for the + ** first iteration of the loop). ** - ** iMem+1 .. iMem+nCol: - ** Number of distinct entries in index considering the - ** left-most N columns only, where N is between 1 and nCol, + ** iMem+1 .. iMem+nCol: + ** Number of distinct index entries seen so far, considering + ** the left-most N columns only, where N is between 1 and nCol, ** inclusive. ** ** iMem+nCol+1 .. Mem+2*nCol: @@ -552,89 +635,192 @@ static void analyzeOneTable( ** Cells iMem through iMem+nCol are initialized to 0. The others are ** initialized to contain an SQL NULL. */ - for(i=0; i<=nCol; i++){ - sqlite3VdbeAddOp2(v, OP_Integer, 0, iMem+i); + regRowid = regStat4+1; /* Rowid argument */ + regEq = regRowid+1; /* First in array of nEq value registers */ + regLt = regEq+nCol; /* First in array of nLt value registers */ + regDLt = regLt+nCol; /* First in array of nDLt value registers */ + regCnt = regDLt+nCol; /* Row counter */ + regPrev = regCnt+1; /* First in array of prev. value registers */ + + if( regPrev+1>pParse->nMem ){ + pParse->nMem = regPrev+1; } - for(i=0; iazColl && pIdx->azColl[i]!=0 ); + pColl = (char*)sqlite3LocateCollSeq(pParse, pIdx->azColl[i]); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regCol); - if( i==0 ){ - /* Always record the very first row */ - addrIfNot = sqlite3VdbeAddOp1(v, OP_IfNot, iMem+1); - } - assert( pIdx->azColl!=0 ); - assert( pIdx->azColl[i]!=0 ); - pColl = sqlite3LocateCollSeq(pParse, pIdx->azColl[i]); - aChngAddr[i] = sqlite3VdbeAddOp4(v, OP_Ne, regCol, 0, iMem+nCol+i+1, - (char*)pColl, P4_COLLSEQ); + sqlite3VdbeAddOp4(v, OP_Eq, regCol, 0, regPrev+i, pColl, P4_COLLSEQ); sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + + sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp2); + sqlite3VdbeChangeP4(v, -1, (char*)&stat4PushFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 2 + 3*nCol); + + aChngAddr[i] = sqlite3VdbeAddOp0(v, OP_Goto); VdbeComment((v, "jump if column %d changed", i)); -#ifdef SQLITE_ENABLE_STAT3 - if( i==0 ){ - sqlite3VdbeAddOp2(v, OP_AddImm, regNumEq, 1); - VdbeComment((v, "incr repeat count")); - } -#endif + + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-3); + sqlite3VdbeAddOp2(v, OP_AddImm, regEq+i, 1); + VdbeComment((v, "incr repeat count")); } - sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfLoop); + + /* Code the "continue" */ + addrGoto = sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfLoop); + + /* And now these: + ** + ** ne_N: + ** regPrev(N) = i(N) + ** if( regEq(N) != N ) regDLt(N) += 1 + ** regLt(N) += regEq(N) + ** regEq(N) = 1 + */ for(i=0; inColumn, regRowid); - sqlite3VdbeAddOp3(v, OP_Add, regNumEq, regNumLt, regNumLt); - sqlite3VdbeAddOp2(v, OP_AddImm, regNumDLt, 1); - sqlite3VdbeAddOp2(v, OP_Integer, 1, regNumEq); -#endif } - sqlite3VdbeAddOp2(v, OP_AddImm, iMem+i+1, 1); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, iMem+nCol+i+1); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regPrev+i); + sqlite3VdbeAddOp2(v, OP_IfNot, regEq+i, sqlite3VdbeCurrentAddr(v)+2); + sqlite3VdbeAddOp2(v, OP_AddImm, regDLt+i, 1); + sqlite3VdbeAddOp3(v, OP_Add, regEq+i, regLt+i, regLt+i); + sqlite3VdbeAddOp2(v, OP_Integer, 1, regEq+i); } sqlite3DbFree(db, aChngAddr); - /* Always jump here after updating the iMem+1...iMem+1+nCol counters */ - sqlite3VdbeResolveLabel(v, endOfLoop); + /* + ** all_eq: + ** regRowid = i(rowid) + */ + sqlite3VdbeJumpHere(v, addrGoto); + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid); + /* The end of the loop that iterates through all index entries. Always + ** jump here after updating the iMem+1...iMem+1+nCol counters. */ + sqlite3VdbeResolveLabel(v, endOfLoop); sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, topOfLoop); sqlite3VdbeAddOp1(v, OP_Close, iIdxCur); -#ifdef SQLITE_ENABLE_STAT3 - sqlite3VdbeAddOp4(v, OP_Function, 1, regNumEq, regTemp2, - (char*)&stat3PushFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 5); + + /* Final invocation of stat4_push() */ + sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp2); + sqlite3VdbeChangeP4(v, -1, (char*)&stat4PushFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 2 + 3*nCol); + + /* Finally: + ** + ** if( regEq(0) != 0 ) regDLt(0) += 1 + */ + for(i=0; iaiColumn[0], regSample); - sqlite3ColumnDefault(v, pTab, pIdx->aiColumn[0], regSample); - sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regNumEq, - (char*)&stat3GetFuncdef, P4_FUNCDEF); + for(i=0; iaiColumn[i]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regPrev+i); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regPrev, nCol, regSample); + sqlite3VdbeChangeP4(v, -1, pIdx->zColAff, 0); + + sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regNumEq); + sqlite3VdbeChangeP4(v, -1, (char*)&stat4GetFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 3); - sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regNumLt, - (char*)&stat3GetFuncdef, P4_FUNCDEF); + + sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regNumLt); + sqlite3VdbeChangeP4(v, -1, (char*)&stat4GetFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 4); - sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regNumDLt, - (char*)&stat3GetFuncdef, P4_FUNCDEF); + + sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regNumDLt); + sqlite3VdbeChangeP4(v, -1, (char*)&stat4GetFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 5); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 6, regRec, "bbbbbb", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regRec, regNewRowid); @@ -660,14 +846,14 @@ static void analyzeOneTable( ** If K>0 then it is always the case the D>0 so division by zero ** is never possible. */ - sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regStat1); - jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, iMem); + sqlite3VdbeAddOp2(v, OP_SCopy, regCnt, regStat1); + jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regCnt); for(i=0; i='0' && c<='9' ){ + v = v*10 + c - '0'; + z++; + } + aOut[i] = v; + if( *z==' ' ) z++; + } + if( pbUnordered && strcmp(z, "unordered")==0 ){ + *pbUnordered = 1; + } +} + /* ** This callback is invoked once for each index when reading the ** sqlite_stat1 table. @@ -851,8 +1069,6 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ analysisInfo *pInfo = (analysisInfo*)pData; Index *pIndex; Table *pTable; - int i, c, n; - tRowcnt v; const char *z; assert( argc==3 ); @@ -870,25 +1086,17 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ }else{ pIndex = 0; } - n = pIndex ? pIndex->nColumn : 0; z = argv[2]; - for(i=0; *z && i<=n; i++){ - v = 0; - while( (c=z[0])>='0' && c<='9' ){ - v = v*10 + c - '0'; - z++; - } - if( i==0 && (pIndex==0 || pIndex->pPartIdxWhere==0) ){ - if( v>0 ) pTable->nRowEst = v; - if( pIndex==0 ) break; - } - pIndex->aiRowEst[i] = v; - if( *z==' ' ) z++; - if( strcmp(z, "unordered")==0 ){ - pIndex->bUnordered = 1; - break; - } + + if( pIndex ){ + int bUnordered = 0; + decodeIntArray((char*)z, pIndex->nColumn, pIndex->aiRowEst, &bUnordered); + if( pIndex->pPartIdxWhere==0 ) pTable->nRowEst = pIndex->aiRowEst[0]; + pIndex->bUnordered = bUnordered; + }else{ + decodeIntArray((char*)z, 1, &pTable->nRowEst, 0); } + return 0; } @@ -897,14 +1105,12 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** and its contents. */ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 if( pIdx->aSample ){ int j; for(j=0; jnSample; j++){ IndexSample *p = &pIdx->aSample[j]; - if( p->eType==SQLITE_TEXT || p->eType==SQLITE_BLOB ){ - sqlite3DbFree(db, p->u.z); - } + sqlite3DbFree(db, p->p); } sqlite3DbFree(db, pIdx->aSample); } @@ -918,27 +1124,26 @@ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ #endif } -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 /* -** Load content from the sqlite_stat3 table into the Index.aSample[] +** Load content from the sqlite_stat4 table into the Index.aSample[] ** arrays of all indices. */ -static int loadStat3(sqlite3 *db, const char *zDb){ +static int loadStat4(sqlite3 *db, const char *zDb){ int rc; /* Result codes from subroutines */ sqlite3_stmt *pStmt = 0; /* An SQL statement being run */ char *zSql; /* Text of the SQL statement */ Index *pPrevIdx = 0; /* Previous index in the loop */ int idx = 0; /* slot in pIdx->aSample[] for next sample */ - int eType; /* Datatype of a sample */ IndexSample *pSample; /* A slot in pIdx->aSample[] */ assert( db->lookaside.bEnabled==0 ); - if( !sqlite3FindTable(db, "sqlite_stat3", zDb) ){ + if( !sqlite3FindTable(db, "sqlite_stat4", zDb) ){ return SQLITE_OK; } zSql = sqlite3MPrintf(db, - "SELECT idx,count(*) FROM %Q.sqlite_stat3" + "SELECT idx,count(*) FROM %Q.sqlite_stat4" " GROUP BY idx", zDb); if( !zSql ){ return SQLITE_NOMEM; @@ -951,6 +1156,9 @@ static int loadStat3(sqlite3 *db, const char *zDb){ char *zIndex; /* Index name */ Index *pIdx; /* Pointer to the index object */ int nSample; /* Number of samples */ + int nByte; /* Bytes of space required */ + int i; /* Bytes of space required */ + tRowcnt *pSpace; zIndex = (char *)sqlite3_column_text(pStmt, 0); if( zIndex==0 ) continue; @@ -959,19 +1167,28 @@ static int loadStat3(sqlite3 *db, const char *zDb){ if( pIdx==0 ) continue; assert( pIdx->nSample==0 ); pIdx->nSample = nSample; - pIdx->aSample = sqlite3DbMallocZero(db, nSample*sizeof(IndexSample)); + nByte = sizeof(IndexSample) * nSample; + nByte += sizeof(tRowcnt) * pIdx->nColumn * 3 * nSample; + + pIdx->aSample = sqlite3DbMallocZero(db, nByte); pIdx->avgEq = pIdx->aiRowEst[1]; if( pIdx->aSample==0 ){ - db->mallocFailed = 1; sqlite3_finalize(pStmt); return SQLITE_NOMEM; } + pSpace = (tRowcnt*)&pIdx->aSample[nSample]; + for(i=0; inSample; i++){ + pIdx->aSample[i].anEq = pSpace; pSpace += pIdx->nColumn; + pIdx->aSample[i].anLt = pSpace; pSpace += pIdx->nColumn; + pIdx->aSample[i].anDLt = pSpace; pSpace += pIdx->nColumn; + } + assert( ((u8*)pSpace)-nByte==(u8*)(pIdx->aSample) ); } rc = sqlite3_finalize(pStmt); if( rc ) return rc; zSql = sqlite3MPrintf(db, - "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat3", zDb); + "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", zDb); if( !zSql ){ return SQLITE_NOMEM; } @@ -984,6 +1201,7 @@ static int loadStat3(sqlite3 *db, const char *zDb){ Index *pIdx; /* Pointer to the index object */ int i; /* Loop counter */ tRowcnt sumEq; /* Sum of the nEq values */ + int nCol; /* Number of columns in index */ zIndex = (char *)sqlite3_column_text(pStmt, 0); if( zIndex==0 ) continue; @@ -997,69 +1215,46 @@ static int loadStat3(sqlite3 *db, const char *zDb){ } assert( idxnSample ); pSample = &pIdx->aSample[idx]; - pSample->nEq = (tRowcnt)sqlite3_column_int64(pStmt, 1); - pSample->nLt = (tRowcnt)sqlite3_column_int64(pStmt, 2); - pSample->nDLt = (tRowcnt)sqlite3_column_int64(pStmt, 3); + + nCol = pIdx->nColumn; + decodeIntArray((char*)sqlite3_column_text(pStmt,1), nCol, pSample->anEq, 0); + decodeIntArray((char*)sqlite3_column_text(pStmt,2), nCol, pSample->anLt, 0); + decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); + if( idx==pIdx->nSample-1 ){ - if( pSample->nDLt>0 ){ - for(i=0, sumEq=0; i<=idx-1; i++) sumEq += pIdx->aSample[i].nEq; - pIdx->avgEq = (pSample->nLt - sumEq)/pSample->nDLt; + if( pSample->anDLt[0]>0 ){ + for(i=0, sumEq=0; i<=idx-1; i++) sumEq += pIdx->aSample[i].anEq[0]; + pIdx->avgEq = (pSample->anLt[0] - sumEq)/pSample->anDLt[0]; } if( pIdx->avgEq<=0 ) pIdx->avgEq = 1; } - eType = sqlite3_column_type(pStmt, 4); - pSample->eType = (u8)eType; - switch( eType ){ - case SQLITE_INTEGER: { - pSample->u.i = sqlite3_column_int64(pStmt, 4); - break; - } - case SQLITE_FLOAT: { - pSample->u.r = sqlite3_column_double(pStmt, 4); - break; - } - case SQLITE_NULL: { - break; - } - default: assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); { - const char *z = (const char *)( - (eType==SQLITE_BLOB) ? - sqlite3_column_blob(pStmt, 4): - sqlite3_column_text(pStmt, 4) - ); - int n = z ? sqlite3_column_bytes(pStmt, 4) : 0; - pSample->nByte = n; - if( n < 1){ - pSample->u.z = 0; - }else{ - pSample->u.z = sqlite3DbMallocRaw(db, n); - if( pSample->u.z==0 ){ - db->mallocFailed = 1; - sqlite3_finalize(pStmt); - return SQLITE_NOMEM; - } - memcpy(pSample->u.z, z, n); - } - } + + pSample->n = sqlite3_column_bytes(pStmt, 4); + pSample->p = sqlite3DbMallocZero(db, pSample->n); + if( pSample->p==0 ){ + sqlite3_finalize(pStmt); + return SQLITE_NOMEM; } + memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); + } return sqlite3_finalize(pStmt); } -#endif /* SQLITE_ENABLE_STAT3 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* -** Load the content of the sqlite_stat1 and sqlite_stat3 tables. The +** Load the content of the sqlite_stat1 and sqlite_stat4 tables. The ** contents of sqlite_stat1 are used to populate the Index.aiRowEst[] -** arrays. The contents of sqlite_stat3 are used to populate the +** arrays. The contents of sqlite_stat4 are used to populate the ** Index.aSample[] arrays. ** ** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR -** is returned. In this case, even if SQLITE_ENABLE_STAT3 was defined -** during compilation and the sqlite_stat3 table is present, no data is +** is returned. In this case, even if SQLITE_ENABLE_STAT4 was defined +** during compilation and the sqlite_stat4 table is present, no data is ** read from it. ** -** If SQLITE_ENABLE_STAT3 was defined during compilation and the -** sqlite_stat3 table is not present in the database, SQLITE_ERROR is +** If SQLITE_ENABLE_STAT4 was defined during compilation and the +** sqlite_stat4 table is not present in the database, SQLITE_ERROR is ** returned. However, in this case, data is read from the sqlite_stat1 ** table (if it is present) before returning. ** @@ -1081,7 +1276,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3DefaultRowEst(pIdx); -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; #endif @@ -1105,12 +1300,12 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ } - /* Load the statistics from the sqlite_stat3 table. */ -#ifdef SQLITE_ENABLE_STAT3 + /* Load the statistics from the sqlite_stat4 table. */ +#ifdef SQLITE_ENABLE_STAT4 if( rc==SQLITE_OK ){ int lookasideEnabled = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; - rc = loadStat3(db, sInfo.zDatabase); + rc = loadStat4(db, sInfo.zDatabase); db->lookaside.bEnabled = lookasideEnabled; } #endif diff --git a/src/btree.c b/src/btree.c index 9b78378791..19a34b0940 100644 --- a/src/btree.c +++ b/src/btree.c @@ -2509,6 +2509,7 @@ static int lockBtree(BtShared *pBt){ assert( pBt->maxLeaf + 23 <= MX_CELL_SIZE(pBt) ); pBt->pPage1 = pPage1; pBt->nPage = nPage; +assert( pPage1->leaf==0 || pPage1->leaf==1 ); return SQLITE_OK; page1_init_failed: diff --git a/src/build.c b/src/build.c index 06ecf6570f..d28689e1ff 100644 --- a/src/build.c +++ b/src/build.c @@ -2024,7 +2024,7 @@ static void sqlite3ClearStatTables( ){ int i; const char *zDbName = pParse->db->aDb[iDb].zName; - for(i=1; i<=3; i++){ + for(i=1; i<=4; i++){ char zTab[24]; sqlite3_snprintf(sizeof(zTab),zTab,"sqlite_stat%d",i); if( sqlite3FindTable(pParse->db, zTab, zDbName) ){ diff --git a/src/ctime.c b/src/ctime.c index 60595ff88d..d69763a304 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -117,8 +117,8 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_ENABLE_RTREE "ENABLE_RTREE", #endif -#ifdef SQLITE_ENABLE_STAT3 - "ENABLE_STAT3", +#ifdef SQLITE_ENABLE_STAT4 + "ENABLE_STAT4", #endif #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY "ENABLE_UNLOCK_NOTIFY", diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 291957db9e..9d33b4c7a7 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1558,16 +1558,11 @@ struct Index { ** analyze.c source file for additional information. */ struct IndexSample { - union { - char *z; /* Value if eType is SQLITE_TEXT or SQLITE_BLOB */ - double r; /* Value if eType is SQLITE_FLOAT */ - i64 i; /* Value if eType is SQLITE_INTEGER */ - } u; - u8 eType; /* SQLITE_NULL, SQLITE_INTEGER ... etc. */ - int nByte; /* Size in byte of text or blob. */ - tRowcnt nEq; /* Est. number of rows where the key equals this sample */ - tRowcnt nLt; /* Est. number of rows where key is less than this sample */ - tRowcnt nDLt; /* Est. number of distinct keys less than this sample */ + void *p; /* Pointer to sampled record */ + int n; /* Size of record in bytes */ + tRowcnt *anEq; /* Est. number of rows where the key equals this sample */ + tRowcnt *anLt; /* Est. number of rows where key is less than this sample */ + tRowcnt *anDLt; /* Est. number of distinct keys less than this sample */ }; /* diff --git a/src/test_config.c b/src/test_config.c index 534727a080..c80fae295a 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -458,10 +458,10 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); Tcl_SetVar2(interp, "sqlite_options", "schema_version", "1", TCL_GLOBAL_ONLY); #endif -#ifdef SQLITE_ENABLE_STAT3 - Tcl_SetVar2(interp, "sqlite_options", "stat3", "1", TCL_GLOBAL_ONLY); +#ifdef SQLITE_ENABLE_STAT4 + Tcl_SetVar2(interp, "sqlite_options", "stat4", "1", TCL_GLOBAL_ONLY); #else - Tcl_SetVar2(interp, "sqlite_options", "stat3", "0", TCL_GLOBAL_ONLY); + Tcl_SetVar2(interp, "sqlite_options", "stat4", "0", TCL_GLOBAL_ONLY); #endif #if !defined(SQLITE_ENABLE_LOCKING_STYLE) diff --git a/src/utf.c b/src/utf.c index 6d5b1bfe40..0d67bf9133 100644 --- a/src/utf.c +++ b/src/utf.c @@ -460,7 +460,7 @@ char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte, u8 enc){ ** If a malloc failure occurs, NULL is returned and the db.mallocFailed ** flag set. */ -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 char *sqlite3Utf8to16(sqlite3 *db, u8 enc, char *z, int n, int *pnOut){ Mem m; memset(&m, 0, sizeof(m)); diff --git a/src/vdbe.c b/src/vdbe.c index 4752c1a882..c12725fd29 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -4816,6 +4816,7 @@ case OP_Clear: { nChange = 0; assert( p->readOnly==0 ); + assert( pOp->p1!=1 ); assert( (p->btreeMask & (((yDbMask)1)<p2))!=0 ); rc = sqlite3BtreeClearTable( db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &nChange : 0) diff --git a/src/vdbemem.c b/src/vdbemem.c index 8fc222e2de..67fc695f3f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1034,11 +1034,11 @@ int sqlite3ValueFromExpr( } op = pExpr->op; - /* op can only be TK_REGISTER if we have compiled with SQLITE_ENABLE_STAT3. + /* op can only be TK_REGISTER if we have compiled with SQLITE_ENABLE_STAT4. ** The ifdef here is to enable us to achieve 100% branch test coverage even - ** when SQLITE_ENABLE_STAT3 is omitted. + ** when SQLITE_ENABLE_STAT4 is omitted. */ -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 if( op==TK_REGISTER ) op = pExpr->op2; #else if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; diff --git a/src/where.c b/src/where.c index 2bf3aaecb7..3bc82b0654 100644 --- a/src/where.c +++ b/src/where.c @@ -284,7 +284,7 @@ struct WhereTerm { #define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */ #define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */ #define TERM_OR_OK 0x40 /* Used during OR-clause processing */ -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 # define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */ #else # define TERM_VNULL 0x00 /* Disabled if not using stat3 */ @@ -1784,7 +1784,7 @@ static void exprAnalyze( } #endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 /* When sqlite_stat3 histogram data is available an operator of the ** form "x IS NOT NULL" can sometimes be evaluated more efficiently ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a @@ -2392,7 +2392,7 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 /* ** Estimate the location of a particular key among all keys in an ** index. Store the results in aStat as follows: @@ -2409,115 +2409,29 @@ static int whereKeyStats( int roundUp, /* Round up if true. Round down if false */ tRowcnt *aStat /* OUT: stats written here */ ){ - tRowcnt n; - IndexSample *aSample; - int i, eType; + IndexSample *aSample = pIdx->aSample; + UnpackedRecord rec; + int i; int isEq = 0; - i64 v; - double r, rS; - assert( roundUp==0 || roundUp==1 ); - assert( pIdx->nSample>0 ); if( pVal==0 ) return SQLITE_ERROR; - n = pIdx->aiRowEst[0]; - aSample = pIdx->aSample; - eType = sqlite3_value_type(pVal); - if( eType==SQLITE_INTEGER ){ - v = sqlite3_value_int64(pVal); - r = (i64)v; - for(i=0; inSample; i++){ - if( aSample[i].eType==SQLITE_NULL ) continue; - if( aSample[i].eType>=SQLITE_TEXT ) break; - if( aSample[i].eType==SQLITE_INTEGER ){ - if( aSample[i].u.i>=v ){ - isEq = aSample[i].u.i==v; - break; - } - }else{ - assert( aSample[i].eType==SQLITE_FLOAT ); - if( aSample[i].u.r>=r ){ - isEq = aSample[i].u.r==r; - break; - } - } - } - }else if( eType==SQLITE_FLOAT ){ - r = sqlite3_value_double(pVal); - for(i=0; inSample; i++){ - if( aSample[i].eType==SQLITE_NULL ) continue; - if( aSample[i].eType>=SQLITE_TEXT ) break; - if( aSample[i].eType==SQLITE_FLOAT ){ - rS = aSample[i].u.r; - }else{ - rS = aSample[i].u.i; - } - if( rS>=r ){ - isEq = rS==r; - break; - } - } - }else if( eType==SQLITE_NULL ){ - i = 0; - if( aSample[0].eType==SQLITE_NULL ) isEq = 1; - }else{ - assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); - for(i=0; inSample; i++){ - if( aSample[i].eType==SQLITE_TEXT || aSample[i].eType==SQLITE_BLOB ){ - break; - } - } - if( inSample ){ - sqlite3 *db = pParse->db; - CollSeq *pColl; - const u8 *z; - if( eType==SQLITE_BLOB ){ - z = (const u8 *)sqlite3_value_blob(pVal); - pColl = db->pDfltColl; - assert( pColl->enc==SQLITE_UTF8 ); - }else{ - pColl = sqlite3GetCollSeq(pParse, SQLITE_UTF8, 0, *pIdx->azColl); - /* If the collating sequence was unavailable, we should have failed - ** long ago and never reached this point. But we'll check just to - ** be doubly sure. */ - if( NEVER(pColl==0) ) return SQLITE_ERROR; - z = (const u8 *)sqlite3ValueText(pVal, pColl->enc); - if( !z ){ - return SQLITE_NOMEM; - } - assert( z && pColl && pColl->xCmp ); - } - n = sqlite3ValueBytes(pVal, pColl->enc); - - for(; inSample; i++){ - int c; - int eSampletype = aSample[i].eType; - if( eSampletypeenc!=SQLITE_UTF8 ){ - int nSample; - char *zSample = sqlite3Utf8to16( - db, pColl->enc, aSample[i].u.z, aSample[i].nByte, &nSample - ); - if( !zSample ){ - assert( db->mallocFailed ); - return SQLITE_NOMEM; - } - c = pColl->xCmp(pColl->pUser, nSample, zSample, n, z); - sqlite3DbFree(db, zSample); - }else -#endif - { - c = pColl->xCmp(pColl->pUser, aSample[i].nByte, aSample[i].u.z, n, z); - } - if( c>=0 ){ - if( c==0 ) isEq = 1; - break; - } - } + memset(&rec, 0, sizeof(UnpackedRecord)); + rec.pKeyInfo = sqlite3IndexKeyinfo(pParse, pIdx); + if( rec.pKeyInfo==0 ) return SQLITE_NOMEM; + rec.pKeyInfo->enc = ENC(pParse->db); + rec.nField = 1; + rec.flags = UNPACKED_PREFIX_MATCH; + rec.aMem = pVal; + + for(i=0; inSample; i++){ + int res = sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, &rec); + if( res>=0 ){ + isEq = (res==0); + break; } } + sqlite3DbFree(pParse->db, rec.pKeyInfo); /* At this point, aSample[i] is the first sample that is greater than ** or equal to pVal. Or if i==pIdx->nSample, then all samples are less @@ -2525,16 +2439,16 @@ static int whereKeyStats( */ if( isEq ){ assert( inSample ); - aStat[0] = aSample[i].nLt; - aStat[1] = aSample[i].nEq; + aStat[0] = aSample[i].anLt[0]; + aStat[1] = aSample[i].anEq[0]; }else{ tRowcnt iLower, iUpper, iGap; if( i==0 ){ iLower = 0; - iUpper = aSample[0].nLt; + iUpper = aSample[0].anLt[0]; }else{ - iUpper = i>=pIdx->nSample ? n : aSample[i].nLt; - iLower = aSample[i-1].nEq + aSample[i-1].nLt; + iUpper = i>=pIdx->nSample ? pIdx->aiRowEst[0] : aSample[i].anLt[0]; + iLower = aSample[i-1].anEq[0] + aSample[i-1].anLt[0]; } aStat[1] = pIdx->avgEq; if( iLower>=iUpper ){ @@ -2551,7 +2465,7 @@ static int whereKeyStats( } return SQLITE_OK; } -#endif /* SQLITE_ENABLE_STAT3 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* ** If expression pExpr represents a literal value, set *pp to point to @@ -2569,7 +2483,7 @@ static int whereKeyStats( ** ** If an error occurs, return an error code. Otherwise, SQLITE_OK. */ -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 static int valueFromExpr( Parse *pParse, Expr *pExpr, @@ -2584,7 +2498,7 @@ static int valueFromExpr( *pp = sqlite3VdbeGetBoundValue(pParse->pReprepare, iVar, aff); return SQLITE_OK; } - return sqlite3ValueFromExpr(pParse->db, pExpr, SQLITE_UTF8, aff, pp); + return sqlite3ValueFromExpr(pParse->db, pExpr, ENC(pParse->db), aff, pp); } #endif @@ -2637,7 +2551,7 @@ static int whereRangeScanEst( ){ int rc = SQLITE_OK; -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 if( nEq==0 && p->nSample && OptimizationEnabled(pParse->db, SQLITE_Stat3) ){ sqlite3_value *pRangeVal; @@ -2699,7 +2613,7 @@ static int whereRangeScanEst( return rc; } -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 /* ** Estimate the number of rows that will be returned based on ** an equality constraint x=VALUE and where that VALUE occurs in @@ -2747,9 +2661,9 @@ whereEqualScanEst_cancel: sqlite3ValueFree(pRhs); return rc; } -#endif /* defined(SQLITE_ENABLE_STAT3) */ +#endif /* defined(SQLITE_ENABLE_STAT4) */ -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 /* ** Estimate the number of rows that will be returned based on ** an IN constraint where the right-hand side of the IN operator @@ -2790,7 +2704,7 @@ static int whereInScanEst( } return rc; } -#endif /* defined(SQLITE_ENABLE_STAT3) */ +#endif /* defined(SQLITE_ENABLE_STAT4) */ /* ** Disable a term in the WHERE clause. Except, do not disable the term @@ -4335,7 +4249,7 @@ static int whereLoopAddBtreeIndex( for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 0; if( pTerm->prereqRight & pNew->maskSelf ) continue; -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 if( (pTerm->wtFlags & TERM_VNULL)!=0 && pSrc->pTab->aCol[iCol].notNull ){ continue; /* skip IS NOT NULL constraints on a NOT NULL column */ } @@ -4401,7 +4315,7 @@ static int whereLoopAddBtreeIndex( pBtm, pTop, &rDiv); pNew->nOut = saved_nOut>rDiv+10 ? saved_nOut - rDiv : 10; } -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 if( pNew->u.btree.nEq==1 && pProbe->nSample && OptimizationEnabled(db, SQLITE_Stat3) ){ tRowcnt nOut = 0; diff --git a/test/alter.test b/test/alter.test index aca71c4405..faa6b0d258 100644 --- a/test/alter.test +++ b/test/alter.test @@ -847,7 +847,7 @@ do_test alter-14.2 { set system_table_list {1 sqlite_master} catchsql ANALYZE ifcapable analyze { lappend system_table_list 2 sqlite_stat1 } -ifcapable stat3 { lappend system_table_list 4 sqlite_stat3 } +ifcapable stat4 { lappend system_table_list 4 sqlite_stat4 } foreach {tn tbl} $system_table_list { do_test alter-15.$tn.1 { diff --git a/test/analyze.test b/test/analyze.test index 362702a9c2..144834924a 100644 --- a/test/analyze.test +++ b/test/analyze.test @@ -288,7 +288,7 @@ do_test analyze-4.3 { } {} # Verify that DROP TABLE and DROP INDEX remove entries from the -# sqlite_stat1 and sqlite_stat3 tables. +# sqlite_stat1 and sqlite_stat4 tables. # do_test analyze-5.0 { execsql { @@ -306,11 +306,11 @@ do_test analyze-5.0 { SELECT DISTINCT tbl FROM sqlite_stat1 ORDER BY 1; } } {t3i1 t3i2 t3i3 t4i1 t4i2 t3 t4} -ifcapable stat3 { +ifcapable stat4 { do_test analyze-5.1 { execsql { - SELECT DISTINCT idx FROM sqlite_stat3 ORDER BY 1; - SELECT DISTINCT tbl FROM sqlite_stat3 ORDER BY 1; + SELECT DISTINCT idx FROM sqlite_stat4 ORDER BY 1; + SELECT DISTINCT tbl FROM sqlite_stat4 ORDER BY 1; } } {t3i1 t3i2 t3i3 t4i1 t4i2 t3 t4} } @@ -321,11 +321,11 @@ do_test analyze-5.2 { SELECT DISTINCT tbl FROM sqlite_stat1 ORDER BY 1; } } {t3i1 t3i3 t4i1 t4i2 t3 t4} -ifcapable stat3 { +ifcapable stat4 { do_test analyze-5.3 { execsql { - SELECT DISTINCT idx FROM sqlite_stat3 ORDER BY 1; - SELECT DISTINCT tbl FROM sqlite_stat3 ORDER BY 1; + SELECT DISTINCT idx FROM sqlite_stat4 ORDER BY 1; + SELECT DISTINCT tbl FROM sqlite_stat4 ORDER BY 1; } } {t3i1 t3i3 t4i1 t4i2 t3 t4} } @@ -336,11 +336,11 @@ do_test analyze-5.4 { SELECT DISTINCT tbl FROM sqlite_stat1 ORDER BY 1; } } {t4i1 t4i2 t4} -ifcapable stat3 { +ifcapable stat4 { do_test analyze-5.5 { execsql { - SELECT DISTINCT idx FROM sqlite_stat3 ORDER BY 1; - SELECT DISTINCT tbl FROM sqlite_stat3 ORDER BY 1; + SELECT DISTINCT idx FROM sqlite_stat4 ORDER BY 1; + SELECT DISTINCT tbl FROM sqlite_stat4 ORDER BY 1; } } {t4i1 t4i2 t4} } @@ -360,5 +360,4 @@ do_test analyze-99.1 { } } {1 {malformed database schema (sqlite_stat1) - near "nonsense": syntax error}} - finish_test diff --git a/test/analyze3.test b/test/analyze3.test index 1e95d591b7..df9e548828 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -17,7 +17,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !stat3 { +ifcapable !stat4 { finish_test return } diff --git a/test/analyze5.test b/test/analyze5.test index c400780fff..9972695abf 100644 --- a/test/analyze5.test +++ b/test/analyze5.test @@ -10,14 +10,14 @@ #*********************************************************************** # # This file implements tests for SQLite library. The focus of the tests -# in this file is the use of the sqlite_stat3 histogram data on tables +# in this file is the use of the sqlite_stat4 histogram data on tables # with many repeated values and only a few distinct values. # set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !stat3 { +ifcapable !stat4 { finish_test return } @@ -28,6 +28,15 @@ proc eqp {sql {db db}} { uplevel execsql [list "EXPLAIN QUERY PLAN $sql"] $db } +proc alpha {blob} { + set ret "" + foreach c [split $blob {}] { + if {[string is alpha $c]} {append ret $c} + } + return $ret +} +db func alpha alpha + unset -nocomplain i t u v w x y z do_test analyze5-1.0 { db eval {CREATE TABLE t1(t,u,v TEXT COLLATE nocase,w,x,y,z)} @@ -55,16 +64,18 @@ do_test analyze5-1.0 { CREATE INDEX t1y ON t1(y); -- integers 0 and very few 1s CREATE INDEX t1z ON t1(z); -- integers 0, 1, 2, and 3 ANALYZE; - SELECT sample FROM sqlite_stat3 WHERE idx='t1u' ORDER BY nlt; + SELECT alpha(sample) FROM sqlite_stat4 WHERE idx='t1u' ORDER BY nlt; } } {alpha bravo charlie delta} do_test analyze5-1.1 { - db eval {SELECT DISTINCT lower(sample) FROM sqlite_stat3 WHERE idx='t1v' - ORDER BY 1} + db eval { + SELECT DISTINCT lower(alpha(sample)) FROM sqlite_stat4 WHERE idx='t1v' + ORDER BY 1 + } } {alpha bravo charlie delta} do_test analyze5-1.2 { - db eval {SELECT idx, count(*) FROM sqlite_stat3 GROUP BY 1 ORDER BY 1} + db eval {SELECT idx, count(*) FROM sqlite_stat4 GROUP BY 1 ORDER BY 1} } {t1t 4 t1u 4 t1v 4 t1w 4 t1x 4 t1y 2 t1z 4} # Verify that range queries generate the correct row count estimates diff --git a/test/analyze6.test b/test/analyze6.test index 55e4347c6b..7c8e653b40 100644 --- a/test/analyze6.test +++ b/test/analyze6.test @@ -17,7 +17,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !stat3 { +ifcapable !stat4 { finish_test return } diff --git a/test/analyze7.test b/test/analyze7.test index ddfe398a9b..07d6ad52a0 100644 --- a/test/analyze7.test +++ b/test/analyze7.test @@ -82,14 +82,14 @@ do_test analyze7-3.1 { do_test analyze7-3.2.1 { execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=?;} } {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=?)}} -ifcapable stat3 { - # If ENABLE_STAT3 is defined, SQLite comes up with a different estimated +ifcapable stat4 { + # If ENABLE_STAT4 is defined, SQLite comes up with a different estimated # row count for (c=2) than it does for (c=?). do_test analyze7-3.2.2 { execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=2;} } {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=?)}} } else { - # If ENABLE_STAT3 is not defined, the expected row count for (c=2) is the + # If ENABLE_STAT4 is not defined, the expected row count for (c=2) is the # same as that for (c=?). do_test analyze7-3.2.3 { execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=2;} @@ -98,7 +98,7 @@ ifcapable stat3 { do_test analyze7-3.3 { execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=123 AND b=123} } {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}} -ifcapable {!stat3} { +ifcapable {!stat4} { do_test analyze7-3.4 { execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=123 AND b=123} } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?)}} diff --git a/test/analyze8.test b/test/analyze8.test index 94054e1c83..f059424ce2 100644 --- a/test/analyze8.test +++ b/test/analyze8.test @@ -16,7 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !stat3 { +ifcapable !stat4 { finish_test return } diff --git a/test/analyze9.test b/test/analyze9.test new file mode 100644 index 0000000000..d5346b9ad6 --- /dev/null +++ b/test/analyze9.test @@ -0,0 +1,55 @@ +# 2013 August 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix analyze9 + +proc s {blob} { + set ret "" + binary scan $blob c* bytes + foreach b $bytes { + set t [binary format c $b] + if {[string is print $t]} { + append ret $t + } else { + append ret . + } + } + return $ret +} +db function s s + +do_test 1.0 { + execsql { CREATE TABLE t1(a TEXT, b TEXT); } + for {set i 0} {$i < 5} {incr i} { + execsql {INSERT INTO t1 VALUES ('('||($i%10)||')', '('||($i%7)||')')} + } + execsql { CREATE INDEX i1 ON t1(a, b) } +} {} + +do_execsql_test 1.1 { + ANALYZE; +} {} + +do_execsql_test 1.2 { + SELECT tbl,idx,nEq,nLt,nDLt,s(sample) FROM sqlite_stat4; +} { + t1 i1 {1 1} {1 1} {1 1} ...(1)(1) + t1 i1 {1 1} {2 2} {2 2} ...(2)(2) + t1 i1 {1 1} {3 3} {3 3} ...(3)(3) + t1 i1 {1 1} {4 4} {4 4} ...(4)(4) +} + + +finish_test + diff --git a/test/auth.test b/test/auth.test index fd402b1d9e..e7d110f2ec 100644 --- a/test/auth.test +++ b/test/auth.test @@ -2325,10 +2325,10 @@ ifcapable compound&&subquery { } } } - ifcapable stat3 { - set stat3 "sqlite_stat3 " + ifcapable stat4 { + set stat4 "sqlite_stat4 " } else { - set stat3 "" + set stat4 "" } do_test auth-5.2 { execsql { @@ -2337,7 +2337,7 @@ ifcapable compound&&subquery { WHERE type='table' ORDER BY name } - } "sqlite_stat1 ${stat3}t1 t2 t3 t4" + } "sqlite_stat1 ${stat4}t1 t2 t3 t4" } # Ticket #3944 diff --git a/test/dbstatus.test b/test/dbstatus.test index 9793df3f4e..ae9a7344b7 100644 --- a/test/dbstatus.test +++ b/test/dbstatus.test @@ -61,7 +61,7 @@ proc lookaside {db} { } } -ifcapable stat3 { +ifcapable stat4 { set STAT3 1 } else { set STAT3 0 @@ -214,7 +214,7 @@ foreach ::lookaside_buffer_size {0 64 120} { # much greater than just that reported by DBSTATUS_SCHEMA_USED in this # case. # - # Some of the memory used for sqlite_stat3 is unaccounted for by + # Some of the memory used for sqlite_stat4 is unaccounted for by # dbstatus. # # Finally, on osx the estimate of memory used by the schema may be diff --git a/test/tkt-cbd054fa6b.test b/test/tkt-cbd054fa6b.test index 51e01991d4..8f16964447 100644 --- a/test/tkt-cbd054fa6b.test +++ b/test/tkt-cbd054fa6b.test @@ -16,11 +16,26 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !stat3 { +ifcapable !stat4 { finish_test return } +proc s {blob} { + set ret "" + binary scan $blob c* bytes + foreach b $bytes { + set t [binary format c $b] + if {[string is print $t]} { + append ret $t + } else { + append ret . + } + } + return $ret +} +db function s s + do_test tkt-cbd05-1.1 { db eval { CREATE TABLE t1(a INTEGER PRIMARY KEY, b TEXT UNIQUE NOT NULL); @@ -45,12 +60,12 @@ do_test tkt-cbd05-1.2 { } {} do_test tkt-cbd05-1.3 { execsql { - SELECT tbl,idx,group_concat(sample,' ') - FROM sqlite_stat3 + SELECT tbl,idx,group_concat(s(sample),' ') + FROM sqlite_stat4 WHERE idx = 't1_x' GROUP BY tbl,idx } -} {/t1 t1_x .[ ABCDEFGHI]{10}./} +} {t1 t1_x {.. ..A ..B ..C ..D ..E ..F ..G ..H ..I}} do_test tkt-cbd05-2.1 { db eval { @@ -77,11 +92,11 @@ do_test tkt-cbd05-2.2 { } {} do_test tkt-cbd05-2.3 { execsql { - SELECT tbl,idx,group_concat(sample,' ') - FROM sqlite_stat3 + SELECT tbl,idx,group_concat(s(sample),' ') + FROM sqlite_stat4 WHERE idx = 't1_x' GROUP BY tbl,idx } -} {/t1 t1_x .[ ABCDEFGHI]{10}./} +} {t1 t1_x {.. ..A ..B ..C ..D ..E ..F ..G ..H ..I}} finish_test diff --git a/test/where9.test b/test/where9.test index 5b000d967e..403178e310 100644 --- a/test/where9.test +++ b/test/where9.test @@ -781,11 +781,11 @@ do_test where9-6.8.2 { OR (b NOT NULL AND c NOT NULL AND d IS NULL) } } {1 {no query solution}} -ifcapable stat3 { +ifcapable stat4 { # When STAT3 is enabled, the "b NOT NULL" terms get translated # into b>NULL, which can be satified by the index t1b. It is a very # expensive way to do the query, but it works, and so a solution is possible. - do_test where9-6.8.3-stat3 { + do_test where9-6.8.3-stat4 { catchsql { UPDATE t1 INDEXED BY t1b SET a=a+100 WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) @@ -793,7 +793,7 @@ ifcapable stat3 { OR (b NOT NULL AND c NOT NULL AND d IS NULL) } } {0 {}} - do_test where9-6.8.4-stat3 { + do_test where9-6.8.4-stat4 { catchsql { DELETE FROM t1 INDEXED BY t1b WHERE (b IS NULL AND c NOT NULL AND d NOT NULL) diff --git a/test/wild001.test b/test/wild001.test index 7fe1404294..e7cfdaae50 100644 --- a/test/wild001.test +++ b/test/wild001.test @@ -42,6 +42,10 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl +# TODO: Reenable this test. + finish_test + return + ifcapable !stat3 { finish_test return From c55521a60ba7e2da8d993883dfcc3567077f62d6 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Aug 2013 05:34:30 +0000 Subject: [PATCH 02/67] Fix a couple of problems in code related to sqlite_stat4. FossilOrigin-Name: badd24d987240db5528b37d1c177431617079f9b --- manifest | 19 ++++++++----------- manifest.uuid | 2 +- src/analyze.c | 11 +++++++++-- test/analyze9.test | 1 + test/table.test | 2 +- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index 7dbbcbff26..e6ce4135d9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Begin\sadding\sexperimental\ssqlite_stat4\stable.\sThis\scommit\sis\sbuggy. -D 2013-08-03T20:24:58.082 +C Fix\sa\scouple\sof\sproblems\sin\scode\srelated\sto\ssqlite_stat4. +D 2013-08-05T05:34:30.270 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 -F src/analyze.c 7c272ae217eb34439ba2ea560b506eec667bd1e0 +F src/analyze.c b9fb000c4dcac4f7f3dbd1a8832ffdacf17a4d35 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -308,7 +308,7 @@ F test/analyze5.test e3eece09761c935ec0b85dc4ed70dbf6cac1ed77 F test/analyze6.test 3c01e084309706a1033f850330ea24f6f7846297 F test/analyze7.test c0af22c5e0140e2e4ac556a21c2b6fff58229c98 F test/analyze8.test 092425439c12f62f9d5c3127e2b4f6e7b3e170cc -F test/analyze9.test 238e3d1fa17e53d80da1d84a151c0465ef8ba666 +F test/analyze9.test 2ffe8f627b8f0309a72c2ff390effa430b1ef2d8 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -810,7 +810,7 @@ F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/table.test a59d985ca366e39b17b175f387f9d5db5a18d4e2 +F test/table.test 03135ef2fd49b2a21894be22afdb3c7de32cecd3 F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c @@ -1106,10 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 213020769f310aec1591d97756b53891d0b64005 -R b9be00b2213e6b6e3de39cb20a1e763e -T *branch * sqlite_stat4 -T *sym-sqlite_stat4 * -T -sym-trunk * +P 2beea303a1d609cd2ff252412c50b966b9e5e8f1 +R d4282412612019a7b27b2fb043ed04d3 U dan -Z 57387cc568421f9309aeb8be1def5190 +Z 13683b9d54d5fd3b76b2568a74c36c12 diff --git a/manifest.uuid b/manifest.uuid index 8f3d3831d4..ced4c6d590 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2beea303a1d609cd2ff252412c50b966b9e5e8f1 \ No newline at end of file +badd24d987240db5528b37d1c177431617079f9b \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index d2057ffc39..8981db789f 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -371,9 +371,16 @@ static void stat4Push( /* Fill in the new Stat4Sample object. */ if( p->nSample==p->mxSample ){ + struct Stat4Sample *pMin = &p->a[iMin]; + tRowcnt *anEq = pMin->anEq; + tRowcnt *anDLt = pMin->anDLt; + tRowcnt *anLt = pMin->anLt; assert( p->nSample - iMin - 1 >= 0 ); - memmove(&p->a[iMin], &p->a[iMin+1], sizeof(p->a[0])*(p->nSample-iMin-1)); + memmove(pMin, &pMin[1], sizeof(p->a[0])*(p->nSample-iMin-1)); pSample = &p->a[p->nSample-1]; + pSample->anEq = anEq; + pSample->anDLt = anDLt; + pSample->anLt = anLt; }else{ pSample = &p->a[p->nSample++]; } @@ -1090,7 +1097,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pIndex ){ int bUnordered = 0; - decodeIntArray((char*)z, pIndex->nColumn, pIndex->aiRowEst, &bUnordered); + decodeIntArray((char*)z, pIndex->nColumn+1, pIndex->aiRowEst, &bUnordered); if( pIndex->pPartIdxWhere==0 ) pTable->nRowEst = pIndex->aiRowEst[0]; pIndex->bUnordered = bUnordered; }else{ diff --git a/test/analyze9.test b/test/analyze9.test index d5346b9ad6..51817893ac 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -44,6 +44,7 @@ do_execsql_test 1.1 { do_execsql_test 1.2 { SELECT tbl,idx,nEq,nLt,nDLt,s(sample) FROM sqlite_stat4; } { + t1 i1 {1 1} {0 0} {0 0} ...(0)(0) t1 i1 {1 1} {1 1} {1 1} ...(1)(1) t1 i1 {1 1} {2 2} {2 2} ...(2)(2) t1 i1 {1 1} {3 3} {3 3} ...(3)(3) diff --git a/test/table.test b/test/table.test index 4826cb927d..9047df23dd 100644 --- a/test/table.test +++ b/test/table.test @@ -267,7 +267,7 @@ do_test table-5.2.1 { ANALYZE; DROP TABLE IF EXISTS sqlite_stat1; DROP TABLE IF EXISTS sqlite_stat2; - DROP TABLE IF EXISTS sqlite_stat3; + DROP TABLE IF EXISTS sqlite_stat4; SELECT name FROM sqlite_master WHERE name GLOB 'sqlite_stat*'; } } {} From e043201d3d58726d86da4fbeb776c258fb7ad971 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Aug 2013 18:00:56 +0000 Subject: [PATCH 03/67] Use N separate cursors when scanning an index with N columns to collect sqlite_stat4 data. This fixes a problem with collecting incorrect nEq values from multi-column indexes. FossilOrigin-Name: 3a71afe67418ce00097cd9714c395fe9ff16f23b --- manifest | 16 +- manifest.uuid | 2 +- src/analyze.c | 366 ++++++++++++++++++++++----------------------- src/test_func.c | 97 ++++++++++++ test/analyze9.test | 61 ++++++++ 5 files changed, 350 insertions(+), 192 deletions(-) diff --git a/manifest b/manifest index e6ce4135d9..1bdf53de0c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sof\sproblems\sin\scode\srelated\sto\ssqlite_stat4. -D 2013-08-05T05:34:30.270 +C Use\sN\sseparate\scursors\swhen\sscanning\san\sindex\swith\sN\scolumns\sto\scollect\ssqlite_stat4\sdata.\sThis\sfixes\sa\sproblem\swith\scollecting\sincorrect\snEq\svalues\sfrom\smulti-column\sindexes. +D 2013-08-05T18:00:56.397 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 -F src/analyze.c b9fb000c4dcac4f7f3dbd1a8832ffdacf17a4d35 +F src/analyze.c a0979f7fdc8cd724f8e646ba9ef6ca1e56fa7491 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -243,7 +243,7 @@ F src/test_config.c 636ecd15a6ba18bf97a590b5a21f47573c8c2b65 F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c 8f786bfd0ad48030cf2a06fb1f050e9c60a150d7 -F src/test_func.c 3a8dd37c08ab43b76d38eea2836e34a3897bf170 +F src/test_func.c fcd238feb694332d5962ee08578ef30ff4ac6559 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61 @@ -308,7 +308,7 @@ F test/analyze5.test e3eece09761c935ec0b85dc4ed70dbf6cac1ed77 F test/analyze6.test 3c01e084309706a1033f850330ea24f6f7846297 F test/analyze7.test c0af22c5e0140e2e4ac556a21c2b6fff58229c98 F test/analyze8.test 092425439c12f62f9d5c3127e2b4f6e7b3e170cc -F test/analyze9.test 2ffe8f627b8f0309a72c2ff390effa430b1ef2d8 +F test/analyze9.test 30479ec9ac395e77cfced52496845500768d45f9 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2beea303a1d609cd2ff252412c50b966b9e5e8f1 -R d4282412612019a7b27b2fb043ed04d3 +P badd24d987240db5528b37d1c177431617079f9b +R 2711e68fa0a11591cfe5a22173f05afc U dan -Z 13683b9d54d5fd3b76b2568a74c36c12 +Z ccb7ab66bd998be3fb0311148b3ca62c diff --git a/manifest.uuid b/manifest.uuid index ced4c6d590..22a684eebe 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -badd24d987240db5528b37d1c177431617079f9b \ No newline at end of file +3a71afe67418ce00097cd9714c395fe9ff16f23b \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 8981db789f..07366b76ab 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -391,7 +391,8 @@ static void stat4Push( for(i=0; inCol; i++){ pSample->anEq[i] = sqlite3_value_int64(aEq[i]); pSample->anLt[i] = sqlite3_value_int64(aLt[i]); - pSample->anDLt[i] = sqlite3_value_int64(aDLt[i]); + pSample->anDLt[i] = sqlite3_value_int64(aDLt[i])-1; + assert( sqlite3_value_int64(aDLt[i])>0 ); } /* Find the new minimum */ @@ -564,11 +565,13 @@ static void analyzeOneTable( } #endif - /* Establish a read-lock on the table at the shared-cache level. */ + /* Establish a read-lock on the table at the shared-cache level. + ** Also open a read-only cursor on the table. */ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - - iIdxCur = pParse->nTab++; + iTabCur = pParse->nTab++; + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns indexed by pIdx */ KeyInfo *pKey; /* KeyInfo structure for pIdx */ @@ -577,12 +580,12 @@ static void analyzeOneTable( int regRowid; /* Register for rowid of current row */ int regPrev; /* First in array of previous values */ - int regDLt; /* First in array of nDlt registers */ + int regDLte; /* First in array of nDlt registers */ int regLt; /* First in array of nLt registers */ int regEq; /* First in array of nEq registers */ int regCnt; /* Number of index entries */ - - int addrGoto; + int regEof; /* True once cursors are all at EOF */ + int endOfScan; /* Label to jump to once scan is finished */ if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0; @@ -592,21 +595,99 @@ static void analyzeOneTable( if( aChngAddr==0 ) continue; pKey = sqlite3IndexKeyinfo(pParse, pIdx); - /* Open a cursor to the index to be analyzed. */ - assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); - sqlite3VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb, - (char *)pKey, P4_KEYINFO_HANDOFF); - VdbeComment((v, "%s", pIdx->zName)); - /* Populate the register containing the index name. */ sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, pIdx->zName, 0); -#ifdef SQLITE_ENABLE_STAT4 - if( once ){ - once = 0; - sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + /* + ** The following pseudo-code demonstrates the way the VM scans an index + ** to call stat4_push() and collect the values for the sqlite_stat1 + ** entry. The code below is for an index with 2 columns. The actual + ** VM code generated may be for any number of columns. + ** + ** One cursor is opened for each column in the index (nCol). All cursors + ** scan concurrently the index from start to end. All variables used in + ** the pseudo-code are initialized to zero. + ** + ** Rewind csr(0) + ** Rewind csr(1) + ** + ** next_0: + ** regPrev(0) = csr(0)[0] + ** regDLte(0) += 1 + ** regLt(0) += regEq(0) + ** regEq(0) = 0 + ** do { + ** regEq(0) += 1 + ** Next csr(0) + ** }while ( csr(0)[0] == regPrev(0) ) + ** + ** next_1: + ** regPrev(1) = csr(1)[1] + ** regDLte(1) += 1 + ** regLt(1) += regEq(1) + ** regEq(1) = 0 + ** regRowid = csr(1)[rowid] // innermost cursor only + ** do { + ** regEq(1) += 1 + ** regCnt += 1 // innermost cursor only + ** Next csr(1) + ** }while ( csr(1)[0..1] == regPrev(0..1) ) + ** + ** stat4_push(regRowid, regEq, regLt, regDLte); + ** + ** if( eof( csr(1) ) ) goto endOfScan + ** if( csr(1)[0] != regPrev(0) ) goto next_0 + ** goto next_1 + ** + ** endOfScan: + ** // done! + ** + ** The last two lines above modify the contents of the regDLte array + ** so that each element contains the number of distinct key prefixes + ** of the corresponding length. As required to calculate the contents + ** of the sqlite_stat1 entry. + ** + ** Currently, the last memory cell allocated (that with the largest + ** integer identifier) is regStat4. Immediately following regStat4 + ** we allocate the following: + ** + ** regRowid - 1 register + ** regEq - nCol registers + ** regLt - nCol registers + ** regDLte - nCol registers + ** regCnt - 1 register + ** regPrev - nCol registers + ** regEof - 1 register + ** + ** The regRowid, regEq, regLt and regDLte registers must be positioned in + ** that order immediately following regStat4 so that they can be passed + ** to the stat4_push() function. + ** + ** All of the above are initialized to contain integer value 0. + */ + regRowid = regStat4+1; /* Rowid argument */ + regEq = regRowid+1; /* First in array of nEq value registers */ + regLt = regEq+nCol; /* First in array of nLt value registers */ + regDLte = regLt+nCol; /* First in array of nDLt value registers */ + regCnt = regDLte+nCol; /* Row counter */ + regPrev = regCnt+1; /* First in array of prev. value registers */ + regEof = regPrev+nCol; /* True once last row read from index */ + if( regEof+1>pParse->nMem ){ + pParse->nMem = regPrev+nCol; } + /* Open a read-only cursor for each column of the index. */ + assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); + iIdxCur = pParse->nTab++; + pParse->nTab += (nCol-1); + for(i=0; itnum, iDb); + sqlite3VdbeChangeP4(v, -1, (char*)pKey, iMode); + VdbeComment((v, "%s", pIdx->zName)); + } + +#ifdef SQLITE_ENABLE_STAT4 /* Invoke the stat4_init() function. The arguments are: ** ** * the number of rows in the index, @@ -621,181 +702,98 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, 3); #endif /* SQLITE_ENABLE_STAT4 */ - /* The block of (1 + 4*nCol) memory cells initialized here is used - ** as follows: - ** - ** TODO: Update this comment: - ** - ** iMem: - ** Loop counter. The number of rows visited so far, including - ** the current row (i.e. this register is set to 1 for the - ** first iteration of the loop). - ** - ** iMem+1 .. iMem+nCol: - ** Number of distinct index entries seen so far, considering - ** the left-most N columns only, where N is between 1 and nCol, - ** inclusive. - ** - ** iMem+nCol+1 .. Mem+2*nCol: - ** Previous value of indexed columns, from left to right. - ** - ** Cells iMem through iMem+nCol are initialized to 0. The others are - ** initialized to contain an SQL NULL. - */ - regRowid = regStat4+1; /* Rowid argument */ - regEq = regRowid+1; /* First in array of nEq value registers */ - regLt = regEq+nCol; /* First in array of nLt value registers */ - regDLt = regLt+nCol; /* First in array of nDLt value registers */ - regCnt = regDLt+nCol; /* Row counter */ - regPrev = regCnt+1; /* First in array of prev. value registers */ - - if( regPrev+1>pParse->nMem ){ - pParse->nMem = regPrev+1; - } - for(i=0; i<2+nCol*4; i++){ - sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowid+i); + /* Initialize all the memory registers allocated above to 0. */ + for(i=regRowid; i<=regEof; i++){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, i); } - /* - ** Loop through all entries in the b-tree index. Pseudo-code for the - ** body of the loop is as follows: - ** - ** foreach i IN index { - ** regCnt += 1 - ** - ** if( regEq(0)==0 ) goto ne_0; - ** - ** if i(0) != regPrev(0) { - ** stat4_push(regRowid, regEq, regLt, regDLt); - ** goto ne_0; - ** } - ** regEq(0) += 1 - ** - ** if i(1) != regPrev(1){ - ** stat4_push(regRowid, regEq, regLt, regDLt); - ** goto ne_1; - ** } - ** regEq(1) += 1 - ** - ** goto all_eq; - ** - ** ne_0: - ** regPrev(0) = i(0) - ** if( regEq(0) != 0 ) regDLt(0) += 1 - ** regLt(0) += regEq(0) - ** regEq(0) = 1 - ** - ** ne_1: - ** regPrev(1) = $i(1) - ** if( regEq(1) != 0 ) regDLt(1) += 1 - ** regLt(1) += regEq(1) - ** regEq(1) = 1 - ** - ** all_eq: - ** regRowid = i(rowid) - ** } - ** - ** stat4_push(regRowid, regEq, regLt, regDLt); - ** - ** if( regEq(0) != 0 ) regDLt(0) += 1 - ** if( regEq(1) != 0 ) regDLt(1) += 1 - ** - ** The last two lines above modify the contents of the regDLt array - ** so that each element contains the number of distinct key prefixes - ** of the corresponding length. As required to calculate the contents - ** of the sqlite_stat1 entry. - ** - ** Note: if regEq(0)==0, stat4_push() is a no-op. - */ - endOfLoop = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp2(v, OP_Rewind, iIdxCur, endOfLoop); - topOfLoop = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_AddImm, regCnt, 1); /* Increment row counter */ - - /* This jump is taken for the first iteration of the loop only. - ** - ** if( regEq(0)==0 ) goto ne_0; - */ - addrIfNot = sqlite3VdbeAddOp1(v, OP_IfNot, regEq); - - /* Code these bits: - ** - ** if i(N) != regPrev(N) { - ** stat4_push(regRowid, regEq, regLt, regDLt); - ** goto ne_N; - ** } - ** regEq(N) += 1 - */ + /* Rewind all cursors open on the index. If the table is entry, this + ** will cause control to jump to address endOfScan immediately. */ + endOfScan = sqlite3VdbeMakeLabel(v); for(i=0; iazColl && pIdx->azColl[i]!=0 ); - pColl = (char*)sqlite3LocateCollSeq(pParse, pIdx->azColl[i]); - - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regCol); - sqlite3VdbeAddOp4(v, OP_Eq, regCol, 0, regPrev+i, pColl, P4_COLLSEQ); - sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); - - sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp2); - sqlite3VdbeChangeP4(v, -1, (char*)&stat4PushFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 2 + 3*nCol); - - aChngAddr[i] = sqlite3VdbeAddOp0(v, OP_Goto); - VdbeComment((v, "jump if column %d changed", i)); - - sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-3); - sqlite3VdbeAddOp2(v, OP_AddImm, regEq+i, 1); - VdbeComment((v, "incr repeat count")); + sqlite3VdbeAddOp2(v, OP_Rewind, iIdxCur+i, endOfScan); } - /* Code the "continue" */ - addrGoto = sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfLoop); - - /* And now these: - ** - ** ne_N: - ** regPrev(N) = i(N) - ** if( regEq(N) != N ) regDLt(N) += 1 - ** regLt(N) += regEq(N) - ** regEq(N) = 1 - */ for(i=0; iazColl[i]); + int iCsr = iIdxCur+i; + int iDo; + int iNe; /* Jump here to exit do{...}while loop */ + int j; + int bInner = (i==(nCol-1)); /* True for innermost cursor */ + + /* Implementation of the following pseudo-code: + ** + ** regPrev(i) = csr(i)[i] + ** regDLte(i) += 1 + ** regLt(i) += regEq(i) + ** regEq(i) = 0 + ** regRowid = csr(i)[rowid] // innermost cursor only + */ + aChngAddr[i] = sqlite3VdbeAddOp3(v, OP_Column, iCsr, i, regPrev+i); + VdbeComment((v, "regPrev(%d) = csr(%d)(%d)", i, i, i)); + sqlite3VdbeAddOp2(v, OP_AddImm, regDLte+i, 1); + VdbeComment((v, "regDLte(%d) += 1", i)); sqlite3VdbeAddOp3(v, OP_Add, regEq+i, regLt+i, regLt+i); - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEq+i); + VdbeComment((v, "regLt(%d) += regEq(%d)", i, i)); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regEq+i); + VdbeComment((v, "regEq(%d) = 0", i)); + if( bInner ) sqlite3VdbeAddOp2(v, OP_IdxRowid, iCsr, regRowid); + + /* This bit: + ** + ** do { + ** regEq(i) += 1 + ** regCnt += 1 // innermost cursor only + ** Next csr(i) + ** if( Eof csr(i) ){ + ** regEof = 1 // innermost cursor only + ** break + ** } + ** }while ( csr(i)[0..i] == regPrev(0..i) ) + */ + iDo = sqlite3VdbeAddOp2(v, OP_AddImm, regEq+i, 1); + VdbeComment((v, "regEq(%d) += 1", i)); + if( bInner ){ + sqlite3VdbeAddOp2(v, OP_AddImm, regCnt, 1); + VdbeComment((v, "regCnt += 1")); + } + sqlite3VdbeAddOp2(v, OP_Next, iCsr, sqlite3VdbeCurrentAddr(v)+2+bInner); + if( bInner ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regEof); + iNe = sqlite3VdbeMakeLabel(v); + sqlite3VdbeAddOp2(v, OP_Goto, 0, iNe); + for(j=0; j<=i; j++){ + sqlite3VdbeAddOp3(v, OP_Column, iCsr, j, regCol); + sqlite3VdbeAddOp4(v, OP_Ne, regCol, iNe, regPrev+j, pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + VdbeComment((v, "if( regPrev(%d) != csr(%d)(%d) )", j, i, j)); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, iDo); + sqlite3VdbeResolveLabel(v, iNe); } - sqlite3DbFree(db, aChngAddr); - /* - ** all_eq: - ** regRowid = i(rowid) - */ - sqlite3VdbeJumpHere(v, addrGoto); - sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid); - - /* The end of the loop that iterates through all index entries. Always - ** jump here after updating the iMem+1...iMem+1+nCol counters. */ - sqlite3VdbeResolveLabel(v, endOfLoop); - sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, topOfLoop); - sqlite3VdbeAddOp1(v, OP_Close, iIdxCur); - - /* Final invocation of stat4_push() */ + /* Invoke stat4_push() */ sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp2); sqlite3VdbeChangeP4(v, -1, (char*)&stat4PushFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2 + 3*nCol); - /* Finally: - ** - ** if( regEq(0) != 0 ) regDLt(0) += 1 - */ + sqlite3VdbeAddOp2(v, OP_If, regEof, endOfScan); + for(i=0; iazColl[i]); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur+nCol-1, i, regCol); + sqlite3VdbeAddOp3(v, OP_Ne, regCol, aChngAddr[i], regPrev+i); + sqlite3VdbeChangeP4(v, -1, pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, aChngAddr[nCol-1]); + sqlite3DbFree(db, aChngAddr); + + sqlite3VdbeResolveLabel(v, endOfScan); + + /* Close all the cursors */ for(i=0; itnum, iDb); VdbeComment((v, "%s", pTab->zName)); - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat1); - sqlite3VdbeAddOp1(v, OP_Close, iIdxCur); + sqlite3VdbeAddOp2(v, OP_Count, iTabCur, regStat1); jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); @@ -888,6 +884,10 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeJumpHere(v, jZeroRows); } + + sqlite3VdbeAddOp1(v, OP_Close, iTabCur); + + /* TODO: Not sure about this... */ if( pParse->nMemnMem = regRec; } diff --git a/src/test_func.c b/src/test_func.c index 6f9bb03dc8..f12e3c200a 100644 --- a/src/test_func.c +++ b/src/test_func.c @@ -18,6 +18,9 @@ #include #include +#include "sqliteInt.h" +#include "vdbeInt.h" + /* ** Allocate nByte bytes of space using sqlite3_malloc(). If the @@ -458,6 +461,99 @@ static void real2hex( sqlite3_result_text(context, zOut, -1, SQLITE_TRANSIENT); } +/* +** tclcmd: test_decode(record) +** +** This function implements an SQL user-function that accepts a blob +** containing a formatted database record as its only argument. It returns +** a tcl list (type SQLITE_TEXT) containing each of the values stored +** in the record. +*/ +static void test_decode( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + u8 *pRec; + u8 *pEndHdr; /* Points to one byte past record header */ + u8 *pHdr; /* Current point in record header */ + u8 *pBody; /* Current point in record data */ + u64 nHdr; /* Bytes in record header */ + Tcl_Obj *pRet; /* Return value */ + + pRet = Tcl_NewObj(); + Tcl_IncrRefCount(pRet); + + assert( argc==1 ); + pRec = (u8*)sqlite3_value_blob(argv[0]); + + pHdr = pRec + sqlite3GetVarint(pRec, &nHdr); + pBody = pEndHdr = &pRec[nHdr]; + while( pHdr> 4) & 0x0F)]; + hex[1] = hexdigit[(z[i] & 0x0F)]; + hex[2] = '\0'; + Tcl_AppendStringsToObj(pVal, hex, 0); + } + Tcl_AppendStringsToObj(pVal, "'", 0); + break; + } + + case SQLITE_FLOAT: + pVal = Tcl_NewDoubleObj(sqlite3_value_double(&mem)); + break; + + case SQLITE_INTEGER: + pVal = Tcl_NewWideIntObj(sqlite3_value_int64(&mem)); + break; + + case SQLITE_NULL: + pVal = Tcl_NewStringObj("NULL", -1); + break; + + default: + assert( 0 ); + } + + Tcl_ListObjAppendElement(0, pRet, pVal); + + if( mem.zMalloc ){ + sqlite3DbFree(db, mem.zMalloc); + } + } + + sqlite3_result_text(context, Tcl_GetString(pRet), -1, SQLITE_TRANSIENT); + Tcl_DecrRefCount(pRet); +} + static int registerTestFunctions(sqlite3 *db){ static const struct { @@ -482,6 +578,7 @@ static int registerTestFunctions(sqlite3 *db){ { "test_isolation", 2, SQLITE_UTF8, test_isolation}, { "test_counter", 1, SQLITE_UTF8, counterFunc}, { "real2hex", 1, SQLITE_UTF8, real2hex}, + { "test_decode", 1, SQLITE_UTF8, test_decode}, }; int i; diff --git a/test/analyze9.test b/test/analyze9.test index 51817893ac..fd83df6031 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -37,10 +37,21 @@ do_test 1.0 { execsql { CREATE INDEX i1 ON t1(a, b) } } {} + do_execsql_test 1.1 { ANALYZE; } {} +do_execsql_test 1.3 { + SELECT tbl,idx,nEq,nLt,nDLt,test_decode(sample) FROM sqlite_stat4; +} { + t1 i1 {1 1} {0 0} {0 0} {(0) (0)} + t1 i1 {1 1} {1 1} {1 1} {(1) (1)} + t1 i1 {1 1} {2 2} {2 2} {(2) (2)} + t1 i1 {1 1} {3 3} {3 3} {(3) (3)} + t1 i1 {1 1} {4 4} {4 4} {(4) (4)} +} + do_execsql_test 1.2 { SELECT tbl,idx,nEq,nLt,nDLt,s(sample) FROM sqlite_stat4; } { @@ -52,5 +63,55 @@ do_execsql_test 1.2 { } +#------------------------------------------------------------------------- +# This is really just to test SQL user function "test_decode". +# +reset_db +do_execsql_test 2.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES('some text', 14, NULL); + INSERT INTO t1 VALUES(22.0, NULL, x'656667'); + CREATE INDEX i1 ON t1(a, b, c); + ANALYZE; + SELECT test_decode(sample) FROM sqlite_stat4; +} { + {22.0 NULL x'656667'} + {{some text} 14 NULL} +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 3.1 { + CREATE TABLE t2(a, b); + CREATE INDEX i2 ON t2(a, b); + BEGIN; +} + +do_test 3.2 { + for {set i 0} {$i < 1000} {incr i} { + set a [expr $i / 10] + set b [expr int(rand() * 15.0)] + execsql { INSERT INTO t2 VALUES($a, $b) } + } + execsql COMMIT +} {} + +db func lindex lindex + +# Each value of "a" occurs exactly 10 times in the table. +# +do_execsql_test 3.3.1 { + SELECT count(*) FROM t2 GROUP BY a; +} [lrange [string repeat "10 " 100] 0 99] + +# The first element in the "nEq" list of all samples should therefore be 10. +# +do_execsql_test 3.3.2 { + ANALYZE; + SELECT lindex(nEq, 0) FROM sqlite_stat4; +} [lrange [string repeat "10 " 100] 0 23] + + finish_test From c612970c9d4d388b3dac6d7a885634f57c1eaee1 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 5 Aug 2013 19:04:07 +0000 Subject: [PATCH 04/67] Modify the vdbe code generated by ANALYZE to use fewer memory cells and cursor slots. FossilOrigin-Name: 4a51cf289fad8aebc637b5f96488de18e861195d --- manifest | 12 ++++----- manifest.uuid | 2 +- src/analyze.c | 69 ++++++++++++++++++--------------------------------- 3 files changed, 31 insertions(+), 52 deletions(-) diff --git a/manifest b/manifest index 1bdf53de0c..dd8eb47156 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sN\sseparate\scursors\swhen\sscanning\san\sindex\swith\sN\scolumns\sto\scollect\ssqlite_stat4\sdata.\sThis\sfixes\sa\sproblem\swith\scollecting\sincorrect\snEq\svalues\sfrom\smulti-column\sindexes. -D 2013-08-05T18:00:56.397 +C Modify\sthe\svdbe\scode\sgenerated\sby\sANALYZE\sto\suse\sfewer\smemory\scells\sand\scursor\sslots. +D 2013-08-05T19:04:07.083 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 -F src/analyze.c a0979f7fdc8cd724f8e646ba9ef6ca1e56fa7491 +F src/analyze.c 726a63b19f7f1478b321b84fc03ae9d754624841 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P badd24d987240db5528b37d1c177431617079f9b -R 2711e68fa0a11591cfe5a22173f05afc +P 3a71afe67418ce00097cd9714c395fe9ff16f23b +R a3a130eab7cfa15d823c14448800974a U dan -Z ccb7ab66bd998be3fb0311148b3ca62c +Z c7783dc63850c26ec6b79b95587c6286 diff --git a/manifest.uuid b/manifest.uuid index 22a684eebe..8b49a29bb2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a71afe67418ce00097cd9714c395fe9ff16f23b \ No newline at end of file +4a51cf289fad8aebc637b5f96488de18e861195d \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 07366b76ab..564edb6139 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -505,15 +505,15 @@ static void analyzeOneTable( Table *pTab, /* Table whose indices are to be analyzed */ Index *pOnlyIdx, /* If not NULL, only analyze this one index */ int iStatCur, /* Index of VdbeCursor that writes the sqlite_stat1 table */ - int iMem /* Available memory locations begin here */ + int iMem, /* Available memory locations begin here */ + int iTab /* Next available cursor */ ){ sqlite3 *db = pParse->db; /* Database handle */ Index *pIdx; /* An index to being analyzed */ int iIdxCur; /* Cursor open on index being analyzed */ + int iTabCur; /* Table cursor */ Vdbe *v; /* The virtual machine being built up */ int i; /* Loop counter */ - int topOfLoop; /* The top of the loop */ - int endOfLoop; /* The end of the loop */ int jZeroRows = -1; /* Jump from here if number of rows is zero */ int iDb; /* Index of database containing pTab */ u8 needTableCnt = 1; /* True to count the table */ @@ -525,23 +525,20 @@ static void analyzeOneTable( int regNumLt = iMem++; /* Number of keys less than regSample */ int regNumDLt = iMem++; /* Number of distinct keys less than regSample */ int regSample = iMem++; /* The next sample value */ - int regRowid = regSample; /* Rowid of a sample */ - int regAccum = iMem++; /* Register to hold Stat4Accum object */ int regLoop = iMem++; /* Loop counter */ - int regCount = iMem++; /* Number of rows in the table or index */ - int regTemp1 = iMem++; /* Intermediate register */ - int regTemp2 = iMem++; /* Intermediate register */ - int once = 1; /* One-time initialization */ int shortJump = 0; /* Instruction address */ - int iTabCur = pParse->nTab++; /* Table cursor */ #endif int regCol = iMem++; /* Content of a column in analyzed table */ int regRec = iMem++; /* Register holding completed record */ int regTemp = iMem++; /* Temporary use register */ int regNewRowid = iMem++; /* Rowid for the inserted record */ + int regEof = iMem++; /* True once cursors are all at EOF */ + int regCnt = iMem++; /* Row counter */ int regStat4 = iMem++; /* Register to hold Stat4Accum object */ + int regRowid = iMem++; /* Rowid argument passed to stat4_push() */ + pParse->nMem = MAX(pParse->nMem, regRowid); v = sqlite3GetVdbe(pParse); if( v==0 || NEVER(pTab==0) ){ return; @@ -568,23 +565,20 @@ static void analyzeOneTable( /* Establish a read-lock on the table at the shared-cache level. ** Also open a read-only cursor on the table. */ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - iTabCur = pParse->nTab++; + iTabCur = iTab++; + pParse->nTab = MAX(pParse->nTab, iTab); sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns indexed by pIdx */ KeyInfo *pKey; /* KeyInfo structure for pIdx */ - int addrIfNot = 0; /* address of OP_IfNot */ int *aChngAddr; /* Array of jump instruction addresses */ - int regRowid; /* Register for rowid of current row */ int regPrev; /* First in array of previous values */ int regDLte; /* First in array of nDlt registers */ int regLt; /* First in array of nLt registers */ int regEq; /* First in array of nEq registers */ - int regCnt; /* Number of index entries */ - int regEof; /* True once cursors are all at EOF */ int endOfScan; /* Label to jump to once scan is finished */ if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; @@ -651,13 +645,10 @@ static void analyzeOneTable( ** integer identifier) is regStat4. Immediately following regStat4 ** we allocate the following: ** - ** regRowid - 1 register ** regEq - nCol registers ** regLt - nCol registers ** regDLte - nCol registers - ** regCnt - 1 register ** regPrev - nCol registers - ** regEof - 1 register ** ** The regRowid, regEq, regLt and regDLte registers must be positioned in ** that order immediately following regStat4 so that they can be passed @@ -665,21 +656,16 @@ static void analyzeOneTable( ** ** All of the above are initialized to contain integer value 0. */ - regRowid = regStat4+1; /* Rowid argument */ regEq = regRowid+1; /* First in array of nEq value registers */ regLt = regEq+nCol; /* First in array of nLt value registers */ regDLte = regLt+nCol; /* First in array of nDLt value registers */ - regCnt = regDLte+nCol; /* Row counter */ - regPrev = regCnt+1; /* First in array of prev. value registers */ - regEof = regPrev+nCol; /* True once last row read from index */ - if( regEof+1>pParse->nMem ){ - pParse->nMem = regPrev+nCol; - } + regPrev = regDLte+nCol; /* First in array of prev. value registers */ + pParse->nMem = MAX(pParse->nMem, regPrev+nCol); /* Open a read-only cursor for each column of the index. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); - iIdxCur = pParse->nTab++; - pParse->nTab += (nCol-1); + iIdxCur = iTab; + pParse->nTab = MAX(pParse->nTab, iTab+nCol); for(i=0; itnum, iDb); @@ -703,9 +689,11 @@ static void analyzeOneTable( #endif /* SQLITE_ENABLE_STAT4 */ /* Initialize all the memory registers allocated above to 0. */ - for(i=regRowid; i<=regEof; i++){ + for(i=regEq; iaiColumn[i]; sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regPrev+i); @@ -884,11 +866,6 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeJumpHere(v, jZeroRows); } - - sqlite3VdbeAddOp1(v, OP_Close, iTabCur); - - /* TODO: Not sure about this... */ - if( pParse->nMemnMem = regRec; } @@ -912,16 +889,18 @@ static void analyzeDatabase(Parse *pParse, int iDb){ HashElem *k; int iStatCur; int iMem; + int iTab; sqlite3BeginWriteOperation(pParse, 0, iDb); iStatCur = pParse->nTab; pParse->nTab += 3; openStatTable(pParse, iDb, iStatCur, 0, 0); iMem = pParse->nMem+1; + iTab = pParse->nTab; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); - analyzeOneTable(pParse, pTab, 0, iStatCur, iMem); + analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); } loadAnalysis(pParse, iDb); } @@ -946,7 +925,7 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){ }else{ openStatTable(pParse, iDb, iStatCur, pTab->zName, "tbl"); } - analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur, pParse->nMem+1); + analyzeOneTable(pParse, pTab, pOnlyIdx, iStatCur,pParse->nMem+1,pParse->nTab); loadAnalysis(pParse, iDb); } From 7a4192358a6aec4cf9828ac7539025fdb659e264 Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 6 Aug 2013 20:01:43 +0000 Subject: [PATCH 05/67] When possible, use the multi-column samples in sqlite_stat4 to estimate the number of index rows scanned by a query plan. FossilOrigin-Name: 2973f5ca736c4a6f13c653d54b6a29d7cae8d0ed --- manifest | 20 ++--- manifest.uuid | 2 +- src/alter.c | 2 +- src/sqliteInt.h | 4 + src/update.c | 2 +- src/vdbemem.c | 106 ++++++++++++++++++++++++++- src/where.c | 189 ++++++++++++++++++++++++++---------------------- 7 files changed, 220 insertions(+), 105 deletions(-) diff --git a/manifest b/manifest index dd8eb47156..0ada153ab5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Modify\sthe\svdbe\scode\sgenerated\sby\sANALYZE\sto\suse\sfewer\smemory\scells\sand\scursor\sslots. -D 2013-08-05T19:04:07.083 +C When\spossible,\suse\sthe\smulti-column\ssamples\sin\ssqlite_stat4\sto\sestimate\sthe\snumber\sof\sindex\srows\sscanned\sby\sa\squery\splan. +D 2013-08-06T20:01:43.152 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -156,7 +156,7 @@ F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad -F src/alter.c f8db986c03eb0bfb221523fc9bbb9d0b70de3168 +F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 F src/analyze.c 726a63b19f7f1478b321b84fc03ae9d754624841 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 @@ -221,7 +221,7 @@ F src/shell.c 52f975eae87c8338c4dfbf4c2842d2a0971f01fd F src/sqlite.h.in 442c109e0c3447c34b1794971ecdb673ce08a843 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 37e2952ce88a64d6572b2c40fcb78956e5aa2f7a +F src/sqliteInt.h 31057687e0ebc9e56a025bcebb252bbd84e5db13 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -273,7 +273,7 @@ F src/test_vfstrace.c 34b544e80ba7fb77be15395a609c669df2e660a2 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 70061085a51f2f4fc15ece94f32c03bcb78e63b2 F src/trigger.c 5c0ea9b8755e7c5e1a700f3e27ac4f8d92dd221e -F src/update.c 7f3fe64d8f3b44c44a1eac293f0f85f87c355b7a +F src/update.c 7d9d38e4f341ada7d79035ea969cdefb8b9014d1 F src/utf.c acd0b6f8beb8df4e0ed178c48c81c693bcc31967 F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 F src/vacuum.c d9c5759f4c5a438bb43c2086f72c5d2edabc36c8 @@ -283,14 +283,14 @@ F src/vdbeInt.h e9b7c6b165a31a4715c5aa97223d20d265515231 F src/vdbeapi.c 4d13580bd058b39623e8fcfc233b7df4b8191e8b F src/vdbeaux.c 4389b3692969b4415fcfd00de36818a02f84df28 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 -F src/vdbemem.c 69c6d1c3ef07f4442e074def9a92d15d02f06eba +F src/vdbemem.c 83f9b6e68a421cfcdde125a66b5ebe8220c12de2 F src/vdbesort.c 3937e06b2a0e354500e17dc206ef4c35770a5017 F src/vdbetrace.c e7ec40e1999ff3c6414424365d5941178966dcbc F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 072abd8556bf1d05676cc14b561da79acb70abb9 +F src/where.c 37c3dc9e9fddc2362c4926c0c64ca2d270c09d26 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3a71afe67418ce00097cd9714c395fe9ff16f23b -R a3a130eab7cfa15d823c14448800974a +P 4a51cf289fad8aebc637b5f96488de18e861195d +R 35ea3b91ab20c676c2ada836f868975b U dan -Z c7783dc63850c26ec6b79b95587c6286 +Z 27d68e9c86b12d5cd9aae1be85713ec1 diff --git a/manifest.uuid b/manifest.uuid index 8b49a29bb2..346421d3a1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4a51cf289fad8aebc637b5f96488de18e861195d \ No newline at end of file +2973f5ca736c4a6f13c653d54b6a29d7cae8d0ed \ No newline at end of file diff --git a/src/alter.c b/src/alter.c index a49d3349d7..9d34b07b0c 100644 --- a/src/alter.c +++ b/src/alter.c @@ -687,7 +687,7 @@ void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ ** can handle (i.e. not CURRENT_TIME etc.) */ if( pDflt ){ - sqlite3_value *pVal; + sqlite3_value *pVal = 0; if( sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_NONE, &pVal) ){ db->mallocFailed = 1; return; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 9d33b4c7a7..63c77085af 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3101,6 +3101,10 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int); void sqlite3BackupRestart(sqlite3_backup *); void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); +int sqlite3Stat4ProbeSetValue(Parse*, UnpackedRecord*, Expr*, u8, int, int*); +void sqlite3Stat4ProbeFree(UnpackedRecord*); +int sqlite3Stat4ProbeNew(Parse*, Index*, UnpackedRecord**); + /* ** The interface to the LEMON-generated parser */ diff --git a/src/update.c b/src/update.c index 4fbefc3b59..5077f64e35 100644 --- a/src/update.c +++ b/src/update.c @@ -61,7 +61,7 @@ static void updateVirtualTable( void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ assert( pTab!=0 ); if( !pTab->pSelect ){ - sqlite3_value *pValue; + sqlite3_value *pValue = 0; u8 enc = ENC(sqlite3VdbeDb(v)); Column *pCol = &pTab->aCol[i]; VdbeComment((v, "%s.%s", pTab->zName, pCol->zName)); diff --git a/src/vdbemem.c b/src/vdbemem.c index 67fc695f3f..8627360113 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1005,6 +1005,11 @@ sqlite3_value *sqlite3ValueNew(sqlite3 *db){ return p; } +static sqlite3_value *valueNew(sqlite3 *db, sqlite3_value *pOld){ + if( pOld ) return pOld; + return sqlite3ValueNew(db); +} + /* ** Create a new sqlite3_value object, containing the value of pExpr. ** @@ -1056,7 +1061,7 @@ int sqlite3ValueFromExpr( } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ - pVal = sqlite3ValueNew(db); + pVal = valueNew(db, *ppVal); if( pVal==0 ) goto no_mem; if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); @@ -1090,7 +1095,7 @@ int sqlite3ValueFromExpr( sqlite3ValueApplyAffinity(pVal, affinity, enc); } }else if( op==TK_NULL ){ - pVal = sqlite3ValueNew(db); + pVal = valueNew(db, *ppVal); if( pVal==0 ) goto no_mem; } #ifndef SQLITE_OMIT_BLOB_LITERAL @@ -1098,7 +1103,7 @@ int sqlite3ValueFromExpr( int nVal; assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); assert( pExpr->u.zToken[1]=='\'' ); - pVal = sqlite3ValueNew(db); + pVal = valueNew(db, *ppVal); if( !pVal ) goto no_mem; zVal = &pExpr->u.zToken[2]; nVal = sqlite3Strlen30(zVal)-1; @@ -1117,11 +1122,104 @@ int sqlite3ValueFromExpr( no_mem: db->mallocFailed = 1; sqlite3DbFree(db, zVal); - sqlite3ValueFree(pVal); + if( *ppVal==0 ) sqlite3ValueFree(pVal); *ppVal = 0; return SQLITE_NOMEM; } +#ifdef SQLITE_ENABLE_STAT4 +int sqlite3Stat4ProbeSetValue( + Parse *pParse, /* Parse context */ + UnpackedRecord *pRec, /* Set field in this probe */ + Expr *pExpr, /* The expression to extract a value from */ + u8 affinity, /* Affinity to use */ + int iVal, /* Array element to populate */ + int *pbOk /* OUT: True if value was extracted */ +){ + int rc = SQLITE_OK; + sqlite3_value *pVal = &pRec->aMem[iVal]; + +#if 0 + if( iVal>0 ){ *pbOk = 0; return SQLITE_OK; } +#endif + + if( !pExpr ){ + sqlite3VdbeMemSetNull((Mem*)pVal); + *pbOk = 1; + }else if( pExpr->op==TK_VARIABLE + || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) + ){ + Vdbe *v; + int iVar = pExpr->iColumn; + sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); + if( v = pParse->pReprepare ){ + rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iVal-1]); + if( rc==SQLITE_OK ){ + sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); + } + pVal->db = pParse->db; + *pbOk = 1; + sqlite3VdbeMemStoreType((Mem*)pVal); + }else{ + *pbOk = 0; + } + }else{ + sqlite3 *db = pRec->aMem[0].db; + rc = sqlite3ValueFromExpr(db, pExpr, ENC(db), affinity, &pVal); + *pbOk = (pVal!=0); + } + assert( pVal==0 || pVal->db==pParse->db ); + return rc; +} + +void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ + if( pRec ){ + int i; + Mem *aMem = pRec->aMem; + sqlite3 *db = aMem[0].db; + for(i=0; ipKeyInfo->nField; i++){ + sqlite3DbFree(db, aMem[i].zMalloc); + } + sqlite3DbFree(db, pRec->pKeyInfo); + sqlite3DbFree(db, pRec); + } +} + +int sqlite3Stat4ProbeNew( + Parse *pParse, /* Parse context */ + Index *pIdx, /* Allocate record for this index */ + UnpackedRecord **ppRec /* OUT: Allocated record */ +){ + sqlite3 *db = pParse->db; /* Database handle */ + UnpackedRecord *pRec; /* Return value */ + int nByte; /* Bytes of space to allocate */ + int i; /* Counter variable */ + + assert( *ppRec==0 ); + if( pIdx->nSample==0 ) return SQLITE_OK; + + nByte = sizeof(Mem) * pIdx->nColumn + sizeof(UnpackedRecord); + *ppRec = pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte); + if( !pRec ) return SQLITE_NOMEM; + pRec->pKeyInfo = sqlite3IndexKeyinfo(pParse, pIdx); + if( !pRec->pKeyInfo ){ + sqlite3DbFree(db, pRec); + *ppRec = 0; + return SQLITE_NOMEM; + } + pRec->pKeyInfo->enc = ENC(pParse->db); + pRec->flags = UNPACKED_PREFIX_MATCH; + pRec->aMem = (Mem *)&pRec[1]; + for(i=0; inColumn; i++){ + pRec->aMem[i].flags = MEM_Null; + pRec->aMem[i].type = SQLITE_NULL; + pRec->aMem[i].db = db; + } + + return SQLITE_OK; +} +#endif /* ifdef SQLITE_ENABLE_STAT4 */ + /* ** Change the string value of an sqlite3_value object */ diff --git a/src/where.c b/src/where.c index 3bc82b0654..ad74928c94 100644 --- a/src/where.c +++ b/src/where.c @@ -390,6 +390,11 @@ struct WhereLoopBuilder { ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ +#ifdef SQLITE_ENABLE_STAT4 + UnpackedRecord *pRec; /* Probe for stat4 (if required) */ + int nRecValid; /* Number of valid fields currently in pRec */ + tRowcnt nMaxRowcnt; /* If !=0, the maximum estimated row count */ +#endif }; /* @@ -2405,33 +2410,21 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ static int whereKeyStats( Parse *pParse, /* Database connection */ Index *pIdx, /* Index to consider domain of */ - sqlite3_value *pVal, /* Value to consider */ + UnpackedRecord *pRec, /* Vector of values to consider */ int roundUp, /* Round up if true. Round down if false */ tRowcnt *aStat /* OUT: stats written here */ ){ IndexSample *aSample = pIdx->aSample; - UnpackedRecord rec; int i; int isEq = 0; - if( pVal==0 ) return SQLITE_ERROR; - - memset(&rec, 0, sizeof(UnpackedRecord)); - rec.pKeyInfo = sqlite3IndexKeyinfo(pParse, pIdx); - if( rec.pKeyInfo==0 ) return SQLITE_NOMEM; - rec.pKeyInfo->enc = ENC(pParse->db); - rec.nField = 1; - rec.flags = UNPACKED_PREFIX_MATCH; - rec.aMem = pVal; - for(i=0; inSample; i++){ - int res = sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, &rec); + int res = sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec); if( res>=0 ){ isEq = (res==0); break; } } - sqlite3DbFree(pParse->db, rec.pKeyInfo); /* At this point, aSample[i] is the first sample that is greater than ** or equal to pVal. Or if i==pIdx->nSample, then all samples are less @@ -2467,41 +2460,6 @@ static int whereKeyStats( } #endif /* SQLITE_ENABLE_STAT4 */ -/* -** If expression pExpr represents a literal value, set *pp to point to -** an sqlite3_value structure containing the same value, with affinity -** aff applied to it, before returning. It is the responsibility of the -** caller to eventually release this structure by passing it to -** sqlite3ValueFree(). -** -** If the current parse is a recompile (sqlite3Reprepare()) and pExpr -** is an SQL variable that currently has a non-NULL value bound to it, -** create an sqlite3_value structure containing this value, again with -** affinity aff applied to it, instead. -** -** If neither of the above apply, set *pp to NULL. -** -** If an error occurs, return an error code. Otherwise, SQLITE_OK. -*/ -#ifdef SQLITE_ENABLE_STAT4 -static int valueFromExpr( - Parse *pParse, - Expr *pExpr, - u8 aff, - sqlite3_value **pp -){ - if( pExpr->op==TK_VARIABLE - || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) - ){ - int iVar = pExpr->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); - *pp = sqlite3VdbeGetBoundValue(pParse->pReprepare, iVar, aff); - return SQLITE_OK; - } - return sqlite3ValueFromExpr(pParse->db, pExpr, ENC(pParse->db), aff, pp); -} -#endif - /* ** This function is used to estimate the number of rows that will be visited ** by scanning an index for a range of values. The range may have an upper @@ -2543,8 +2501,7 @@ static int valueFromExpr( */ static int whereRangeScanEst( Parse *pParse, /* Parsing & code generating context */ - Index *p, /* The index containing the range-compared column; "x" */ - int nEq, /* index into p->aCol[] of the range-compared column */ + WhereLoopBuilder *pBuilder, WhereTerm *pLower, /* Lower bound on the range. ex: "x>123" Might be NULL */ WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ WhereCost *pRangeDiv /* OUT: Reduce search space by this divisor */ @@ -2552,44 +2509,54 @@ static int whereRangeScanEst( int rc = SQLITE_OK; #ifdef SQLITE_ENABLE_STAT4 + Index *p = pBuilder->pNew->u.btree.pIndex; + int nEq = pBuilder->pNew->u.btree.nEq; - if( nEq==0 && p->nSample && OptimizationEnabled(pParse->db, SQLITE_Stat3) ){ - sqlite3_value *pRangeVal; + if( nEq==pBuilder->nRecValid + && p->nSample + && OptimizationEnabled(pParse->db, SQLITE_Stat3) + ){ + UnpackedRecord *pRec = pBuilder->pRec; tRowcnt iLower = 0; tRowcnt iUpper = p->aiRowEst[0]; tRowcnt a[2]; u8 aff = p->pTable->aCol[p->aiColumn[0]].affinity; - if( pLower ){ + int bOk; /* True if value is extracted from pExpr */ Expr *pExpr = pLower->pExpr->pRight; - rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); - if( rc==SQLITE_OK - && whereKeyStats(pParse, p, pRangeVal, 0, a)==SQLITE_OK + rc = sqlite3Stat4ProbeSetValue(pParse, pRec, pExpr, aff, nEq, &bOk); + pRec->nField = nEq+1; + if( rc==SQLITE_OK && bOk + && whereKeyStats(pParse, p, pRec, 0, a)==SQLITE_OK ){ iLower = a[0]; if( (pLower->eOperator & WO_GT)!=0 ) iLower += a[1]; } - sqlite3ValueFree(pRangeVal); } if( rc==SQLITE_OK && pUpper ){ + int bOk; /* True if value is extracted from pExpr */ Expr *pExpr = pUpper->pExpr->pRight; - rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal); assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); - if( rc==SQLITE_OK - && whereKeyStats(pParse, p, pRangeVal, 1, a)==SQLITE_OK + rc = sqlite3Stat4ProbeSetValue(pParse, pRec, pExpr, aff, nEq, &bOk); + pRec->nField = nEq+1; + if( rc==SQLITE_OK && bOk + && whereKeyStats(pParse, p, pRec, 1, a)==SQLITE_OK ){ iUpper = a[0]; if( (pUpper->eOperator & WO_LE)!=0 ) iUpper += a[1]; } - sqlite3ValueFree(pRangeVal); } if( rc==SQLITE_OK ){ WhereCost iBase = whereCost(p->aiRowEst[0]); if( iUpper>iLower ){ iBase -= whereCost(iUpper - iLower); } - *pRangeDiv = iBase; + if( pBuilder->nMaxRowcnt && iBasenMaxRowcnt ){ + *pRangeDiv = pBuilder->nMaxRowcnt; + }else{ + *pRangeDiv = iBase; + } WHERETRACE(0x100, ("range scan regions: %u..%u div=%d\n", (u32)iLower, (u32)iUpper, *pRangeDiv)); return SQLITE_OK; @@ -2597,7 +2564,7 @@ static int whereRangeScanEst( } #else UNUSED_PARAMETER(pParse); - UNUSED_PARAMETER(p); + UNUSED_PARAMETER(pBuilder); UNUSED_PARAMETER(nEq); #endif assert( pLower || pUpper ); @@ -2633,32 +2600,52 @@ static int whereRangeScanEst( */ static int whereEqualScanEst( Parse *pParse, /* Parsing & code generating context */ - Index *p, /* The index whose left-most column is pTerm */ + WhereLoopBuilder *pBuilder, Expr *pExpr, /* Expression for VALUE in the x=VALUE constraint */ tRowcnt *pnRow /* Write the revised row estimate here */ ){ - sqlite3_value *pRhs = 0; /* VALUE on right-hand side of pTerm */ + Index *p = pBuilder->pNew->u.btree.pIndex; + int nEq = pBuilder->pNew->u.btree.nEq; + UnpackedRecord *pRec = pBuilder->pRec; u8 aff; /* Column affinity */ int rc; /* Subfunction return code */ tRowcnt a[2]; /* Statistics */ + int bOk; + assert( nEq>=1 ); + assert( nEq<=(p->nColumn+1) ); assert( p->aSample!=0 ); assert( p->nSample>0 ); - aff = p->pTable->aCol[p->aiColumn[0]].affinity; - if( pExpr ){ - rc = valueFromExpr(pParse, pExpr, aff, &pRhs); - if( rc ) goto whereEqualScanEst_cancel; - }else{ - pRhs = sqlite3ValueNew(pParse->db); + assert( pBuilder->nRecValidnRecValid<(nEq-1) ){ + return SQLITE_NOTFOUND; } - if( pRhs==0 ) return SQLITE_NOTFOUND; - rc = whereKeyStats(pParse, p, pRhs, 0, a); + + if( nEq>p->nColumn ){ + *pnRow = 1; + return SQLITE_OK; + } + + aff = p->pTable->aCol[p->aiColumn[0]].affinity; + rc = sqlite3Stat4ProbeSetValue(pParse, pRec, pExpr, aff, nEq-1, &bOk); + if( rc!=SQLITE_OK ) return rc; + if( bOk==0 ) return SQLITE_NOTFOUND; + + pBuilder->nRecValid = nEq; + pRec->nField = nEq; + + rc = whereKeyStats(pParse, p, pRec, 0, a); if( rc==SQLITE_OK ){ WHERETRACE(0x100,("equality scan regions: %d\n", (int)a[1])); *pnRow = a[1]; + if( pBuilder->nMaxRowcnt && *pnRow>pBuilder->nMaxRowcnt ){ + *pnRow = pBuilder->nMaxRowcnt; + } } -whereEqualScanEst_cancel: - sqlite3ValueFree(pRhs); + return rc; } #endif /* defined(SQLITE_ENABLE_STAT4) */ @@ -2682,10 +2669,12 @@ whereEqualScanEst_cancel: */ static int whereInScanEst( Parse *pParse, /* Parsing & code generating context */ - Index *p, /* The index whose left-most column is pTerm */ + WhereLoopBuilder *pBuilder, ExprList *pList, /* The value list on the RHS of "x IN (v1,v2,v3,...)" */ tRowcnt *pnRow /* Write the revised row estimate here */ ){ + Index *p = pBuilder->pNew->u.btree.pIndex; + int nRecValid = pBuilder->nRecValid; int rc = SQLITE_OK; /* Subfunction return code */ tRowcnt nEst; /* Number of rows for a single term */ tRowcnt nRowEst = 0; /* New estimate of the number of rows */ @@ -2694,14 +2683,21 @@ static int whereInScanEst( assert( p->aSample!=0 ); for(i=0; rc==SQLITE_OK && inExpr; i++){ nEst = p->aiRowEst[0]; - rc = whereEqualScanEst(pParse, p, pList->a[i].pExpr, &nEst); + rc = whereEqualScanEst(pParse, pBuilder, pList->a[i].pExpr, &nEst); nRowEst += nEst; + pBuilder->nRecValid = nRecValid; } + if( rc==SQLITE_OK ){ if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; - *pnRow = nRowEst; + if( pBuilder->nMaxRowcnt && nRowEst>pBuilder->nMaxRowcnt ){ + *pnRow = pBuilder->nMaxRowcnt; + }else{ + *pnRow = nRowEst; + } WHERETRACE(0x100,("IN row estimate: est=%g\n", nRowEst)); } + assert( pBuilder->nRecValid==nRecValid ); return rc; } #endif /* defined(SQLITE_ENABLE_STAT4) */ @@ -4248,12 +4244,15 @@ static int whereLoopAddBtreeIndex( rLogSize = estLog(whereCost(pProbe->aiRowEst[0])); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 0; - if( pTerm->prereqRight & pNew->maskSelf ) continue; #ifdef SQLITE_ENABLE_STAT4 + int nRecValid = pBuilder->nRecValid; + int nMaxRowcnt = pBuilder->nMaxRowcnt; if( (pTerm->wtFlags & TERM_VNULL)!=0 && pSrc->pTab->aCol[iCol].notNull ){ continue; /* skip IS NOT NULL constraints on a NOT NULL column */ } #endif + if( pTerm->prereqRight & pNew->maskSelf ) continue; + pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->nLTerm = saved_nLTerm; @@ -4311,21 +4310,22 @@ static int whereLoopAddBtreeIndex( if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ /* Adjust nOut and rRun for STAT3 range values */ WhereCost rDiv; - whereRangeScanEst(pParse, pProbe, pNew->u.btree.nEq, - pBtm, pTop, &rDiv); + whereRangeScanEst(pParse, pBuilder, pBtm, pTop, &rDiv); pNew->nOut = saved_nOut>rDiv+10 ? saved_nOut - rDiv : 10; } #ifdef SQLITE_ENABLE_STAT4 - if( pNew->u.btree.nEq==1 && pProbe->nSample - && OptimizationEnabled(db, SQLITE_Stat3) ){ + if( nInMul==0 && pProbe->nSample && OptimizationEnabled(db, SQLITE_Stat3) ){ + Expr *pExpr = pTerm->pExpr; tRowcnt nOut = 0; if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ testcase( pTerm->eOperator & WO_EQ ); testcase( pTerm->eOperator & WO_ISNULL ); - rc = whereEqualScanEst(pParse, pProbe, pTerm->pExpr->pRight, &nOut); + rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); + assert( nOut==0||pBuilder->nMaxRowcnt==0||nOut<=pBuilder->nMaxRowcnt); + if( nOut ) pBuilder->nMaxRowcnt = nOut; }else if( (pTerm->eOperator & WO_IN) - && !ExprHasProperty(pTerm->pExpr, EP_xIsSelect) ){ - rc = whereInScanEst(pParse, pProbe, pTerm->pExpr->x.pList, &nOut); + && !ExprHasProperty(pExpr, EP_xIsSelect) ){ + rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); } assert( nOut==0 || rc==SQLITE_OK ); if( nOut ) pNew->nOut = whereCost(nOut); @@ -4345,6 +4345,10 @@ static int whereLoopAddBtreeIndex( ){ whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } +#ifdef SQLITE_ENABLE_STAT4 + pBuilder->nRecValid = nRecValid; + pBuilder->nMaxRowcnt = nMaxRowcnt; +#endif } pNew->prereq = saved_prereq; pNew->u.btree.nEq = saved_nEq; @@ -4571,7 +4575,16 @@ static int whereLoopAddBtree( if( rc ) break; } } - rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); + + assert( pBuilder->pRec==0 ); + rc = sqlite3Stat4ProbeNew(pWInfo->pParse, pProbe, &pBuilder->pRec); + if( rc==SQLITE_OK ){ + rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); + sqlite3Stat4ProbeFree(pBuilder->pRec); + pBuilder->nRecValid = 0; + pBuilder->pRec = 0; + } + assert( pBuilder->pRec==0 ); /* If there was an INDEXED BY clause, then only that one index is ** considered. */ From ddc2d6e8f51e2b6b184894eecf99dc739f2ed64a Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 6 Aug 2013 20:15:06 +0000 Subject: [PATCH 06/67] Fixes for builds without SQLITE_ENABLE_STAT4. FossilOrigin-Name: 84999e27cc0d14b89d9fe024e29d287c69285369 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 2 ++ src/where.c | 5 ++++- test/analyze9.test | 5 +++++ 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 0ada153ab5..539c78a499 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\spossible,\suse\sthe\smulti-column\ssamples\sin\ssqlite_stat4\sto\sestimate\sthe\snumber\sof\sindex\srows\sscanned\sby\sa\squery\splan. -D 2013-08-06T20:01:43.152 +C Fixes\sfor\sbuilds\swithout\sSQLITE_ENABLE_STAT4. +D 2013-08-06T20:15:06.692 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 726a63b19f7f1478b321b84fc03ae9d754624841 +F src/analyze.c b8b0d15347ae34772f0df9b6d02848dea9dfe8a3 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -290,7 +290,7 @@ F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 37c3dc9e9fddc2362c4926c0c64ca2d270c09d26 +F src/where.c 142525786b7855a0997e113f3932833bf5c1f63c F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -308,7 +308,7 @@ F test/analyze5.test e3eece09761c935ec0b85dc4ed70dbf6cac1ed77 F test/analyze6.test 3c01e084309706a1033f850330ea24f6f7846297 F test/analyze7.test c0af22c5e0140e2e4ac556a21c2b6fff58229c98 F test/analyze8.test 092425439c12f62f9d5c3127e2b4f6e7b3e170cc -F test/analyze9.test 30479ec9ac395e77cfced52496845500768d45f9 +F test/analyze9.test 1ed4e7d95d8e1e1923766281b20870d61730450c F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4a51cf289fad8aebc637b5f96488de18e861195d -R 35ea3b91ab20c676c2ada836f868975b +P 2973f5ca736c4a6f13c653d54b6a29d7cae8d0ed +R 3fc350a341f72f5a912cf9e1ccf56bc1 U dan -Z 27d68e9c86b12d5cd9aae1be85713ec1 +Z 97c03093fe0c6a197e3d6120177c182a diff --git a/manifest.uuid b/manifest.uuid index 346421d3a1..3d5a8cb3bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2973f5ca736c4a6f13c653d54b6a29d7cae8d0ed \ No newline at end of file +84999e27cc0d14b89d9fe024e29d287c69285369 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 564edb6139..17cecb43d9 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -761,9 +761,11 @@ static void analyzeOneTable( } /* Invoke stat4_push() */ +#ifdef SQLITE_ENABLE_STAT4 sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp); sqlite3VdbeChangeP4(v, -1, (char*)&stat4PushFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2 + 3*nCol); +#endif sqlite3VdbeAddOp2(v, OP_If, regEof, endOfScan); for(i=0; ipRec==0 ); rc = sqlite3Stat4ProbeNew(pWInfo->pParse, pProbe, &pBuilder->pRec); if( rc==SQLITE_OK ){ +#endif rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); +#ifdef SQLITE_ENABLE_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); pBuilder->nRecValid = 0; pBuilder->pRec = 0; } assert( pBuilder->pRec==0 ); +#endif /* If there was an INDEXED BY clause, then only that one index is ** considered. */ diff --git a/test/analyze9.test b/test/analyze9.test index fd83df6031..1e0171c03b 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -14,6 +14,11 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix analyze9 +ifcapable !stat4 { + finish_test + return +} + proc s {blob} { set ret "" binary scan $blob c* bytes From 87cd93215e5f877005ae7a545f7a156286faa176 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 Aug 2013 15:52:41 +0000 Subject: [PATCH 07/67] When estimating the number of rows scanned using data from the sqlite_stat4 table, avoid allocating UnpackedRecord and KeyInfo structures until they are definitely required. FossilOrigin-Name: 353950a5269fa439cc3e57b62e16558a84ea2557 --- manifest | 16 ++-- manifest.uuid | 2 +- src/sqliteInt.h | 5 +- src/vdbemem.c | 230 ++++++++++++++++++++++++++++++++++-------------- src/where.c | 27 ++---- 5 files changed, 183 insertions(+), 97 deletions(-) diff --git a/manifest b/manifest index 539c78a499..4bba71c21f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fixes\sfor\sbuilds\swithout\sSQLITE_ENABLE_STAT4. -D 2013-08-06T20:15:06.692 +C When\sestimating\sthe\snumber\sof\srows\sscanned\susing\sdata\sfrom\sthe\ssqlite_stat4\stable,\savoid\sallocating\sUnpackedRecord\sand\sKeyInfo\sstructures\suntil\sthey\sare\sdefinitely\srequired. +D 2013-08-07T15:52:41.738 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -221,7 +221,7 @@ F src/shell.c 52f975eae87c8338c4dfbf4c2842d2a0971f01fd F src/sqlite.h.in 442c109e0c3447c34b1794971ecdb673ce08a843 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 31057687e0ebc9e56a025bcebb252bbd84e5db13 +F src/sqliteInt.h f56ec862ae37cf1b6e699ab5a2b53d40c8ec5068 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -283,14 +283,14 @@ F src/vdbeInt.h e9b7c6b165a31a4715c5aa97223d20d265515231 F src/vdbeapi.c 4d13580bd058b39623e8fcfc233b7df4b8191e8b F src/vdbeaux.c 4389b3692969b4415fcfd00de36818a02f84df28 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 -F src/vdbemem.c 83f9b6e68a421cfcdde125a66b5ebe8220c12de2 +F src/vdbemem.c f0512045147702adec3ca6388663e243c17d2ea4 F src/vdbesort.c 3937e06b2a0e354500e17dc206ef4c35770a5017 F src/vdbetrace.c e7ec40e1999ff3c6414424365d5941178966dcbc F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 142525786b7855a0997e113f3932833bf5c1f63c +F src/where.c 4e188dc4a1f668d761750eb27e603616179806d0 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2973f5ca736c4a6f13c653d54b6a29d7cae8d0ed -R 3fc350a341f72f5a912cf9e1ccf56bc1 +P 84999e27cc0d14b89d9fe024e29d287c69285369 +R b05f226dd58169f35c18db270697f169 U dan -Z 97c03093fe0c6a197e3d6120177c182a +Z 7ab1efe6195a9307bc618357bb65cf2b diff --git a/manifest.uuid b/manifest.uuid index 3d5a8cb3bf..48fb3c47e0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -84999e27cc0d14b89d9fe024e29d287c69285369 \ No newline at end of file +353950a5269fa439cc3e57b62e16558a84ea2557 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 63c77085af..d3825a597c 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1545,7 +1545,7 @@ struct Index { unsigned autoIndex:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ unsigned bUnordered:1; /* Use this index for == or IN queries only */ unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ -#ifdef SQLITE_ENABLE_STAT3 +#ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ tRowcnt avgEq; /* Average nEq value for key values not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ @@ -3101,9 +3101,8 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int); void sqlite3BackupRestart(sqlite3_backup *); void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); -int sqlite3Stat4ProbeSetValue(Parse*, UnpackedRecord*, Expr*, u8, int, int*); +int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*); void sqlite3Stat4ProbeFree(UnpackedRecord*); -int sqlite3Stat4ProbeNew(Parse*, Index*, UnpackedRecord**); /* ** The interface to the LEMON-generated parser diff --git a/src/vdbemem.c b/src/vdbemem.c index 8627360113..9c7f579b1f 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1005,27 +1005,30 @@ sqlite3_value *sqlite3ValueNew(sqlite3 *db){ return p; } -static sqlite3_value *valueNew(sqlite3 *db, sqlite3_value *pOld){ - if( pOld ) return pOld; - return sqlite3ValueNew(db); +/* +** Argument pCtx is actually a pointer to a database handle. Allocate and +** return an sqlite3_value object associated with this database handle. +** +** This function used as the xAlloc callback for valueFromExpr() when +** it is called by sqlite3ValueFromExpr(). +*/ +static sqlite3_value *valueNew(void *pCtx){ + return sqlite3ValueNew((sqlite3*)pCtx); } /* -** Create a new sqlite3_value object, containing the value of pExpr. -** -** This only works for very simple expressions that consist of one constant -** token (i.e. "5", "5.1", "'a string'"). If the expression can -** be converted directly into a value, then the value is allocated and -** a pointer written to *ppVal. The caller is responsible for deallocating -** the value by passing it to sqlite3ValueFree() later on. If the expression -** cannot be converted to a value, then *ppVal is set to NULL. +** This function is the same as sqlite3ValueFromExpr(), except that instead +** of allocating any required sqlite3_value object by calling +** sqlite3ValueNew(), it does so by calling the supplied xAlloc hook. */ -int sqlite3ValueFromExpr( - sqlite3 *db, /* The database connection */ - Expr *pExpr, /* The expression to evaluate */ - u8 enc, /* Encoding to use */ - u8 affinity, /* Affinity to use */ - sqlite3_value **ppVal /* Write the new value here */ +int valueFromExpr( + sqlite3 *db, /* The database connection */ + Expr *pExpr, /* The expression to evaluate */ + u8 enc, /* Encoding to use */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal, /* Write the new value here */ + sqlite3_value *(*xAlloc)(void*), /* Used to allocate new sqlite3_value */ + void *pAlloc /* Argument passed to xAlloc */ ){ int op; char *zVal = 0; @@ -1061,7 +1064,7 @@ int sqlite3ValueFromExpr( } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ - pVal = valueNew(db, *ppVal); + pVal = xAlloc(pAlloc); if( pVal==0 ) goto no_mem; if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); @@ -1095,7 +1098,7 @@ int sqlite3ValueFromExpr( sqlite3ValueApplyAffinity(pVal, affinity, enc); } }else if( op==TK_NULL ){ - pVal = valueNew(db, *ppVal); + pVal = xAlloc(pAlloc); if( pVal==0 ) goto no_mem; } #ifndef SQLITE_OMIT_BLOB_LITERAL @@ -1103,7 +1106,7 @@ int sqlite3ValueFromExpr( int nVal; assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); assert( pExpr->u.zToken[1]=='\'' ); - pVal = valueNew(db, *ppVal); + pVal = xAlloc(pAlloc); if( !pVal ) goto no_mem; zVal = &pExpr->u.zToken[2]; nVal = sqlite3Strlen30(zVal)-1; @@ -1127,51 +1130,178 @@ no_mem: return SQLITE_NOMEM; } +/* +** Create a new sqlite3_value object, containing the value of pExpr. +** +** This only works for very simple expressions that consist of one constant +** token (i.e. "5", "5.1", "'a string'"). If the expression can +** be converted directly into a value, then the value is allocated and +** a pointer written to *ppVal. The caller is responsible for deallocating +** the value by passing it to sqlite3ValueFree() later on. If the expression +** cannot be converted to a value, then *ppVal is set to NULL. +*/ +int sqlite3ValueFromExpr( + sqlite3 *db, /* The database connection */ + Expr *pExpr, /* The expression to evaluate */ + u8 enc, /* Encoding to use */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal /* Write the new value here */ +){ + return valueFromExpr(db, pExpr, enc, affinity, ppVal, valueNew, (void*)db); +} + #ifdef SQLITE_ENABLE_STAT4 +/* +** A pointer to an instance of this object is passed as the context +** pointer to valueNewStat4() (see below. +*/ +struct ValueNewStat4Ctx { + Parse *pParse; + Index *pIdx; + UnpackedRecord **ppRec; + int iVal; +}; + +/* +** This function is used as the xAlloc function with valueFromExpr() when +** it is called by sqlite3Stat4ProbeSetValue(). The argument points to +** an object of type ValueNewStat4Ctx (see above). +** +** If it has not already been allocated, this function allocates an +** UnpackedRecord structure and space for up to N values, where N is the +** number of columns in the index being probed. +*/ +static sqlite3_value *valueNewStat4(void *pCtx){ + struct ValueNewStat4Ctx *p = (struct ValueNewStat4Ctx*)pCtx; + UnpackedRecord *pRec = p->ppRec[0]; + + if( pRec==0 ){ + sqlite3 *db = p->pParse->db; /* Database handle */ + Index *pIdx = p->pIdx; /* Index being probed */ + int nByte; /* Bytes of space to allocate */ + int i; /* Counter variable */ + + nByte = sizeof(Mem) * pIdx->nColumn + sizeof(UnpackedRecord); + pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte); + if( pRec ){ + pRec->pKeyInfo = sqlite3IndexKeyinfo(p->pParse, pIdx); + if( pRec->pKeyInfo ){ + pRec->pKeyInfo->enc = ENC(db); + pRec->flags = UNPACKED_PREFIX_MATCH; + pRec->aMem = (Mem *)&pRec[1]; + for(i=0; inColumn; i++){ + pRec->aMem[i].flags = MEM_Null; + pRec->aMem[i].type = SQLITE_NULL; + pRec->aMem[i].db = db; + } + }else{ + sqlite3DbFree(db, pRec); + pRec = 0; + } + } + if( pRec==0 ) return 0; + p->ppRec[0] = pRec; + } + + pRec->nField = p->iVal+1; + return &pRec->aMem[p->iVal]; +} + +/* +** This function is used to allocate and populate UnpackedRecord +** structures intended to be compared against sample index keys stored +** in the sqlite_stat4 table. +** +** A single call to this function attempts to populates field iVal (leftmost +** is 0 etc.) of the unpacked record with a value extracted from expression +** pExpr. Extraction of values is possible if: +** +** * (pExpr==0). In this case the value is assumed to be an SQL NULL, +** +** * The expression is a bound variable, and this is a reprepare, or +** +** * The sqlite3ValueFromExpr() function is able to extract a value +** from the expression (i.e. the expression is a literal value). +** +** If a value can be extracted, the affinity passed as the 5th argument +** is applied to it before it is copied into the UnpackedRecord. Output +** parameter *pbOk is set to true if a value is extracted, or false +** otherwise. +** +** When this function is called, *ppRec must either point to an object +** allocated by an earlier call to this function, or must be NULL. If it +** is NULL and a value can be successfully extracted, a new UnpackedRecord +** is allocated (and *ppRec set to point to it) before returning. +** +** Unless an error is encountered, SQLITE_OK is returned. It is not an +** error if a value cannot be extracted from pExpr. If an error does +** occur, an SQLite error code is returned. +*/ int sqlite3Stat4ProbeSetValue( Parse *pParse, /* Parse context */ - UnpackedRecord *pRec, /* Set field in this probe */ + Index *pIdx, /* Index being probed */ + UnpackedRecord **ppRec, /* IN/OUT: Probe record */ Expr *pExpr, /* The expression to extract a value from */ u8 affinity, /* Affinity to use */ int iVal, /* Array element to populate */ int *pbOk /* OUT: True if value was extracted */ ){ int rc = SQLITE_OK; - sqlite3_value *pVal = &pRec->aMem[iVal]; + sqlite3_value *pVal = 0; + + struct ValueNewStat4Ctx alloc; + alloc.pParse = pParse; + alloc.pIdx = pIdx; + alloc.ppRec = ppRec; + alloc.iVal = iVal; #if 0 if( iVal>0 ){ *pbOk = 0; return SQLITE_OK; } #endif if( !pExpr ){ - sqlite3VdbeMemSetNull((Mem*)pVal); - *pbOk = 1; + pVal = valueNewStat4((void*)&alloc); + if( pVal ){ + sqlite3VdbeMemSetNull((Mem*)pVal); + *pbOk = 1; + } }else if( pExpr->op==TK_VARIABLE || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) ){ Vdbe *v; int iVar = pExpr->iColumn; sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); - if( v = pParse->pReprepare ){ - rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iVal-1]); - if( rc==SQLITE_OK ){ - sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); + if( (v = pParse->pReprepare) ){ + pVal = valueNewStat4((void*)&alloc); + if( pVal ){ + rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iVal-1]); + if( rc==SQLITE_OK ){ + sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); + } + pVal->db = pParse->db; + *pbOk = 1; + sqlite3VdbeMemStoreType((Mem*)pVal); } - pVal->db = pParse->db; - *pbOk = 1; - sqlite3VdbeMemStoreType((Mem*)pVal); }else{ *pbOk = 0; } }else{ - sqlite3 *db = pRec->aMem[0].db; - rc = sqlite3ValueFromExpr(db, pExpr, ENC(db), affinity, &pVal); + sqlite3 *db = pParse->db; + rc = valueFromExpr( + db, pExpr, ENC(db), affinity, &pVal, valueNewStat4, (void*)&alloc + ); *pbOk = (pVal!=0); } + assert( pVal==0 || pVal->db==pParse->db ); return rc; } +/* +** Unless it is NULL, the argument must be an UnpackedRecord object returned +** by an earlier call to sqlite3Stat4ProbeSetValue(). This call deletes +** the object. +*/ void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ if( pRec ){ int i; @@ -1184,40 +1314,6 @@ void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ sqlite3DbFree(db, pRec); } } - -int sqlite3Stat4ProbeNew( - Parse *pParse, /* Parse context */ - Index *pIdx, /* Allocate record for this index */ - UnpackedRecord **ppRec /* OUT: Allocated record */ -){ - sqlite3 *db = pParse->db; /* Database handle */ - UnpackedRecord *pRec; /* Return value */ - int nByte; /* Bytes of space to allocate */ - int i; /* Counter variable */ - - assert( *ppRec==0 ); - if( pIdx->nSample==0 ) return SQLITE_OK; - - nByte = sizeof(Mem) * pIdx->nColumn + sizeof(UnpackedRecord); - *ppRec = pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte); - if( !pRec ) return SQLITE_NOMEM; - pRec->pKeyInfo = sqlite3IndexKeyinfo(pParse, pIdx); - if( !pRec->pKeyInfo ){ - sqlite3DbFree(db, pRec); - *ppRec = 0; - return SQLITE_NOMEM; - } - pRec->pKeyInfo->enc = ENC(pParse->db); - pRec->flags = UNPACKED_PREFIX_MATCH; - pRec->aMem = (Mem *)&pRec[1]; - for(i=0; inColumn; i++){ - pRec->aMem[i].flags = MEM_Null; - pRec->aMem[i].type = SQLITE_NULL; - pRec->aMem[i].db = db; - } - - return SQLITE_OK; -} #endif /* ifdef SQLITE_ENABLE_STAT4 */ /* diff --git a/src/where.c b/src/where.c index a8437d843c..2af52cbb30 100644 --- a/src/where.c +++ b/src/where.c @@ -2525,8 +2525,7 @@ static int whereRangeScanEst( int bOk; /* True if value is extracted from pExpr */ Expr *pExpr = pLower->pExpr->pRight; assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); - rc = sqlite3Stat4ProbeSetValue(pParse, pRec, pExpr, aff, nEq, &bOk); - pRec->nField = nEq+1; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); if( rc==SQLITE_OK && bOk && whereKeyStats(pParse, p, pRec, 0, a)==SQLITE_OK ){ @@ -2538,8 +2537,7 @@ static int whereRangeScanEst( int bOk; /* True if value is extracted from pExpr */ Expr *pExpr = pUpper->pExpr->pRight; assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); - rc = sqlite3Stat4ProbeSetValue(pParse, pRec, pExpr, aff, nEq, &bOk); - pRec->nField = nEq+1; + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); if( rc==SQLITE_OK && bOk && whereKeyStats(pParse, p, pRec, 1, a)==SQLITE_OK ){ @@ -2547,6 +2545,7 @@ static int whereRangeScanEst( if( (pUpper->eOperator & WO_LE)!=0 ) iUpper += a[1]; } } + pBuilder->pRec = pRec; if( rc==SQLITE_OK ){ WhereCost iBase = whereCost(p->aiRowEst[0]); if( iUpper>iLower ){ @@ -2629,12 +2628,11 @@ static int whereEqualScanEst( } aff = p->pTable->aCol[p->aiColumn[0]].affinity; - rc = sqlite3Stat4ProbeSetValue(pParse, pRec, pExpr, aff, nEq-1, &bOk); + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); + pBuilder->pRec = pRec; if( rc!=SQLITE_OK ) return rc; if( bOk==0 ) return SQLITE_NOTFOUND; - pBuilder->nRecValid = nEq; - pRec->nField = nEq; rc = whereKeyStats(pParse, p, pRec, 0, a); if( rc==SQLITE_OK ){ @@ -4575,18 +4573,11 @@ static int whereLoopAddBtree( } } + rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); #ifdef SQLITE_ENABLE_STAT4 - assert( pBuilder->pRec==0 ); - rc = sqlite3Stat4ProbeNew(pWInfo->pParse, pProbe, &pBuilder->pRec); - if( rc==SQLITE_OK ){ -#endif - rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); -#ifdef SQLITE_ENABLE_STAT4 - sqlite3Stat4ProbeFree(pBuilder->pRec); - pBuilder->nRecValid = 0; - pBuilder->pRec = 0; - } - assert( pBuilder->pRec==0 ); + sqlite3Stat4ProbeFree(pBuilder->pRec); + pBuilder->nRecValid = 0; + pBuilder->pRec = 0; #endif /* If there was an INDEXED BY clause, then only that one index is From 4bfd4ad9a7f5fd1440594205d31dad69340701de Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 7 Aug 2013 15:57:24 +0000 Subject: [PATCH 08/67] Remove the unused sqlite3Utf8to16() utility function. FossilOrigin-Name: 9159b43eb2cb5d6ed18a5ad168fa27134fec2553 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/sqliteInt.h | 3 --- src/utf.c | 26 -------------------------- 4 files changed, 9 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index 4bba71c21f..c52dcb9af8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C When\sestimating\sthe\snumber\sof\srows\sscanned\susing\sdata\sfrom\sthe\ssqlite_stat4\stable,\savoid\sallocating\sUnpackedRecord\sand\sKeyInfo\sstructures\suntil\sthey\sare\sdefinitely\srequired. -D 2013-08-07T15:52:41.738 +C Remove\sthe\sunused\ssqlite3Utf8to16()\sutility\sfunction. +D 2013-08-07T15:57:24.962 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -221,7 +221,7 @@ F src/shell.c 52f975eae87c8338c4dfbf4c2842d2a0971f01fd F src/sqlite.h.in 442c109e0c3447c34b1794971ecdb673ce08a843 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h f56ec862ae37cf1b6e699ab5a2b53d40c8ec5068 +F src/sqliteInt.h 42adcd08a254fe14d8a369cf6381cf15e2e2e971 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -274,7 +274,7 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9 F src/tokenize.c 70061085a51f2f4fc15ece94f32c03bcb78e63b2 F src/trigger.c 5c0ea9b8755e7c5e1a700f3e27ac4f8d92dd221e F src/update.c 7d9d38e4f341ada7d79035ea969cdefb8b9014d1 -F src/utf.c acd0b6f8beb8df4e0ed178c48c81c693bcc31967 +F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 F src/vacuum.c d9c5759f4c5a438bb43c2086f72c5d2edabc36c8 F src/vdbe.c 91fa72a040cb1065045fce5e84196be093e29918 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 84999e27cc0d14b89d9fe024e29d287c69285369 -R b05f226dd58169f35c18db270697f169 -U dan -Z 7ab1efe6195a9307bc618357bb65cf2b +P 353950a5269fa439cc3e57b62e16558a84ea2557 +R 09ec399fdfd502185b4c383f87ff6d6c +U drh +Z 7e0f07555c210e239e313baee1455f2b diff --git a/manifest.uuid b/manifest.uuid index 48fb3c47e0..cd6114b430 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -353950a5269fa439cc3e57b62e16558a84ea2557 \ No newline at end of file +9159b43eb2cb5d6ed18a5ad168fa27134fec2553 \ No newline at end of file diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d3825a597c..6bda6f05fe 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3033,9 +3033,6 @@ void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void sqlite3ValueFree(sqlite3_value*); sqlite3_value *sqlite3ValueNew(sqlite3 *); char *sqlite3Utf16to8(sqlite3 *, const void*, int, u8); -#ifdef SQLITE_ENABLE_STAT3 -char *sqlite3Utf8to16(sqlite3 *, u8, char *, int, int *); -#endif int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value **); void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8); #ifndef SQLITE_AMALGAMATION diff --git a/src/utf.c b/src/utf.c index 0d67bf9133..ecb3ea03b8 100644 --- a/src/utf.c +++ b/src/utf.c @@ -450,32 +450,6 @@ char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte, u8 enc){ return m.z; } -/* -** Convert a UTF-8 string to the UTF-16 encoding specified by parameter -** enc. A pointer to the new string is returned, and the value of *pnOut -** is set to the length of the returned string in bytes. The call should -** arrange to call sqlite3DbFree() on the returned pointer when it is -** no longer required. -** -** If a malloc failure occurs, NULL is returned and the db.mallocFailed -** flag set. -*/ -#ifdef SQLITE_ENABLE_STAT4 -char *sqlite3Utf8to16(sqlite3 *db, u8 enc, char *z, int n, int *pnOut){ - Mem m; - memset(&m, 0, sizeof(m)); - m.db = db; - sqlite3VdbeMemSetStr(&m, z, n, SQLITE_UTF8, SQLITE_STATIC); - if( sqlite3VdbeMemTranslate(&m, enc) ){ - assert( db->mallocFailed ); - return 0; - } - assert( m.z==m.zMalloc ); - *pnOut = m.n; - return m.z; -} -#endif - /* ** zIn is a UTF-16 encoded unicode string at least nChar characters long. ** Return the number of bytes in the first nChar unicode characters From 7ed103210d0bd95300998e4b8f6d993d415ec9ce Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 7 Aug 2013 16:04:27 +0000 Subject: [PATCH 09/67] Fix the ".dump" command on the command-line shell so that it works for "sqlite_stat4" in addition to "sqlite_stat1". FossilOrigin-Name: 1e80c4b12dbb5beab422e2a33a8782ac9d767321 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/shell.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index c52dcb9af8..bce64529b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\sthe\sunused\ssqlite3Utf8to16()\sutility\sfunction. -D 2013-08-07T15:57:24.962 +C Fix\sthe\s".dump"\scommand\son\sthe\scommand-line\sshell\sso\sthat\sit\sworks\sfor\n"sqlite_stat4"\sin\saddition\sto\s"sqlite_stat1". +D 2013-08-07T16:04:27.058 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50 F src/resolve.c 17e670996729ac41aadf6a31f57b4e6f29b3d819 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 20369c82dc38eb4a77b458c8f6e353ef550580c9 -F src/shell.c 52f975eae87c8338c4dfbf4c2842d2a0971f01fd +F src/shell.c cb075e24f125e08cc6deb4d8837b0b7ff394e65d F src/sqlite.h.in 442c109e0c3447c34b1794971ecdb673ce08a843 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 353950a5269fa439cc3e57b62e16558a84ea2557 -R 09ec399fdfd502185b4c383f87ff6d6c +P 9159b43eb2cb5d6ed18a5ad168fa27134fec2553 +R f3f1a72dd300dcd2969e458028afbec5 U drh -Z 7e0f07555c210e239e313baee1455f2b +Z 194a793cc38dc0b3add1e00828469db6 diff --git a/manifest.uuid b/manifest.uuid index cd6114b430..1c323591df 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9159b43eb2cb5d6ed18a5ad168fa27134fec2553 \ No newline at end of file +1e80c4b12dbb5beab422e2a33a8782ac9d767321 \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index 7b8a8ad7f3..7b01d45c84 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1285,7 +1285,7 @@ static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ if( strcmp(zTable, "sqlite_sequence")==0 ){ zPrepStmt = "DELETE FROM sqlite_sequence;\n"; - }else if( strcmp(zTable, "sqlite_stat1")==0 ){ + }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){ fprintf(p->out, "ANALYZE sqlite_master;\n"); }else if( strncmp(zTable, "sqlite_", 7)==0 ){ return 0; From 1f28eaddc4903c3b204a87089631a9ffb0c6c9c8 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 Aug 2013 16:15:32 +0000 Subject: [PATCH 10/67] Change the way samples for the sqlite_stat4 table are collected. FossilOrigin-Name: 13ed5ac13562e7a39905d70fd47059f4d8001bba --- manifest | 14 +++++------ manifest.uuid | 2 +- src/analyze.c | 66 ++++++++++++++++++++++++++++++++------------------- 3 files changed, 49 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index bce64529b7..ac641a833e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sthe\s".dump"\scommand\son\sthe\scommand-line\sshell\sso\sthat\sit\sworks\sfor\n"sqlite_stat4"\sin\saddition\sto\s"sqlite_stat1". -D 2013-08-07T16:04:27.058 +C Change\sthe\sway\ssamples\sfor\sthe\ssqlite_stat4\stable\sare\scollected. +D 2013-08-07T16:15:32.765 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c b8b0d15347ae34772f0df9b6d02848dea9dfe8a3 +F src/analyze.c 8dfc6a70274314bcb0ba41d218e840a5591dd895 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9159b43eb2cb5d6ed18a5ad168fa27134fec2553 -R f3f1a72dd300dcd2969e458028afbec5 -U drh -Z 194a793cc38dc0b3add1e00828469db6 +P 1e80c4b12dbb5beab422e2a33a8782ac9d767321 +R 872f61a83df4a029ddba5de7545c74a2 +U dan +Z ad2d3dd84452de28ba71d37d31a2a3f2 diff --git a/manifest.uuid b/manifest.uuid index 1c323591df..503648f119 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1e80c4b12dbb5beab422e2a33a8782ac9d767321 \ No newline at end of file +13ed5ac13562e7a39905d70fd47059f4d8001bba \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 17cecb43d9..49a34a04b7 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -218,14 +218,13 @@ typedef struct Stat4Accum Stat4Accum; struct Stat4Accum { tRowcnt nRow; /* Number of rows in the entire table */ tRowcnt nPSample; /* How often to do a periodic sample */ - int iMin; /* Index of entry with minimum nSumEq and hash */ + int iMin; /* Index of entry with minimum nEq and hash */ int mxSample; /* Maximum number of samples to accumulate */ int nSample; /* Current number of samples */ int nCol; /* Number of columns in the index */ u32 iPrn; /* Pseudo-random number used for sampling */ struct Stat4Sample { i64 iRowid; /* Rowid in main table of the key */ - tRowcnt nSumEq; /* Sum of anEq[] values */ tRowcnt *anEq; /* sqlite_stat4.nEq */ tRowcnt *anLt; /* sqlite_stat4.nLt */ tRowcnt *anDLt; /* sqlite_stat4.nDLt */ @@ -349,23 +348,38 @@ static void stat4Push( assert( p->nCol>0 ); assert( argc==(2 + 3*p->nCol) ); - /* Set nSumEq to the sum of all nEq parameters. */ - for(i=0; inCol; i++){ - nSumEq += sqlite3_value_int64(aEq[i]); - } - if( nSumEq==0 ) return; - - /* Figure out if this sample will be used. Set isPSample to true if this - ** is a periodic sample, or false if it is being captured because of a - ** large nSumEq value. If the sample will not be used, return early. */ + /* Figure out if this sample will be used. There are two reasons a + ** sample may be used: + ** + ** 1. It may be a periodic sample. In this case set isPSample to true + ** as well. Or, + ** + ** 2. Less than p->mxSample samples have been collected so far, or + ** + ** 3. It is more desirable than some other non-periodic sample that has + ** already been collected. Samples are compared based on the values + ** in the anEq array, starting from last (right-most index column) + ** to first (left-most index column). If all elements of the anEq + ** array are equal, samples are compared by hash value. + */ h = p->iPrn = p->iPrn*1103515245 + 12345; if( (nLt/p->nPSample)!=((nEq+nLt)/p->nPSample) ){ doInsert = isPSample = 1; - }else if( (p->nSamplemxSample) - || (nSumEq>p->a[iMin].nSumEq) - || (nSumEq==p->a[iMin].nSumEq && h>p->a[iMin].iHash) - ){ + }else if( p->nSamplemxSample ){ doInsert = 1; + }else{ + tRowcnt *aMinEq = p->a[iMin].anEq; + for(i=p->nCol-1; i>=0; i--){ + i64 nEq = sqlite3_value_int64(aEq[i]); + if( nEqaMinEq[i] ){ + doInsert = 1; + break; + } + } + if( i<0 && h>p->a[iMin].iHash ){ + doInsert = 1; + } } if( !doInsert ) return; @@ -387,7 +401,6 @@ static void stat4Push( pSample->iRowid = rowid; pSample->iHash = h; pSample->isPSample = isPSample; - pSample->nSumEq = nSumEq; for(i=0; inCol; i++){ pSample->anEq[i] = sqlite3_value_int64(aEq[i]); pSample->anLt[i] = sqlite3_value_int64(aLt[i]); @@ -397,18 +410,21 @@ static void stat4Push( /* Find the new minimum */ if( p->nSample==p->mxSample ){ - u32 iHash = 0; /* Hash corresponding to iMin/nSumEq entry */ - i64 nMinEq = LARGEST_INT64; /* Smallest nSumEq seen so far */ - assert( iMin = -1 ); - + iMin = -1; for(i=0; imxSample; i++){ if( p->a[i].isPSample ) continue; - if( (p->a[i].nSumEqa[i].nSumEq==nMinEq && p->a[i].iHasha[i].nSumEq; - iHash = p->a[i].iHash; + }else{ + int j; + for(j=p->nCol-1; j>=0; j++){ + i64 iCmp = (p->a[iMin].anEq[j] - p->a[i].anEq[j]); + if( iCmp<0 ){ iMin = i; } + if( iCmp ) break; + } + if( j==0 && p->a[iMin].iHasha[i].iHash ){ + iMin = i; + } } } assert( iMin>=0 ); From 32c693a6e6b8e3e66a6e1911b3e9e699d1fec11e Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 Aug 2013 16:38:33 +0000 Subject: [PATCH 11/67] Fix typos in a comment in analyze.c. No code changes. FossilOrigin-Name: 812ed0c58fc5f729a2d4f16775fad6724cc367a6 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 5 ++++- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manifest b/manifest index ac641a833e..a0b0db322a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sway\ssamples\sfor\sthe\ssqlite_stat4\stable\sare\scollected. -D 2013-08-07T16:15:32.765 +C Fix\stypos\sin\sa\scomment\sin\sanalyze.c.\sNo\scode\schanges. +D 2013-08-07T16:38:33.619 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 8dfc6a70274314bcb0ba41d218e840a5591dd895 +F src/analyze.c fcc42c3a9f4c3a048b20e138e3530b7f3d8470f6 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1e80c4b12dbb5beab422e2a33a8782ac9d767321 -R 872f61a83df4a029ddba5de7545c74a2 +P 13ed5ac13562e7a39905d70fd47059f4d8001bba +R 084c38143e1dba830587618d10f058e0 U dan -Z ad2d3dd84452de28ba71d37d31a2a3f2 +Z 1a4859355b309a8c52cef488ffe4e611 diff --git a/manifest.uuid b/manifest.uuid index 503648f119..c1071fb6ce 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -13ed5ac13562e7a39905d70fd47059f4d8001bba \ No newline at end of file +812ed0c58fc5f729a2d4f16775fad6724cc367a6 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 49a34a04b7..a17ff4f1b0 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -348,7 +348,7 @@ static void stat4Push( assert( p->nCol>0 ); assert( argc==(2 + 3*p->nCol) ); - /* Figure out if this sample will be used. There are two reasons a + /* Figure out if this sample will be used. There are three reasons a ** sample may be used: ** ** 1. It may be a periodic sample. In this case set isPSample to true @@ -361,6 +361,9 @@ static void stat4Push( ** in the anEq array, starting from last (right-most index column) ** to first (left-most index column). If all elements of the anEq ** array are equal, samples are compared by hash value. + ** + ** For both the contents of the anEq[] array and the hash value, + ** larger values are considered more desirable. */ h = p->iPrn = p->iPrn*1103515245 + 12345; if( (nLt/p->nPSample)!=((nEq+nLt)/p->nPSample) ){ From eea568d68e4fefeb4409abfe51a058322549c624 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 7 Aug 2013 19:46:15 +0000 Subject: [PATCH 12/67] Replace variable Index.avgEq (average number of rows in keys for which there is no sample in sqlite_stat4) with vector Index.aAvgEq. FossilOrigin-Name: 7b70b419c43b2c3b2daf11d833a1d60245bfaef5 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 18 ++++++++++++------ src/sqliteInt.h | 2 +- src/where.c | 18 ++++++++++-------- 5 files changed, 32 insertions(+), 24 deletions(-) diff --git a/manifest b/manifest index 924a46707c..d23d9a5f36 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\slatest\strunk\schanges\swith\sthis\sbranch. -D 2013-08-07T18:42:27.752 +C Replace\svariable\sIndex.avgEq\s(average\snumber\sof\srows\sin\skeys\sfor\swhich\sthere\sis\sno\ssample\sin\ssqlite_stat4)\swith\svector\sIndex.aAvgEq. +D 2013-08-07T19:46:15.623 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c fcc42c3a9f4c3a048b20e138e3530b7f3d8470f6 +F src/analyze.c 0f54320fb58eedf3690dd9609e6176fff3f8c157 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -221,7 +221,7 @@ F src/shell.c 128eb16ccec68509a4a2f1948f2483819bf63425 F src/sqlite.h.in bd1451ba1ab681022a53bccc3c39580ba094a3ff F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h dfb02189a27dac993ee6635017dd3d6926508f5a +F src/sqliteInt.h 0ff47977058e1babf0c4265f1791c379d172b02f F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -290,7 +290,7 @@ F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 0051a3640491a67ce2d4abcbffc687a818bb49f7 +F src/where.c c973297fc29c5dae03a07d6deb479af432d24005 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 812ed0c58fc5f729a2d4f16775fad6724cc367a6 0ad83ceb79767738bd06a28840cf84da0464ab4f -R d674ab11a2de1d9766d2ac27cc923b51 +P 08f74c45ecf711a2373af578d44470add9082377 +R 4d9cb44bdbdf4b1edd0f4ec9caccbe56 U dan -Z 85dc674af92c540fb0c7cdb1bc5adf5a +Z a128130e52d0e5080bab7e97885213b4 diff --git a/manifest.uuid b/manifest.uuid index 3eba261736..843b9772c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -08f74c45ecf711a2373af578d44470add9082377 \ No newline at end of file +7b70b419c43b2c3b2daf11d833a1d60245bfaef5 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index a17ff4f1b0..f5fe93ead8 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -327,7 +327,6 @@ static void stat4Push( ){ Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); i64 rowid = sqlite3_value_int64(argv[1]); - i64 nSumEq = 0; /* Sum of all nEq parameters */ struct Stat4Sample *pSample; u32 h; int iMin = p->iMin; @@ -1176,14 +1175,15 @@ static int loadStat4(sqlite3 *db, const char *zDb){ pIdx->nSample = nSample; nByte = sizeof(IndexSample) * nSample; nByte += sizeof(tRowcnt) * pIdx->nColumn * 3 * nSample; + nByte += pIdx->nColumn * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ pIdx->aSample = sqlite3DbMallocZero(db, nByte); - pIdx->avgEq = pIdx->aiRowEst[1]; if( pIdx->aSample==0 ){ sqlite3_finalize(pStmt); return SQLITE_NOMEM; } pSpace = (tRowcnt*)&pIdx->aSample[nSample]; + pIdx->aAvgEq = pSpace; pSpace += pIdx->nColumn; for(i=0; inSample; i++){ pIdx->aSample[i].anEq = pSpace; pSpace += pIdx->nColumn; pIdx->aSample[i].anLt = pSpace; pSpace += pIdx->nColumn; @@ -1229,11 +1229,17 @@ static int loadStat4(sqlite3 *db, const char *zDb){ decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); if( idx==pIdx->nSample-1 ){ - if( pSample->anDLt[0]>0 ){ - for(i=0, sumEq=0; i<=idx-1; i++) sumEq += pIdx->aSample[i].anEq[0]; - pIdx->avgEq = (pSample->anLt[0] - sumEq)/pSample->anDLt[0]; + int iCol; + for(iCol=0; iColnColumn; iCol++){ + tRowcnt avgEq = 0; + tRowcnt nDLt = pSample->anDLt[iCol]; + if( nDLt>idx ){ + for(i=0, sumEq=0; iaSample[i].anEq[iCol]; + avgEq = (pSample->anLt[iCol] - sumEq)/(nDLt - idx); + } + if( avgEq==0 ) avgEq = 1; + pIdx->aAvgEq[iCol] = avgEq; } - if( pIdx->avgEq<=0 ) pIdx->avgEq = 1; } pSample->n = sqlite3_column_bytes(pStmt, 4); diff --git a/src/sqliteInt.h b/src/sqliteInt.h index e2bcaafd2c..19b0d5560d 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1551,7 +1551,7 @@ struct Index { unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ #ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ - tRowcnt avgEq; /* Average nEq value for key values not in aSample */ + tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ #endif }; diff --git a/src/where.c b/src/where.c index 28568bb0d6..0a300ca2f4 100644 --- a/src/where.c +++ b/src/where.c @@ -2418,7 +2418,9 @@ static int whereKeyStats( IndexSample *aSample = pIdx->aSample; int i; int isEq = 0; + int iCol = pRec->nField-1; + assert( pRec->nField>0 && iColnColumn ); for(i=0; inSample; i++){ int res = sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec); if( res>=0 ){ @@ -2433,18 +2435,18 @@ static int whereKeyStats( */ if( isEq ){ assert( inSample ); - aStat[0] = aSample[i].anLt[0]; - aStat[1] = aSample[i].anEq[0]; + aStat[0] = aSample[i].anLt[iCol]; + aStat[1] = aSample[i].anEq[iCol]; }else{ tRowcnt iLower, iUpper, iGap; if( i==0 ){ iLower = 0; - iUpper = aSample[0].anLt[0]; + iUpper = aSample[0].anLt[iCol]; }else{ - iUpper = i>=pIdx->nSample ? pIdx->aiRowEst[0] : aSample[i].anLt[0]; - iLower = aSample[i-1].anEq[0] + aSample[i-1].anLt[0]; + iUpper = i>=pIdx->nSample ? pIdx->aiRowEst[0] : aSample[i].anLt[iCol]; + iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; } - aStat[1] = pIdx->avgEq; + aStat[1] = pIdx->aAvgEq[iCol]; if( iLower>=iUpper ){ iGap = 0; }else{ @@ -2628,7 +2630,7 @@ static int whereEqualScanEst( return SQLITE_OK; } - aff = p->pTable->aCol[p->aiColumn[0]].affinity; + aff = p->pTable->aCol[p->aiColumn[nEq-1]].affinity; rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); pBuilder->pRec = pRec; if( rc!=SQLITE_OK ) return rc; @@ -2643,7 +2645,7 @@ static int whereEqualScanEst( *pnRow = pBuilder->nMaxRowcnt; } } - + return rc; } #endif /* defined(SQLITE_ENABLE_STAT4) */ From 6cb8d76ccb7ae39b3a62ca6f407bdcc7124536aa Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 8 Aug 2013 11:48:57 +0000 Subject: [PATCH 13/67] Fix a bug in using stat4 data to estimate the number of rows selected by a range constraint. FossilOrigin-Name: f783938ea999731ea073cd2c78e278095f7bea6d --- manifest | 14 ++++---- manifest.uuid | 2 +- src/where.c | 85 ++++++++++++++++++++++------------------------ test/analyze8.test | 17 ++++++++-- 4 files changed, 63 insertions(+), 55 deletions(-) diff --git a/manifest b/manifest index d23d9a5f36..44a7e1f774 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Replace\svariable\sIndex.avgEq\s(average\snumber\sof\srows\sin\skeys\sfor\swhich\sthere\sis\sno\ssample\sin\ssqlite_stat4)\swith\svector\sIndex.aAvgEq. -D 2013-08-07T19:46:15.623 +C Fix\sa\sbug\sin\susing\sstat4\sdata\sto\sestimate\sthe\snumber\sof\srows\sselected\sby\sa\srange\sconstraint. +D 2013-08-08T11:48:57.819 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c c973297fc29c5dae03a07d6deb479af432d24005 +F src/where.c 5ea698bd91c8c264bd00fb9c6aafc30043a3873b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -307,7 +307,7 @@ F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 F test/analyze5.test e3eece09761c935ec0b85dc4ed70dbf6cac1ed77 F test/analyze6.test 3c01e084309706a1033f850330ea24f6f7846297 F test/analyze7.test c0af22c5e0140e2e4ac556a21c2b6fff58229c98 -F test/analyze8.test 092425439c12f62f9d5c3127e2b4f6e7b3e170cc +F test/analyze8.test 8d1f76ff1e47c4093bb7be3971ba08fa56dc470d F test/analyze9.test 1ed4e7d95d8e1e1923766281b20870d61730450c F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 08f74c45ecf711a2373af578d44470add9082377 -R 4d9cb44bdbdf4b1edd0f4ec9caccbe56 +P 7b70b419c43b2c3b2daf11d833a1d60245bfaef5 +R 57059d9c69ace9dccd9c4ceedb6afa1e U dan -Z a128130e52d0e5080bab7e97885213b4 +Z 9ec2673c689c33df3101a910ffd90aa8 diff --git a/manifest.uuid b/manifest.uuid index 843b9772c2..5cea5a9336 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -7b70b419c43b2c3b2daf11d833a1d60245bfaef5 \ No newline at end of file +f783938ea999731ea073cd2c78e278095f7bea6d \ No newline at end of file diff --git a/src/where.c b/src/where.c index 0a300ca2f4..d230758ed2 100644 --- a/src/where.c +++ b/src/where.c @@ -394,7 +394,6 @@ struct WhereLoopBuilder { #ifdef SQLITE_ENABLE_STAT4 UnpackedRecord *pRec; /* Probe for stat4 (if required) */ int nRecValid; /* Number of valid fields currently in pRec */ - tRowcnt nMaxRowcnt; /* If !=0, the maximum estimated row count */ #endif }; @@ -2478,38 +2477,40 @@ static int whereKeyStats( ** If either of the upper or lower bound is not present, then NULL is passed in ** place of the corresponding WhereTerm. ** -** The nEq parameter is passed the index of the index column subject to the -** range constraint. Or, equivalently, the number of equality constraints -** optimized by the proposed index scan. For example, assuming index p is -** on t1(a, b), and the SQL query is: +** The value in (pBuilder->pNew->u.btree.nEq) is the index of the index +** column subject to the range constraint. Or, equivalently, the number of +** equality constraints optimized by the proposed index scan. For example, +** assuming index p is on t1(a, b), and the SQL query is: ** ** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... ** -** then nEq should be passed the value 1 (as the range restricted column, -** b, is the second left-most column of the index). Or, if the query is: +** then nEq is set to 1 (as the range restricted column, b, is the second +** left-most column of the index). Or, if the query is: ** ** ... FROM t1 WHERE a > ? AND a < ? ... ** -** then nEq should be passed 0. +** then nEq is set to 0. ** -** The returned value is an integer divisor to reduce the estimated -** search space. A return value of 1 means that range constraints are -** no help at all. A return value of 2 means range constraints are -** expected to reduce the search space by half. And so forth... -** -** In the absence of sqlite_stat3 ANALYZE data, each range inequality -** reduces the search space by a factor of 4. Hence a single constraint (x>?) -** results in a return of 4 and a range constraint (x>? AND x? AND x123" Might be NULL */ WhereTerm *pUpper, /* Upper bound on the range. ex: "x<455" Might be NULL */ - WhereCost *pRangeDiv /* OUT: Reduce search space by this divisor */ + WhereCost *pnOut /* IN/OUT: Number of rows visited */ ){ int rc = SQLITE_OK; + int nOut = (int)*pnOut; #ifdef SQLITE_ENABLE_STAT4 Index *p = pBuilder->pNew->u.btree.pIndex; @@ -2550,17 +2551,18 @@ static int whereRangeScanEst( } pBuilder->pRec = pRec; if( rc==SQLITE_OK ){ - WhereCost iBase = whereCost(p->aiRowEst[0]); + WhereCost nNew; if( iUpper>iLower ){ - iBase -= whereCost(iUpper - iLower); - } - if( pBuilder->nMaxRowcnt && iBasenMaxRowcnt ){ - *pRangeDiv = pBuilder->nMaxRowcnt; + nNew = whereCost(iUpper - iLower); }else{ - *pRangeDiv = iBase; + nNew = whereCost(2); /* Small number */ } - WHERETRACE(0x100, ("range scan regions: %u..%u div=%d\n", - (u32)iLower, (u32)iUpper, *pRangeDiv)); + if( nNewwtFlags & TERM_VNULL)==0 ){ - *pRangeDiv += 20; assert( 20==whereCost(4) ); + nOut -= 20; assert( 20==whereCost(4) ); } if( pUpper ){ - *pRangeDiv += 20; assert( 20==whereCost(4) ); + nOut -= 20; assert( 20==whereCost(4) ); } + if( nOut<10 ) nOut = 10; + *pnOut = (WhereCost)nOut; return rc; } @@ -2641,9 +2644,6 @@ static int whereEqualScanEst( if( rc==SQLITE_OK ){ WHERETRACE(0x100,("equality scan regions: %d\n", (int)a[1])); *pnRow = a[1]; - if( pBuilder->nMaxRowcnt && *pnRow>pBuilder->nMaxRowcnt ){ - *pnRow = pBuilder->nMaxRowcnt; - } } return rc; @@ -2690,11 +2690,7 @@ static int whereInScanEst( if( rc==SQLITE_OK ){ if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0]; - if( pBuilder->nMaxRowcnt && nRowEst>pBuilder->nMaxRowcnt ){ - *pnRow = pBuilder->nMaxRowcnt; - }else{ - *pnRow = nRowEst; - } + *pnRow = nRowEst; WHERETRACE(0x100,("IN row estimate: est=%g\n", nRowEst)); } assert( pBuilder->nRecValid==nRecValid ); @@ -4246,7 +4242,6 @@ static int whereLoopAddBtreeIndex( int nIn = 0; #ifdef SQLITE_ENABLE_STAT4 int nRecValid = pBuilder->nRecValid; - int nMaxRowcnt = pBuilder->nMaxRowcnt; if( (pTerm->wtFlags & TERM_VNULL)!=0 && pSrc->pTab->aCol[iCol].notNull ){ continue; /* skip IS NOT NULL constraints on a NOT NULL column */ } @@ -4309,9 +4304,8 @@ static int whereLoopAddBtreeIndex( } if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ /* Adjust nOut and rRun for STAT3 range values */ - WhereCost rDiv; - whereRangeScanEst(pParse, pBuilder, pBtm, pTop, &rDiv); - pNew->nOut = saved_nOut>rDiv+10 ? saved_nOut - rDiv : 10; + assert( pNew->nOut==saved_nOut ); + whereRangeScanEst(pParse, pBuilder, pBtm, pTop, &pNew->nOut); } #ifdef SQLITE_ENABLE_STAT4 if( nInMul==0 && pProbe->nSample && OptimizationEnabled(db, SQLITE_Stat3) ){ @@ -4321,14 +4315,15 @@ static int whereLoopAddBtreeIndex( testcase( pTerm->eOperator & WO_EQ ); testcase( pTerm->eOperator & WO_ISNULL ); rc = whereEqualScanEst(pParse, pBuilder, pExpr->pRight, &nOut); - assert( nOut==0||pBuilder->nMaxRowcnt==0||nOut<=pBuilder->nMaxRowcnt); - if( nOut ) pBuilder->nMaxRowcnt = nOut; }else if( (pTerm->eOperator & WO_IN) && !ExprHasProperty(pExpr, EP_xIsSelect) ){ rc = whereInScanEst(pParse, pBuilder, pExpr->x.pList, &nOut); } assert( nOut==0 || rc==SQLITE_OK ); - if( nOut ) pNew->nOut = whereCost(nOut); + if( nOut ){ + nOut = whereCost(nOut); + pNew->nOut = MIN(nOut, saved_nOut); + } } #endif if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ @@ -4347,7 +4342,7 @@ static int whereLoopAddBtreeIndex( } #ifdef SQLITE_ENABLE_STAT4 pBuilder->nRecValid = nRecValid; - pBuilder->nMaxRowcnt = nMaxRowcnt; + pNew->nOut = saved_nOut; #endif } pNew->prereq = saved_prereq; diff --git a/test/analyze8.test b/test/analyze8.test index f059424ce2..3a00e69e53 100644 --- a/test/analyze8.test +++ b/test/analyze8.test @@ -84,14 +84,27 @@ do_test 2.1 { # There are many more values of c between 0 and 100000 than there are # between 800000 and 900000. So t1c is more selective for the latter # range. +# +# Test 3.2 is a little unstable. It depends on the planner estimating +# that (b BETWEEN 40 AND 44) will match more rows than (c BETWEEN +# 800000 AND 900000). Which is a pretty close call (50 vs. 32), so +# the planner could get it wrong with an unlucky set of samples. This +# case happens to work, but others ("b BETWEEN 50 AND 54" for example) +# will fail. # +do_execsql_test 3.0 { + SELECT count(*) FROM t1 WHERE b BETWEEN 40 AND 44; + SELECT count(*) FROM t1 WHERE c BETWEEN 0 AND 100000; + SELECT count(*) FROM t1 WHERE c BETWEEN 800000 AND 900000; +} {50 376 32} do_test 3.1 { - eqp {SELECT * FROM t1 WHERE b BETWEEN 50 AND 54 AND c BETWEEN 0 AND 100000} + eqp {SELECT * FROM t1 WHERE b BETWEEN 40 AND 44 AND c BETWEEN 0 AND 100000} } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b? AND c Date: Thu, 8 Aug 2013 12:21:32 +0000 Subject: [PATCH 14/67] Fix a segfault in "ALTER TABLE t1 ADD COLUMN b DEFAULT (-+1)". Also an assert() failure that could occur if SQLITE_ENABLE_STAT4 were not defined. FossilOrigin-Name: 9fec3e38287067d60874530300fbeb602958c951 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbemem.c | 4 +++- src/where.c | 5 ++++- test/alter4.test | 5 +++++ 5 files changed, 21 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 44a7e1f774..2ecaf9c19f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\susing\sstat4\sdata\sto\sestimate\sthe\snumber\sof\srows\sselected\sby\sa\srange\sconstraint. -D 2013-08-08T11:48:57.819 +C Fix\sa\ssegfault\sin\s"ALTER\sTABLE\st1\sADD\sCOLUMN\sb\sDEFAULT\s(-+1)".\sAlso\san\sassert()\sfailure\sthat\scould\soccur\sif\sSQLITE_ENABLE_STAT4\swere\snot\sdefined. +D 2013-08-08T12:21:32.556 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,14 +283,14 @@ F src/vdbeInt.h e9b7c6b165a31a4715c5aa97223d20d265515231 F src/vdbeapi.c 4d13580bd058b39623e8fcfc233b7df4b8191e8b F src/vdbeaux.c a6ea36a9dc714e1128a0173249a0532ddcab0489 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 -F src/vdbemem.c f0512045147702adec3ca6388663e243c17d2ea4 +F src/vdbemem.c 4aff02f52e95cad546b47c15a7145a9940a61b67 F src/vdbesort.c 3937e06b2a0e354500e17dc206ef4c35770a5017 F src/vdbetrace.c e7ec40e1999ff3c6414424365d5941178966dcbc F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 5ea698bd91c8c264bd00fb9c6aafc30043a3873b +F src/where.c a14294548b55404e9f6c082c0e63bc6d24926c8d F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -299,7 +299,7 @@ F test/all.test 6ff7b43c2b4b905c74dc4a813d201d0fa64c5783 F test/alter.test 7e771c3c3f401198557dbbcf4a2c21950ba934f3 F test/alter2.test 7ea05c7d92ac99349a802ef7ada17294dd647060 F test/alter3.test 49c9d9fba2b8fcdce2dedeca97bbf1f369cc548d -F test/alter4.test b2debc14d8cbe4c1d12ccd6a41eef88a8c1f15d5 +F test/alter4.test 8e93bf7a7e6919b14b0c9a6c1e4908bcf21b0165 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/analyze.test 4d08a739c5ec28db93e0465e3b5a468befdf145f F test/analyze3.test 4532e5475d2aa68d752627548bdcaf70aff51010 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 7b70b419c43b2c3b2daf11d833a1d60245bfaef5 -R 57059d9c69ace9dccd9c4ceedb6afa1e +P f783938ea999731ea073cd2c78e278095f7bea6d +R 5118870e9b33149ca23f1597a77ab495 U dan -Z 9ec2673c689c33df3101a910ffd90aa8 +Z e9558b330772b69c8c1b515706fc9347 diff --git a/manifest.uuid b/manifest.uuid index 5cea5a9336..347dc90377 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f783938ea999731ea073cd2c78e278095f7bea6d \ No newline at end of file +9fec3e38287067d60874530300fbeb602958c951 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 9c7f579b1f..de0361e64c 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1085,7 +1085,9 @@ int valueFromExpr( } }else if( op==TK_UMINUS ) { /* This branch happens for multiple negative signs. Ex: -(-5) */ - if( SQLITE_OK==sqlite3ValueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal) ){ + if( SQLITE_OK==sqlite3ValueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal) + && pVal!=0 + ){ sqlite3VdbeMemNumerify(pVal); if( pVal->u.i==SMALLEST_INT64 ){ pVal->flags &= MEM_Int; diff --git a/src/where.c b/src/where.c index d230758ed2..c2a5e81b3c 100644 --- a/src/where.c +++ b/src/where.c @@ -4242,12 +4242,15 @@ static int whereLoopAddBtreeIndex( int nIn = 0; #ifdef SQLITE_ENABLE_STAT4 int nRecValid = pBuilder->nRecValid; + assert( pNew->nOut==saved_nOut ); if( (pTerm->wtFlags & TERM_VNULL)!=0 && pSrc->pTab->aCol[iCol].notNull ){ continue; /* skip IS NOT NULL constraints on a NOT NULL column */ } #endif if( pTerm->prereqRight & pNew->maskSelf ) continue; + assert( pNew->nOut==saved_nOut ); + pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; pNew->nLTerm = saved_nLTerm; @@ -4340,9 +4343,9 @@ static int whereLoopAddBtreeIndex( ){ whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } + pNew->nOut = saved_nOut; #ifdef SQLITE_ENABLE_STAT4 pBuilder->nRecValid = nRecValid; - pNew->nOut = saved_nOut; #endif } pNew->prereq = saved_prereq; diff --git a/test/alter4.test b/test/alter4.test index cda45533c6..59704fed4c 100644 --- a/test/alter4.test +++ b/test/alter4.test @@ -143,6 +143,11 @@ do_test alter4-2.6 { alter table t1 add column d DEFAULT CURRENT_TIME; } } {1 {Cannot add a column with non-constant default}} +do_test alter4-2.7 { + catchsql { + alter table t1 add column d default (-+1); + } +} {1 {Cannot add a column with non-constant default}} do_test alter4-2.99 { execsql { DROP TABLE t1; From 84c309b6ae488c3fd766e25f25c3902f58490b17 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 8 Aug 2013 16:17:12 +0000 Subject: [PATCH 15/67] Use a binary search instead of a linear scan when comparing a sample key against data from the sqlite_stat4 table. FossilOrigin-Name: e50dc30523210ba12324d5d8379503610f13aa34 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 44 +++++++++++++++++++++++++++++++++----------- 3 files changed, 40 insertions(+), 18 deletions(-) diff --git a/manifest b/manifest index 2ecaf9c19f..f1338e1786 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\ssegfault\sin\s"ALTER\sTABLE\st1\sADD\sCOLUMN\sb\sDEFAULT\s(-+1)".\sAlso\san\sassert()\sfailure\sthat\scould\soccur\sif\sSQLITE_ENABLE_STAT4\swere\snot\sdefined. -D 2013-08-08T12:21:32.556 +C Use\sa\sbinary\ssearch\sinstead\sof\sa\slinear\sscan\swhen\scomparing\sa\ssample\skey\sagainst\sdata\sfrom\sthe\ssqlite_stat4\stable. +D 2013-08-08T16:17:12.293 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c a14294548b55404e9f6c082c0e63bc6d24926c8d +F src/where.c 2323663d074a94fa1609d65d7175948d490e560b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P f783938ea999731ea073cd2c78e278095f7bea6d -R 5118870e9b33149ca23f1597a77ab495 +P 9fec3e38287067d60874530300fbeb602958c951 +R 05ac6975552968743dc313f6e2204120 U dan -Z e9558b330772b69c8c1b515706fc9347 +Z 09be040e1ce68baa088fb590dab65772 diff --git a/manifest.uuid b/manifest.uuid index 347dc90377..d8543e7110 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9fec3e38287067d60874530300fbeb602958c951 \ No newline at end of file +e50dc30523210ba12324d5d8379503610f13aa34 \ No newline at end of file diff --git a/src/where.c b/src/where.c index c2a5e81b3c..a8559bce86 100644 --- a/src/where.c +++ b/src/where.c @@ -2415,25 +2415,47 @@ static int whereKeyStats( tRowcnt *aStat /* OUT: stats written here */ ){ IndexSample *aSample = pIdx->aSample; - int i; - int isEq = 0; - int iCol = pRec->nField-1; + int iCol = pRec->nField-1; /* Index of required stats in anEq[] etc. */ + int iMin = 0; /* Smallest sample not yet tested */ + int i = pIdx->nSample; /* Smallest sample larger than or equal to pRec */ + int iTest; /* Next sample to test */ + int res; /* Result of comparison operation */ + assert( pIdx->nSample>0 ); assert( pRec->nField>0 && iColnColumn ); - for(i=0; inSample; i++){ - int res = sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec); - if( res>=0 ){ - isEq = (res==0); - break; + do{ + iTest = (iMin+i)/2; + res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec); + if( res<0 ){ + iMin = iTest+1; + }else{ + i = iTest; } + }while( res && iMinnSample ); + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) ); + }else{ + /* Otherwise, pRec must be smaller than sample $i and larger than + ** sample ($i-1). */ + assert( i==pIdx->nSample + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 ); + assert( i==0 + || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 ); } +#endif /* ifdef SQLITE_DEBUG */ /* At this point, aSample[i] is the first sample that is greater than ** or equal to pVal. Or if i==pIdx->nSample, then all samples are less - ** than pVal. If aSample[i]==pVal, then isEq==1. + ** than pVal. If aSample[i]==pVal, then res==0. */ - if( isEq ){ - assert( inSample ); + if( res==0 ){ aStat[0] = aSample[i].anLt[iCol]; aStat[1] = aSample[i].anEq[iCol]; }else{ From b3c02e210f6cab7124252de36f1e44a03137cd01 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 8 Aug 2013 19:38:40 +0000 Subject: [PATCH 16/67] Fix problems in estimating the number of rows visited by a range query using sqlite_stat4 data when the column subject to the range query is not the leftmost of the index. FossilOrigin-Name: 9228aaf54dd2700c4f460f94f9c2309407578983 --- manifest | 12 ++++----- manifest.uuid | 2 +- src/where.c | 74 ++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 60 insertions(+), 28 deletions(-) diff --git a/manifest b/manifest index f1338e1786..1d393f93d1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Use\sa\sbinary\ssearch\sinstead\sof\sa\slinear\sscan\swhen\scomparing\sa\ssample\skey\sagainst\sdata\sfrom\sthe\ssqlite_stat4\stable. -D 2013-08-08T16:17:12.293 +C Fix\sproblems\sin\sestimating\sthe\snumber\sof\srows\svisited\sby\sa\srange\squery\susing\ssqlite_stat4\sdata\swhen\sthe\scolumn\ssubject\sto\sthe\srange\squery\sis\snot\sthe\sleftmost\sof\sthe\sindex. +D 2013-08-08T19:38:40.828 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 2323663d074a94fa1609d65d7175948d490e560b +F src/where.c 0e058c33d0f4e2616ecd0996c04f3f1b7e3f7afa F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9fec3e38287067d60874530300fbeb602958c951 -R 05ac6975552968743dc313f6e2204120 +P e50dc30523210ba12324d5d8379503610f13aa34 +R b712a80bcc08bb0cfbaad8e54099473e U dan -Z 09be040e1ce68baa088fb590dab65772 +Z 6384d8e74fabf5ee0daae5c0c2206cf9 diff --git a/manifest.uuid b/manifest.uuid index d8543e7110..944774c90e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -e50dc30523210ba12324d5d8379503610f13aa34 \ No newline at end of file +9228aaf54dd2700c4f460f94f9c2309407578983 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a8559bce86..cde57a1687 100644 --- a/src/where.c +++ b/src/where.c @@ -2407,7 +2407,7 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ ** ** Return SQLITE_OK on success. */ -static int whereKeyStats( +static void whereKeyStats( Parse *pParse, /* Database connection */ Index *pIdx, /* Index to consider domain of */ UnpackedRecord *pRec, /* Vector of values to consider */ @@ -2480,7 +2480,6 @@ static int whereKeyStats( } aStat[0] = iLower + iGap; } - return SQLITE_OK; } #endif /* SQLITE_ENABLE_STAT4 */ @@ -2543,41 +2542,76 @@ static int whereRangeScanEst( && OptimizationEnabled(pParse->db, SQLITE_Stat3) ){ UnpackedRecord *pRec = pBuilder->pRec; - tRowcnt iLower = 0; - tRowcnt iUpper = p->aiRowEst[0]; tRowcnt a[2]; u8 aff = p->pTable->aCol[p->aiColumn[0]].affinity; + + /* Variable iLower will be set to the estimate of the number of rows in + ** the index that are less than the lower bound of the range query. The + ** lower bound being the concatenation of $P and $L, where $P is the + ** key-prefix formed by the nEq values matched against the nEq left-most + ** columns of the index, and $L is the value in pLower. + ** + ** Or, if pLower is NULL or $L cannot be extracted from it (because it + ** is not a simple variable or literal value), the lower bound of the + ** range is $P. Due to a quirk in the way whereKeyStats() works, even + ** if $L is available, whereKeyStats() is called for both ($P) and + ** ($P:$L) and the larger of the two returned values used. + ** + ** Similarly, iUpper is to be set to the estimate of the number of rows + ** less than the upper bound of the range query. Where the upper bound + ** is either ($P) or ($P:$U). Again, even if $U is available, both values + ** of iUpper are requested of whereKeyStats() and the smaller used. + */ + tRowcnt iLower; + tRowcnt iUpper; + + /* Determine iLower and iUpper using ($P) only. */ + if( nEq==0 ){ + iLower = 0; + iUpper = p->aiRowEst[0]; + }else{ + /* Note: this call could be optimized away - since the same values must + ** have been requested when testing key $P in whereEqualScanEst(). */ + whereKeyStats(pParse, p, pRec, 0, a); + iLower = a[0]; + iUpper = a[0] + a[1]; + } + + /* If possible, improve on the iLower estimate using ($P:$L). */ if( pLower ){ int bOk; /* True if value is extracted from pExpr */ Expr *pExpr = pLower->pExpr->pRight; assert( (pLower->eOperator & (WO_GT|WO_GE))!=0 ); rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk - && whereKeyStats(pParse, p, pRec, 0, a)==SQLITE_OK - ){ - iLower = a[0]; - if( (pLower->eOperator & WO_GT)!=0 ) iLower += a[1]; + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + whereKeyStats(pParse, p, pRec, 0, a); + iNew = a[0] + ((pLower->eOperator & WO_GT) ? a[1] : 0); + if( iNew>iLower ) iLower = iNew; } } - if( rc==SQLITE_OK && pUpper ){ + + /* If possible, improve on the iUpper estimate using ($P:$U). */ + if( pUpper ){ int bOk; /* True if value is extracted from pExpr */ Expr *pExpr = pUpper->pExpr->pRight; assert( (pUpper->eOperator & (WO_LT|WO_LE))!=0 ); rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk - && whereKeyStats(pParse, p, pRec, 1, a)==SQLITE_OK - ){ - iUpper = a[0]; - if( (pUpper->eOperator & WO_LE)!=0 ) iUpper += a[1]; + if( rc==SQLITE_OK && bOk ){ + tRowcnt iNew; + whereKeyStats(pParse, p, pRec, 1, a); + iNew = a[0] + ((pUpper->eOperator & WO_LE) ? a[1] : 0); + if( iNewpRec = pRec; if( rc==SQLITE_OK ){ WhereCost nNew; if( iUpper>iLower ){ nNew = whereCost(iUpper - iLower); }else{ - nNew = whereCost(2); /* Small number */ + nNew = 10; assert( 10==whereCost(2) ); } if( nNewnRecValid = nEq; - rc = whereKeyStats(pParse, p, pRec, 0, a); - if( rc==SQLITE_OK ){ - WHERETRACE(0x100,("equality scan regions: %d\n", (int)a[1])); - *pnRow = a[1]; - } + whereKeyStats(pParse, p, pRec, 0, a); + WHERETRACE(0x100,("equality scan regions: %d\n", (int)a[1])); + *pnRow = a[1]; return rc; } From c8af8504798bb23681681c792ed5e7eb6ee9edc8 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 9 Aug 2013 14:07:55 +0000 Subject: [PATCH 17/67] Update the header comment on analyze.c to describe the sqlite_stat4 table format. FossilOrigin-Name: 4d97809d6b29809f12d753043bda1976bdb1bd3b --- manifest | 14 +++++----- manifest.uuid | 2 +- src/analyze.c | 74 ++++++++++++++++++++++++++++++++++----------------- 3 files changed, 57 insertions(+), 33 deletions(-) diff --git a/manifest b/manifest index 1d393f93d1..086cfdf3f2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sproblems\sin\sestimating\sthe\snumber\sof\srows\svisited\sby\sa\srange\squery\susing\ssqlite_stat4\sdata\swhen\sthe\scolumn\ssubject\sto\sthe\srange\squery\sis\snot\sthe\sleftmost\sof\sthe\sindex. -D 2013-08-08T19:38:40.828 +C Update\sthe\sheader\scomment\son\sanalyze.c\sto\sdescribe\sthe\ssqlite_stat4\stable\nformat. +D 2013-08-09T14:07:55.264 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 0f54320fb58eedf3690dd9609e6176fff3f8c157 +F src/analyze.c 9fced5c126f9655d8cea2be534da6caef93f203c F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P e50dc30523210ba12324d5d8379503610f13aa34 -R b712a80bcc08bb0cfbaad8e54099473e -U dan -Z 6384d8e74fabf5ee0daae5c0c2206cf9 +P 9228aaf54dd2700c4f460f94f9c2309407578983 +R 8f8b5382a2066b3e84011ccbfba51880 +U drh +Z c6ed6d4032cb2df263d6c33284a68607 diff --git a/manifest.uuid b/manifest.uuid index 944774c90e..611d0fe358 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9228aaf54dd2700c4f460f94f9c2309407578983 \ No newline at end of file +4d97809d6b29809f12d753043bda1976bdb1bd3b \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index f5fe93ead8..bf418c6e48 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -20,6 +20,7 @@ ** CREATE TABLE sqlite_stat1(tbl, idx, stat); ** CREATE TABLE sqlite_stat2(tbl, idx, sampleno, sample); ** CREATE TABLE sqlite_stat3(tbl, idx, nEq, nLt, nDLt, sample); +** CREATE TABLE sqlite_stat4(tbl, idx, nEq, nLt, nDLt, sample); ** ** Additional tables might be added in future releases of SQLite. ** The sqlite_stat2 table is not created or used unless the SQLite version @@ -27,8 +28,13 @@ ** with SQLITE_ENABLE_STAT2. The sqlite_stat2 table is deprecated. ** The sqlite_stat2 table is superseded by sqlite_stat3, which is only ** created and used by SQLite versions 3.7.9 and later and with -** SQLITE_ENABLE_STAT3 defined. The fucntionality of sqlite_stat3 -** is a superset of sqlite_stat2. +** SQLITE_ENABLE_STAT3 defined. The functionality of sqlite_stat3 +** is a superset of sqlite_stat2. The sqlite_stat4 is an enhanced +** version of sqlite_stat3 and is only available when compiled with +** SQLITE_ENABLE_STAT4 and in SQLite versions 3.8.0 and later. +** +** For most applications, sqlite_stat1 provides all the statisics required +** for the query planner to make good choices. ** ** Format of sqlite_stat1: ** @@ -36,7 +42,8 @@ ** name in the idx column. The tbl column is the name of the table to ** which the index belongs. In each such row, the stat column will be ** a string consisting of a list of integers. The first integer in this -** list is the number of rows in the index and in the table. The second +** list is the number of rows in the index. (This is the same as the +** number of rows in the table, except for partial indices.) The second ** integer is the average number of rows in the index that have the same ** value in the first column of the index. The third integer is the average ** number of rows in the index that have the same value for the first two @@ -83,35 +90,52 @@ ** ** Format for sqlite_stat3: ** -** The sqlite_stat3 is an enhancement to sqlite_stat2. A new name is -** used to avoid compatibility problems. +** The sqlite_stat3 format is a subset of sqlite_stat4. Hence, the +** sqlite_stat4 format will be described first. Further information +** about sqlite_stat3 follows the sqlite_stat4 description. ** -** The format of the sqlite_stat3 table is similar to the format of -** the sqlite_stat2 table. There are multiple entries for each index. +** Format for sqlite_stat4: +** +** As with sqlite_stat2, the sqlite_stat4 table contains histogram data +** to aid the query planner in choosing good indices based on the values +** that indexed columns are compared against in the WHERE clauses of +** queries. +** +** The sqlite_stat4 table contains multiple entries for each index. ** The idx column names the index and the tbl column is the table of the ** index. If the idx and tbl columns are the same, then the sample is -** of the INTEGER PRIMARY KEY. The sample column is a value taken from -** the left-most column of the index. The nEq column is the approximate -** number of entires in the index whose left-most column exactly matches -** the sample. nLt is the approximate number of entires whose left-most -** column is less than the sample. The nDLt column is the approximate -** number of distinct left-most entries in the index that are less than -** the sample. +** of the INTEGER PRIMARY KEY. The sample column is a blob which is the +** binary encoding of a key from the index, with the trailing rowid +** omitted. The nEq column is a list of integers. The first integer +** is the approximate number of entires in the index whose left-most +** column exactly matches the left-most column the sample. The second +** integer in nEq is the approximate number of entires in the index where +** the first two columns match the first two columns of the sample. +** And so forth. nLt is another list of integer that show the approximate +** number of entires that are strictly less than the sample. The first +** integer in nLt contains the number of entries in the index where the +** left-most column is less than the left-most column of the sample. +** The K-th integer in the nLt entry is the number of index entries +** where the first K columns are less than the first K columns of the +** sample. The nDLt column is like nLt except that it contains the +** number of distinct entries in the index that are less than the +** sample. ** -** Future versions of SQLite might change to store a string containing -** multiple integers values in the nDLt column of sqlite_stat3. The first -** integer will be the number of prior index entires that are distinct in -** the left-most column. The second integer will be the number of prior index -** entries that are distinct in the first two columns. The third integer -** will be the number of prior index entries that are distinct in the first -** three columns. And so forth. With that extension, the nDLt field is -** similar in function to the sqlite_stat1.stat field. -** -** There can be an arbitrary number of sqlite_stat3 entries per index. +** There can be an arbitrary number of sqlite_stat4 entries per index. ** The ANALYZE command will typically generate sqlite_stat3 tables ** that contain between 10 and 40 samples which are distributed across ** the key space, though not uniformly, and which include samples with -** largest possible nEq values. +** large nEq values. +** +** Format for sqlite_stat3 redux: +** +** The sqlite_stat3 table is like sqlite_stat4 except that it only +** looks at the left-most column of the index. The sqlite_stat3.sample +** column contains the actual value of the left-most column instead +** of a blob encoding of the complete index key as is found in +** sqlite_stat4.sample. The nEq, nLt, and nDLt entries of sqlite_stat3 +** all contain just a single integer which is the same as the first +** integer in the equivalent columns in sqlite_stat4. */ #ifndef SQLITE_OMIT_ANALYZE #include "sqliteInt.h" From 84d4fcc52d2856fbef5ee9d3e48129fe54dfa7bd Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 9 Aug 2013 19:04:07 +0000 Subject: [PATCH 18/67] Fix a couple of typos in a comment in analyze.c. No code changes. FossilOrigin-Name: 5bcccb93df98f5dfee0ea4d797b07fe0257258a9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 12 ++++++------ 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 086cfdf3f2..004ced59a3 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\sthe\sheader\scomment\son\sanalyze.c\sto\sdescribe\sthe\ssqlite_stat4\stable\nformat. -D 2013-08-09T14:07:55.264 +C Fix\sa\scouple\sof\stypos\sin\sa\scomment\sin\sanalyze.c.\sNo\scode\schanges. +D 2013-08-09T19:04:07.214 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 9fced5c126f9655d8cea2be534da6caef93f203c +F src/analyze.c aba0749f8900cef07bb05324b624fd4eb19bf7c6 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9228aaf54dd2700c4f460f94f9c2309407578983 -R 8f8b5382a2066b3e84011ccbfba51880 -U drh -Z c6ed6d4032cb2df263d6c33284a68607 +P 4d97809d6b29809f12d753043bda1976bdb1bd3b +R b0844c4d270fc066470bde75f744fcb8 +U dan +Z 863ef3ffe27da5a538c7139e648347ba diff --git a/manifest.uuid b/manifest.uuid index 611d0fe358..0857e7e7bf 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4d97809d6b29809f12d753043bda1976bdb1bd3b \ No newline at end of file +5bcccb93df98f5dfee0ea4d797b07fe0257258a9 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index bf418c6e48..7ce3dceb81 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -107,12 +107,12 @@ ** of the INTEGER PRIMARY KEY. The sample column is a blob which is the ** binary encoding of a key from the index, with the trailing rowid ** omitted. The nEq column is a list of integers. The first integer -** is the approximate number of entires in the index whose left-most -** column exactly matches the left-most column the sample. The second -** integer in nEq is the approximate number of entires in the index where +** is the approximate number of entries in the index whose left-most +** column exactly matches the left-most column of the sample. The second +** integer in nEq is the approximate number of entries in the index where ** the first two columns match the first two columns of the sample. -** And so forth. nLt is another list of integer that show the approximate -** number of entires that are strictly less than the sample. The first +** And so forth. nLt is another list of integers that show the approximate +** number of entries that are strictly less than the sample. The first ** integer in nLt contains the number of entries in the index where the ** left-most column is less than the left-most column of the sample. ** The K-th integer in the nLt entry is the number of index entries @@ -122,7 +122,7 @@ ** sample. ** ** There can be an arbitrary number of sqlite_stat4 entries per index. -** The ANALYZE command will typically generate sqlite_stat3 tables +** The ANALYZE command will typically generate sqlite_stat4 tables ** that contain between 10 and 40 samples which are distributed across ** the key space, though not uniformly, and which include samples with ** large nEq values. From dd6e1f193e4efdebc226defbaa217be157b81de5 Mon Sep 17 00:00:00 2001 From: dan Date: Sat, 10 Aug 2013 19:08:30 +0000 Subject: [PATCH 19/67] Add the rowid field to the end of sample records stored in the sqlite_stat4 table. FossilOrigin-Name: 3a5e8ab7ddbe1d943b35ef329fe4e5a1bfdb0d9d --- manifest | 24 ++--- manifest.uuid | 2 +- src/analyze.c | 207 ++++++++++++++++++++++++--------------- src/vdbemem.c | 9 +- src/where.c | 4 +- test/analyze3.test | 6 +- test/analyze5.test | 11 ++- test/analyze9.test | 62 ++++++++---- test/tkt-cbd054fa6b.test | 4 +- 9 files changed, 208 insertions(+), 121 deletions(-) diff --git a/manifest b/manifest index 004ced59a3..b5c7ade0f5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\sof\stypos\sin\sa\scomment\sin\sanalyze.c.\sNo\scode\schanges. -D 2013-08-09T19:04:07.214 +C Add\sthe\srowid\sfield\sto\sthe\send\sof\ssample\srecords\sstored\sin\sthe\ssqlite_stat4\stable. +D 2013-08-10T19:08:30.794 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c aba0749f8900cef07bb05324b624fd4eb19bf7c6 +F src/analyze.c 178c33a77551d4a0d72831c71c9b93908f7c6a20 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -283,14 +283,14 @@ F src/vdbeInt.h e9b7c6b165a31a4715c5aa97223d20d265515231 F src/vdbeapi.c 4d13580bd058b39623e8fcfc233b7df4b8191e8b F src/vdbeaux.c a6ea36a9dc714e1128a0173249a0532ddcab0489 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 -F src/vdbemem.c 4aff02f52e95cad546b47c15a7145a9940a61b67 +F src/vdbemem.c b16ba7b3d1ead79c081d1f79e157e6b2efd13ca5 F src/vdbesort.c 3937e06b2a0e354500e17dc206ef4c35770a5017 F src/vdbetrace.c e7ec40e1999ff3c6414424365d5941178966dcbc F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 0e058c33d0f4e2616ecd0996c04f3f1b7e3f7afa +F src/where.c adf476146fcd78af6ebc7dea50853bcbb14ba2b6 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -302,13 +302,13 @@ F test/alter3.test 49c9d9fba2b8fcdce2dedeca97bbf1f369cc548d F test/alter4.test 8e93bf7a7e6919b14b0c9a6c1e4908bcf21b0165 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc F test/analyze.test 4d08a739c5ec28db93e0465e3b5a468befdf145f -F test/analyze3.test 4532e5475d2aa68d752627548bdcaf70aff51010 +F test/analyze3.test ea4cba3277eb89d16dfeada7259ea437e7b00f3b F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 -F test/analyze5.test e3eece09761c935ec0b85dc4ed70dbf6cac1ed77 +F test/analyze5.test 96ac783a56142bbbedb58a7c1eebd1808b49cfae F test/analyze6.test 3c01e084309706a1033f850330ea24f6f7846297 F test/analyze7.test c0af22c5e0140e2e4ac556a21c2b6fff58229c98 F test/analyze8.test 8d1f76ff1e47c4093bb7be3971ba08fa56dc470d -F test/analyze9.test 1ed4e7d95d8e1e1923766281b20870d61730450c +F test/analyze9.test 3e1bd0209354bb987832fba580c754cf77dc6ba3 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -865,7 +865,7 @@ F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 F test/tkt-bdc6bbbb38.test fc38bb09bdd440e3513a1f5f98fc60a075182d7d F test/tkt-c48d99d690.test ba61977d62ab612fc515b3c488a6fbd6464a2447 -F test/tkt-cbd054fa6b.test 2fd674fda943346a31cd020883f70bf6c037e98c +F test/tkt-cbd054fa6b.test 6595eac9c561b0aa46c1e4f4bae3f876acf38ce3 F test/tkt-d11f09d36e.test d999b548fef885d1d1afa49a0e8544ecf436869d F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4d97809d6b29809f12d753043bda1976bdb1bd3b -R b0844c4d270fc066470bde75f744fcb8 +P 5bcccb93df98f5dfee0ea4d797b07fe0257258a9 +R 96e0779963a0fd7aab46f6c8c7627c84 U dan -Z 863ef3ffe27da5a538c7139e648347ba +Z ee31c385e2b881dc7d9f322f79a33eaa diff --git a/manifest.uuid b/manifest.uuid index 0857e7e7bf..84bca67c98 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5bcccb93df98f5dfee0ea4d797b07fe0257258a9 \ No newline at end of file +3a5e8ab7ddbe1d943b35ef329fe4e5a1bfdb0d9d \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 7ce3dceb81..89d7dee958 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -237,6 +237,16 @@ static void openStatTable( ** Three SQL functions - stat4_init(), stat4_push(), and stat4_pop() - ** share an instance of the following structure to hold their state ** information. +** +** bHaveP, bHaveNonP: +** The stat4_push() user-defined-function may be invoked multiple +** times with index keys that are identical except for the rowid +** field. An argument is passed to stat4_push() to indicate if this +** is the case or not. +** +** bHaveP is set to true if a periodic sample corresponding to the +** current index key has already been added. bHaveNonP is true if a +** non-periodic sample has been added. */ typedef struct Stat4Accum Stat4Accum; struct Stat4Accum { @@ -245,8 +255,10 @@ struct Stat4Accum { int iMin; /* Index of entry with minimum nEq and hash */ int mxSample; /* Maximum number of samples to accumulate */ int nSample; /* Current number of samples */ - int nCol; /* Number of columns in the index */ + int nCol; /* Number of columns in the index including rowid */ u32 iPrn; /* Pseudo-random number used for sampling */ + int bHaveP; + int bHaveNonP; struct Stat4Sample { i64 iRowid; /* Rowid in main table of the key */ tRowcnt *anEq; /* sqlite_stat4.nEq */ @@ -285,7 +297,7 @@ static void stat4Init( nRow = (tRowcnt)sqlite3_value_int64(argv[0]); nCol = sqlite3_value_int(argv[1]); mxSample = sqlite3_value_int(argv[2]); - assert( nCol>0 ); + assert( nCol>1 ); /* >1 because it includes the rowid column */ /* Allocate the space required for the Stat4Accum object */ n = sizeof(*p) + (sizeof(p->a[0]) + 3*sizeof(tRowcnt)*nCol)*mxSample; @@ -351,51 +363,65 @@ static void stat4Push( ){ Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); i64 rowid = sqlite3_value_int64(argv[1]); + int bNewKey = sqlite3_value_int(argv[2]); struct Stat4Sample *pSample; - u32 h; + u32 h; /* Hash value for this key */ int iMin = p->iMin; int i; u8 isPSample = 0; u8 doInsert = 0; - sqlite3_value **aEq = &argv[2]; - sqlite3_value **aLt = &argv[2+p->nCol]; - sqlite3_value **aDLt = &argv[2+p->nCol+p->nCol]; + sqlite3_value **aEq = &argv[3]; + sqlite3_value **aLt = &argv[3+p->nCol]; + sqlite3_value **aDLt = &argv[3+p->nCol+p->nCol]; - i64 nEq = sqlite3_value_int64(aEq[p->nCol-1]); i64 nLt = sqlite3_value_int64(aLt[p->nCol-1]); UNUSED_PARAMETER(context); UNUSED_PARAMETER(argc); - assert( p->nCol>0 ); - assert( argc==(2 + 3*p->nCol) ); + assert( argc==(3 + 3*p->nCol) ); + assert( p->bHaveNonP==0 || p->bHaveP==0 ); - /* Figure out if this sample will be used. There are three reasons a - ** sample may be used: - ** - ** 1. It may be a periodic sample. In this case set isPSample to true - ** as well. Or, - ** - ** 2. Less than p->mxSample samples have been collected so far, or - ** - ** 3. It is more desirable than some other non-periodic sample that has - ** already been collected. Samples are compared based on the values - ** in the anEq array, starting from last (right-most index column) - ** to first (left-most index column). If all elements of the anEq - ** array are equal, samples are compared by hash value. - ** - ** For both the contents of the anEq[] array and the hash value, - ** larger values are considered more desirable. - */ + if( bNewKey ){ + p->bHaveP = 0; + p->bHaveNonP = 0; + } h = p->iPrn = p->iPrn*1103515245 + 12345; - if( (nLt/p->nPSample)!=((nEq+nLt)/p->nPSample) ){ + + /* Check if this should be a periodic sample. If this is a periodic + ** sample and there is already a non-periodic sample for this key, + ** replace it. */ + if( (nLt/p->nPSample) != (nLt+1)/p->nPSample ){ doInsert = isPSample = 1; + if( p->bHaveNonP ){ + p->nSample--; + p->bHaveNonP = 0; + p->bHaveP = 1; + assert( p->nSamplemxSample ); + assert( p->a[p->nSample].isPSample==0 ); + } + + /* Or, if this is not a periodic sample, and there is already at least one + ** periodic sample, return early. */ + }else if( p->bHaveP ){ + /* no-op */ + + /* If there is already a non-periodic sample for the key, but this one + ** has a higher hash score, replace the existing sample. */ + }else if( p->bHaveNonP ){ + if( p->a[p->nSample-1].iHashnSample--; + doInsert = 1; + } + + /* Finally, check if this should be added as a non-periodic sample. */ }else if( p->nSamplemxSample ){ doInsert = 1; + p->bHaveNonP = 1; }else{ tRowcnt *aMinEq = p->a[iMin].anEq; - for(i=p->nCol-1; i>=0; i--){ + for(i=p->nCol-2; i>=0; i--){ i64 nEq = sqlite3_value_int64(aEq[i]); if( nEqaMinEq[i] ){ @@ -406,8 +432,9 @@ static void stat4Push( if( i<0 && h>p->a[iMin].iHash ){ doInsert = 1; } + p->bHaveNonP = doInsert; } - if( !doInsert ) return; + if( doInsert==0 ) return; /* Fill in the new Stat4Sample object. */ if( p->nSample==p->mxSample ){ @@ -579,8 +606,9 @@ static void analyzeOneTable( int regStat4 = iMem++; /* Register to hold Stat4Accum object */ int regRowid = iMem++; /* Rowid argument passed to stat4_push() */ + int regKeychng = iMem++; /* True if key has changed */ - pParse->nMem = MAX(pParse->nMem, regRowid); + pParse->nMem = MAX(pParse->nMem, regKeychng); v = sqlite3GetVdbe(pParse); if( v==0 || NEVER(pTab==0) ){ return; @@ -627,7 +655,7 @@ static void analyzeOneTable( if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0; VdbeNoopComment((v, "Begin analysis of %s", pIdx->zName)); nCol = pIdx->nColumn; - aChngAddr = sqlite3DbMallocRaw(db, sizeof(int)*nCol); + aChngAddr = sqlite3DbMallocRaw(db, sizeof(int)*(nCol+1)); if( aChngAddr==0 ) continue; pKey = sqlite3IndexKeyinfo(pParse, pIdx); @@ -640,12 +668,14 @@ static void analyzeOneTable( ** entry. The code below is for an index with 2 columns. The actual ** VM code generated may be for any number of columns. ** - ** One cursor is opened for each column in the index (nCol). All cursors - ** scan concurrently the index from start to end. All variables used in - ** the pseudo-code are initialized to zero. + ** One cursor is opened for each column in the index and one for the + ** rowid column (nCol+1 in total). All cursors scan concurrently the + ** index from start to end. All variables used in the pseudo-code are + ** initialized to zero. ** ** Rewind csr(0) ** Rewind csr(1) + ** Rewind csr(2) ** ** next_0: ** regPrev(0) = csr(0)[0] @@ -662,18 +692,26 @@ static void analyzeOneTable( ** regDLte(1) += 1 ** regLt(1) += regEq(1) ** regEq(1) = 0 - ** regRowid = csr(1)[rowid] // innermost cursor only ** do { ** regEq(1) += 1 - ** regCnt += 1 // innermost cursor only ** Next csr(1) ** }while ( csr(1)[0..1] == regPrev(0..1) ) ** - ** stat4_push(regRowid, regEq, regLt, regDLte); + ** regKeychng = 1 + ** next_row: + ** regRowid = csr(2)[rowid] + ** regEq(2) = 1 + ** regLt(2) = regCnt + ** regCnt += 1 + ** regDLte(2) = regCnt + ** stat4_push(regRowid, regKeychng, regEq, regLt, regDLte); + ** regKeychng = 0 + ** Next csr(2) + ** if( eof( csr(2) ) ) goto endOfScan ** - ** if( eof( csr(1) ) ) goto endOfScan - ** if( csr(1)[0] != regPrev(0) ) goto next_0 - ** goto next_1 + ** if( csr(2)[0] != regPrev(0) ) goto next_0 + ** if( csr(2)[1] != regPrev(1) ) goto next_1 + ** goto next_row ** ** endOfScan: ** // done! @@ -683,32 +721,31 @@ static void analyzeOneTable( ** of the corresponding length. As required to calculate the contents ** of the sqlite_stat1 entry. ** - ** Currently, the last memory cell allocated (that with the largest - ** integer identifier) is regStat4. Immediately following regStat4 + ** At this point, the last memory cell allocated (that with the largest + ** integer identifier) is regKeychng. Immediately following regKeychng ** we allocate the following: ** ** regEq - nCol registers - ** regLt - nCol registers - ** regDLte - nCol registers - ** regPrev - nCol registers + ** regLt - nCol+1 registers + ** regDLte - nCol+1 registers + ** regPrev - nCol+1 registers ** - ** The regRowid, regEq, regLt and regDLte registers must be positioned in - ** that order immediately following regStat4 so that they can be passed - ** to the stat4_push() function. + ** can be passed to the stat4_push() function. ** ** All of the above are initialized to contain integer value 0. */ - regEq = regRowid+1; /* First in array of nEq value registers */ - regLt = regEq+nCol; /* First in array of nLt value registers */ - regDLte = regLt+nCol; /* First in array of nDLt value registers */ - regPrev = regDLte+nCol; /* First in array of prev. value registers */ + regEq = regKeychng+1; /* First in array of nEq value registers */ + regLt = regEq+nCol+1; /* First in array of nLt value registers */ + regDLte = regLt+nCol+1; /* First in array of nDLt value registers */ + regPrev = regDLte+nCol+1; /* First in array of prev. value registers */ pParse->nMem = MAX(pParse->nMem, regPrev+nCol); - /* Open a read-only cursor for each column of the index. */ + /* Open a read-only cursor for each column of the index. And one for + ** the rowid column. A total of (nCol+1) cursors. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); iIdxCur = iTab; - pParse->nTab = MAX(pParse->nTab, iTab+nCol); - for(i=0; inTab = MAX(pParse->nTab, iTab+nCol+1); + for(i=0; i<(nCol+1); i++){ int iMode = (i==0 ? P4_KEYINFO_HANDOFF : P4_KEYINFO); sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur+i, pIdx->tnum, iDb); sqlite3VdbeChangeP4(v, -1, (char*)pKey, iMode); @@ -719,11 +756,11 @@ static void analyzeOneTable( /* Invoke the stat4_init() function. The arguments are: ** ** * the number of rows in the index, - ** * the number of columns in the index, + ** * the number of columns in the index including the rowid, ** * the recommended number of samples for the stat4 table. */ sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+1); - sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+2); + sqlite3VdbeAddOp2(v, OP_Integer, nCol+1, regStat4+2); sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_STAT4_SAMPLES, regStat4+3); sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); sqlite3VdbeChangeP4(v, -1, (char*)&stat4InitFuncdef, P4_FUNCDEF); @@ -740,7 +777,7 @@ static void analyzeOneTable( /* Rewind all cursors open on the index. If the table is entry, this ** will cause control to jump to address endOfScan immediately. */ endOfScan = sqlite3VdbeMakeLabel(v); - for(i=0; iazColl[i]); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur+nCol-1, i, regCol); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur+nCol, i, regCol); sqlite3VdbeAddOp3(v, OP_Ne, regCol, aChngAddr[i], regPrev+i); sqlite3VdbeChangeP4(v, -1, pColl, P4_COLLSEQ); sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); } - sqlite3VdbeAddOp2(v, OP_Goto, 0, aChngAddr[nCol-1]); + sqlite3VdbeAddOp2(v, OP_Goto, 0, aChngAddr[nCol]); sqlite3DbFree(db, aChngAddr); sqlite3VdbeResolveLabel(v, endOfScan); @@ -827,17 +878,17 @@ static void analyzeOneTable( regLoop = regStat4+1; sqlite3VdbeAddOp2(v, OP_Integer, -1, regLoop); shortJump = sqlite3VdbeAddOp2(v, OP_AddImm, regLoop, 1); - sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4, regTemp); + sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4, regEq+nCol); sqlite3VdbeChangeP4(v, -1, (char*)&stat4GetFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 2); - sqlite3VdbeAddOp1(v, OP_IsNull, regTemp); + sqlite3VdbeAddOp1(v, OP_IsNull, regEq+nCol); - sqlite3VdbeAddOp3(v, OP_NotExists, iTabCur, shortJump, regTemp); + sqlite3VdbeAddOp3(v, OP_NotExists, iTabCur, shortJump, regEq+nCol); for(i=0; iaiColumn[i]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regPrev+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regEq+i); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regPrev, nCol, regSample); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regEq, nCol+1, regSample); sqlite3VdbeChangeP4(v, -1, pIdx->zColAff, 0); sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regNumEq); diff --git a/src/vdbemem.c b/src/vdbemem.c index de0361e64c..8eef88ed3b 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1182,16 +1182,18 @@ static sqlite3_value *valueNewStat4(void *pCtx){ Index *pIdx = p->pIdx; /* Index being probed */ int nByte; /* Bytes of space to allocate */ int i; /* Counter variable */ + int nCol = pIdx->nColumn+1; /* Number of index columns including rowid */ - nByte = sizeof(Mem) * pIdx->nColumn + sizeof(UnpackedRecord); + nByte = sizeof(Mem) * nCol + sizeof(UnpackedRecord); pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte); if( pRec ){ pRec->pKeyInfo = sqlite3IndexKeyinfo(p->pParse, pIdx); if( pRec->pKeyInfo ){ + assert( pRec->pKeyInfo->nField+1==nCol ); pRec->pKeyInfo->enc = ENC(db); pRec->flags = UNPACKED_PREFIX_MATCH; pRec->aMem = (Mem *)&pRec[1]; - for(i=0; inColumn; i++){ + for(i=0; iaMem[i].flags = MEM_Null; pRec->aMem[i].type = SQLITE_NULL; pRec->aMem[i].db = db; @@ -1307,9 +1309,10 @@ int sqlite3Stat4ProbeSetValue( void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ if( pRec ){ int i; + int nCol = pRec->pKeyInfo->nField+1; Mem *aMem = pRec->aMem; sqlite3 *db = aMem[0].db; - for(i=0; ipKeyInfo->nField; i++){ + for(i=0; ipKeyInfo); diff --git a/src/where.c b/src/where.c index cde57a1687..a33605954a 100644 --- a/src/where.c +++ b/src/where.c @@ -2422,7 +2422,7 @@ static void whereKeyStats( int res; /* Result of comparison operation */ assert( pIdx->nSample>0 ); - assert( pRec->nField>0 && iColnColumn ); + assert( pRec->nField>0 && iCol<=pIdx->nColumn ); do{ iTest = (iMin+i)/2; res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec); @@ -2684,6 +2684,8 @@ static int whereEqualScanEst( return SQLITE_NOTFOUND; } + /* This is an optimization only. The call to sqlite3Stat4ProbeSetValue() + ** below would return the same value. */ if( nEq>p->nColumn ){ *pnRow = 1; return SQLITE_OK; diff --git a/test/analyze3.test b/test/analyze3.test index df9e548828..394d321b97 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -95,7 +95,10 @@ do_test analyze3-1.1.1 { COMMIT; ANALYZE; } -} {} + execsql { + SELECT count(*)>0 FROM sqlite_stat4; + } +} {1} do_eqp_test analyze3-1.1.2 { SELECT sum(y) FROM t1 WHERE x>200 AND x<300 @@ -312,7 +315,6 @@ do_test analyze3-3.1 { execsql COMMIT execsql ANALYZE } {} - do_test analyze3-3.2.1 { set S [sqlite3_prepare_v2 db "SELECT * FROM t1 WHERE b>?" -1 dummy] sqlite3_expired $S diff --git a/test/analyze5.test b/test/analyze5.test index 9972695abf..048e6a5ef7 100644 --- a/test/analyze5.test +++ b/test/analyze5.test @@ -37,6 +37,8 @@ proc alpha {blob} { } db func alpha alpha +db func lindex lindex + unset -nocomplain i t u v w x y z do_test analyze5-1.0 { db eval {CREATE TABLE t1(t,u,v TEXT COLLATE nocase,w,x,y,z)} @@ -64,19 +66,20 @@ do_test analyze5-1.0 { CREATE INDEX t1y ON t1(y); -- integers 0 and very few 1s CREATE INDEX t1z ON t1(z); -- integers 0, 1, 2, and 3 ANALYZE; - SELECT alpha(sample) FROM sqlite_stat4 WHERE idx='t1u' ORDER BY nlt; + SELECT DISTINCT lindex(test_decode(sample),0) + FROM sqlite_stat4 WHERE idx='t1u' ORDER BY nlt; } } {alpha bravo charlie delta} do_test analyze5-1.1 { db eval { - SELECT DISTINCT lower(alpha(sample)) FROM sqlite_stat4 WHERE idx='t1v' - ORDER BY 1 + SELECT DISTINCT lower(lindex(test_decode(sample), 0)) + FROM sqlite_stat4 WHERE idx='t1v' ORDER BY 1 } } {alpha bravo charlie delta} do_test analyze5-1.2 { db eval {SELECT idx, count(*) FROM sqlite_stat4 GROUP BY 1 ORDER BY 1} -} {t1t 4 t1u 4 t1v 4 t1w 4 t1x 4 t1y 2 t1z 4} +} {t1t 8 t1u 8 t1v 8 t1w 8 t1x 8 t1y 9 t1z 8} # Verify that range queries generate the correct row count estimates # diff --git a/test/analyze9.test b/test/analyze9.test index 1e0171c03b..0323b860ba 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -34,12 +34,14 @@ proc s {blob} { } db function s s -do_test 1.0 { - execsql { CREATE TABLE t1(a TEXT, b TEXT); } - for {set i 0} {$i < 5} {incr i} { - execsql {INSERT INTO t1 VALUES ('('||($i%10)||')', '('||($i%7)||')')} - } - execsql { CREATE INDEX i1 ON t1(a, b) } +do_execsql_test 1.0 { + CREATE TABLE t1(a TEXT, b TEXT); + INSERT INTO t1 VALUES('(0)', '(0)'); + INSERT INTO t1 VALUES('(1)', '(1)'); + INSERT INTO t1 VALUES('(2)', '(2)'); + INSERT INTO t1 VALUES('(3)', '(3)'); + INSERT INTO t1 VALUES('(4)', '(4)'); + CREATE INDEX i1 ON t1(a, b); } {} @@ -50,21 +52,21 @@ do_execsql_test 1.1 { do_execsql_test 1.3 { SELECT tbl,idx,nEq,nLt,nDLt,test_decode(sample) FROM sqlite_stat4; } { - t1 i1 {1 1} {0 0} {0 0} {(0) (0)} - t1 i1 {1 1} {1 1} {1 1} {(1) (1)} - t1 i1 {1 1} {2 2} {2 2} {(2) (2)} - t1 i1 {1 1} {3 3} {3 3} {(3) (3)} - t1 i1 {1 1} {4 4} {4 4} {(4) (4)} + t1 i1 {1 1 1} {0 0 0} {0 0 0} {(0) (0) 1} + t1 i1 {1 1 1} {1 1 1} {1 1 1} {(1) (1) 2} + t1 i1 {1 1 1} {2 2 2} {2 2 2} {(2) (2) 3} + t1 i1 {1 1 1} {3 3 3} {3 3 3} {(3) (3) 4} + t1 i1 {1 1 1} {4 4 4} {4 4 4} {(4) (4) 5} } do_execsql_test 1.2 { SELECT tbl,idx,nEq,nLt,nDLt,s(sample) FROM sqlite_stat4; } { - t1 i1 {1 1} {0 0} {0 0} ...(0)(0) - t1 i1 {1 1} {1 1} {1 1} ...(1)(1) - t1 i1 {1 1} {2 2} {2 2} ...(2)(2) - t1 i1 {1 1} {3 3} {3 3} ...(3)(3) - t1 i1 {1 1} {4 4} {4 4} ...(4)(4) + t1 i1 {1 1 1} {0 0 0} {0 0 0} ....(0)(0) + t1 i1 {1 1 1} {1 1 1} {1 1 1} ....(1)(1). + t1 i1 {1 1 1} {2 2 2} {2 2 2} ....(2)(2). + t1 i1 {1 1 1} {3 3 3} {3 3 3} ....(3)(3). + t1 i1 {1 1 1} {4 4 4} {4 4 4} ....(4)(4). } @@ -80,8 +82,8 @@ do_execsql_test 2.1 { ANALYZE; SELECT test_decode(sample) FROM sqlite_stat4; } { - {22.0 NULL x'656667'} - {{some text} 14 NULL} + {22.0 NULL x'656667' 2} + {{some text} 14 NULL 1} } #------------------------------------------------------------------------- @@ -117,6 +119,30 @@ do_execsql_test 3.3.2 { SELECT lindex(nEq, 0) FROM sqlite_stat4; } [lrange [string repeat "10 " 100] 0 23] +#------------------------------------------------------------------------- +# +do_execsql_test 3.4 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 1, 'one-a'); + INSERT INTO t1 VALUES(11, 1, 'one-b'); + INSERT INTO t1 VALUES(21, 1, 'one-c'); + INSERT INTO t1 VALUES(31, 1, 'one-d'); + INSERT INTO t1 VALUES(41, 1, 'one-e'); + INSERT INTO t1 VALUES(51, 1, 'one-f'); + INSERT INTO t1 VALUES(61, 1, 'one-g'); + INSERT INTO t1 VALUES(71, 1, 'one-h'); + INSERT INTO t1 VALUES(81, 1, 'one-i'); + INSERT INTO t1 VALUES(91, 1, 'one-j'); + INSERT INTO t1 SELECT a+1,2,'two' || substr(c,4) FROM t1; + INSERT INTO t1 SELECT a+2,3,'three'||substr(c,4) FROM t1 WHERE c GLOB 'one-*'; + INSERT INTO t1 SELECT a+3,4,'four'||substr(c,4) FROM t1 WHERE c GLOB 'one-*'; + INSERT INTO t1 SELECT a+4,5,'five'||substr(c,4) FROM t1 WHERE c GLOB 'one-*'; + INSERT INTO t1 SELECT a+5,6,'six'||substr(c,4) FROM t1 WHERE c GLOB 'one-*'; + CREATE INDEX t1b ON t1(b); + ANALYZE; + SELECT c FROM t1 WHERE b=3 AND a BETWEEN 30 AND 60; +} {three-d three-e three-f} finish_test diff --git a/test/tkt-cbd054fa6b.test b/test/tkt-cbd054fa6b.test index 8f16964447..92b367e700 100644 --- a/test/tkt-cbd054fa6b.test +++ b/test/tkt-cbd054fa6b.test @@ -65,7 +65,7 @@ do_test tkt-cbd05-1.3 { WHERE idx = 't1_x' GROUP BY tbl,idx } -} {t1 t1_x {.. ..A ..B ..C ..D ..E ..F ..G ..H ..I}} +} {t1 t1_x {... ...A. ...B. ...C. ...D. ...E. ...F. ...G. ...H. ...I.}} do_test tkt-cbd05-2.1 { db eval { @@ -97,6 +97,6 @@ do_test tkt-cbd05-2.3 { WHERE idx = 't1_x' GROUP BY tbl,idx } -} {t1 t1_x {.. ..A ..B ..C ..D ..E ..F ..G ..H ..I}} +} {t1 t1_x {... ...A. ...B. ...C. ...D. ...E. ...F. ...G. ...H. ...I.}} finish_test From 568cd51b79eb1193b7ea93cb16663f01e13d14c5 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Aug 2013 09:29:04 +0000 Subject: [PATCH 20/67] Fix minor problems caused by adding the rowid to the records in stat4. FossilOrigin-Name: 088d1ff94890ada50d43e6a366a58167ec5a8e96 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 11 +++++------ src/where.c | 2 +- 4 files changed, 14 insertions(+), 15 deletions(-) diff --git a/manifest b/manifest index b5c7ade0f5..f847eb0805 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\srowid\sfield\sto\sthe\send\sof\ssample\srecords\sstored\sin\sthe\ssqlite_stat4\stable. -D 2013-08-10T19:08:30.794 +C Fix\sminor\sproblems\scaused\sby\sadding\sthe\srowid\sto\sthe\srecords\sin\sstat4. +D 2013-08-12T09:29:04.530 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 178c33a77551d4a0d72831c71c9b93908f7c6a20 +F src/analyze.c 49729c117665fc927280c0bcd10bffa838ff29d1 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -290,7 +290,7 @@ F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c adf476146fcd78af6ebc7dea50853bcbb14ba2b6 +F src/where.c c1090a2769c6e47b88aa79cd34f2e763af5282f8 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5bcccb93df98f5dfee0ea4d797b07fe0257258a9 -R 96e0779963a0fd7aab46f6c8c7627c84 +P 3a5e8ab7ddbe1d943b35ef329fe4e5a1bfdb0d9d +R 61c9bc8ff2fa4dfd3e5854cb245baf85 U dan -Z ee31c385e2b881dc7d9f322f79a33eaa +Z 74fdc1c4224404aea1a5d8ea05036c15 diff --git a/manifest.uuid b/manifest.uuid index 84bca67c98..4bb020c7db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3a5e8ab7ddbe1d943b35ef329fe4e5a1bfdb0d9d \ No newline at end of file +088d1ff94890ada50d43e6a366a58167ec5a8e96 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 89d7dee958..07e2cbc20f 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1249,7 +1249,7 @@ static int loadStat4(sqlite3 *db, const char *zDb){ assert( pIdx->nSample==0 ); pIdx->nSample = nSample; nByte = sizeof(IndexSample) * nSample; - nByte += sizeof(tRowcnt) * pIdx->nColumn * 3 * nSample; + nByte += sizeof(tRowcnt) * (pIdx->nColumn+1) * 3 * nSample; nByte += pIdx->nColumn * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ pIdx->aSample = sqlite3DbMallocZero(db, nByte); @@ -1260,9 +1260,9 @@ static int loadStat4(sqlite3 *db, const char *zDb){ pSpace = (tRowcnt*)&pIdx->aSample[nSample]; pIdx->aAvgEq = pSpace; pSpace += pIdx->nColumn; for(i=0; inSample; i++){ - pIdx->aSample[i].anEq = pSpace; pSpace += pIdx->nColumn; - pIdx->aSample[i].anLt = pSpace; pSpace += pIdx->nColumn; - pIdx->aSample[i].anDLt = pSpace; pSpace += pIdx->nColumn; + pIdx->aSample[i].anEq = pSpace; pSpace += pIdx->nColumn+1; + pIdx->aSample[i].anLt = pSpace; pSpace += pIdx->nColumn+1; + pIdx->aSample[i].anDLt = pSpace; pSpace += pIdx->nColumn+1; } assert( ((u8*)pSpace)-nByte==(u8*)(pIdx->aSample) ); } @@ -1298,7 +1298,7 @@ static int loadStat4(sqlite3 *db, const char *zDb){ assert( idxnSample ); pSample = &pIdx->aSample[idx]; - nCol = pIdx->nColumn; + nCol = pIdx->nColumn+1; decodeIntArray((char*)sqlite3_column_text(pStmt,1), nCol, pSample->anEq, 0); decodeIntArray((char*)sqlite3_column_text(pStmt,2), nCol, pSample->anLt, 0); decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); @@ -1324,7 +1324,6 @@ static int loadStat4(sqlite3 *db, const char *zDb){ return SQLITE_NOMEM; } memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); - } return sqlite3_finalize(pStmt); } diff --git a/src/where.c b/src/where.c index a33605954a..1fd185d418 100644 --- a/src/where.c +++ b/src/where.c @@ -2467,7 +2467,7 @@ static void whereKeyStats( iUpper = i>=pIdx->nSample ? pIdx->aiRowEst[0] : aSample[i].anLt[iCol]; iLower = aSample[i-1].anEq[iCol] + aSample[i-1].anLt[iCol]; } - aStat[1] = pIdx->aAvgEq[iCol]; + aStat[1] = (pIdx->nColumn>iCol ? pIdx->aAvgEq[iCol] : 1); if( iLower>=iUpper ){ iGap = 0; }else{ From 5133c78cae3b3a0cc8977efe7603ba5b30f91325 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Aug 2013 11:21:10 +0000 Subject: [PATCH 21/67] Fix a bug in calculating the average number of entries for keys not present in the sqlite_stat4 table. FossilOrigin-Name: ec3ffb174844406a6186c3dcc41b76d0331b502c --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 21 +++++++++++++++++---- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index f847eb0805..c08ca1d77c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sminor\sproblems\scaused\sby\sadding\sthe\srowid\sto\sthe\srecords\sin\sstat4. -D 2013-08-12T09:29:04.530 +C Fix\sa\sbug\sin\scalculating\sthe\saverage\snumber\sof\sentries\sfor\skeys\snot\spresent\sin\sthe\ssqlite_stat4\stable. +D 2013-08-12T11:21:10.969 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 49729c117665fc927280c0bcd10bffa838ff29d1 +F src/analyze.c fd1bcb9bc4ca29cd36f60c620cc501c933048c28 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3a5e8ab7ddbe1d943b35ef329fe4e5a1bfdb0d9d -R 61c9bc8ff2fa4dfd3e5854cb245baf85 +P 088d1ff94890ada50d43e6a366a58167ec5a8e96 +R 4b6821682980828caa2dcfc6f067a6e7 U dan -Z 74fdc1c4224404aea1a5d8ea05036c15 +Z 150d3ee65bd4be58d3670346ad74798d diff --git a/manifest.uuid b/manifest.uuid index 4bb020c7db..4e199eec40 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -088d1ff94890ada50d43e6a366a58167ec5a8e96 \ No newline at end of file +ec3ffb174844406a6186c3dcc41b76d0331b502c \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 07e2cbc20f..641d416aa1 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1282,7 +1282,6 @@ static int loadStat4(sqlite3 *db, const char *zDb){ char *zIndex; /* Index name */ Index *pIdx; /* Pointer to the index object */ int i; /* Loop counter */ - tRowcnt sumEq; /* Sum of the nEq values */ int nCol; /* Number of columns in index */ zIndex = (char *)sqlite3_column_text(pStmt, 0); @@ -1304,13 +1303,27 @@ static int loadStat4(sqlite3 *db, const char *zDb){ decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); if( idx==pIdx->nSample-1 ){ + IndexSample *aSample = pIdx->aSample; int iCol; for(iCol=0; iColnColumn; iCol++){ + tRowcnt sumEq = 0; /* Sum of the nEq values */ + int nSum = 0; /* Number of terms contributing to sumEq */ tRowcnt avgEq = 0; tRowcnt nDLt = pSample->anDLt[iCol]; - if( nDLt>idx ){ - for(i=0, sumEq=0; iaSample[i].anEq[iCol]; - avgEq = (pSample->anLt[iCol] - sumEq)/(nDLt - idx); + + /* Set nSum to the number of distinct (iCol+1) field prefixes that + ** occur in the stat4 table for this index before pSample. Set + ** sumEq to the sum of the nEq values for column iCol for the same + ** set (adding the value only once where there exist dupicate + ** prefixes). */ + for(i=0; i<(pIdx->nSample-1); i++){ + if( aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] ){ + sumEq += aSample[i].anEq[iCol]; + nSum++; + } + } + if( nDLt>nSum ){ + avgEq = (pSample->anLt[iCol] - sumEq)/(nDLt - nSum); } if( avgEq==0 ) avgEq = 1; pIdx->aAvgEq[iCol] = avgEq; From 0106e378f180a16ff2446ab37ef5791c2d8deab9 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Aug 2013 16:34:32 +0000 Subject: [PATCH 22/67] Re-enable reading from the sqlite_stat3 table (as well as sqlite_stat4). FossilOrigin-Name: 6d45078e621526fc2bac0eaefbb0f9602b9a8ec5 --- manifest | 21 ++++--- manifest.uuid | 2 +- src/analyze.c | 150 +++++++++++++++++++++++++++++++++++++-------- src/func.c | 3 + src/sqliteInt.h | 1 + src/test_func.c | 49 +++++++++++++++ test/analyze9.test | 3 + test/analyzeA.test | 132 +++++++++++++++++++++++++++++++++++++++ 8 files changed, 326 insertions(+), 35 deletions(-) create mode 100644 test/analyzeA.test diff --git a/manifest b/manifest index c08ca1d77c..3b395f8f6e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\sbug\sin\scalculating\sthe\saverage\snumber\sof\sentries\sfor\skeys\snot\spresent\sin\sthe\ssqlite_stat4\stable. -D 2013-08-12T11:21:10.969 +C Re-enable\sreading\sfrom\sthe\ssqlite_stat3\stable\s(as\swell\sas\ssqlite_stat4). +D 2013-08-12T16:34:32.514 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c fd1bcb9bc4ca29cd36f60c620cc501c933048c28 +F src/analyze.c e5ce42f04f58003fa45908791a5ad06bdd2c2ff1 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -175,7 +175,7 @@ F src/delete.c 2317c814866d9aa71fea16b3faf4fdd4d6a49b94 F src/expr.c 0bbb44462a19169189b2709fbbd800950521b5ae F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 914a6bbd987d857c41ac9d244efa6641f36faadb -F src/func.c 5c50c1ea31fd864b0fe921fe1a8d4c55acd609ef +F src/func.c 78c371ddfb0bb2d4c4356e8d7336b582d4ca96a9 F src/global.c 5caf4deab621abb45b4c607aad1bd21c20aac759 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22 @@ -221,7 +221,7 @@ F src/shell.c 128eb16ccec68509a4a2f1948f2483819bf63425 F src/sqlite.h.in bd1451ba1ab681022a53bccc3c39580ba094a3ff F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 0ff47977058e1babf0c4265f1791c379d172b02f +F src/sqliteInt.h 646063fc1564842fd8e54eee00f8b8b429e2eb1f F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -243,7 +243,7 @@ F src/test_config.c 636ecd15a6ba18bf97a590b5a21f47573c8c2b65 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f -F src/test_func.c fcd238feb694332d5962ee08578ef30ff4ac6559 +F src/test_func.c 338a6e5ade3560ad36280881bbcf45f28d06cb68 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61 @@ -308,7 +308,8 @@ F test/analyze5.test 96ac783a56142bbbedb58a7c1eebd1808b49cfae F test/analyze6.test 3c01e084309706a1033f850330ea24f6f7846297 F test/analyze7.test c0af22c5e0140e2e4ac556a21c2b6fff58229c98 F test/analyze8.test 8d1f76ff1e47c4093bb7be3971ba08fa56dc470d -F test/analyze9.test 3e1bd0209354bb987832fba580c754cf77dc6ba3 +F test/analyze9.test 1b419d03407f2a6f4f1485620d54cb3e1bab3a71 +F test/analyzeA.test 949c3344280e0ca6de0b49805e4f291cdc1daa43 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -1106,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 088d1ff94890ada50d43e6a366a58167ec5a8e96 -R 4b6821682980828caa2dcfc6f067a6e7 +P ec3ffb174844406a6186c3dcc41b76d0331b502c +R e94619aadc884e84bf22674359c1bbe0 U dan -Z 150d3ee65bd4be58d3670346ad74798d +Z c424468455b7fdc35c2aa42e6130568e diff --git a/manifest.uuid b/manifest.uuid index 4e199eec40..4b73be3aaa 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec3ffb174844406a6186c3dcc41b76d0331b502c \ No newline at end of file +6d45078e621526fc2bac0eaefbb0f9602b9a8ec5 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 641d416aa1..acdf1fde76 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -168,8 +168,10 @@ static void openStatTable( const char *zCols; } aTable[] = { { "sqlite_stat1", "tbl,idx,stat" }, -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) { "sqlite_stat4", "tbl,idx,neq,nlt,ndlt,sample" }, +#elif defined(SQLITE_ENABLE_STAT3) + { "sqlite_stat3", "tbl,idx,neq,nlt,ndlt,sample" }, #endif }; @@ -1206,11 +1208,81 @@ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ } #ifdef SQLITE_ENABLE_STAT4 + /* -** Load content from the sqlite_stat4 table into the Index.aSample[] -** arrays of all indices. +** The implementation of the sqlite_record() function. This function accepts +** a single argument of any type. The return value is a formatted database +** record (a blob) containing the argument value. +** +** This is used to convert the value stored in the 'sample' column of the +** sqlite_stat3 table to the record format SQLite uses internally. */ -static int loadStat4(sqlite3 *db, const char *zDb){ +static void recordFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const int file_format = 1; + int iSerial; /* Serial type */ + int nSerial; /* Bytes of space for iSerial as varint */ + int nVal; /* Bytes of space required for argv[0] */ + int nRet; + sqlite3 *db; + u8 *aRet; + + iSerial = sqlite3VdbeSerialType(argv[0], file_format); + nSerial = sqlite3VarintLen(iSerial); + nVal = sqlite3VdbeSerialTypeLen(iSerial); + db = sqlite3_context_db_handle(context); + + nRet = 1 + nSerial + nVal; + aRet = sqlite3DbMallocRaw(db, nRet); + if( aRet==0 ){ + sqlite3_result_error_nomem(context); + }else{ + aRet[0] = nSerial+1; + sqlite3PutVarint(&aRet[1], iSerial); + sqlite3VdbeSerialPut(&aRet[1+nSerial], nVal, argv[0], file_format); + sqlite3_result_blob(context, aRet, nRet, SQLITE_TRANSIENT); + sqlite3DbFree(db, aRet); + } +} + +/* +** Register built-in functions used to help read ANALYZE data. +*/ +void sqlite3AnalyzeFunctions(void){ + static SQLITE_WSD FuncDef aAnalyzeTableFuncs[] = { + FUNCTION(sqlite_record, 1, 0, 0, recordFunc), + }; + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAnalyzeTableFuncs); + for(i=0; iaSample[] */ assert( db->lookaside.bEnabled==0 ); - if( !sqlite3FindTable(db, "sqlite_stat4", zDb) ){ - return SQLITE_OK; - } - - zSql = sqlite3MPrintf(db, - "SELECT idx,count(*) FROM %Q.sqlite_stat4" - " GROUP BY idx", zDb); + zSql = sqlite3MPrintf(db, zSql1, zDb); if( !zSql ){ return SQLITE_NOMEM; } @@ -1234,6 +1300,9 @@ static int loadStat4(sqlite3 *db, const char *zDb){ if( rc ) return rc; while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int nIdxCol = 1; /* Number of columns in stat4 records */ + int nAvgCol = 1; /* Number of entries in Index.aAvgEq */ + char *zIndex; /* Index name */ Index *pIdx; /* Pointer to the index object */ int nSample; /* Number of samples */ @@ -1247,10 +1316,14 @@ static int loadStat4(sqlite3 *db, const char *zDb){ pIdx = sqlite3FindIndex(db, zIndex, zDb); if( pIdx==0 ) continue; assert( pIdx->nSample==0 ); + if( bStat3==0 ){ + nIdxCol = pIdx->nColumn+1; + nAvgCol = pIdx->nColumn; + } pIdx->nSample = nSample; nByte = sizeof(IndexSample) * nSample; - nByte += sizeof(tRowcnt) * (pIdx->nColumn+1) * 3 * nSample; - nByte += pIdx->nColumn * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ + nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; + nByte += nAvgCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ pIdx->aSample = sqlite3DbMallocZero(db, nByte); if( pIdx->aSample==0 ){ @@ -1258,19 +1331,18 @@ static int loadStat4(sqlite3 *db, const char *zDb){ return SQLITE_NOMEM; } pSpace = (tRowcnt*)&pIdx->aSample[nSample]; - pIdx->aAvgEq = pSpace; pSpace += pIdx->nColumn; + pIdx->aAvgEq = pSpace; pSpace += nAvgCol; for(i=0; inSample; i++){ - pIdx->aSample[i].anEq = pSpace; pSpace += pIdx->nColumn+1; - pIdx->aSample[i].anLt = pSpace; pSpace += pIdx->nColumn+1; - pIdx->aSample[i].anDLt = pSpace; pSpace += pIdx->nColumn+1; + pIdx->aSample[i].anEq = pSpace; pSpace += nIdxCol; + pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol; + pIdx->aSample[i].anDLt = pSpace; pSpace += nIdxCol; } assert( ((u8*)pSpace)-nByte==(u8*)(pIdx->aSample) ); } rc = sqlite3_finalize(pStmt); if( rc ) return rc; - zSql = sqlite3MPrintf(db, - "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", zDb); + zSql = sqlite3MPrintf(db, zSql2, zDb); if( !zSql ){ return SQLITE_NOMEM; } @@ -1279,10 +1351,10 @@ static int loadStat4(sqlite3 *db, const char *zDb){ if( rc ) return rc; while( sqlite3_step(pStmt)==SQLITE_ROW ){ - char *zIndex; /* Index name */ - Index *pIdx; /* Pointer to the index object */ - int i; /* Loop counter */ - int nCol; /* Number of columns in index */ + char *zIndex; /* Index name */ + Index *pIdx; /* Pointer to the index object */ + int i; /* Loop counter */ + int nCol = 1; /* Number of columns in index */ zIndex = (char *)sqlite3_column_text(pStmt, 0); if( zIndex==0 ) continue; @@ -1297,7 +1369,9 @@ static int loadStat4(sqlite3 *db, const char *zDb){ assert( idxnSample ); pSample = &pIdx->aSample[idx]; - nCol = pIdx->nColumn+1; + if( bStat3==0 ){ + nCol = pIdx->nColumn+1; + } decodeIntArray((char*)sqlite3_column_text(pStmt,1), nCol, pSample->anEq, 0); decodeIntArray((char*)sqlite3_column_text(pStmt,2), nCol, pSample->anLt, 0); decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); @@ -1327,6 +1401,7 @@ static int loadStat4(sqlite3 *db, const char *zDb){ } if( avgEq==0 ) avgEq = 1; pIdx->aAvgEq[iCol] = avgEq; + if( bStat3 ) break; } } @@ -1340,6 +1415,33 @@ static int loadStat4(sqlite3 *db, const char *zDb){ } return sqlite3_finalize(pStmt); } + +/* +** Load content from the sqlite_stat4 and sqlite_stat3 tables into +** the Index.aSample[] arrays of all indices. +*/ +static int loadStat4(sqlite3 *db, const char *zDb){ + int rc = SQLITE_OK; /* Result codes from subroutines */ + + assert( db->lookaside.bEnabled==0 ); + if( sqlite3FindTable(db, "sqlite_stat4", zDb) ){ + rc = loadStatTbl(db, 0, + "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", + "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", + zDb + ); + } + + if( rc==SQLITE_OK && sqlite3FindTable(db, "sqlite_stat3", zDb) ){ + rc = loadStatTbl(db, 1, + "SELECT idx,count(*) FROM %Q.sqlite_stat3 GROUP BY idx", + "SELECT idx,neq,nlt,ndlt,sqlite_record(sample) FROM %Q.sqlite_stat3", + zDb + ); + } + + return rc; +} #endif /* SQLITE_ENABLE_STAT4 */ /* diff --git a/src/func.c b/src/func.c index 49f6c892be..c6f70e18fa 100644 --- a/src/func.c +++ b/src/func.c @@ -1715,4 +1715,7 @@ void sqlite3RegisterGlobalFunctions(void){ #ifndef SQLITE_OMIT_ALTERTABLE sqlite3AlterFunctions(); #endif +#if defined(SQLITE_ENABLE_STAT3) || defined(SQLITE_ENABLE_STAT4) + sqlite3AnalyzeFunctions(); +#endif } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 19b0d5560d..2b8c1ad55e 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3055,6 +3055,7 @@ extern int sqlite3PendingByte; void sqlite3RootPageMoved(sqlite3*, int, int, int); void sqlite3Reindex(Parse*, Token*, Token*); void sqlite3AlterFunctions(void); +void sqlite3AnalyzeFunctions(void); void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); int sqlite3GetToken(const unsigned char *, int *); void sqlite3NestedParse(Parse*, const char*, ...); diff --git a/src/test_func.c b/src/test_func.c index f12e3c200a..cc4596d488 100644 --- a/src/test_func.c +++ b/src/test_func.c @@ -461,6 +461,54 @@ static void real2hex( sqlite3_result_text(context, zOut, -1, SQLITE_TRANSIENT); } +/* +** tclcmd: test_extract(record, field) +** +** This function implements an SQL user-function that accepts a blob +** containing a formatted database record as the first argument. The +** second argument is the index of the field within that record to +** extract and return. +*/ +static void test_extract( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + u8 *pRec; + u8 *pEndHdr; /* Points to one byte past record header */ + u8 *pHdr; /* Current point in record header */ + u8 *pBody; /* Current point in record data */ + u64 nHdr; /* Bytes in record header */ + int iIdx; /* Required field */ + int iCurrent = 0; /* Current field */ + + assert( argc==2 ); + pRec = (u8*)sqlite3_value_blob(argv[0]); + iIdx = sqlite3_value_int(argv[1]); + + pHdr = pRec + sqlite3GetVarint(pRec, &nHdr); + pBody = pEndHdr = &pRec[nHdr]; + + for(iCurrent=0; pHdr? AND b? AND c Date: Mon, 12 Aug 2013 17:00:08 +0000 Subject: [PATCH 23/67] Handle a NULL input to decodeIntArray() that can result from a prior OOM. FossilOrigin-Name: fa1588adab6759fd3d1be02524aa19a0d1c6adaa --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 3b395f8f6e..9d210df0ad 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Re-enable\sreading\sfrom\sthe\ssqlite_stat3\stable\s(as\swell\sas\ssqlite_stat4). -D 2013-08-12T16:34:32.514 +C Handle\sa\sNULL\sinput\sto\sdecodeIntArray()\sthat\scan\sresult\sfrom\sa\sprior\sOOM. +D 2013-08-12T17:00:08.943 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c e5ce42f04f58003fa45908791a5ad06bdd2c2ff1 +F src/analyze.c cfbf5b631aa15d122941eb2370589b2700a41c45 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ec3ffb174844406a6186c3dcc41b76d0331b502c -R e94619aadc884e84bf22674359c1bbe0 -U dan -Z c424468455b7fdc35c2aa42e6130568e +P 6d45078e621526fc2bac0eaefbb0f9602b9a8ec5 +R a5e2ac56df6a571538e82bfa8985f662 +U drh +Z ac0a7d3fec5c7e4a46461a76c725cc26 diff --git a/manifest.uuid b/manifest.uuid index 4b73be3aaa..2ca3618189 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6d45078e621526fc2bac0eaefbb0f9602b9a8ec5 \ No newline at end of file +fa1588adab6759fd3d1be02524aa19a0d1c6adaa \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index acdf1fde76..15ff54059f 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1122,7 +1122,8 @@ static void decodeIntArray( tRowcnt v; assert( pbUnordered==0 || *pbUnordered==0 ); - + + if( z==0 ) z = ""; for(i=0; *z && i='0' && c<='9' ){ From 86f69d98d209ab41e06d04813b6c6f3cdadc95a4 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Aug 2013 17:31:32 +0000 Subject: [PATCH 24/67] If there is data in both the sqlite_stat4 and sqlite_stat3 tables for a single index, ignore the sqlite_stat3 records. FossilOrigin-Name: 2a41736728d83a777ea8112da927cb047ec6684e --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 9 +++++++-- test/analyzeA.test | 40 ++++++++++++++++++++++++++++++++++------ 4 files changed, 50 insertions(+), 17 deletions(-) diff --git a/manifest b/manifest index 9d210df0ad..d7c9529c7f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Handle\sa\sNULL\sinput\sto\sdecodeIntArray()\sthat\scan\sresult\sfrom\sa\sprior\sOOM. -D 2013-08-12T17:00:08.943 +C If\sthere\sis\sdata\sin\sboth\sthe\ssqlite_stat4\sand\ssqlite_stat3\stables\sfor\sa\ssingle\sindex,\signore\sthe\ssqlite_stat3\srecords. +D 2013-08-12T17:31:32.368 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c cfbf5b631aa15d122941eb2370589b2700a41c45 +F src/analyze.c c020f2ff9991412d85d8c5c736097de82c38ea51 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -309,7 +309,7 @@ F test/analyze6.test 3c01e084309706a1033f850330ea24f6f7846297 F test/analyze7.test c0af22c5e0140e2e4ac556a21c2b6fff58229c98 F test/analyze8.test 8d1f76ff1e47c4093bb7be3971ba08fa56dc470d F test/analyze9.test 1b419d03407f2a6f4f1485620d54cb3e1bab3a71 -F test/analyzeA.test 949c3344280e0ca6de0b49805e4f291cdc1daa43 +F test/analyzeA.test 3a24600dd50e4a8815a952e1470ecb610046b069 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6d45078e621526fc2bac0eaefbb0f9602b9a8ec5 -R a5e2ac56df6a571538e82bfa8985f662 -U drh -Z ac0a7d3fec5c7e4a46461a76c725cc26 +P fa1588adab6759fd3d1be02524aa19a0d1c6adaa +R f7aa978164035b98da3c8ba701a7b74c +U dan +Z fe069e15f895d55c37e51ff5b63e6804 diff --git a/manifest.uuid b/manifest.uuid index 2ca3618189..3d11676d20 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fa1588adab6759fd3d1be02524aa19a0d1c6adaa \ No newline at end of file +2a41736728d83a777ea8112da927cb047ec6684e \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 15ff54059f..c631854b31 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1315,8 +1315,10 @@ static int loadStatTbl( if( zIndex==0 ) continue; nSample = sqlite3_column_int(pStmt, 1); pIdx = sqlite3FindIndex(db, zIndex, zDb); - if( pIdx==0 ) continue; - assert( pIdx->nSample==0 ); + assert( pIdx==0 || bStat3 || pIdx->nSample==0 ); + /* Index.nSample is non-zero at this point if data has already been + ** loaded from the stat4 table. In this case ignore stat3 data. */ + if( pIdx==0 || pIdx->nSample ) continue; if( bStat3==0 ){ nIdxCol = pIdx->nColumn+1; nAvgCol = pIdx->nColumn; @@ -1368,6 +1370,9 @@ static int loadStatTbl( idx = 0; } assert( idxnSample ); + /* This next condition is true if data has already been loaded from + ** the sqlite_stat4 table. In this case ignore stat3 data. */ + if( bStat3 && pIdx->aSample[idx].anEq[0] ) continue; pSample = &pIdx->aSample[idx]; if( bStat3==0 ){ diff --git a/test/analyzeA.test b/test/analyzeA.test index c1257a2aa9..47fd2f0dd6 100644 --- a/test/analyzeA.test +++ b/test/analyzeA.test @@ -23,7 +23,9 @@ ifcapable !stat4&&!stat3 { return } -proc populate_stat3 {} { +# Populate the stat3 table according to the current contents of the db +# +proc populate_stat3 {{bDropTable 1}} { # Open a second connection on database "test.db" and run ANALYZE. If this # is an ENABLE_STAT3 build, this is all that is required to create and # populate the sqlite_stat3 table. @@ -42,11 +44,11 @@ proc populate_stat3 {} { CREATE TABLE sqlite_stat3(tbl,idx,neq,nlt,ndlt,sample); INSERT INTO sqlite_stat3 SELECT DISTINCT tbl, idx, - lindex(neq,0), lindex(nlt,0), lindex(ndlt,0), test_extract(sample, 0) + lindex(neq,0), lindex(nlt,0), lindex(ndlt,0), test_extract(sample, 0) FROM sqlite_stat4; - DROP TABLE sqlite_stat4; - PRAGMA writable_schema = off; } db2 + if {$bDropTable} { execsql {DROP TABLE sqlite_stat4} db2 } + execsql { PRAGMA writable_schema = off } } # Modify the database schema cookie to ensure that the other connection @@ -59,7 +61,30 @@ proc populate_stat3 {} { db2 close } +# Populate the stat4 table according to the current contents of the db. +# Leave deceptive data in the stat3 table. This data should be ignored +# in favour of that from the stat4 table. +# +proc populate_both {} { + populate_stat3 0 + sqlite3 db2 test.db + execsql { + PRAGMA writable_schema = on; + UPDATE sqlite_stat3 SET idx = + CASE idx WHEN 't1b' THEN 't1c' ELSE 't1b' + END; + PRAGMA writable_schema = off; + CREATE TABLE obscure_tbl_nm(x); + DROP TABLE obscure_tbl_nm; + } db2 + + db2 close +} + + +# Populate the stat4 table according to the current contents of the db +# proc populate_stat4 {} { execsql { ANALYZE } # ifcapable stat3 { @@ -77,7 +102,11 @@ proc populate_stat4 {} { # } } -foreach {tn analyze_cmd} {1 populate_stat4 2 populate_stat3} { +foreach {tn analyze_cmd} { + 1 populate_stat4 + 2 populate_stat3 + 3 populate_both +} { reset_db do_test 1.$tn.1 { execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } @@ -110,7 +139,6 @@ foreach {tn analyze_cmd} {1 populate_stat4 2 populate_stat3} { do_execsql_test 1.$tn.3.2 { SELECT count(*) FROM t1 WHERE c BETWEEN 0 AND 50 } {90} - do_execsql_test 1.$tn.3.3 { SELECT count(*) FROM t1 WHERE b BETWEEN 75 AND 125 } {90} From 8ad169abb47ec64471fa7d3bdcb09b2f9080b965 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 12 Aug 2013 20:14:04 +0000 Subject: [PATCH 25/67] If ENABLE_STAT3 is defined but ENABLE_STAT4 is not, have ANALYZE create and populate the sqlite_stat3 table instead of sqlite_stat4. FossilOrigin-Name: cca8bf4372ab7a0258aa5c9397818415c6cf0abf --- manifest | 48 +++++----- manifest.uuid | 2 +- src/analyze.c | 192 ++++++++++++++++++--------------------- src/ctime.c | 4 +- src/sqliteInt.h | 3 +- src/test_config.c | 5 + src/vdbemem.c | 58 +++++++++++- src/where.c | 33 ++++--- test/analyze.test | 33 +++---- test/analyze3.test | 9 +- test/analyze5.test | 38 ++++++-- test/analyze6.test | 2 +- test/analyze7.test | 5 +- test/analyze8.test | 2 +- test/analyzeA.test | 53 ++++++----- test/auth.test | 6 +- test/dbstatus.test | 2 +- test/index6.test | 2 +- test/table.test | 1 + test/tkt-cbd054fa6b.test | 26 ++++-- test/where9.test | 7 +- 21 files changed, 320 insertions(+), 211 deletions(-) diff --git a/manifest b/manifest index d7c9529c7f..2b09ccada8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sthere\sis\sdata\sin\sboth\sthe\ssqlite_stat4\sand\ssqlite_stat3\stables\sfor\sa\ssingle\sindex,\signore\sthe\ssqlite_stat3\srecords. -D 2013-08-12T17:31:32.368 +C If\sENABLE_STAT3\sis\sdefined\sbut\sENABLE_STAT4\sis\snot,\shave\sANALYZE\screate\sand\spopulate\sthe\ssqlite_stat3\stable\sinstead\sof\ssqlite_stat4. +D 2013-08-12T20:14:04.167 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c c020f2ff9991412d85d8c5c736097de82c38ea51 +F src/analyze.c cbd13a1b3c7122729814d34245ce2b4007262f5e F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -169,7 +169,7 @@ F src/btreeInt.h eecc84f02375b2bb7a44abbcbbe3747dde73edb2 F src/build.c c2e4d057c833b616c6e32e690c29c03ba949b571 F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac -F src/ctime.c 177fa0cbf28b8deda3f216603beee0b883408a40 +F src/ctime.c ea4b7f3623a0fcb1146e7f245d7410033e86859c F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c 2317c814866d9aa71fea16b3faf4fdd4d6a49b94 F src/expr.c 0bbb44462a19169189b2709fbbd800950521b5ae @@ -221,7 +221,7 @@ F src/shell.c 128eb16ccec68509a4a2f1948f2483819bf63425 F src/sqlite.h.in bd1451ba1ab681022a53bccc3c39580ba094a3ff F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 646063fc1564842fd8e54eee00f8b8b429e2eb1f +F src/sqliteInt.h 35ee14455ae3cb21b166fea7777867a42c60d0d5 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -239,7 +239,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16 -F src/test_config.c 636ecd15a6ba18bf97a590b5a21f47573c8c2b65 +F src/test_config.c 3d148e338b575bd937f7746824f36a9c6682d238 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f @@ -283,14 +283,14 @@ F src/vdbeInt.h e9b7c6b165a31a4715c5aa97223d20d265515231 F src/vdbeapi.c 4d13580bd058b39623e8fcfc233b7df4b8191e8b F src/vdbeaux.c a6ea36a9dc714e1128a0173249a0532ddcab0489 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 -F src/vdbemem.c b16ba7b3d1ead79c081d1f79e157e6b2efd13ca5 +F src/vdbemem.c 7ec9a78d6d4b2d4ebb4d3fd9b706065146616019 F src/vdbesort.c 3937e06b2a0e354500e17dc206ef4c35770a5017 F src/vdbetrace.c e7ec40e1999ff3c6414424365d5941178966dcbc F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c c1090a2769c6e47b88aa79cd34f2e763af5282f8 +F src/where.c d97b5cb215c2e3e5b172b84000d2f3ee67ca0e86 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -301,15 +301,15 @@ F test/alter2.test 7ea05c7d92ac99349a802ef7ada17294dd647060 F test/alter3.test 49c9d9fba2b8fcdce2dedeca97bbf1f369cc548d F test/alter4.test 8e93bf7a7e6919b14b0c9a6c1e4908bcf21b0165 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc -F test/analyze.test 4d08a739c5ec28db93e0465e3b5a468befdf145f -F test/analyze3.test ea4cba3277eb89d16dfeada7259ea437e7b00f3b +F test/analyze.test 1772936d66471c65221e437b6d1999c3a03166c4 +F test/analyze3.test 412f690dfe95b337475e3e78a84a85d25f6f125d F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 -F test/analyze5.test 96ac783a56142bbbedb58a7c1eebd1808b49cfae -F test/analyze6.test 3c01e084309706a1033f850330ea24f6f7846297 -F test/analyze7.test c0af22c5e0140e2e4ac556a21c2b6fff58229c98 -F test/analyze8.test 8d1f76ff1e47c4093bb7be3971ba08fa56dc470d +F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 +F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab +F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f +F test/analyze8.test f3afd910c09f7d9b76f853e3f2f2fb169196d0b1 F test/analyze9.test 1b419d03407f2a6f4f1485620d54cb3e1bab3a71 -F test/analyzeA.test 3a24600dd50e4a8815a952e1470ecb610046b069 +F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7 @@ -321,7 +321,7 @@ F test/attach2.test e54436ed956d3d88bdee61221da59bf3935a0966 F test/attach3.test d89ccfe4fe6e2b5e368d480fcdfe4b496c54cf4e F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0 -F test/auth.test cb43e31c14b8f5849d04edd6d3a1f844687cc855 +F test/auth.test 9bea29041871807d9f289ee679d05d3ed103642f F test/auth2.test a2a371aa6df15f8b0c8109b33d3d7f0f73e4c9aa F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5 F test/autoinc.test bd30d372d00045252f6c2e41b5f41455e1975acf @@ -413,7 +413,7 @@ F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8 F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c F test/ctime.test 7bd009071e242aac4f18521581536b652b789a47 F test/date.test f3228180c87bbe5d39c9397bf001c0095c3821b9 -F test/dbstatus.test 1e64356a8c0407d7aeead201852fc4de9418196a +F test/dbstatus.test aee30c3f337e6c217ff06df59fb8fe6e6448dce2 F test/dbstatus2.test 10418e62b3db5dca070f0c3eef3ea13946f339c2 F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701 @@ -592,7 +592,7 @@ F test/index2.test ee83c6b5e3173a3d7137140d945d9a5d4fdfb9d6 F test/index3.test 423a25c789fc8cc51aaf2a4370bbdde2d9e9eed7 F test/index4.test 2983216eb8c86ee62d9ed7cb206b5cc3331c0026 F test/index5.test fc07c14193c0430814e7a08b5da46888ee795c33 -F test/index6.test 5be279e46e5f40e2dcc67d9b2aecdb5b0031db18 +F test/index6.test d84c03a7c16222b874877e8f4efb2bb7fed5d29e F test/indexedby.test 0e959308707c808515c3a51363f7a9835027108c F test/indexfault.test 31d4ab9a7d2f6e9616933eb079722362a883eb1d F test/init.test 15c823093fdabbf7b531fe22cf037134d09587a7 @@ -811,7 +811,7 @@ F test/superlock.test 1cde669f68d2dd37d6c9bd35eee1d95491ae3fc2 F test/sync.test a34cd43e98b7fb84eabbf38f7ed8f7349b3f3d85 F test/syscall.test a653783d985108c4912cc64d341ffbbb55ad2806 F test/sysfault.test fa776e60bf46bdd3ae69f0b73e46ee3977a58ae6 -F test/table.test 03135ef2fd49b2a21894be22afdb3c7de32cecd3 +F test/table.test 30423211108121884588d24d6776c7f38702ad7b F test/tableapi.test 2674633fa95d80da917571ebdd759a14d9819126 F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c @@ -866,7 +866,7 @@ F test/tkt-b72787b1.test a95e8cdad0b98af1853ac7f0afd4ab27b77bf5f3 F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898 F test/tkt-bdc6bbbb38.test fc38bb09bdd440e3513a1f5f98fc60a075182d7d F test/tkt-c48d99d690.test ba61977d62ab612fc515b3c488a6fbd6464a2447 -F test/tkt-cbd054fa6b.test 6595eac9c561b0aa46c1e4f4bae3f876acf38ce3 +F test/tkt-cbd054fa6b.test 06ccd57af3c0c7895d0f7dc844f13c51f8258885 F test/tkt-d11f09d36e.test d999b548fef885d1d1afa49a0e8544ecf436869d F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09 F test/tkt-d82e3f3721.test bcc0dfba658d15bab30fd4a9320c9e35d214ce30 @@ -1049,7 +1049,7 @@ F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5a4b0abc207d71da4deecd734ad8579e8dd40aa8 F test/where8.test 6f95896633cf2d307b5263145b942b7d33e837c6 F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 -F test/where9.test 71aa15cc17cb3343566a2de3aef47d4548610e4d +F test/where9.test 74245dea86592a744b758dff2e7daf0a07bade7d F test/whereA.test 24c234263c8fe358f079d5e57d884fb569d2da0a F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fa1588adab6759fd3d1be02524aa19a0d1c6adaa -R f7aa978164035b98da3c8ba701a7b74c +P 2a41736728d83a777ea8112da927cb047ec6684e +R ebabceacbf319f11921788de02fa2c3b U dan -Z fe069e15f895d55c37e51ff5b63e6804 +Z 6caae0dd3ca2077b483e5c3e88d20d9e diff --git a/manifest.uuid b/manifest.uuid index 3d11676d20..5f36a65cc7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2a41736728d83a777ea8112da927cb047ec6684e \ No newline at end of file +cca8bf4372ab7a0258aa5c9397818415c6cf0abf \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index c631854b31..be6b9586f1 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -140,6 +140,12 @@ #ifndef SQLITE_OMIT_ANALYZE #include "sqliteInt.h" +#ifdef SQLITE_ENABLE_STAT4 +# define IsStat3 0 +#else +# define IsStat3 1 +#endif + /* ** This routine generates code that opens the sqlite_stat1 table for ** writing with cursor iStatCur. If the library was built with the @@ -170,8 +176,10 @@ static void openStatTable( { "sqlite_stat1", "tbl,idx,stat" }, #if defined(SQLITE_ENABLE_STAT4) { "sqlite_stat4", "tbl,idx,neq,nlt,ndlt,sample" }, + { "sqlite_stat3", 0 }, #elif defined(SQLITE_ENABLE_STAT3) { "sqlite_stat3", "tbl,idx,neq,nlt,ndlt,sample" }, + { "sqlite_stat4", 0 }, #endif }; @@ -194,15 +202,17 @@ static void openStatTable( const char *zTab = aTable[i].zName; Table *pStat; if( (pStat = sqlite3FindTable(db, zTab, pDb->zName))==0 ){ - /* The sqlite_stat[12] table does not exist. Create it. Note that a - ** side-effect of the CREATE TABLE statement is to leave the rootpage - ** of the new table in register pParse->regRoot. This is important - ** because the OpenWrite opcode below will be needing it. */ - sqlite3NestedParse(pParse, - "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols - ); - aRoot[i] = pParse->regRoot; - aCreateTbl[i] = OPFLAG_P2ISREG; + if( aTable[i].zCols ){ + /* The sqlite_stat[12] table does not exist. Create it. Note that a + ** side-effect of the CREATE TABLE statement is to leave the rootpage + ** of the new table in register pParse->regRoot. This is important + ** because the OpenWrite opcode below will be needing it. */ + sqlite3NestedParse(pParse, + "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols + ); + aRoot[i] = pParse->regRoot; + aCreateTbl[i] = OPFLAG_P2ISREG; + } }else{ /* The table already exists. If zWhere is not NULL, delete all entries ** associated with the table zWhere. If zWhere is NULL, delete the @@ -214,14 +224,14 @@ static void openStatTable( "DELETE FROM %Q.%s WHERE %s=%Q", pDb->zName, zTab, zWhereType, zWhere ); }else{ - /* The sqlite_stat[12] table already exists. Delete all rows. */ + /* The sqlite_stat[134] table already exists. Delete all rows. */ sqlite3VdbeAddOp2(v, OP_Clear, aRoot[i], iDb); } } } /* Open the sqlite_stat[14] tables for writing. */ - for(i=0; i1 ); /* >1 because it includes the rowid column */ /* Allocate the space required for the Stat4Accum object */ @@ -370,14 +380,16 @@ static void stat4Push( u32 h; /* Hash value for this key */ int iMin = p->iMin; int i; - u8 isPSample = 0; + int nSampleCol; /* Number of fields in samples */ + u8 isPSample = 0; /* True if this is a periodic sample */ u8 doInsert = 0; sqlite3_value **aEq = &argv[3]; sqlite3_value **aLt = &argv[3+p->nCol]; sqlite3_value **aDLt = &argv[3+p->nCol+p->nCol]; - i64 nLt = sqlite3_value_int64(aLt[p->nCol-1]); + i64 nLt; + i64 nEq; UNUSED_PARAMETER(context); UNUSED_PARAMETER(argc); @@ -385,6 +397,18 @@ static void stat4Push( assert( argc==(3 + 3*p->nCol) ); assert( p->bHaveNonP==0 || p->bHaveP==0 ); + if( IsStat3 ){ + /* Stat3 builds ignore any call with bNewKey==0. And consider only + ** the first column of the index keys. */ + if( bNewKey==0 ) return; + nEq = sqlite3_value_int64(aEq[0]); + nSampleCol = 1; + }else{ + nEq = 1; + nSampleCol = p->nCol; + } + nLt = sqlite3_value_int64(aLt[nSampleCol-1]); + if( bNewKey ){ p->bHaveP = 0; p->bHaveNonP = 0; @@ -394,7 +418,7 @@ static void stat4Push( /* Check if this should be a periodic sample. If this is a periodic ** sample and there is already a non-periodic sample for this key, ** replace it. */ - if( (nLt/p->nPSample) != (nLt+1)/p->nPSample ){ + if( (nLt/p->nPSample) != (nLt+nEq)/p->nPSample ){ doInsert = isPSample = 1; if( p->bHaveNonP ){ p->nSample--; @@ -423,7 +447,7 @@ static void stat4Push( p->bHaveNonP = 1; }else{ tRowcnt *aMinEq = p->a[iMin].anEq; - for(i=p->nCol-2; i>=0; i--){ + for(i=(IsStat3 ? 0 : p->nCol-2); i>=0; i--){ i64 nEq = sqlite3_value_int64(aEq[i]); if( nEqaMinEq[i] ){ @@ -456,12 +480,12 @@ static void stat4Push( pSample->iRowid = rowid; pSample->iHash = h; pSample->isPSample = isPSample; - for(i=0; inCol; i++){ + for(i=0; ianEq[i] = sqlite3_value_int64(aEq[i]); pSample->anLt[i] = sqlite3_value_int64(aLt[i]); pSample->anDLt[i] = sqlite3_value_int64(aDLt[i])-1; assert( sqlite3_value_int64(aDLt[i])>0 ); - } + } /* Find the new minimum */ if( p->nSample==p->mxSample ){ @@ -472,7 +496,7 @@ static void stat4Push( iMin = i; }else{ int j; - for(j=p->nCol-1; j>=0; j++){ + for(j=nSampleCol-1; j>=0; j++){ i64 iCmp = (p->a[iMin].anEq[j] - p->a[i].anEq[j]); if( iCmp<0 ){ iMin = i; } if( iCmp ) break; @@ -533,19 +557,23 @@ static void stat4Get( default: aCnt = p->a[n].anDLt; break; } - zRet = sqlite3MallocZero(p->nCol * 25); - if( zRet==0 ){ - sqlite3_result_error_nomem(context); + if( IsStat3 ){ + sqlite3_result_int64(context, (i64)aCnt[0]); }else{ - int i; - char *z = zRet; - for(i=0; inCol; i++){ - sqlite3_snprintf(24, z, "%lld ", aCnt[i]); - z += sqlite3Strlen30(z); + zRet = sqlite3MallocZero(p->nCol * 25); + if( zRet==0 ){ + sqlite3_result_error_nomem(context); + }else{ + int i; + char *z = zRet; + for(i=0; inCol; i++){ + sqlite3_snprintf(24, z, "%lld ", aCnt[i]); + z += sqlite3Strlen30(z); + } + assert( z[0]=='\0' && z>zRet ); + z[-1] = '\0'; + sqlite3_result_text(context, zRet, -1, sqlite3_free); } - assert( z[0]=='\0' && z>zRet ); - z[-1] = '\0'; - sqlite3_result_text(context, zRet, -1, sqlite3_free); } } } @@ -591,7 +619,7 @@ static void analyzeOneTable( int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ int regStat1 = iMem++; /* The stat column of sqlite_stat1 */ -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) int regNumEq = regStat1; /* Number of instances. Same as regStat1 */ int regNumLt = iMem++; /* Number of keys less than regSample */ int regNumDLt = iMem++; /* Number of distinct keys less than regSample */ @@ -646,7 +674,6 @@ static void analyzeOneTable( int nCol; /* Number of columns indexed by pIdx */ KeyInfo *pKey; /* KeyInfo structure for pIdx */ int *aChngAddr; /* Array of jump instruction addresses */ - int regPrev; /* First in array of previous values */ int regDLte; /* First in array of nDlt registers */ int regLt; /* First in array of nLt registers */ @@ -688,6 +715,7 @@ static void analyzeOneTable( ** regEq(0) += 1 ** Next csr(0) ** }while ( csr(0)[0] == regPrev(0) ) + ** if( IsStat3 ) regKeychng = 1 ** ** next_1: ** regPrev(1) = csr(1)[1] @@ -699,7 +727,7 @@ static void analyzeOneTable( ** Next csr(1) ** }while ( csr(1)[0..1] == regPrev(0..1) ) ** - ** regKeychng = 1 + ** if( IsStat3==0 ) regKeychng = 1 ** next_row: ** regRowid = csr(2)[rowid] ** regEq(2) = 1 @@ -754,12 +782,15 @@ static void analyzeOneTable( VdbeComment((v, "%s", pIdx->zName)); } -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) /* Invoke the stat4_init() function. The arguments are: ** ** * the number of rows in the index, ** * the number of columns in the index including the rowid, ** * the recommended number of samples for the stat4 table. + ** + ** If this is a stat3 build, the number of columns in the index is + ** set to 1 (as this is the number of index fields gathered). */ sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+1); sqlite3VdbeAddOp2(v, OP_Integer, nCol+1, regStat4+2); @@ -830,6 +861,10 @@ static void analyzeOneTable( } sqlite3VdbeAddOp2(v, OP_Goto, 0, iDo); sqlite3VdbeResolveLabel(v, iNe); + + if( IsStat3 && i==0 ){ + sqlite3VdbeAddOp2(v, OP_Integer, 1, regKeychng); + } } /* This stuff: @@ -846,8 +881,10 @@ static void analyzeOneTable( ** Next csr(2) ** if( eof( csr(2) ) ) goto endOfScan */ -#ifdef SQLITE_ENABLE_STAT4 - sqlite3VdbeAddOp2(v, OP_Integer, 1, regKeychng); +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) + if( 0==IsStat3 ){ + sqlite3VdbeAddOp2(v, OP_Integer, 1, regKeychng); + } aChngAddr[nCol] = sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur+nCol, regRowid); sqlite3VdbeAddOp2(v, OP_Integer, 1, regEq+nCol); @@ -875,7 +912,7 @@ static void analyzeOneTable( sqlite3VdbeResolveLabel(v, endOfScan); -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) /* Add rows to the sqlite_stat4 table */ regLoop = regStat4+1; sqlite3VdbeAddOp2(v, OP_Integer, -1, regLoop); @@ -886,12 +923,17 @@ static void analyzeOneTable( sqlite3VdbeAddOp1(v, OP_IsNull, regEq+nCol); sqlite3VdbeAddOp3(v, OP_NotExists, iTabCur, shortJump, regEq+nCol); - for(i=0; iaiColumn[i]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regEq+i); + if( IsStat3==0 ){ + for(i=0; iaiColumn[i]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regEq+i); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regEq, nCol+1, regSample); + sqlite3VdbeChangeP4(v, -1, pIdx->zColAff, 0); + }else{ + int iCol = pIdx->aiColumn[0]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regSample); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regEq, nCol+1, regSample); - sqlite3VdbeChangeP4(v, -1, pIdx->zColAff, 0); sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regNumEq); sqlite3VdbeChangeP4(v, -1, (char*)&stat4GetFuncdef, P4_FUNCDEF); @@ -1189,7 +1231,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** and its contents. */ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) if( pIdx->aSample ){ int j; for(j=0; jnSample; j++){ @@ -1208,62 +1250,7 @@ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ #endif } -#ifdef SQLITE_ENABLE_STAT4 - -/* -** The implementation of the sqlite_record() function. This function accepts -** a single argument of any type. The return value is a formatted database -** record (a blob) containing the argument value. -** -** This is used to convert the value stored in the 'sample' column of the -** sqlite_stat3 table to the record format SQLite uses internally. -*/ -static void recordFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const int file_format = 1; - int iSerial; /* Serial type */ - int nSerial; /* Bytes of space for iSerial as varint */ - int nVal; /* Bytes of space required for argv[0] */ - int nRet; - sqlite3 *db; - u8 *aRet; - - iSerial = sqlite3VdbeSerialType(argv[0], file_format); - nSerial = sqlite3VarintLen(iSerial); - nVal = sqlite3VdbeSerialTypeLen(iSerial); - db = sqlite3_context_db_handle(context); - - nRet = 1 + nSerial + nVal; - aRet = sqlite3DbMallocRaw(db, nRet); - if( aRet==0 ){ - sqlite3_result_error_nomem(context); - }else{ - aRet[0] = nSerial+1; - sqlite3PutVarint(&aRet[1], iSerial); - sqlite3VdbeSerialPut(&aRet[1+nSerial], nVal, argv[0], file_format); - sqlite3_result_blob(context, aRet, nRet, SQLITE_TRANSIENT); - sqlite3DbFree(db, aRet); - } -} - -/* -** Register built-in functions used to help read ANALYZE data. -*/ -void sqlite3AnalyzeFunctions(void){ - static SQLITE_WSD FuncDef aAnalyzeTableFuncs[] = { - FUNCTION(sqlite_record, 1, 0, 0, recordFunc), - }; - int i; - FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); - FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAnalyzeTableFuncs); - for(i=0; inColumn+1; nAvgCol = pIdx->nColumn; } + pIdx->nSampleCol = nIdxCol; pIdx->nSample = nSample; nByte = sizeof(IndexSample) * nSample; nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; @@ -1484,7 +1472,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3DefaultRowEst(pIdx); -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; #endif @@ -1509,7 +1497,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat4 table. */ -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) if( rc==SQLITE_OK ){ int lookasideEnabled = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; diff --git a/src/ctime.c b/src/ctime.c index d69763a304..7c915d58c6 100644 --- a/src/ctime.c +++ b/src/ctime.c @@ -117,8 +117,10 @@ static const char * const azCompileOpt[] = { #ifdef SQLITE_ENABLE_RTREE "ENABLE_RTREE", #endif -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) "ENABLE_STAT4", +#elif defined(SQLITE_ENABLE_STAT3) + "ENABLE_STAT3", #endif #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY "ENABLE_UNLOCK_NOTIFY", diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 2b8c1ad55e..658ebc2941 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -1549,8 +1549,9 @@ struct Index { unsigned autoIndex:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ unsigned bUnordered:1; /* Use this index for == or IN queries only */ unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) int nSample; /* Number of elements in aSample[] */ + int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ #endif diff --git a/src/test_config.c b/src/test_config.c index c80fae295a..de1822e3b1 100644 --- a/src/test_config.c +++ b/src/test_config.c @@ -463,6 +463,11 @@ Tcl_SetVar2(interp, "sqlite_options", "mergesort", "1", TCL_GLOBAL_ONLY); #else Tcl_SetVar2(interp, "sqlite_options", "stat4", "0", TCL_GLOBAL_ONLY); #endif +#if defined(SQLITE_ENABLE_STAT3) && !defined(SQLITE_ENABLE_STAT4) + Tcl_SetVar2(interp, "sqlite_options", "stat3", "1", TCL_GLOBAL_ONLY); +#else + Tcl_SetVar2(interp, "sqlite_options", "stat3", "0", TCL_GLOBAL_ONLY); +#endif #if !defined(SQLITE_ENABLE_LOCKING_STYLE) # if defined(__APPLE__) diff --git a/src/vdbemem.c b/src/vdbemem.c index 8eef88ed3b..db3e6b262e 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1046,7 +1046,7 @@ int valueFromExpr( ** The ifdef here is to enable us to achieve 100% branch test coverage even ** when SQLITE_ENABLE_STAT4 is omitted. */ -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) if( op==TK_REGISTER ) op = pExpr->op2; #else if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; @@ -1152,7 +1152,61 @@ int sqlite3ValueFromExpr( return valueFromExpr(db, pExpr, enc, affinity, ppVal, valueNew, (void*)db); } -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +/* +** The implementation of the sqlite_record() function. This function accepts +** a single argument of any type. The return value is a formatted database +** record (a blob) containing the argument value. +** +** This is used to convert the value stored in the 'sample' column of the +** sqlite_stat3 table to the record format SQLite uses internally. +*/ +static void recordFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const int file_format = 1; + int iSerial; /* Serial type */ + int nSerial; /* Bytes of space for iSerial as varint */ + int nVal; /* Bytes of space required for argv[0] */ + int nRet; + sqlite3 *db; + u8 *aRet; + + iSerial = sqlite3VdbeSerialType(argv[0], file_format); + nSerial = sqlite3VarintLen(iSerial); + nVal = sqlite3VdbeSerialTypeLen(iSerial); + db = sqlite3_context_db_handle(context); + + nRet = 1 + nSerial + nVal; + aRet = sqlite3DbMallocRaw(db, nRet); + if( aRet==0 ){ + sqlite3_result_error_nomem(context); + }else{ + aRet[0] = nSerial+1; + sqlite3PutVarint(&aRet[1], iSerial); + sqlite3VdbeSerialPut(&aRet[1+nSerial], nVal, argv[0], file_format); + sqlite3_result_blob(context, aRet, nRet, SQLITE_TRANSIENT); + sqlite3DbFree(db, aRet); + } +} + +/* +** Register built-in functions used to help read ANALYZE data. +*/ +void sqlite3AnalyzeFunctions(void){ + static SQLITE_WSD FuncDef aAnalyzeTableFuncs[] = { + FUNCTION(sqlite_record, 1, 0, 0, recordFunc), + }; + int i; + FuncDefHash *pHash = &GLOBAL(FuncDefHash, sqlite3GlobalFunctions); + FuncDef *aFunc = (FuncDef*)&GLOBAL(FuncDef, aAnalyzeTableFuncs); + for(i=0; iNULL or x<=NULL term */ #else # define TERM_VNULL 0x00 /* Disabled if not using stat3 */ @@ -391,7 +391,7 @@ struct WhereLoopBuilder { ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) UnpackedRecord *pRec; /* Probe for stat4 (if required) */ int nRecValid; /* Number of valid fields currently in pRec */ #endif @@ -1789,7 +1789,7 @@ static void exprAnalyze( } #endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) /* When sqlite_stat3 histogram data is available an operator of the ** form "x IS NOT NULL" can sometimes be evaluated more efficiently ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a @@ -2397,7 +2397,7 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) /* ** Estimate the location of a particular key among all keys in an ** index. Store the results in aStat as follows: @@ -2422,7 +2422,7 @@ static void whereKeyStats( int res; /* Result of comparison operation */ assert( pIdx->nSample>0 ); - assert( pRec->nField>0 && iCol<=pIdx->nColumn ); + assert( pRec->nField>0 && iColnSampleCol ); do{ iTest = (iMin+i)/2; res = sqlite3VdbeRecordCompare(aSample[iTest].n, aSample[iTest].p, pRec); @@ -2533,11 +2533,12 @@ static int whereRangeScanEst( int rc = SQLITE_OK; int nOut = (int)*pnOut; -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) Index *p = pBuilder->pNew->u.btree.pIndex; int nEq = pBuilder->pNew->u.btree.nEq; - if( nEq==pBuilder->nRecValid + if( nEq==pBuilder->nRecValid + && nEqnSampleCol && p->nSample && OptimizationEnabled(pParse->db, SQLITE_Stat3) ){ @@ -2640,7 +2641,7 @@ static int whereRangeScanEst( return rc; } -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) /* ** Estimate the number of rows that will be returned based on ** an equality constraint x=VALUE and where that VALUE occurs in @@ -2706,7 +2707,7 @@ static int whereEqualScanEst( } #endif /* defined(SQLITE_ENABLE_STAT4) */ -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) /* ** Estimate the number of rows that will be returned based on ** an IN constraint where the right-hand side of the IN operator @@ -4296,7 +4297,7 @@ static int whereLoopAddBtreeIndex( rLogSize = estLog(whereCost(pProbe->aiRowEst[0])); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 0; -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) int nRecValid = pBuilder->nRecValid; assert( pNew->nOut==saved_nOut ); if( (pTerm->wtFlags & TERM_VNULL)!=0 && pSrc->pTab->aCol[iCol].notNull ){ @@ -4366,8 +4367,12 @@ static int whereLoopAddBtreeIndex( assert( pNew->nOut==saved_nOut ); whereRangeScanEst(pParse, pBuilder, pBtm, pTop, &pNew->nOut); } -#ifdef SQLITE_ENABLE_STAT4 - if( nInMul==0 && pProbe->nSample && OptimizationEnabled(db, SQLITE_Stat3) ){ +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) + if( nInMul==0 + && pProbe->nSample + && pNew->u.btree.nEq<=pProbe->nSampleCol + && OptimizationEnabled(db, SQLITE_Stat3) + ){ Expr *pExpr = pTerm->pExpr; tRowcnt nOut = 0; if( (pTerm->eOperator & (WO_EQ|WO_ISNULL))!=0 ){ @@ -4400,7 +4405,7 @@ static int whereLoopAddBtreeIndex( whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } pNew->nOut = saved_nOut; -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) pBuilder->nRecValid = nRecValid; #endif } @@ -4631,7 +4636,7 @@ static int whereLoopAddBtree( } rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) sqlite3Stat4ProbeFree(pBuilder->pRec); pBuilder->nRecValid = 0; pBuilder->pRec = 0; diff --git a/test/analyze.test b/test/analyze.test index 144834924a..c445084263 100644 --- a/test/analyze.test +++ b/test/analyze.test @@ -288,7 +288,7 @@ do_test analyze-4.3 { } {} # Verify that DROP TABLE and DROP INDEX remove entries from the -# sqlite_stat1 and sqlite_stat4 tables. +# sqlite_stat1, sqlite_stat3 and sqlite_stat4 tables. # do_test analyze-5.0 { execsql { @@ -306,12 +306,13 @@ do_test analyze-5.0 { SELECT DISTINCT tbl FROM sqlite_stat1 ORDER BY 1; } } {t3i1 t3i2 t3i3 t4i1 t4i2 t3 t4} -ifcapable stat4 { +ifcapable stat4||stat3 { + ifcapable stat4 {set stat sqlite_stat4} else {set stat sqlite_stat3} do_test analyze-5.1 { - execsql { - SELECT DISTINCT idx FROM sqlite_stat4 ORDER BY 1; - SELECT DISTINCT tbl FROM sqlite_stat4 ORDER BY 1; - } + execsql " + SELECT DISTINCT idx FROM $stat ORDER BY 1; + SELECT DISTINCT tbl FROM $stat ORDER BY 1; + " } {t3i1 t3i2 t3i3 t4i1 t4i2 t3 t4} } do_test analyze-5.2 { @@ -321,12 +322,12 @@ do_test analyze-5.2 { SELECT DISTINCT tbl FROM sqlite_stat1 ORDER BY 1; } } {t3i1 t3i3 t4i1 t4i2 t3 t4} -ifcapable stat4 { +ifcapable stat4||stat3 { do_test analyze-5.3 { - execsql { - SELECT DISTINCT idx FROM sqlite_stat4 ORDER BY 1; - SELECT DISTINCT tbl FROM sqlite_stat4 ORDER BY 1; - } + execsql " + SELECT DISTINCT idx FROM $stat ORDER BY 1; + SELECT DISTINCT tbl FROM $stat ORDER BY 1; + " } {t3i1 t3i3 t4i1 t4i2 t3 t4} } do_test analyze-5.4 { @@ -336,12 +337,12 @@ do_test analyze-5.4 { SELECT DISTINCT tbl FROM sqlite_stat1 ORDER BY 1; } } {t4i1 t4i2 t4} -ifcapable stat4 { +ifcapable stat4||stat3 { do_test analyze-5.5 { - execsql { - SELECT DISTINCT idx FROM sqlite_stat4 ORDER BY 1; - SELECT DISTINCT tbl FROM sqlite_stat4 ORDER BY 1; - } + execsql " + SELECT DISTINCT idx FROM $stat ORDER BY 1; + SELECT DISTINCT tbl FROM $stat ORDER BY 1; + " } {t4i1 t4i2 t4} } diff --git a/test/analyze3.test b/test/analyze3.test index 394d321b97..fb26303ee3 100644 --- a/test/analyze3.test +++ b/test/analyze3.test @@ -17,7 +17,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !stat4 { +ifcapable !stat4&&!stat3 { finish_test return } @@ -95,8 +95,11 @@ do_test analyze3-1.1.1 { COMMIT; ANALYZE; } - execsql { - SELECT count(*)>0 FROM sqlite_stat4; + + ifcapable stat4 { + execsql { SELECT count(*)>0 FROM sqlite_stat4; } + } else { + execsql { SELECT count(*)>0 FROM sqlite_stat3; } } } {1} diff --git a/test/analyze5.test b/test/analyze5.test index 048e6a5ef7..ac175c07b5 100644 --- a/test/analyze5.test +++ b/test/analyze5.test @@ -17,7 +17,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !stat4 { +ifcapable !stat4&&!stat3 { finish_test return } @@ -66,20 +66,40 @@ do_test analyze5-1.0 { CREATE INDEX t1y ON t1(y); -- integers 0 and very few 1s CREATE INDEX t1z ON t1(z); -- integers 0, 1, 2, and 3 ANALYZE; - SELECT DISTINCT lindex(test_decode(sample),0) - FROM sqlite_stat4 WHERE idx='t1u' ORDER BY nlt; + } + ifcapable stat4 { + db eval { + SELECT DISTINCT lindex(test_decode(sample),0) + FROM sqlite_stat4 WHERE idx='t1u' ORDER BY nlt; + } + } else { + db eval { + SELECT sample FROM sqlite_stat3 WHERE idx='t1u' ORDER BY nlt; + } } } {alpha bravo charlie delta} do_test analyze5-1.1 { - db eval { - SELECT DISTINCT lower(lindex(test_decode(sample), 0)) - FROM sqlite_stat4 WHERE idx='t1v' ORDER BY 1 + ifcapable stat4 { + db eval { + SELECT DISTINCT lower(lindex(test_decode(sample), 0)) + FROM sqlite_stat4 WHERE idx='t1v' ORDER BY 1 + } + } else { + db eval { + SELECT lower(sample) FROM sqlite_stat3 WHERE idx='t1v' ORDER BY 1 + } } } {alpha bravo charlie delta} -do_test analyze5-1.2 { - db eval {SELECT idx, count(*) FROM sqlite_stat4 GROUP BY 1 ORDER BY 1} -} {t1t 8 t1u 8 t1v 8 t1w 8 t1x 8 t1y 9 t1z 8} +ifcapable stat4 { + do_test analyze5-1.2 { + db eval {SELECT idx, count(*) FROM sqlite_stat4 GROUP BY 1 ORDER BY 1} + } {t1t 8 t1u 8 t1v 8 t1w 8 t1x 8 t1y 9 t1z 8} +} else { + do_test analyze5-1.2 { + db eval {SELECT idx, count(*) FROM sqlite_stat3 GROUP BY 1 ORDER BY 1} + } {t1t 4 t1u 4 t1v 4 t1w 4 t1x 4 t1y 2 t1z 4} +} # Verify that range queries generate the correct row count estimates # diff --git a/test/analyze6.test b/test/analyze6.test index 7c8e653b40..d2c96518be 100644 --- a/test/analyze6.test +++ b/test/analyze6.test @@ -17,7 +17,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !stat4 { +ifcapable !stat4&&!stat3 { finish_test return } diff --git a/test/analyze7.test b/test/analyze7.test index 07d6ad52a0..76664546a5 100644 --- a/test/analyze7.test +++ b/test/analyze7.test @@ -82,7 +82,7 @@ do_test analyze7-3.1 { do_test analyze7-3.2.1 { execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=?;} } {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=?)}} -ifcapable stat4 { +ifcapable stat4||stat3 { # If ENABLE_STAT4 is defined, SQLite comes up with a different estimated # row count for (c=2) than it does for (c=?). do_test analyze7-3.2.2 { @@ -98,7 +98,8 @@ ifcapable stat4 { do_test analyze7-3.3 { execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=123 AND b=123} } {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?)}} -ifcapable {!stat4} { + +ifcapable {!stat4 && !stat3} { do_test analyze7-3.4 { execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=123 AND b=123} } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?)}} diff --git a/test/analyze8.test b/test/analyze8.test index 3a00e69e53..b46d38651f 100644 --- a/test/analyze8.test +++ b/test/analyze8.test @@ -16,7 +16,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -ifcapable !stat4 { +ifcapable !stat4&&!stat3 { finish_test return } diff --git a/test/analyzeA.test b/test/analyzeA.test index 47fd2f0dd6..d9ca2c0f3b 100644 --- a/test/analyzeA.test +++ b/test/analyzeA.test @@ -61,12 +61,41 @@ proc populate_stat3 {{bDropTable 1}} { db2 close } +# Populate the stat4 table according to the current contents of the db +# +proc populate_stat4 {{bDropTable 1}} { + sqlite3 db2 test.db + execsql { ANALYZE } + + ifcapable stat3 { + execsql { + PRAGMA writable_schema = on; + CREATE TABLE sqlite_stat4(tbl,idx,neq,nlt,ndlt,sample); + INSERT INTO sqlite_stat4 + SELECT tbl, idx, neq, nlt, ndlt, sqlite_record(sample) + FROM sqlite_stat3; + } db2 + if {$bDropTable} { execsql {DROP TABLE sqlite_stat3} db2 } + execsql { PRAGMA writable_schema = off } + } + + # Modify the database schema cookie to ensure that the other connection + # reloads the schema. + # + execsql { + CREATE TABLE obscure_tbl_nm(x); + DROP TABLE obscure_tbl_nm; + } db2 + db2 close +} + # Populate the stat4 table according to the current contents of the db. # Leave deceptive data in the stat3 table. This data should be ignored # in favour of that from the stat4 table. # proc populate_both {} { - populate_stat3 0 + ifcapable stat4 { populate_stat3 0 } + ifcapable stat3 { populate_stat4 0 } sqlite3 db2 test.db execsql { @@ -78,30 +107,9 @@ proc populate_both {} { CREATE TABLE obscure_tbl_nm(x); DROP TABLE obscure_tbl_nm; } db2 - db2 close } - -# Populate the stat4 table according to the current contents of the db -# -proc populate_stat4 {} { - execsql { ANALYZE } -# ifcapable stat3 { -# execsql { -# PRAGMA writable_schema = on; -# CREATE TABLE sqlite_stat4(tbl,idx,neq,nlt,ndlt,sample); -# INSERT INTO sqlite_stat4 SELECT -# tbl, idx, nlt, neq, ndlt, -# test_extract(sample, 1) -# FROM sqlite_stat4; -# DROP TABLE sqlite_stat4; -# PRAGMA writable_schema = off; -# ANALYZE sqlite_master; -# } -# } -} - foreach {tn analyze_cmd} { 1 populate_stat4 2 populate_stat3 @@ -155,6 +163,5 @@ foreach {tn analyze_cmd} { } {0 0 0 {SEARCH TABLE t1 USING INDEX t1c (c>? AND cNULL, which can be satified by the index t1b. It is a very # expensive way to do the query, but it works, and so a solution is possible. @@ -851,6 +851,11 @@ do_test where9-7.0 { INSERT INTO t6 SELECT * FROM t5; ANALYZE t5; } + ifcapable stat3 { + sqlite3 db2 test.db + db2 eval { DROP TABLE IF EXISTS sqlite_stat3 } + db2 close + } } {} do_test where9-7.1.1 { count_steps { From f00e902580c1a0cb60395be63f50cc9575a857f5 Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 14 Aug 2013 19:54:12 +0000 Subject: [PATCH 26/67] Change the way ANALYZE works to use a single cursor when scanning indices. FossilOrigin-Name: bdce612b35193abf72de1a563ea7962375b3574e --- manifest | 14 +- manifest.uuid | 2 +- src/analyze.c | 1041 ++++++++++++++++++++++---------------------- test/analyze8.test | 11 +- 4 files changed, 539 insertions(+), 529 deletions(-) diff --git a/manifest b/manifest index 2b09ccada8..35822cf8b7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C If\sENABLE_STAT3\sis\sdefined\sbut\sENABLE_STAT4\sis\snot,\shave\sANALYZE\screate\sand\spopulate\sthe\ssqlite_stat3\stable\sinstead\sof\ssqlite_stat4. -D 2013-08-12T20:14:04.167 +C Change\sthe\sway\sANALYZE\sworks\sto\suse\sa\ssingle\scursor\swhen\sscanning\sindices. +D 2013-08-14T19:54:12.120 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c cbd13a1b3c7122729814d34245ce2b4007262f5e +F src/analyze.c 37b78257fcf73790fd443b2c62b7ce62b78c3c98 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -307,7 +307,7 @@ F test/analyze4.test eff2df19b8dd84529966420f29ea52edc6b56213 F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f -F test/analyze8.test f3afd910c09f7d9b76f853e3f2f2fb169196d0b1 +F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 F test/analyze9.test 1b419d03407f2a6f4f1485620d54cb3e1bab3a71 F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2a41736728d83a777ea8112da927cb047ec6684e -R ebabceacbf319f11921788de02fa2c3b +P cca8bf4372ab7a0258aa5c9397818415c6cf0abf +R 798b6e107d9cfc4282515cb67b7b0a49 U dan -Z 6caae0dd3ca2077b483e5c3e88d20d9e +Z 0758340c46c18a215a897e2672d1ddd5 diff --git a/manifest.uuid b/manifest.uuid index 5f36a65cc7..c82b437d55 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -cca8bf4372ab7a0258aa5c9397818415c6cf0abf \ No newline at end of file +bdce612b35193abf72de1a563ea7962375b3574e \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index be6b9586f1..d8edadf885 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -140,10 +140,15 @@ #ifndef SQLITE_OMIT_ANALYZE #include "sqliteInt.h" -#ifdef SQLITE_ENABLE_STAT4 +#if defined(SQLITE_ENABLE_STAT4) +# define IsStat4 1 # define IsStat3 0 -#else +#elif defined(SQLITE_ENABLE_STAT3) +# define IsStat4 0 # define IsStat3 1 +#else +# define IsStat4 0 +# define IsStat3 0 #endif /* @@ -230,11 +235,12 @@ static void openStatTable( } } - /* Open the sqlite_stat[14] tables for writing. */ + /* Open the sqlite_stat[134] tables for writing. */ for(i=0; i1 ); /* >1 because it includes the rowid column */ /* Allocate the space required for the Stat4Accum object */ - n = sizeof(*p) + (sizeof(p->a[0]) + 3*sizeof(tRowcnt)*nCol)*mxSample; - p = sqlite3MallocZero( n ); + n = sizeof(*p) + + sizeof(tRowcnt)*nCol /* Stat4Accum.anEq */ + + sizeof(tRowcnt)*nCol /* Stat4Accum.anLt */ + + sizeof(tRowcnt)*nCol /* Stat4Accum.anDLt */ + + sizeof(Stat4Sample)*(nCol+mxSample) /* Stat4Accum.aBest[], a[] */ + + sizeof(tRowcnt)*3*nCol*(nCol+mxSample); + p = sqlite3MallocZero(n); if( p==0 ){ sqlite3_result_error_nomem(context); return; } - /* Populate the new Stat4Accum object */ p->nRow = nRow; p->nCol = nCol; p->mxSample = mxSample; p->nPSample = p->nRow/(mxSample/3+1) + 1; + p->iGet = -1; + + p->current.anDLt = (tRowcnt*)&p[1]; + p->current.anEq = &p->current.anDLt[nCol]; + p->current.anLt = &p->current.anEq[nCol]; sqlite3_randomness(sizeof(p->iPrn), &p->iPrn); - p->a = (struct Stat4Sample*)&p[1]; - pSpace = (u8*)(&p->a[mxSample]); - for(i=0; ia = (struct Stat4Sample*)&p->current.anLt[nCol]; + p->aBest = &p->a[mxSample]; + pSpace = (u8*)(&p->a[mxSample+nCol]); + for(i=0; i<(mxSample+nCol); i++){ p->a[i].anEq = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nCol); p->a[i].anLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nCol); p->a[i].anDLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nCol); } assert( (pSpace - (u8*)p)==n ); + for(i=0; iaBest[i].iCol = i; + } + /* Return a pointer to the allocated object to the caller */ sqlite3_result_blob(context, p, sizeof(p), sqlite3_free); } -static const FuncDef stat4InitFuncdef = { - 3, /* nArg */ - SQLITE_UTF8, /* iPrefEnc */ - 0, /* flags */ - 0, /* pUserData */ - 0, /* pNext */ - stat4Init, /* xFunc */ - 0, /* xStep */ - 0, /* xFinalize */ - "stat4_init", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ +static const FuncDef statInitFuncdef = { + 3, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + statInit, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat_init", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ }; +/* +** Return true if pNew is to be preferred over pOld. +*/ +static int sampleIsBetter(Stat4Sample *pNew, Stat4Sample *pOld){ + tRowcnt nEqNew = pNew->anEq[pNew->iCol]; + tRowcnt nEqOld = pOld->anEq[pOld->iCol]; + + assert( pOld->isPSample==0 && pNew->isPSample==0 ); + assert( IsStat4 || (pNew->iCol==0 && pOld->iCol==0) ); + + if( (nEqNew>nEqOld) + || (nEqNew==nEqOld && pNew->iColiCol) + || (nEqNew==nEqOld && pNew->iCol==pOld->iCol && pNew->iHash>pOld->iHash) + ){ + return 1; + } + return 0; +} /* -** Implementation of the stat4_push SQL function. The arguments describe a -** single key instance. This routine makes the decision about whether or -** not to retain this key for the sqlite_stat4 table. -** -** The calling convention is: -** -** stat4_push(P, rowid, ...nEq args..., ...nLt args..., ...nDLt args...) -** -** where each instance of the "...nXX args..." is replaced by an array of -** nCol arguments, where nCol is the number of columns in the index being -** sampled (if the index being sampled is "CREATE INDEX i ON t(a, b)", a -** total of 8 arguments are passed when this function is invoked). -** -** The return value is always NULL. +** Copy the contents of object (*pFrom) into (*pTo). */ -static void stat4Push( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); - i64 rowid = sqlite3_value_int64(argv[1]); - int bNewKey = sqlite3_value_int(argv[2]); - struct Stat4Sample *pSample; - u32 h; /* Hash value for this key */ - int iMin = p->iMin; +void sampleCopy(Stat4Accum *p, Stat4Sample *pTo, Stat4Sample *pFrom){ + pTo->iRowid = pFrom->iRowid; + pTo->isPSample = pFrom->isPSample; + pTo->iCol = pFrom->iCol; + pTo->iHash = pFrom->iHash; + memcpy(pTo->anEq, pFrom->anEq, sizeof(tRowcnt)*p->nCol); + memcpy(pTo->anLt, pFrom->anLt, sizeof(tRowcnt)*p->nCol); + memcpy(pTo->anDLt, pFrom->anDLt, sizeof(tRowcnt)*p->nCol); +} + +/* +** Copy the contents of sample *pNew into the p->a[] array. If necessary, +** remove the least desirable sample from p->a[] to make room. +*/ +static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ + Stat4Sample *pSample; int i; - int nSampleCol; /* Number of fields in samples */ - u8 isPSample = 0; /* True if this is a periodic sample */ - u8 doInsert = 0; + i64 iSeq; + i64 iPos; - sqlite3_value **aEq = &argv[3]; - sqlite3_value **aLt = &argv[3+p->nCol]; - sqlite3_value **aDLt = &argv[3+p->nCol+p->nCol]; + assert( IsStat4 || nEqZero==0 ); - i64 nLt; - i64 nEq; + if( pNew->isPSample==0 ){ + assert( pNew->anEq[pNew->iCol]>0 ); - UNUSED_PARAMETER(context); - UNUSED_PARAMETER(argc); - assert( p->nCol>0 ); - assert( argc==(3 + 3*p->nCol) ); - assert( p->bHaveNonP==0 || p->bHaveP==0 ); - - if( IsStat3 ){ - /* Stat3 builds ignore any call with bNewKey==0. And consider only - ** the first column of the index keys. */ - if( bNewKey==0 ) return; - nEq = sqlite3_value_int64(aEq[0]); - nSampleCol = 1; - }else{ - nEq = 1; - nSampleCol = p->nCol; - } - nLt = sqlite3_value_int64(aLt[nSampleCol-1]); - - if( bNewKey ){ - p->bHaveP = 0; - p->bHaveNonP = 0; - } - h = p->iPrn = p->iPrn*1103515245 + 12345; - - /* Check if this should be a periodic sample. If this is a periodic - ** sample and there is already a non-periodic sample for this key, - ** replace it. */ - if( (nLt/p->nPSample) != (nLt+nEq)/p->nPSample ){ - doInsert = isPSample = 1; - if( p->bHaveNonP ){ - p->nSample--; - p->bHaveNonP = 0; - p->bHaveP = 1; - assert( p->nSamplemxSample ); - assert( p->a[p->nSample].isPSample==0 ); - } - - /* Or, if this is not a periodic sample, and there is already at least one - ** periodic sample, return early. */ - }else if( p->bHaveP ){ - /* no-op */ - - /* If there is already a non-periodic sample for the key, but this one - ** has a higher hash score, replace the existing sample. */ - }else if( p->bHaveNonP ){ - if( p->a[p->nSample-1].iHashnSample--; - doInsert = 1; - } - - /* Finally, check if this should be added as a non-periodic sample. */ - }else if( p->nSamplemxSample ){ - doInsert = 1; - p->bHaveNonP = 1; - }else{ - tRowcnt *aMinEq = p->a[iMin].anEq; - for(i=(IsStat3 ? 0 : p->nCol-2); i>=0; i--){ - i64 nEq = sqlite3_value_int64(aEq[i]); - if( nEqaMinEq[i] ){ - doInsert = 1; - break; + /* This sample is being added because the prefix that ends in column + ** iCol occurs many times in the table. However, if we have already + ** added a sample that shares this prefix, there is no need to add + ** this one. Instead, upgrade the priority of the existing sample. */ + for(i=p->nSample-1; i>=0; i--){ + Stat4Sample *pOld = &p->a[i]; + if( pOld->anEq[pNew->iCol]==0 ){ + if( pOld->isPSample==0 ){ + assert( sampleIsBetter(pNew, pOld) ); + assert( pOld->iCol>pNew->iCol ); + pOld->iCol = pNew->iCol; + } + goto find_new_min; } } - if( i<0 && h>p->a[iMin].iHash ){ - doInsert = 1; - } - p->bHaveNonP = doInsert; } - if( doInsert==0 ) return; - /* Fill in the new Stat4Sample object. */ - if( p->nSample==p->mxSample ){ - struct Stat4Sample *pMin = &p->a[iMin]; + /* If necessary, remove sample iMin to make room for the new sample. */ + if( p->nSample>=p->mxSample ){ + Stat4Sample *pMin = &p->a[p->iMin]; tRowcnt *anEq = pMin->anEq; - tRowcnt *anDLt = pMin->anDLt; tRowcnt *anLt = pMin->anLt; - assert( p->nSample - iMin - 1 >= 0 ); - memmove(pMin, &pMin[1], sizeof(p->a[0])*(p->nSample-iMin-1)); + tRowcnt *anDLt = pMin->anDLt; + memmove(pMin, &pMin[1], sizeof(p->a[0])*(p->nSample-p->iMin-1)); pSample = &p->a[p->nSample-1]; pSample->anEq = anEq; pSample->anDLt = anDLt; pSample->anLt = anLt; - }else{ - pSample = &p->a[p->nSample++]; - } - pSample->iRowid = rowid; - pSample->iHash = h; - pSample->isPSample = isPSample; - for(i=0; ianEq[i] = sqlite3_value_int64(aEq[i]); - pSample->anLt[i] = sqlite3_value_int64(aLt[i]); - pSample->anDLt[i] = sqlite3_value_int64(aDLt[i])-1; - assert( sqlite3_value_int64(aDLt[i])>0 ); + p->nSample = p->mxSample-1; } - /* Find the new minimum */ - if( p->nSample==p->mxSample ){ - iMin = -1; + /* Figure out where in the a[] array the new sample should be inserted. */ + iSeq = pNew->anLt[p->nCol-1]; + for(iPos=p->nSample; iPos>0; iPos--){ + if( iSeq>p->a[iPos-1].anLt[p->nCol-1] ) break; + } + + /* Insert the new sample */ + pSample = &p->a[iPos]; + if( iPos!=p->nSample ){ + Stat4Sample *pEnd = &p->a[p->nSample]; + tRowcnt *anEq = pEnd->anEq; + tRowcnt *anLt = pEnd->anLt; + tRowcnt *anDLt = pEnd->anDLt; + memmove(&p->a[iPos], &p->a[iPos+1], (p->nSample-iPos)*sizeof(p->a[0])); + pSample->anEq = anEq; + pSample->anDLt = anDLt; + pSample->anLt = anLt; + } + p->nSample++; + sampleCopy(p, pSample, pNew); + + /* Zero the first nEqZero entries in the anEq[] array. */ + memset(pSample->anEq, 0, sizeof(tRowcnt)*nEqZero); + + find_new_min: + if( p->nSample>=p->mxSample ){ + int iMin = -1; for(i=0; imxSample; i++){ if( p->a[i].isPSample ) continue; - if( iMin<0 ){ + if( iMin<0 || sampleIsBetter(&p->a[iMin], &p->a[i]) ){ iMin = i; - }else{ - int j; - for(j=nSampleCol-1; j>=0; j++){ - i64 iCmp = (p->a[iMin].anEq[j] - p->a[i].anEq[j]); - if( iCmp<0 ){ iMin = i; } - if( iCmp ) break; - } - if( j==0 && p->a[iMin].iHasha[i].iHash ){ - iMin = i; - } } } assert( iMin>=0 ); p->iMin = iMin; } } -static const FuncDef stat4PushFuncdef = { - -1, /* nArg */ - SQLITE_UTF8, /* iPrefEnc */ - 0, /* flags */ - 0, /* pUserData */ - 0, /* pNext */ - stat4Push, /* xFunc */ - 0, /* xStep */ - 0, /* xFinalize */ - "stat4_push", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ + +/* +** Field iChng of the index being scanned has changed. So at this point +** p->current contains a sample that reflects the previous row of the +** index. The value of anEq[iChng] and subsequent anEq[] elements are +** correct at this point. +*/ +static void samplePushPrevious(Stat4Accum *p, int iChng){ + if( IsStat4 ){ + int i; + + /* Check if any samples from the aBest[] array should be pushed + ** into IndexSample.a[] at this point. */ + for(i=(p->nCol-2); i>=iChng; i--){ + Stat4Sample *pBest = &p->aBest[i]; + if( p->nSamplemxSample + || sampleIsBetter(pBest, &p->a[p->iMin]) + ){ + sampleInsert(p, pBest, i); + } + } + + /* Update the anEq[] fields of any samples already collected. */ + for(i=p->nSample-1; i>=0; i--){ + int j; + for(j=iChng; jnCol; j++){ + if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + } + } + } + + if( IsStat3 && iChng==0 ){ + tRowcnt nLt = p->current.anLt[0]; + tRowcnt nEq = p->current.anEq[0]; + + /* Check if this is to be a periodic sample. If so, add it. */ + if( (nLt/p->nPSample)!=(nLt+nEq)/p->nPSample ){ + p->current.isPSample = 1; + sampleInsert(p, &p->current, 0); + p->current.isPSample = 0; + }else + + /* Or if it is a non-periodic sample. Add it in this case too. */ + if( p->nSamplemxSample || sampleIsBetter(&p->current, &p->a[p->iMin]) ){ + sampleInsert(p, &p->current, 0); + } + } +} + +/* +** Implementation of the stat_push SQL function. +** +** stat_push(P,R,C) +** +** The return value is always NULL. +*/ +static void statPush( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + + /* The three function arguments */ + Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); + i64 rowid = sqlite3_value_int64(argv[1]); + int iChng = sqlite3_value_int(argv[2]); + + assert( p->nCol>1 ); /* Includes rowid field */ + assert( iChngnCol ); + + /* p->current.anEq[0] is false the first time this function is called. */ + if( p->current.anEq[0] ){ + + samplePushPrevious(p, iChng); + + /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply + ** to the current row of the index. */ + for(i=0; icurrent.anEq[i]++; + } + for(i=iChng; inCol; i++){ + p->current.anDLt[i]++; + p->current.anLt[i] += p->current.anEq[i]; + p->current.anEq[i] = 1; + } + + }else{ + for(i=0; inCol; i++) p->current.anEq[i] = 1; + } + + if( IsStat4 || IsStat3 ){ + p->current.iRowid = rowid; + p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; + } + + if( IsStat4 ){ + tRowcnt nLt = p->current.anLt[p->nCol-1]; + + /* Check if this is to be a periodic sample. If so, add it. */ + if( (nLt/p->nPSample)!=(nLt+1)/p->nPSample ){ + p->current.isPSample = 1; + p->current.iCol = 0; + sampleInsert(p, &p->current, p->nCol-1); + p->current.isPSample = 0; + } + + /* Update the aBest[] array. */ + for(i=0; i<(p->nCol-1); i++){ + p->current.iCol = i; + if( i>=iChng || sampleIsBetter(&p->current, &p->aBest[i]) ){ + sampleCopy(p, &p->aBest[i], &p->current); + } + } + } +} +static const FuncDef statPushFuncdef = { + 3, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + statPush, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat_push", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ }; +#define STAT_GET_STAT1 0 /* "stat" column of stat1 table */ +#define STAT_GET_ROWID 1 /* "rowid" column of stat[34] entry */ +#define STAT_GET_NEQ 2 /* "neq" column of stat[34] entry */ +#define STAT_GET_NLT 3 /* "nlt" column of stat[34] entry */ +#define STAT_GET_NDLT 4 /* "ndlt" column of stat[34] entry */ + /* ** Implementation of the stat3_get(P,N,...) SQL function. This routine is ** used to query the results. Content is returned for the Nth sqlite_stat3 @@ -535,32 +628,87 @@ static const FuncDef stat4PushFuncdef = { ** argc==4 result: nLt ** argc==5 result: nDLt */ -static void stat4Get( +static void statGet( sqlite3_context *context, int argc, sqlite3_value **argv ){ Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); - int n = sqlite3_value_int(argv[1]); + int eCall = sqlite3_value_int(argv[1]); + assert( eCall==STAT_GET_STAT1 || eCall==STAT_GET_NEQ + || eCall==STAT_GET_ROWID || eCall==STAT_GET_NLT + || eCall==STAT_GET_NDLT + ); - assert( p!=0 ); - if( nnSample ){ + if( eCall==STAT_GET_STAT1 ){ + /* Return the value to store in the "stat" column of the sqlite_stat1 + ** table for this index. + ** + ** The value is a string composed of a list of integers describing + ** the index. The first integer in the list is the total number of + ** entries in the index. There is one additional integer in the list + ** for each indexed column. This additional integer is an estimate of + ** the number of rows matched by a stabbing query on the index using + ** a key with the corresponding number of fields. In other words, + ** if the index is on columns (a,b) and the sqlite_stat1 value is + ** "100 10 2", then SQLite estimates that: + ** + ** * the index contains 100 rows, + ** * "WHERE a=?" matches 10 rows, and + ** * "WHERE a=? AND b=?" matches 2 rows. + ** + ** If D is the count of distinct values and K is the total number of + ** rows, then each estimate is computed as: + ** + ** I = (K+D-1)/D + */ + char *z; + int i; + + char *zRet = sqlite3MallocZero(p->nCol * 25); + if( zRet==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + sqlite3_snprintf(24, zRet, "%lld", p->nRow); + z = zRet + sqlite3Strlen30(zRet); + for(i=0; i<(p->nCol-1); i++){ + i64 nDistinct = p->current.anDLt[i] + 1; + i64 iVal = (p->nRow + nDistinct - 1) / nDistinct; + sqlite3_snprintf(24, z, " %lld", iVal); + z += sqlite3Strlen30(z); + assert( p->current.anEq[i] ); + } + assert( z[0]=='\0' && z>zRet ); + + sqlite3_result_text(context, zRet, -1, sqlite3_free); + }else if( eCall==STAT_GET_ROWID ){ + if( p->iGet<0 ){ + samplePushPrevious(p, 0); + p->iGet = 0; + } + if( p->iGetnSample ){ + sqlite3_result_int64(context, p->a[p->iGet].iRowid); + } + }else{ tRowcnt *aCnt = 0; - char *zRet; - - switch( argc ){ - case 2: - sqlite3_result_int64(context, p->a[n].iRowid); - return; - case 3: aCnt = p->a[n].anEq; break; - case 4: aCnt = p->a[n].anLt; break; - default: aCnt = p->a[n].anDLt; break; + + assert( p->iGetnSample ); + switch( eCall ){ + case STAT_GET_NEQ: aCnt = p->a[p->iGet].anEq; break; + case STAT_GET_NLT: aCnt = p->a[p->iGet].anLt; break; + default: { + aCnt = p->a[p->iGet].anDLt; + p->iGet++; + break; + } } if( IsStat3 ){ sqlite3_result_int64(context, (i64)aCnt[0]); }else{ - zRet = sqlite3MallocZero(p->nCol * 25); + char *zRet = sqlite3MallocZero(p->nCol * 25); if( zRet==0 ){ sqlite3_result_error_nomem(context); }else{ @@ -577,23 +725,27 @@ static void stat4Get( } } } -static const FuncDef stat4GetFuncdef = { - -1, /* nArg */ - SQLITE_UTF8, /* iPrefEnc */ - 0, /* flags */ - 0, /* pUserData */ - 0, /* pNext */ - stat4Get, /* xFunc */ - 0, /* xStep */ - 0, /* xFinalize */ - "stat4_get", /* zName */ - 0, /* pHash */ - 0 /* pDestructor */ +static const FuncDef statGetFuncdef = { + 2, /* nArg */ + SQLITE_UTF8, /* iPrefEnc */ + 0, /* flags */ + 0, /* pUserData */ + 0, /* pNext */ + statGet, /* xFunc */ + 0, /* xStep */ + 0, /* xFinalize */ + "stat_get", /* zName */ + 0, /* pHash */ + 0 /* pDestructor */ }; -#endif /* SQLITE_ENABLE_STAT4 */ - - +static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ + assert( regOut!=regStat4 && regOut!=regStat4+1 ); + sqlite3VdbeAddOp2(v, OP_Integer, iParam, regStat4+1); + sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4, regOut); + sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 2); +} /* ** Generate code to do an analysis of all indices associated with @@ -616,29 +768,17 @@ static void analyzeOneTable( int jZeroRows = -1; /* Jump from here if number of rows is zero */ int iDb; /* Index of database containing pTab */ u8 needTableCnt = 1; /* True to count the table */ + int regNewRowid = iMem++; /* Rowid for the inserted record */ + int regStat4 = iMem++; /* Register to hold Stat4Accum object */ + int regRowid = iMem++; /* Rowid argument passed to stat_push() */ + int regChng = iMem++; /* Index of changed index field */ + int regTemp = iMem++; /* Temporary use register */ int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ - int regStat1 = iMem++; /* The stat column of sqlite_stat1 */ -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) - int regNumEq = regStat1; /* Number of instances. Same as regStat1 */ - int regNumLt = iMem++; /* Number of keys less than regSample */ - int regNumDLt = iMem++; /* Number of distinct keys less than regSample */ - int regSample = iMem++; /* The next sample value */ - int regLoop = iMem++; /* Loop counter */ - int shortJump = 0; /* Instruction address */ -#endif - int regCol = iMem++; /* Content of a column in analyzed table */ - int regRec = iMem++; /* Register holding completed record */ - int regTemp = iMem++; /* Temporary use register */ - int regNewRowid = iMem++; /* Rowid for the inserted record */ - int regEof = iMem++; /* True once cursors are all at EOF */ - int regCnt = iMem++; /* Row counter */ + int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ + int regPrev = iMem; /* MUST BE LAST (see below) */ - int regStat4 = iMem++; /* Register to hold Stat4Accum object */ - int regRowid = iMem++; /* Rowid argument passed to stat4_push() */ - int regKeychng = iMem++; /* True if key has changed */ - - pParse->nMem = MAX(pParse->nMem, regKeychng); + pParse->nMem = MAX(pParse->nMem, regChng); v = sqlite3GetVdbe(pParse); if( v==0 || NEVER(pTab==0) ){ return; @@ -663,9 +803,12 @@ static void analyzeOneTable( #endif /* Establish a read-lock on the table at the shared-cache level. - ** Also open a read-only cursor on the table. */ + ** Open a read-only cursor on the table. Also allocate a cursor number + ** to use for scanning indexes (iIdxCur). No index cursor is opened at + ** this time though. */ sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); iTabCur = iTab++; + iIdxCur = iTab++; pParse->nTab = MAX(pParse->nTab, iTab); sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp4(v, OP_String8, 0, regTabname, 0, pTab->zName, 0); @@ -673,324 +816,192 @@ static void analyzeOneTable( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns indexed by pIdx */ KeyInfo *pKey; /* KeyInfo structure for pIdx */ - int *aChngAddr; /* Array of jump instruction addresses */ - int regPrev; /* First in array of previous values */ - int regDLte; /* First in array of nDlt registers */ - int regLt; /* First in array of nLt registers */ - int regEq; /* First in array of nEq registers */ - int endOfScan; /* Label to jump to once scan is finished */ + int *aGotoChng; /* Array of jump instruction addresses */ + int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrGotoChng0; /* Address of "Goto addr_chng_0" */ + int addrNextRow; /* Address of "next_row:" */ if( pOnlyIdx && pOnlyIdx!=pIdx ) continue; if( pIdx->pPartIdxWhere==0 ) needTableCnt = 0; VdbeNoopComment((v, "Begin analysis of %s", pIdx->zName)); nCol = pIdx->nColumn; - aChngAddr = sqlite3DbMallocRaw(db, sizeof(int)*(nCol+1)); - if( aChngAddr==0 ) continue; + aGotoChng = sqlite3DbMallocRaw(db, sizeof(int)*(nCol+1)); + if( aGotoChng==0 ) continue; pKey = sqlite3IndexKeyinfo(pParse, pIdx); /* Populate the register containing the index name. */ sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, pIdx->zName, 0); /* - ** The following pseudo-code demonstrates the way the VM scans an index - ** to call stat4_push() and collect the values for the sqlite_stat1 - ** entry. The code below is for an index with 2 columns. The actual - ** VM code generated may be for any number of columns. + ** Pseudo-code for loop that calls stat_push(): ** - ** One cursor is opened for each column in the index and one for the - ** rowid column (nCol+1 in total). All cursors scan concurrently the - ** index from start to end. All variables used in the pseudo-code are - ** initialized to zero. + ** Rewind csr + ** if eof(csr) goto end_of_scan; + ** regChng = 0 + ** goto chng_addr_0; ** - ** Rewind csr(0) - ** Rewind csr(1) - ** Rewind csr(2) - ** - ** next_0: - ** regPrev(0) = csr(0)[0] - ** regDLte(0) += 1 - ** regLt(0) += regEq(0) - ** regEq(0) = 0 - ** do { - ** regEq(0) += 1 - ** Next csr(0) - ** }while ( csr(0)[0] == regPrev(0) ) - ** if( IsStat3 ) regKeychng = 1 - ** - ** next_1: - ** regPrev(1) = csr(1)[1] - ** regDLte(1) += 1 - ** regLt(1) += regEq(1) - ** regEq(1) = 0 - ** do { - ** regEq(1) += 1 - ** Next csr(1) - ** }while ( csr(1)[0..1] == regPrev(0..1) ) - ** - ** if( IsStat3==0 ) regKeychng = 1 ** next_row: - ** regRowid = csr(2)[rowid] - ** regEq(2) = 1 - ** regLt(2) = regCnt - ** regCnt += 1 - ** regDLte(2) = regCnt - ** stat4_push(regRowid, regKeychng, regEq, regLt, regDLte); - ** regKeychng = 0 - ** Next csr(2) - ** if( eof( csr(2) ) ) goto endOfScan - ** - ** if( csr(2)[0] != regPrev(0) ) goto next_0 - ** if( csr(2)[1] != regPrev(1) ) goto next_1 - ** goto next_row + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto chng_addr_N ** - ** endOfScan: - ** // done! + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... ** - ** The last two lines above modify the contents of the regDLte array - ** so that each element contains the number of distinct key prefixes - ** of the corresponding length. As required to calculate the contents - ** of the sqlite_stat1 entry. + ** chng_addr_N: + ** regRowid = idx(rowid) + ** stat_push(P, regRowid, regChng) + ** Next csr + ** if !eof(csr) goto next_row; ** - ** At this point, the last memory cell allocated (that with the largest - ** integer identifier) is regKeychng. Immediately following regKeychng - ** we allocate the following: - ** - ** regEq - nCol registers - ** regLt - nCol+1 registers - ** regDLte - nCol+1 registers - ** regPrev - nCol+1 registers - ** - ** can be passed to the stat4_push() function. - ** - ** All of the above are initialized to contain integer value 0. + ** end_of_scan: */ - regEq = regKeychng+1; /* First in array of nEq value registers */ - regLt = regEq+nCol+1; /* First in array of nLt value registers */ - regDLte = regLt+nCol+1; /* First in array of nDLt value registers */ - regPrev = regDLte+nCol+1; /* First in array of prev. value registers */ + + /* Make sure there are enough memory cells allocated to accommodate + ** the regPrev array and a trailing rowid (the rowid slot is required + ** when building a record to insert into the sample column of + ** the sqlite_stat4 table. */ pParse->nMem = MAX(pParse->nMem, regPrev+nCol); - /* Open a read-only cursor for each column of the index. And one for - ** the rowid column. A total of (nCol+1) cursors. */ + /* Open a read-only cursor on the index being analyzed. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); - iIdxCur = iTab; - pParse->nTab = MAX(pParse->nTab, iTab+nCol+1); - for(i=0; i<(nCol+1); i++){ - int iMode = (i==0 ? P4_KEYINFO_HANDOFF : P4_KEYINFO); - sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur+i, pIdx->tnum, iDb); - sqlite3VdbeChangeP4(v, -1, (char*)pKey, iMode); - VdbeComment((v, "%s", pIdx->zName)); - } + sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb); + sqlite3VdbeChangeP4(v, -1, (char*)pKey, P4_KEYINFO_HANDOFF); + VdbeComment((v, "%s", pIdx->zName)); -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) - /* Invoke the stat4_init() function. The arguments are: + /* Invoke the stat_init() function. The arguments are: ** ** * the number of rows in the index, ** * the number of columns in the index including the rowid, - ** * the recommended number of samples for the stat4 table. - ** - ** If this is a stat3 build, the number of columns in the index is - ** set to 1 (as this is the number of index fields gathered). + ** * the recommended number of samples for the stat3/stat4 table. */ sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+1); sqlite3VdbeAddOp2(v, OP_Integer, nCol+1, regStat4+2); sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_STAT4_SAMPLES, regStat4+3); sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); - sqlite3VdbeChangeP4(v, -1, (char*)&stat4InitFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); sqlite3VdbeChangeP5(v, 3); -#endif /* SQLITE_ENABLE_STAT4 */ - /* Initialize all the memory registers allocated above to 0. */ - for(i=regEq; iazColl[i]); - int iCsr = iIdxCur+i; - int iDo; - int iNe; /* Jump here to exit do{...}while loop */ - int j; - - /* Implementation of the following pseudo-code: - ** - ** regPrev(i) = csr(i)[i] - ** regDLte(i) += 1 - ** regLt(i) += regEq(i) - ** regEq(i) = 0 - ** regRowid = csr(i)[rowid] // innermost cursor only - */ - aChngAddr[i] = sqlite3VdbeAddOp3(v, OP_Column, iCsr, i, regPrev+i); - VdbeComment((v, "regPrev(%d) = csr(%d)(%d)", i, i, i)); - sqlite3VdbeAddOp2(v, OP_AddImm, regDLte+i, 1); - VdbeComment((v, "regDLte(%d) += 1", i)); - sqlite3VdbeAddOp3(v, OP_Add, regEq+i, regLt+i, regLt+i); - VdbeComment((v, "regLt(%d) += regEq(%d)", i, i)); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regEq+i); - VdbeComment((v, "regEq(%d) = 0", i)); - - /* This bit: - ** - ** do { - ** regEq(i) += 1 - ** Next csr(i) - ** if( Eof csr(i) ){ - ** break - ** } - ** }while ( csr(i)[0..i] == regPrev(0..i) ) - */ - iDo = sqlite3VdbeAddOp2(v, OP_AddImm, regEq+i, 1); - VdbeComment((v, "regEq(%d) += 1", i)); - sqlite3VdbeAddOp2(v, OP_Next, iCsr, sqlite3VdbeCurrentAddr(v)+2); - iNe = sqlite3VdbeMakeLabel(v); - sqlite3VdbeAddOp2(v, OP_Goto, 0, iNe); - for(j=0; j<=i; j++){ - sqlite3VdbeAddOp3(v, OP_Column, iCsr, j, regCol); - sqlite3VdbeAddOp4(v, OP_Ne, regCol, iNe, regPrev+j, pColl, P4_COLLSEQ); - sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); - VdbeComment((v, "if( regPrev(%d) != csr(%d)(%d) )", j, i, j)); - } - sqlite3VdbeAddOp2(v, OP_Goto, 0, iDo); - sqlite3VdbeResolveLabel(v, iNe); - - if( IsStat3 && i==0 ){ - sqlite3VdbeAddOp2(v, OP_Integer, 1, regKeychng); - } - } - - /* This stuff: - ** - ** regKeychng = 1 - ** next_row: - ** regRowid = csr(2)[rowid] - ** regEq(2) = 1 - ** regLt(2) = regCnt - ** regCnt += 1 - ** regDLte(2) = regCnt - ** stat4_push(regRowid, regKeychng, regEq, regLt, regDLte); - ** regKeychng = 0 - ** Next csr(2) - ** if( eof( csr(2) ) ) goto endOfScan + /* Implementation of the following: + ** + ** Rewind csr + ** if eof(csr) goto end_of_scan; + ** regChng = 0 + ** goto next_push_0; + ** */ -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) - if( 0==IsStat3 ){ - sqlite3VdbeAddOp2(v, OP_Integer, 1, regKeychng); - } - aChngAddr[nCol] = - sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur+nCol, regRowid); - sqlite3VdbeAddOp2(v, OP_Integer, 1, regEq+nCol); - sqlite3VdbeAddOp2(v, OP_Copy, regCnt, regLt+nCol); - sqlite3VdbeAddOp2(v, OP_AddImm, regCnt, 1); - sqlite3VdbeAddOp2(v, OP_Copy, regCnt, regDLte+nCol); - sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp); - sqlite3VdbeChangeP4(v, -1, (char*)&stat4PushFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 3 + 3*(nCol+1)); - sqlite3VdbeAddOp2(v, OP_Integer, 0, regKeychng); - sqlite3VdbeAddOp2(v, OP_Next, iIdxCur+nCol, sqlite3VdbeCurrentAddr(v)+2); - sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfScan); -#endif + addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); + addrGotoChng0 = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeAddOp2(v, OP_If, regEof, endOfScan); + /* + ** next_row: + ** regChng = 0 + ** if( idx(0) != regPrev(0) ) goto chng_addr_0 + ** regChng = 1 + ** if( idx(1) != regPrev(1) ) goto chng_addr_1 + ** ... + ** regChng = N + ** goto chng_addr_N + */ + addrNextRow = sqlite3VdbeCurrentAddr(v); for(i=0; iazColl[i]); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur+nCol, i, regCol); - sqlite3VdbeAddOp3(v, OP_Ne, regCol, aChngAddr[i], regPrev+i); - sqlite3VdbeChangeP4(v, -1, pColl, P4_COLLSEQ); + sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); + aGotoChng[i] = + sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); } - sqlite3VdbeAddOp2(v, OP_Goto, 0, aChngAddr[nCol]); - sqlite3DbFree(db, aChngAddr); + sqlite3VdbeAddOp2(v, OP_Integer, nCol, regChng); + aGotoChng[nCol] = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeResolveLabel(v, endOfScan); - -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) - /* Add rows to the sqlite_stat4 table */ - regLoop = regStat4+1; - sqlite3VdbeAddOp2(v, OP_Integer, -1, regLoop); - shortJump = sqlite3VdbeAddOp2(v, OP_AddImm, regLoop, 1); - sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4, regEq+nCol); - sqlite3VdbeChangeP4(v, -1, (char*)&stat4GetFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 2); - sqlite3VdbeAddOp1(v, OP_IsNull, regEq+nCol); - - sqlite3VdbeAddOp3(v, OP_NotExists, iTabCur, shortJump, regEq+nCol); - if( IsStat3==0 ){ - for(i=0; iaiColumn[i]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regEq+i); - } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regEq, nCol+1, regSample); - sqlite3VdbeChangeP4(v, -1, pIdx->zColAff, 0); - }else{ - int iCol = pIdx->aiColumn[0]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regSample); - } - - sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regNumEq); - sqlite3VdbeChangeP4(v, -1, (char*)&stat4GetFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 3); - - sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regNumLt); - sqlite3VdbeChangeP4(v, -1, (char*)&stat4GetFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 4); - - sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regNumDLt); - sqlite3VdbeChangeP4(v, -1, (char*)&stat4GetFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 5); - - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 6, regRec, "bbbbbb", 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); - sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regRec, regNewRowid); - sqlite3VdbeAddOp2(v, OP_Goto, 0, shortJump); - sqlite3VdbeJumpHere(v, shortJump+2); -#endif - - /* Store the results in sqlite_stat1. - ** - ** The result is a single row of the sqlite_stat1 table. The first - ** two columns are the names of the table and index. The third column - ** is a string composed of a list of integer statistics about the - ** index. The first integer in the list is the total number of entries - ** in the index. There is one additional integer in the list for each - ** column of the table. This additional integer is a guess of how many - ** rows of the table the index will select. If D is the count of distinct - ** values and K is the total number of rows, then the integer is computed - ** as: - ** - ** I = (K+D-1)/D - ** - ** If K==0 then no entry is made into the sqlite_stat1 table. - ** If K>0 then it is always the case the D>0 so division by zero - ** is never possible. + /* + ** chng_addr_0: + ** regPrev(0) = idx(0) + ** chng_addr_1: + ** regPrev(1) = idx(1) + ** ... */ - sqlite3VdbeAddOp2(v, OP_SCopy, regCnt, regStat1); - jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regCnt); + sqlite3VdbeJumpHere(v, addrGotoChng0); for(i=0; ipPartIdxWhere!=0 ) sqlite3VdbeJumpHere(v, jZeroRows); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); + + /* + ** chng_addr_N: + ** regRowid = idx(rowid) + ** stat_push(P, regRowid, regChng) + ** Next csr + ** if !eof(csr) goto next_row; + */ + sqlite3VdbeJumpHere(v, aGotoChng[nCol]); + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid); + sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp); + sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, 3); + assert( regRowid==(regStat4+1) && regChng==(regStat4+2) ); + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); + + /* Add the entry to the stat1 table. */ + callStatGet(v, regStat4, STAT_GET_STAT1, regStat1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "aaa", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); - sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - if( pIdx->pPartIdxWhere==0 ) sqlite3VdbeJumpHere(v, jZeroRows); + + /* Add the entries to the stat3 or stat4 table. */ + if( IsStat3 || IsStat4 ){ + int regEq = regStat1; + int regLt = regStat1+1; + int regDLt = regStat1+2; + int regSample = regStat1+3; + int regCol = regStat1+4; + int regSampleRowid = regCol + nCol; + int addrNext; + int addrIsNull; + + pParse->nMem = MAX(pParse->nMem, regCol+nCol+1); + + addrNext = sqlite3VdbeCurrentAddr(v); + callStatGet(v, regStat4, STAT_GET_ROWID, regSampleRowid); + addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regSampleRowid); + callStatGet(v, regStat4, STAT_GET_NEQ, regEq); + callStatGet(v, regStat4, STAT_GET_NLT, regLt); + callStatGet(v, regStat4, STAT_GET_NDLT, regDLt); + sqlite3VdbeAddOp3(v, OP_NotExists, iTabCur, addrNext, regSampleRowid); + if( IsStat3 ){ + int iCol = pIdx->aiColumn[0]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regSample); + }else{ + for(i=0; iaiColumn[i]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol+1, regSample); + } + + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 6, regTemp, "bbbbbb", 0); + sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regTemp, regNewRowid); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNext); + sqlite3VdbeJumpHere(v, addrIsNull); + } + + /* Jump here if the index is empty */ + sqlite3VdbeJumpHere(v, addrRewind); + sqlite3DbFree(db, aGotoChng); } + /* Create a single sqlite_stat1 entry containing NULL as the index ** name and the row count as the content. */ @@ -999,9 +1010,9 @@ static void analyzeOneTable( sqlite3VdbeAddOp2(v, OP_Count, iTabCur, regStat1); jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "aaa", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); - sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regNewRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); sqlite3VdbeJumpHere(v, jZeroRows); } diff --git a/test/analyze8.test b/test/analyze8.test index b46d38651f..4384c39676 100644 --- a/test/analyze8.test +++ b/test/analyze8.test @@ -86,25 +86,24 @@ do_test 2.1 { # range. # # Test 3.2 is a little unstable. It depends on the planner estimating -# that (b BETWEEN 40 AND 44) will match more rows than (c BETWEEN +# that (b BETWEEN 50 AND 54) will match more rows than (c BETWEEN # 800000 AND 900000). Which is a pretty close call (50 vs. 32), so # the planner could get it wrong with an unlucky set of samples. This -# case happens to work, but others ("b BETWEEN 50 AND 54" for example) +# case happens to work, but others ("b BETWEEN 40 AND 44" for example) # will fail. # do_execsql_test 3.0 { - SELECT count(*) FROM t1 WHERE b BETWEEN 40 AND 44; + SELECT count(*) FROM t1 WHERE b BETWEEN 50 AND 54; SELECT count(*) FROM t1 WHERE c BETWEEN 0 AND 100000; SELECT count(*) FROM t1 WHERE c BETWEEN 800000 AND 900000; } {50 376 32} do_test 3.1 { - eqp {SELECT * FROM t1 WHERE b BETWEEN 40 AND 44 AND c BETWEEN 0 AND 100000} + eqp {SELECT * FROM t1 WHERE b BETWEEN 50 AND 54 AND c BETWEEN 0 AND 100000} } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b? AND c Date: Thu, 15 Aug 2013 14:39:09 +0000 Subject: [PATCH 27/67] Add tests for sqlite_stat4 sample selection. And a fix for the same. FossilOrigin-Name: 1fb4d9d6f2675515feb8e3d971bbd54716372549 --- manifest | 14 +++--- manifest.uuid | 2 +- src/analyze.c | 18 +++++--- test/analyze9.test | 109 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 129 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 35822cf8b7..7dbb565005 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sthe\sway\sANALYZE\sworks\sto\suse\sa\ssingle\scursor\swhen\sscanning\sindices. -D 2013-08-14T19:54:12.120 +C Add\stests\sfor\ssqlite_stat4\ssample\sselection.\sAnd\sa\sfix\sfor\sthe\ssame. +D 2013-08-15T14:39:09.873 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 37b78257fcf73790fd443b2c62b7ce62b78c3c98 +F src/analyze.c f7494349f8c26cbffff3fab198834d7583f52c4f F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -308,7 +308,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test 1b419d03407f2a6f4f1485620d54cb3e1bab3a71 +F test/analyze9.test f75d1f2edd7707c8bbf5703be998e53212312abe F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P cca8bf4372ab7a0258aa5c9397818415c6cf0abf -R 798b6e107d9cfc4282515cb67b7b0a49 +P bdce612b35193abf72de1a563ea7962375b3574e +R cae3925a5ea9d2d41d5f5a62ca3fba06 U dan -Z 0758340c46c18a215a897e2672d1ddd5 +Z 071feb0454ff17e1bc198cd08a86a76d diff --git a/manifest.uuid b/manifest.uuid index c82b437d55..40e0dcd524 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -bdce612b35193abf72de1a563ea7962375b3574e \ No newline at end of file +1fb4d9d6f2675515feb8e3d971bbd54716372549 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index d8edadf885..ac9ba5edeb 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -411,23 +411,29 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ assert( IsStat4 || nEqZero==0 ); if( pNew->isPSample==0 ){ + Stat4Sample *pUpgrade = 0; assert( pNew->anEq[pNew->iCol]>0 ); /* This sample is being added because the prefix that ends in column ** iCol occurs many times in the table. However, if we have already ** added a sample that shares this prefix, there is no need to add - ** this one. Instead, upgrade the priority of the existing sample. */ + ** this one. Instead, upgrade the priority of the highest priority + ** existing sample that shares this prefix. */ for(i=p->nSample-1; i>=0; i--){ Stat4Sample *pOld = &p->a[i]; if( pOld->anEq[pNew->iCol]==0 ){ - if( pOld->isPSample==0 ){ - assert( sampleIsBetter(pNew, pOld) ); - assert( pOld->iCol>pNew->iCol ); - pOld->iCol = pNew->iCol; + if( pOld->isPSample ) return; + assert( sampleIsBetter(pNew, pOld) ); + if( pUpgrade==0 || sampleIsBetter(pOld, pUpgrade) ){ + pUpgrade = pOld; } - goto find_new_min; } } + if( pUpgrade ){ + pUpgrade->iCol = pNew->iCol; + pUpgrade->anEq[pUpgrade->iCol] = pNew->anEq[pUpgrade->iCol]; + goto find_new_min; + } } /* If necessary, remove sample iMin to make room for the new sample. */ diff --git a/test/analyze9.test b/test/analyze9.test index cbb2319f8c..51531c5895 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -147,5 +147,114 @@ do_execsql_test 3.4 { SELECT c FROM t1 WHERE b=3 AND a BETWEEN 30 AND 60; } {three-d three-e three-f} +#------------------------------------------------------------------------- +# These tests verify that the sample selection for stat4 appears to be +# working as designed. +# + +reset_db +db func lindex lindex +db func lrange lrange + +do_execsql_test 4.0 { + DROP TABLE IF EXISTS t1; + CREATE TABLE t1(a, b, c); + CREATE INDEX i1 ON t1(c, b, a); +} + + +proc insert_filler_rows_n {iStart args} { + set A(-ncopy) 1 + set A(-nval) 1 + + foreach {k v} $args { + if {[info exists A($k)]==0} { error "no such option: $k" } + set A($k) $v + } + if {[llength $args] % 2} { + error "option requires an argument: [lindex $args end]" + } + + for {set i 0} {$i < $A(-nval)} {incr i} { + set iVal [expr $iStart+$i] + for {set j 0} {$j < $A(-ncopy)} {incr j} { + execsql { INSERT INTO t1 VALUES($iVal, $iVal, $iVal) } + } + } +} + +do_test 4.1 { + execsql { BEGIN } + insert_filler_rows_n 0 -ncopy 10 -nval 19 + insert_filler_rows_n 20 -ncopy 1 -nval 100 + + execsql { + INSERT INTO t1(c, b, a) VALUES(200, 1, 'a'); + INSERT INTO t1(c, b, a) VALUES(200, 1, 'b'); + INSERT INTO t1(c, b, a) VALUES(200, 1, 'c'); + + INSERT INTO t1(c, b, a) VALUES(200, 2, 'e'); + INSERT INTO t1(c, b, a) VALUES(200, 2, 'f'); + + INSERT INTO t1(c, b, a) VALUES(201, 3, 'g'); + INSERT INTO t1(c, b, a) VALUES(201, 4, 'h'); + + ANALYZE; + SELECT count(*) FROM sqlite_stat4; + SELECT count(*) FROM t1; + } +} {24 297} + +do_execsql_test 4.2 { + SELECT + neq, + lrange(nlt, 0, 2), + lrange(ndlt, 0, 2), + lrange(test_decode(sample), 0, 2) + FROM sqlite_stat4 + ORDER BY rowid LIMIT 16; +} { + {10 10 10 1} {0 0 0} {0 0 0} {0 0 0} + {10 10 10 1} {10 10 10} {1 1 1} {1 1 1} + {10 10 10 1} {20 20 20} {2 2 2} {2 2 2} + {10 10 10 1} {30 30 30} {3 3 3} {3 3 3} + {10 10 10 1} {40 40 40} {4 4 4} {4 4 4} + {10 10 10 1} {50 50 50} {5 5 5} {5 5 5} + {10 10 10 1} {60 60 60} {6 6 6} {6 6 6} + {10 10 10 1} {70 70 70} {7 7 7} {7 7 7} + {10 10 10 1} {80 80 80} {8 8 8} {8 8 8} + {10 10 10 1} {90 90 90} {9 9 9} {9 9 9} + {10 10 10 1} {100 100 100} {10 10 10} {10 10 10} + {10 10 10 1} {110 110 110} {11 11 11} {11 11 11} + {10 10 10 1} {120 120 120} {12 12 12} {12 12 12} + {10 10 10 1} {130 130 130} {13 13 13} {13 13 13} + {10 10 10 1} {140 140 140} {14 14 14} {14 14 14} + {10 10 10 1} {150 150 150} {15 15 15} {15 15 15} +} + +do_execsql_test 4.3 { + SELECT + neq, + lrange(nlt, 0, 2), + lrange(ndlt, 0, 2), + lrange(test_decode(sample), 0, 1) + FROM sqlite_stat4 + ORDER BY rowid DESC LIMIT 2; +} { + {2 1 1 1} {295 296 296} {120 122 125} {201 4} + {5 3 1 1} {290 290 292} {119 119 121} {200 1} +} + +do_execsql_test 4.4 { SELECT count(DISTINCT c) FROM t1 WHERE c<201 } 120 +do_execsql_test 4.5 { SELECT count(DISTINCT c) FROM t1 WHERE c<200 } 119 + +# Check that the perioidic samples are present. +do_execsql_test 4.6 { + SELECT count(*) FROM sqlite_stat4 + WHERE lindex(test_decode(sample), 3) IN + ('34', '68', '102', '136', '170', '204', '238', '272') +} {8} + + finish_test From f0459fc410175a33aa6a80f618664b24151e9c39 Mon Sep 17 00:00:00 2001 From: drh Date: Thu, 15 Aug 2013 16:15:00 +0000 Subject: [PATCH 28/67] Make sure the ANALYZE command allocates enough VDBE registers. FossilOrigin-Name: 46fec9b1a1c4616df5a634dbf9235bd13408d3a9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/analyze.c | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 7dbb565005..dfe5f6d40f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\stests\sfor\ssqlite_stat4\ssample\sselection.\sAnd\sa\sfix\sfor\sthe\ssame. -D 2013-08-15T14:39:09.873 +C Make\ssure\sthe\sANALYZE\scommand\sallocates\senough\sVDBE\sregisters. +D 2013-08-15T16:15:00.074 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c f7494349f8c26cbffff3fab198834d7583f52c4f +F src/analyze.c e60d6329f77e84bac0a300d5227c28a05f5ff5de F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bdce612b35193abf72de1a563ea7962375b3574e -R cae3925a5ea9d2d41d5f5a62ca3fba06 -U dan -Z 071feb0454ff17e1bc198cd08a86a76d +P 1fb4d9d6f2675515feb8e3d971bbd54716372549 +R 9628ea795791e781c0796fcdc3f77669 +U drh +Z 31fc7cf422d15cf81f1695579e8c285d diff --git a/manifest.uuid b/manifest.uuid index 40e0dcd524..0644d9cb4c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -1fb4d9d6f2675515feb8e3d971bbd54716372549 \ No newline at end of file +46fec9b1a1c4616df5a634dbf9235bd13408d3a9 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index ac9ba5edeb..2c7a142fd0 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -784,7 +784,7 @@ static void analyzeOneTable( int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ int regPrev = iMem; /* MUST BE LAST (see below) */ - pParse->nMem = MAX(pParse->nMem, regChng); + pParse->nMem = MAX(pParse->nMem, iMem); v = sqlite3GetVdbe(pParse); if( v==0 || NEVER(pTab==0) ){ return; From 3bc9f74fe9a9b250f0445d29626ef99867847a25 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 Aug 2013 16:18:39 +0000 Subject: [PATCH 29/67] Change some assert() statements in vdbe.c to ensure that a memory cell used to store a VdbeCursor object is not also used for some other purpose. FossilOrigin-Name: 71070c9fce86103f174220e07771df99b2e01405 --- manifest | 16 ++++++------- manifest.uuid | 2 +- src/vdbe.c | 58 +++++++++++++++++++++++----------------------- test/analyze9.test | 10 ++++++++ 4 files changed, 48 insertions(+), 38 deletions(-) diff --git a/manifest b/manifest index dfe5f6d40f..4f54fd9f5d 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\ssure\sthe\sANALYZE\scommand\sallocates\senough\sVDBE\sregisters. -D 2013-08-15T16:15:00.074 +C Change\ssome\sassert()\sstatements\sin\svdbe.c\sto\sensure\sthat\sa\smemory\scell\sused\sto\sstore\sa\sVdbeCursor\sobject\sis\snot\salso\sused\sfor\ssome\sother\spurpose. +D 2013-08-15T16:18:39.664 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -277,7 +277,7 @@ F src/update.c 7d9d38e4f341ada7d79035ea969cdefb8b9014d1 F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269 F src/util.c f566b5138099a2df8533b190d0dcc74b7dfbe0c9 F src/vacuum.c d9c5759f4c5a438bb43c2086f72c5d2edabc36c8 -F src/vdbe.c a3c98ec54d27b30b67b6247ed0a29268de11dc17 +F src/vdbe.c c9488a255843c815a4bbb58ca7ad2fa871150d69 F src/vdbe.h 4f554b5627f26710c4c36d919110a3fc611ca5c4 F src/vdbeInt.h e9b7c6b165a31a4715c5aa97223d20d265515231 F src/vdbeapi.c 4d13580bd058b39623e8fcfc233b7df4b8191e8b @@ -308,7 +308,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test f75d1f2edd7707c8bbf5703be998e53212312abe +F test/analyze9.test b018c837164ada65f4a80dadbbcdc89cb1fff362 F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 1fb4d9d6f2675515feb8e3d971bbd54716372549 -R 9628ea795791e781c0796fcdc3f77669 -U drh -Z 31fc7cf422d15cf81f1695579e8c285d +P 46fec9b1a1c4616df5a634dbf9235bd13408d3a9 +R e1ce93ce8d017990abe28f4c9f5b58f9 +U dan +Z 9abc3408e2534a3305d7999552dece52 diff --git a/manifest.uuid b/manifest.uuid index 0644d9cb4c..e48df0d900 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -46fec9b1a1c4616df5a634dbf9235bd13408d3a9 \ No newline at end of file +71070c9fce86103f174220e07771df99b2e01405 \ No newline at end of file diff --git a/src/vdbe.c b/src/vdbe.c index e9453e976e..810650489a 100644 --- a/src/vdbe.c +++ b/src/vdbe.c @@ -651,7 +651,7 @@ int sqlite3VdbeExec( assert( pOp->opflags==sqlite3OpcodeProperty[pOp->opcode] ); if( pOp->opflags & OPFLG_OUT2_PRERELEASE ){ assert( pOp->p2>0 ); - assert( pOp->p2<=p->nMem ); + assert( pOp->p2<=(p->nMem-p->nCursor) ); pOut = &aMem[pOp->p2]; memAboutToChange(p, pOut); VdbeMemRelease(pOut); @@ -662,30 +662,30 @@ int sqlite3VdbeExec( #ifdef SQLITE_DEBUG if( (pOp->opflags & OPFLG_IN1)!=0 ){ assert( pOp->p1>0 ); - assert( pOp->p1<=p->nMem ); + assert( pOp->p1<=(p->nMem-p->nCursor) ); assert( memIsValid(&aMem[pOp->p1]) ); REGISTER_TRACE(pOp->p1, &aMem[pOp->p1]); } if( (pOp->opflags & OPFLG_IN2)!=0 ){ assert( pOp->p2>0 ); - assert( pOp->p2<=p->nMem ); + assert( pOp->p2<=(p->nMem-p->nCursor) ); assert( memIsValid(&aMem[pOp->p2]) ); REGISTER_TRACE(pOp->p2, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_IN3)!=0 ){ assert( pOp->p3>0 ); - assert( pOp->p3<=p->nMem ); + assert( pOp->p3<=(p->nMem-p->nCursor) ); assert( memIsValid(&aMem[pOp->p3]) ); REGISTER_TRACE(pOp->p3, &aMem[pOp->p3]); } if( (pOp->opflags & OPFLG_OUT2)!=0 ){ assert( pOp->p2>0 ); - assert( pOp->p2<=p->nMem ); + assert( pOp->p2<=(p->nMem-p->nCursor) ); memAboutToChange(p, &aMem[pOp->p2]); } if( (pOp->opflags & OPFLG_OUT3)!=0 ){ assert( pOp->p3>0 ); - assert( pOp->p3<=p->nMem ); + assert( pOp->p3<=(p->nMem-p->nCursor) ); memAboutToChange(p, &aMem[pOp->p3]); } #endif @@ -778,7 +778,7 @@ check_for_interrupt: ** and then jump to address P2. */ case OP_Gosub: { /* jump */ - assert( pOp->p1>0 && pOp->p1<=p->nMem ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); pIn1 = &aMem[pOp->p1]; assert( (pIn1->flags & MEM_Dyn)==0 ); memAboutToChange(p, pIn1); @@ -990,7 +990,7 @@ case OP_Null: { /* out2-prerelease */ int cnt; u16 nullFlag; cnt = pOp->p3-pOp->p2; - assert( pOp->p3<=p->nMem ); + assert( pOp->p3<=(p->nMem-p->nCursor) ); pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null; while( cnt>0 ){ pOut++; @@ -1059,8 +1059,8 @@ case OP_Move: { pIn1 = &aMem[p1]; pOut = &aMem[p2]; while( n-- ){ - assert( pOut<=&aMem[p->nMem] ); - assert( pIn1<=&aMem[p->nMem] ); + assert( pOut<=&aMem[(p->nMem-p->nCursor)] ); + assert( pIn1<=&aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); memAboutToChange(p, pOut); zMalloc = pOut->zMalloc; @@ -1144,7 +1144,7 @@ case OP_ResultRow: { int i; assert( p->nResColumn==pOp->p2 ); assert( pOp->p1>0 ); - assert( pOp->p1+pOp->p2<=p->nMem+1 ); + assert( pOp->p1+pOp->p2<=(p->nMem-p->nCursor)+1 ); /* If this statement has violated immediate foreign key constraints, do ** not return the number of rows modified. And do not RELEASE the statement @@ -1418,11 +1418,11 @@ case OP_Function: { n = pOp->p5; apVal = p->apArg; assert( apVal || n==0 ); - assert( pOp->p3>0 && pOp->p3<=p->nMem ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pOut = &aMem[pOp->p3]; memAboutToChange(p, pOut); - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=p->nMem+1) ); + assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem-p->nCursor)+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); pArg = &aMem[pOp->p2]; for(i=0; imx ) mx = aPermute[k]; - assert( p1>0 && p1+mx<=p->nMem+1 ); - assert( p2>0 && p2+mx<=p->nMem+1 ); + assert( p1>0 && p1+mx<=(p->nMem-p->nCursor)+1 ); + assert( p2>0 && p2+mx<=(p->nMem-p->nCursor)+1 ); }else{ - assert( p1>0 && p1+n<=p->nMem+1 ); - assert( p2>0 && p2+n<=p->nMem+1 ); + assert( p1>0 && p1+n<=(p->nMem-p->nCursor)+1 ); + assert( p2>0 && p2+n<=(p->nMem-p->nCursor)+1 ); } #endif /* SQLITE_DEBUG */ for(i=0; inCursor ); - assert( pOp->p3>0 && pOp->p3<=p->nMem ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); zRec = 0; @@ -2507,7 +2507,7 @@ case OP_Affinity: { assert( zAffinity[pOp->p2]==0 ); pIn1 = &aMem[pOp->p1]; while( (cAff = *(zAffinity++))!=0 ){ - assert( pIn1 <= &p->aMem[p->nMem] ); + assert( pIn1 <= &p->aMem[(p->nMem-p->nCursor)] ); assert( memIsValid(pIn1) ); ExpandBlob(pIn1); applyAffinity(pIn1, cAff, encoding); @@ -2568,7 +2568,7 @@ case OP_MakeRecord: { nZero = 0; /* Number of zero bytes at the end of the record */ nField = pOp->p1; zAffinity = pOp->p4.z; - assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=p->nMem+1 ); + assert( nField>0 && pOp->p2>0 && pOp->p2+nField<=(p->nMem-p->nCursor)+1 ); pData0 = &aMem[nField]; nField = pOp->p2; pLast = &pData0[nField-1]; @@ -2634,7 +2634,7 @@ case OP_MakeRecord: { } assert( i==nByte ); - assert( pOp->p3>0 && pOp->p3<=p->nMem ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pOut->n = (int)nByte; pOut->flags = MEM_Blob | MEM_Dyn; pOut->xDel = 0; @@ -3214,7 +3214,7 @@ case OP_OpenWrite: { } if( pOp->p5 & OPFLAG_P2ISREG ){ assert( p2>0 ); - assert( p2<=p->nMem ); + assert( p2<=(p->nMem-p->nCursor) ); pIn2 = &aMem[p2]; assert( memIsValid(pIn2) ); assert( (pIn2->flags & MEM_Int)!=0 ); @@ -3751,7 +3751,7 @@ case OP_IsUnique: { /* jump, in3 */ aMx = &aMem[pOp->p4.i]; /* Assert that the values of parameters P1 and P4 are in range. */ assert( pOp->p4type==P4_INT32 ); - assert( pOp->p4.i>0 && pOp->p4.i<=p->nMem ); + assert( pOp->p4.i>0 && pOp->p4.i<=(p->nMem-p->nCursor) ); assert( pOp->p1>=0 && pOp->p1nCursor ); /* Find the index cursor. */ @@ -3954,7 +3954,7 @@ case OP_NewRowid: { /* out2-prerelease */ pMem = &pFrame->aMem[pOp->p3]; }else{ /* Assert that P3 is a valid memory cell. */ - assert( pOp->p3<=p->nMem ); + assert( pOp->p3<=(p->nMem-p->nCursor) ); pMem = &aMem[pOp->p3]; memAboutToChange(p, pMem); } @@ -4608,7 +4608,7 @@ case OP_IdxDelete: { UnpackedRecord r; assert( pOp->p3>0 ); - assert( pOp->p2>0 && pOp->p2+pOp->p3<=p->nMem+1 ); + assert( pOp->p2>0 && pOp->p2+pOp->p3<=(p->nMem-p->nCursor)+1 ); assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); @@ -5017,7 +5017,7 @@ case OP_IntegrityCk: { assert( nRoot>0 ); aRoot = sqlite3DbMallocRaw(db, sizeof(int)*(nRoot+1) ); if( aRoot==0 ) goto no_mem; - assert( pOp->p3>0 && pOp->p3<=p->nMem ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pnErr = &aMem[pOp->p3]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); @@ -5441,7 +5441,7 @@ case OP_AggStep: { sqlite3VdbeMemStoreType(pRec); } ctx.pFunc = pOp->p4.pFunc; - assert( pOp->p3>0 && pOp->p3<=p->nMem ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); ctx.pMem = pMem = &aMem[pOp->p3]; pMem->n++; ctx.s.flags = MEM_Null; @@ -5488,7 +5488,7 @@ case OP_AggStep: { */ case OP_AggFinal: { Mem *pMem; - assert( pOp->p1>0 && pOp->p1<=p->nMem ); + assert( pOp->p1>0 && pOp->p1<=(p->nMem-p->nCursor) ); pMem = &aMem[pOp->p1]; assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); @@ -5905,7 +5905,7 @@ case OP_VColumn: { VdbeCursor *pCur = p->apCsr[pOp->p1]; assert( pCur->pVtabCursor ); - assert( pOp->p3>0 && pOp->p3<=p->nMem ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem-p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); if( pCur->nullRow ){ diff --git a/test/analyze9.test b/test/analyze9.test index 51531c5895..08e8126b24 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -147,6 +147,7 @@ do_execsql_test 3.4 { SELECT c FROM t1 WHERE b=3 AND a BETWEEN 30 AND 60; } {three-d three-e three-f} + #------------------------------------------------------------------------- # These tests verify that the sample selection for stat4 appears to be # working as designed. @@ -255,6 +256,15 @@ do_execsql_test 4.6 { ('34', '68', '102', '136', '170', '204', '238', '272') } {8} +#------------------------------------------------------------------------- +# The following would cause a crash at one point. +# +reset_db +do_execsql_test 5.1 { + PRAGMA encoding = 'utf-16'; + CREATE TABLE t0(v); + ANALYZE; +} finish_test From af2583c83c9860619edb432835bdbc7cc8b5257c Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 Aug 2013 18:43:21 +0000 Subject: [PATCH 30/67] Fix a crash that can occur following an OOM fault. FossilOrigin-Name: 9f80b2687012ab7c4d6d654fe19f40878bd78bd8 --- manifest | 16 ++--- manifest.uuid | 2 +- src/vdbemem.c | 172 +++++++++++++++++++++------------------------ test/analyze9.test | 24 +++++++ test/mallocA.test | 32 ++++++++- 5 files changed, 146 insertions(+), 100 deletions(-) diff --git a/manifest b/manifest index 4f54fd9f5d..a2529d2cb2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\ssome\sassert()\sstatements\sin\svdbe.c\sto\sensure\sthat\sa\smemory\scell\sused\sto\sstore\sa\sVdbeCursor\sobject\sis\snot\salso\sused\sfor\ssome\sother\spurpose. -D 2013-08-15T16:18:39.664 +C Fix\sa\scrash\sthat\scan\soccur\sfollowing\san\sOOM\sfault. +D 2013-08-15T18:43:21.395 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/vdbeInt.h e9b7c6b165a31a4715c5aa97223d20d265515231 F src/vdbeapi.c 4d13580bd058b39623e8fcfc233b7df4b8191e8b F src/vdbeaux.c a6ea36a9dc714e1128a0173249a0532ddcab0489 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 -F src/vdbemem.c 7ec9a78d6d4b2d4ebb4d3fd9b706065146616019 +F src/vdbemem.c c08dd81009fd6708dbb961d8ec7f82549765d680 F src/vdbesort.c 3937e06b2a0e354500e17dc206ef4c35770a5017 F src/vdbetrace.c e7ec40e1999ff3c6414424365d5941178966dcbc F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 @@ -308,7 +308,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test b018c837164ada65f4a80dadbbcdc89cb1fff362 +F test/analyze9.test 83e74db42a49bb185e05f3a44a5d65323aba8a40 F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b @@ -651,7 +651,7 @@ F test/malloc6.test 2f039d9821927eacae43e1831f815e157659a151 F test/malloc7.test 7c68a32942858bc715284856c5507446bba88c3a F test/malloc8.test 9b7a3f8cb9cf0b12fff566e80a980b1767bd961d F test/malloc9.test 2307c6ee3703b0a21391f3ea92388b4b73f9105e -F test/mallocA.test 47006c8d70f29b030652e251cb9d35ba60289198 +F test/mallocA.test 71e4b57e640c017cf2833e51fe6e8e43e8575b73 F test/mallocAll.test 98f1be74bc9f49a858bc4f361fc58e26486798be F test/mallocB.test bc475ab850cda896142ab935bbfbc74c24e51ed6 F test/mallocC.test 3dffe16532f109293ce1ccecd0c31dca55ef08c4 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 46fec9b1a1c4616df5a634dbf9235bd13408d3a9 -R e1ce93ce8d017990abe28f4c9f5b58f9 +P 71070c9fce86103f174220e07771df99b2e01405 +R fd0e4baaf9a9ffd0fe23644f24c090cf U dan -Z 9abc3408e2534a3305d7999552dece52 +Z 1c4d6e6edcc7281c470c7559e77faff9 diff --git a/manifest.uuid b/manifest.uuid index e48df0d900..f0c5e4a7e1 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -71070c9fce86103f174220e07771df99b2e01405 \ No newline at end of file +9f80b2687012ab7c4d6d654fe19f40878bd78bd8 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index db3e6b262e..b75e86b035 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1006,29 +1006,85 @@ sqlite3_value *sqlite3ValueNew(sqlite3 *db){ } /* -** Argument pCtx is actually a pointer to a database handle. Allocate and -** return an sqlite3_value object associated with this database handle. -** -** This function used as the xAlloc callback for valueFromExpr() when -** it is called by sqlite3ValueFromExpr(). +** Context object passed by sqlite3Stat4ProbeSetValue() through to +** valueNew(). See comments above valueNew() for details. */ -static sqlite3_value *valueNew(void *pCtx){ - return sqlite3ValueNew((sqlite3*)pCtx); +struct ValueNewStat4Ctx { + Parse *pParse; + Index *pIdx; + UnpackedRecord **ppRec; + int iVal; +}; + +/* +** Allocate and return a pointer to a new sqlite3_value object. If +** the second argument to this function is NULL, the object is allocated +** by calling sqlite3ValueNew(). +** +** Otherwise, if the second argument is non-zero, then this function is +** being called indirectly by sqlite3Stat4ProbeSetValue(). If it has not +** already been allocated, allocate the UnpackedRecord structure that +** that function will return to its caller here. Then return a pointer +** an sqlite3_value within the UnpackedRecord.a[] array. +*/ +static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) + if( p ){ + UnpackedRecord *pRec = p->ppRec[0]; + + if( pRec==0 ){ + Index *pIdx = p->pIdx; /* Index being probed */ + int nByte; /* Bytes of space to allocate */ + int i; /* Counter variable */ + int nCol = pIdx->nColumn+1; /* Number of index columns including rowid */ + + nByte = sizeof(Mem) * nCol + sizeof(UnpackedRecord); + pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte); + if( pRec ){ + pRec->pKeyInfo = sqlite3IndexKeyinfo(p->pParse, pIdx); + if( pRec->pKeyInfo ){ + assert( pRec->pKeyInfo->nField+1==nCol ); + pRec->pKeyInfo->enc = ENC(db); + pRec->flags = UNPACKED_PREFIX_MATCH; + pRec->aMem = (Mem *)&pRec[1]; + for(i=0; iaMem[i].flags = MEM_Null; + pRec->aMem[i].type = SQLITE_NULL; + pRec->aMem[i].db = db; + } + }else{ + sqlite3DbFree(db, pRec); + pRec = 0; + } + } + if( pRec==0 ) return 0; + p->ppRec[0] = pRec; + } + + pRec->nField = p->iVal+1; + return &pRec->aMem[p->iVal]; + } +#endif + return sqlite3ValueNew(db); } /* -** This function is the same as sqlite3ValueFromExpr(), except that instead -** of allocating any required sqlite3_value object by calling -** sqlite3ValueNew(), it does so by calling the supplied xAlloc hook. +** Extract a value from the supplied expression in the manner described +** above sqlite3ValueFromExpr(). Allocate the sqlite3_value object +** using valueNew(). +** +** If pCtx is NULL and an error occurs after the sqlite3_value object +** has been allocated, it is freed before returning. Or, if pCtx is not +** NULL, it is assumed that the caller will free any allocated object +** in all cases. */ int valueFromExpr( - sqlite3 *db, /* The database connection */ - Expr *pExpr, /* The expression to evaluate */ - u8 enc, /* Encoding to use */ - u8 affinity, /* Affinity to use */ - sqlite3_value **ppVal, /* Write the new value here */ - sqlite3_value *(*xAlloc)(void*), /* Used to allocate new sqlite3_value */ - void *pAlloc /* Argument passed to xAlloc */ + sqlite3 *db, /* The database connection */ + Expr *pExpr, /* The expression to evaluate */ + u8 enc, /* Encoding to use */ + u8 affinity, /* Affinity to use */ + sqlite3_value **ppVal, /* Write the new value here */ + struct ValueNewStat4Ctx *pCtx /* Second argument for valueNew() */ ){ int op; char *zVal = 0; @@ -1064,7 +1120,7 @@ int valueFromExpr( } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ - pVal = xAlloc(pAlloc); + pVal = valueNew(db, pCtx); if( pVal==0 ) goto no_mem; if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); @@ -1100,7 +1156,7 @@ int valueFromExpr( sqlite3ValueApplyAffinity(pVal, affinity, enc); } }else if( op==TK_NULL ){ - pVal = xAlloc(pAlloc); + pVal = valueNew(db, pCtx); if( pVal==0 ) goto no_mem; } #ifndef SQLITE_OMIT_BLOB_LITERAL @@ -1108,7 +1164,7 @@ int valueFromExpr( int nVal; assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); assert( pExpr->u.zToken[1]=='\'' ); - pVal = xAlloc(pAlloc); + pVal = valueNew(db, pCtx); if( !pVal ) goto no_mem; zVal = &pExpr->u.zToken[2]; nVal = sqlite3Strlen30(zVal)-1; @@ -1127,8 +1183,8 @@ int valueFromExpr( no_mem: db->mallocFailed = 1; sqlite3DbFree(db, zVal); - if( *ppVal==0 ) sqlite3ValueFree(pVal); - *ppVal = 0; + assert( *ppVal==0 ); + if( pCtx==0 ) sqlite3ValueFree(pVal); return SQLITE_NOMEM; } @@ -1149,7 +1205,7 @@ int sqlite3ValueFromExpr( u8 affinity, /* Affinity to use */ sqlite3_value **ppVal /* Write the new value here */ ){ - return valueFromExpr(db, pExpr, enc, affinity, ppVal, valueNew, (void*)db); + return valueFromExpr(db, pExpr, enc, affinity, ppVal, 0); } #if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) @@ -1207,64 +1263,6 @@ void sqlite3AnalyzeFunctions(void){ } } -/* -** A pointer to an instance of this object is passed as the context -** pointer to valueNewStat4() (see below. -*/ -struct ValueNewStat4Ctx { - Parse *pParse; - Index *pIdx; - UnpackedRecord **ppRec; - int iVal; -}; - -/* -** This function is used as the xAlloc function with valueFromExpr() when -** it is called by sqlite3Stat4ProbeSetValue(). The argument points to -** an object of type ValueNewStat4Ctx (see above). -** -** If it has not already been allocated, this function allocates an -** UnpackedRecord structure and space for up to N values, where N is the -** number of columns in the index being probed. -*/ -static sqlite3_value *valueNewStat4(void *pCtx){ - struct ValueNewStat4Ctx *p = (struct ValueNewStat4Ctx*)pCtx; - UnpackedRecord *pRec = p->ppRec[0]; - - if( pRec==0 ){ - sqlite3 *db = p->pParse->db; /* Database handle */ - Index *pIdx = p->pIdx; /* Index being probed */ - int nByte; /* Bytes of space to allocate */ - int i; /* Counter variable */ - int nCol = pIdx->nColumn+1; /* Number of index columns including rowid */ - - nByte = sizeof(Mem) * nCol + sizeof(UnpackedRecord); - pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte); - if( pRec ){ - pRec->pKeyInfo = sqlite3IndexKeyinfo(p->pParse, pIdx); - if( pRec->pKeyInfo ){ - assert( pRec->pKeyInfo->nField+1==nCol ); - pRec->pKeyInfo->enc = ENC(db); - pRec->flags = UNPACKED_PREFIX_MATCH; - pRec->aMem = (Mem *)&pRec[1]; - for(i=0; iaMem[i].flags = MEM_Null; - pRec->aMem[i].type = SQLITE_NULL; - pRec->aMem[i].db = db; - } - }else{ - sqlite3DbFree(db, pRec); - pRec = 0; - } - } - if( pRec==0 ) return 0; - p->ppRec[0] = pRec; - } - - pRec->nField = p->iVal+1; - return &pRec->aMem[p->iVal]; -} - /* ** This function is used to allocate and populate UnpackedRecord ** structures intended to be compared against sample index keys stored @@ -1313,12 +1311,8 @@ int sqlite3Stat4ProbeSetValue( alloc.ppRec = ppRec; alloc.iVal = iVal; -#if 0 - if( iVal>0 ){ *pbOk = 0; return SQLITE_OK; } -#endif - if( !pExpr ){ - pVal = valueNewStat4((void*)&alloc); + pVal = valueNew(pParse->db, &alloc); if( pVal ){ sqlite3VdbeMemSetNull((Mem*)pVal); *pbOk = 1; @@ -1330,7 +1324,7 @@ int sqlite3Stat4ProbeSetValue( int iVar = pExpr->iColumn; sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); if( (v = pParse->pReprepare) ){ - pVal = valueNewStat4((void*)&alloc); + pVal = valueNew(pParse->db, &alloc); if( pVal ){ rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iVal-1]); if( rc==SQLITE_OK ){ @@ -1345,9 +1339,7 @@ int sqlite3Stat4ProbeSetValue( } }else{ sqlite3 *db = pParse->db; - rc = valueFromExpr( - db, pExpr, ENC(db), affinity, &pVal, valueNewStat4, (void*)&alloc - ); + rc = valueFromExpr(db, pExpr, ENC(db), affinity, &pVal, &alloc); *pbOk = (pVal!=0); } diff --git a/test/analyze9.test b/test/analyze9.test index 08e8126b24..21b354792b 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -256,6 +256,30 @@ do_execsql_test 4.6 { ('34', '68', '102', '136', '170', '204', '238', '272') } {8} +reset_db +do_test 4.7 { + execsql { + BEGIN; + CREATE TABLE t1(o,t INTEGER PRIMARY KEY); + CREATE INDEX i1 ON t1(o); + } + for {set i 0} {$i<10000} {incr i [expr (($i<1000)?1:10)]} { + execsql { INSERT INTO t1 VALUES('x', $i) } + } + execsql { + COMMIT; + ANALYZE; + SELECT count(*) FROM sqlite_stat4; + } +} {8} +do_execsql_test 4.8 { + SELECT test_decode(sample) FROM sqlite_stat4; +} { + {x 211} {x 423} {x 635} {x 847} + {x 1590} {x 3710} {x 5830} {x 7950} +} + + #------------------------------------------------------------------------- # The following would cause a crash at one point. # diff --git a/test/mallocA.test b/test/mallocA.test index 89951276f8..1e05723e13 100644 --- a/test/mallocA.test +++ b/test/mallocA.test @@ -15,6 +15,7 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl source $testdir/malloc_common.tcl +set testprefix mallocA # Only run these tests if memory debugging is turned on. # @@ -40,7 +41,6 @@ db eval { db close copy_file test.db test.db.bu - do_malloc_test mallocA-1 -testdb test.db.bu -sqlbody { ANALYZE } @@ -53,6 +53,7 @@ do_malloc_test mallocA-1.2 -testdb test.db.bu -sqlbody { do_malloc_test mallocA-1.3 -testdb test.db.bu -sqlbody { ANALYZE main.t1 } + ifcapable reindex { do_malloc_test mallocA-2 -testdb test.db.bu -sqlbody { REINDEX; @@ -68,6 +69,35 @@ ifcapable reindex { } } +reset_db +sqlite3_db_config_lookaside db 0 0 0 +do_execsql_test 6-prep { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + INSERT INTO t1 VALUES('abc', 'w'); -- rowid=1 + INSERT INTO t1 VALUES('abc', 'x'); -- rowid=2 + INSERT INTO t1 VALUES('abc', 'y'); -- rowid=3 + INSERT INTO t1 VALUES('abc', 'z'); -- rowid=4 + + INSERT INTO t1 VALUES('def', 'w'); -- rowid=5 + INSERT INTO t1 VALUES('def', 'x'); -- rowid=6 + INSERT INTO t1 VALUES('def', 'y'); -- rowid=7 + INSERT INTO t1 VALUES('def', 'z'); -- rowid=8 + + ANALYZE; +} + +do_faultsim_test 6.1 -faults oom* -body { + execsql { SELECT rowid FROM t1 WHERE a='abc' AND b='x' } +} -test { + faultsim_test_result [list 0 2] +} +do_faultsim_test 6.2 -faults oom* -body { + execsql { SELECT rowid FROM t1 WHERE a='abc' AND b<'y' } +} -test { + faultsim_test_result [list 0 {1 2}] +} + # Ensure that no file descriptors were leaked. do_test malloc-99.X { catch {db close} From 0adbed8a6014e4d3af66b8c25a73566ddcae66cb Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 15 Aug 2013 19:56:32 +0000 Subject: [PATCH 31/67] Fix a crash that can occur if the sqlite_stat3 or sqlite_stat4 table is corrupt. FossilOrigin-Name: d51df8a8fcc31c37f6e1c9612204af5738ed865e --- manifest | 14 +++---- manifest.uuid | 2 +- src/analyze.c | 91 +++++++++++++++++++++++----------------------- test/analyze9.test | 27 ++++++++++++++ 4 files changed, 81 insertions(+), 53 deletions(-) diff --git a/manifest b/manifest index a2529d2cb2..55b717508b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scrash\sthat\scan\soccur\sfollowing\san\sOOM\sfault. -D 2013-08-15T18:43:21.395 +C Fix\sa\scrash\sthat\scan\soccur\sif\sthe\ssqlite_stat3\sor\ssqlite_stat4\stable\sis\scorrupt. +D 2013-08-15T19:56:32.997 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c e60d6329f77e84bac0a300d5227c28a05f5ff5de +F src/analyze.c 9533c7e948398bf9628178d6bf9a7845d7e29046 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -308,7 +308,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test 83e74db42a49bb185e05f3a44a5d65323aba8a40 +F test/analyze9.test b73f9514af962a139d2c61d7741b0ba090789ea2 F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 71070c9fce86103f174220e07771df99b2e01405 -R fd0e4baaf9a9ffd0fe23644f24c090cf +P 9f80b2687012ab7c4d6d654fe19f40878bd78bd8 +R abb1a9d20998265ab3bcabc4adb13b2c U dan -Z 1c4d6e6edcc7281c470c7559e77faff9 +Z 4cffc531446b09bd876c8b206583a719 diff --git a/manifest.uuid b/manifest.uuid index f0c5e4a7e1..3e8d68298e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f80b2687012ab7c4d6d654fe19f40878bd78bd8 \ No newline at end of file +d51df8a8fcc31c37f6e1c9612204af5738ed865e \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 2c7a142fd0..79fd421a78 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1268,6 +1268,44 @@ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ } #if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) + +/* +** Populate the pIdx->aAvgEq[] array based on the samples currently +** stored in pIdx->aSample[]. +*/ +static void initAvgEq(Index *pIdx){ + if( pIdx ){ + IndexSample *aSample = pIdx->aSample; + IndexSample *pFinal = &aSample[pIdx->nSample-1]; + int iCol; + for(iCol=0; iColnColumn; iCol++){ + int i; /* Used to iterate through samples */ + tRowcnt sumEq = 0; /* Sum of the nEq values */ + int nSum = 0; /* Number of terms contributing to sumEq */ + tRowcnt avgEq = 0; + tRowcnt nDLt = pFinal->anDLt[iCol]; + + /* Set nSum to the number of distinct (iCol+1) field prefixes that + ** occur in the stat4 table for this index before pFinal. Set + ** sumEq to the sum of the nEq values for column iCol for the same + ** set (adding the value only once where there exist dupicate + ** prefixes). */ + for(i=0; i<(pIdx->nSample-1); i++){ + if( aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] ){ + sumEq += aSample[i].anEq[iCol]; + nSum++; + } + } + if( nDLt>nSum ){ + avgEq = (pFinal->anLt[iCol] - sumEq)/(nDLt - nSum); + } + if( avgEq==0 ) avgEq = 1; + pIdx->aAvgEq[iCol] = avgEq; + if( pIdx->nSampleCol==1 ) break; + } + } +} + /* ** Load the content from either the sqlite_stat4 or sqlite_stat3 table ** into the relevant Index.aSample[] arrays. @@ -1292,7 +1330,6 @@ static int loadStatTbl( sqlite3_stmt *pStmt = 0; /* An SQL statement being run */ char *zSql; /* Text of the SQL statement */ Index *pPrevIdx = 0; /* Previous index in the loop */ - int idx = 0; /* slot in pIdx->aSample[] for next sample */ IndexSample *pSample; /* A slot in pIdx->aSample[] */ assert( db->lookaside.bEnabled==0 ); @@ -1328,7 +1365,6 @@ static int loadStatTbl( nAvgCol = pIdx->nColumn; } pIdx->nSampleCol = nIdxCol; - pIdx->nSample = nSample; nByte = sizeof(IndexSample) * nSample; nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; nByte += nAvgCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ @@ -1340,7 +1376,7 @@ static int loadStatTbl( } pSpace = (tRowcnt*)&pIdx->aSample[nSample]; pIdx->aAvgEq = pSpace; pSpace += nAvgCol; - for(i=0; inSample; i++){ + for(i=0; iaSample[i].anEq = pSpace; pSpace += nIdxCol; pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol; pIdx->aSample[i].anDLt = pSpace; pSpace += nIdxCol; @@ -1361,61 +1397,25 @@ static int loadStatTbl( while( sqlite3_step(pStmt)==SQLITE_ROW ){ char *zIndex; /* Index name */ Index *pIdx; /* Pointer to the index object */ - int i; /* Loop counter */ int nCol = 1; /* Number of columns in index */ zIndex = (char *)sqlite3_column_text(pStmt, 0); if( zIndex==0 ) continue; pIdx = sqlite3FindIndex(db, zIndex, zDb); if( pIdx==0 ) continue; - if( pIdx==pPrevIdx ){ - idx++; - }else{ - pPrevIdx = pIdx; - idx = 0; - } - assert( idxnSample ); /* This next condition is true if data has already been loaded from ** the sqlite_stat4 table. In this case ignore stat3 data. */ - if( bStat3 && pIdx->aSample[idx].anEq[0] ) continue; - pSample = &pIdx->aSample[idx]; - - if( bStat3==0 ){ - nCol = pIdx->nColumn+1; + nCol = pIdx->nSampleCol; + if( bStat3 && nCol>1 ) continue; + if( pIdx!=pPrevIdx ){ + initAvgEq(pPrevIdx); + pPrevIdx = pIdx; } + pSample = &pIdx->aSample[pIdx->nSample++]; decodeIntArray((char*)sqlite3_column_text(pStmt,1), nCol, pSample->anEq, 0); decodeIntArray((char*)sqlite3_column_text(pStmt,2), nCol, pSample->anLt, 0); decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); - if( idx==pIdx->nSample-1 ){ - IndexSample *aSample = pIdx->aSample; - int iCol; - for(iCol=0; iColnColumn; iCol++){ - tRowcnt sumEq = 0; /* Sum of the nEq values */ - int nSum = 0; /* Number of terms contributing to sumEq */ - tRowcnt avgEq = 0; - tRowcnt nDLt = pSample->anDLt[iCol]; - - /* Set nSum to the number of distinct (iCol+1) field prefixes that - ** occur in the stat4 table for this index before pSample. Set - ** sumEq to the sum of the nEq values for column iCol for the same - ** set (adding the value only once where there exist dupicate - ** prefixes). */ - for(i=0; i<(pIdx->nSample-1); i++){ - if( aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] ){ - sumEq += aSample[i].anEq[iCol]; - nSum++; - } - } - if( nDLt>nSum ){ - avgEq = (pSample->anLt[iCol] - sumEq)/(nDLt - nSum); - } - if( avgEq==0 ) avgEq = 1; - pIdx->aAvgEq[iCol] = avgEq; - if( bStat3 ) break; - } - } - pSample->n = sqlite3_column_bytes(pStmt, 4); pSample->p = sqlite3DbMallocZero(db, pSample->n); if( pSample->p==0 ){ @@ -1424,6 +1424,7 @@ static int loadStatTbl( } memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); } + initAvgEq(pPrevIdx); return sqlite3_finalize(pStmt); } diff --git a/test/analyze9.test b/test/analyze9.test index 21b354792b..7e67c5f832 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -290,5 +290,32 @@ do_execsql_test 5.1 { ANALYZE; } +#------------------------------------------------------------------------- +# This was also crashing. +# +reset_db +do_execsql_test 6.1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a); + CREATE INDEX i2 ON t1(b); + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 2); + INSERT INTO t1 VALUES(3, 3); + INSERT INTO t1 VALUES(4, 4); + INSERT INTO t1 VALUES(5, 5); + ANALYZE; + PRAGMA writable_schema = 1; + CREATE TEMP TABLE x1 AS + SELECT tbl,idx,neq,nlt,ndlt,sample FROM sqlite_stat4 + ORDER BY (rowid%5), rowid; + DELETE FROM sqlite_stat4; + INSERT INTO sqlite_stat4 SELECT * FROM x1; + PRAGMA writable_schema = 0; + ANALYZE sqlite_master; +} +do_execsql_test 6.2 { + SELECT * FROM t1 WHERE a = 'abc'; +} + finish_test From 61b34406032bf89435cf92ee5b6d706fb2a0af94 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 Aug 2013 13:34:50 +0000 Subject: [PATCH 32/67] Fix a potential segfault following an OOM while running ANALYZE. FossilOrigin-Name: 0118797823c9093e68113578f3f3aae65de41453 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 5 +++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index bb48b65cf5..8b743d0d4c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\srecent\strunk\schanges\sinto\sthe\sSTAT4\sbranch. -D 2013-08-16T12:26:33.317 +C Fix\sa\spotential\ssegfault\sfollowing\san\sOOM\swhile\srunning\sANALYZE. +D 2013-08-16T13:34:50.978 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 9533c7e948398bf9628178d6bf9a7845d7e29046 +F src/analyze.c 0427f9ea0b74fa90fc9b372b7f4ca136b48bb6b5 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P d51df8a8fcc31c37f6e1c9612204af5738ed865e f2d175f975cd0be63425424ec322a98fb650019e -R a01801631ed4ae55018a7d07c406419f +P c69b512af276a438399747af22659415af3a5d4d +R 1d895af259827a89f1c38733ddab2eab U drh -Z 2e42de2ef767503ad01b3c8e93c6139b +Z 7fb242ddc62e8a326599f76c33fd34fe diff --git a/manifest.uuid b/manifest.uuid index 564eb232af..acd4f03a69 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c69b512af276a438399747af22659415af3a5d4d \ No newline at end of file +0118797823c9093e68113578f3f3aae65de41453 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 79fd421a78..b1931914a8 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1424,8 +1424,9 @@ static int loadStatTbl( } memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); } - initAvgEq(pPrevIdx); - return sqlite3_finalize(pStmt); + rc = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) initAvgEq(pPrevIdx); + return rc; } /* From c367d4c05207333872408cf822eafc45ab281ee7 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 Aug 2013 14:09:43 +0000 Subject: [PATCH 33/67] Avoid buffer overreads and false OOM error reports that could be caused by corrupted sample records in the sqlite_stat4 table. FossilOrigin-Name: 9f85b6a52a0b9b1524daa6f24d85257e7f591e95 --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/analyze.c | 11 +++++++++-- test/analyze9.test | 28 +++++++++++++++++++++++++++- 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 8b743d0d4c..3c5429d527 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\spotential\ssegfault\sfollowing\san\sOOM\swhile\srunning\sANALYZE. -D 2013-08-16T13:34:50.978 +C Avoid\sbuffer\soverreads\sand\sfalse\sOOM\serror\sreports\sthat\scould\sbe\scaused\sby\scorrupted\ssample\srecords\sin\sthe\ssqlite_stat4\stable. +D 2013-08-16T14:09:43.262 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 0427f9ea0b74fa90fc9b372b7f4ca136b48bb6b5 +F src/analyze.c 30dd8962a6b95ed62793dc1e488ddec440c8a589 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -308,7 +308,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test b73f9514af962a139d2c61d7741b0ba090789ea2 +F test/analyze9.test baeb1ac3f2a6ee8bc740721f3e85b84f5ec35002 F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c69b512af276a438399747af22659415af3a5d4d -R 1d895af259827a89f1c38733ddab2eab -U drh -Z 7fb242ddc62e8a326599f76c33fd34fe +P 0118797823c9093e68113578f3f3aae65de41453 +R be7b1ba9d391582f66fe114108e327e7 +U dan +Z daee3e03dcffd75664c2937dc08ce654 diff --git a/manifest.uuid b/manifest.uuid index acd4f03a69..82cf5035bc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0118797823c9093e68113578f3f3aae65de41453 \ No newline at end of file +9f85b6a52a0b9b1524daa6f24d85257e7f591e95 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index b1931914a8..421890034b 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1411,18 +1411,25 @@ static int loadStatTbl( initAvgEq(pPrevIdx); pPrevIdx = pIdx; } - pSample = &pIdx->aSample[pIdx->nSample++]; + pSample = &pIdx->aSample[pIdx->nSample]; decodeIntArray((char*)sqlite3_column_text(pStmt,1), nCol, pSample->anEq, 0); decodeIntArray((char*)sqlite3_column_text(pStmt,2), nCol, pSample->anLt, 0); decodeIntArray((char*)sqlite3_column_text(pStmt,3), nCol, pSample->anDLt,0); + /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. + ** This is in case the sample record is corrupted. In that case, the + ** sqlite3VdbeRecordCompare() may read up to two varints past the + ** end of the allocated buffer before it realizes it is dealing with + ** a corrupt record. Adding the two 0x00 bytes prevents this from causing + ** a buffer overread. */ pSample->n = sqlite3_column_bytes(pStmt, 4); - pSample->p = sqlite3DbMallocZero(db, pSample->n); + pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); if( pSample->p==0 ){ sqlite3_finalize(pStmt); return SQLITE_NOMEM; } memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); + pIdx->nSample++; } rc = sqlite3_finalize(pStmt); if( rc==SQLITE_OK ) initAvgEq(pPrevIdx); diff --git a/test/analyze9.test b/test/analyze9.test index 7e67c5f832..33d823ba99 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -291,7 +291,7 @@ do_execsql_test 5.1 { } #------------------------------------------------------------------------- -# This was also crashing. +# This was also crashing (corrupt sqlite_stat4 table). # reset_db do_execsql_test 6.1 { @@ -317,5 +317,31 @@ do_execsql_test 6.2 { SELECT * FROM t1 WHERE a = 'abc'; } +#------------------------------------------------------------------------- +# The following tests experiment with adding corrupted records to the +# 'sample' column of the sqlite_stat4 table. +# +reset_db +sqlite3_db_config_lookaside db 0 0 0 + +do_execsql_test 7.1 { + CREATE TABLE t1(a, b); + CREATE INDEX i1 ON t1(a, b); + INSERT INTO t1 VALUES(1, 1); + INSERT INTO t1 VALUES(2, 2); + INSERT INTO t1 VALUES(3, 3); + INSERT INTO t1 VALUES(4, 4); + INSERT INTO t1 VALUES(5, 5); + ANALYZE; + UPDATE sqlite_stat4 SET sample = X'' WHERE rowid = 1; + ANALYZE sqlite_master; +} + +do_execsql_test 7.2 { + UPDATE sqlite_stat4 SET sample = X'FFFF'; + ANALYZE sqlite_master; + SELECT * FROM t1 WHERE a = 1; +} {1 1} + finish_test From 585c823256617e06ca826d1c7420c4c4896cb8b0 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 Aug 2013 14:23:02 +0000 Subject: [PATCH 34/67] Add extra tests. No code changes. FossilOrigin-Name: 949127d53e0a907ed6f73c6828637d94b265468c --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/analyze9.test | 22 ++++++++++++++++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 3c5429d527..2f0577a5e2 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Avoid\sbuffer\soverreads\sand\sfalse\sOOM\serror\sreports\sthat\scould\sbe\scaused\sby\scorrupted\ssample\srecords\sin\sthe\ssqlite_stat4\stable. -D 2013-08-16T14:09:43.262 +C Add\sextra\stests.\sNo\scode\schanges. +D 2013-08-16T14:23:02.198 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -308,7 +308,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test baeb1ac3f2a6ee8bc740721f3e85b84f5ec35002 +F test/analyze9.test 0e25a960081391909659844ac87fdc26542630e4 F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0118797823c9093e68113578f3f3aae65de41453 -R be7b1ba9d391582f66fe114108e327e7 +P 9f85b6a52a0b9b1524daa6f24d85257e7f591e95 +R 236823b823655ed29e083b8eac1f75f3 U dan -Z daee3e03dcffd75664c2937dc08ce654 +Z b297cf1029f061cc4748c85c1d9ef2e3 diff --git a/manifest.uuid b/manifest.uuid index 82cf5035bc..070ba78f13 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9f85b6a52a0b9b1524daa6f24d85257e7f591e95 \ No newline at end of file +949127d53e0a907ed6f73c6828637d94b265468c \ No newline at end of file diff --git a/test/analyze9.test b/test/analyze9.test index 33d823ba99..710f66793d 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -343,5 +343,27 @@ do_execsql_test 7.2 { SELECT * FROM t1 WHERE a = 1; } {1 1} +do_execsql_test 7.3 { + ANALYZE; + UPDATE sqlite_stat4 SET neq = '0 0 0'; + ANALYZE sqlite_master; + SELECT * FROM t1 WHERE a = 1; +} {1 1} + +do_execsql_test 7.4 { + ANALYZE; + UPDATE sqlite_stat4 SET ndlt = '0 0 0'; + ANALYZE sqlite_master; + SELECT * FROM t1 WHERE a = 3; +} {3 3} + +do_execsql_test 7.5 { + ANALYZE; + UPDATE sqlite_stat4 SET nlt = '0 0 0'; + ANALYZE sqlite_master; + SELECT * FROM t1 WHERE a = 5; +} {5 5} + + finish_test From 22d73b1cbed9e72c93c5f38e8173459644568b60 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 Aug 2013 14:48:23 +0000 Subject: [PATCH 35/67] Fixes for test code that was not working with utf16 databases. Run the analyze*.test scripts as part of the 'utf16' permutation test. FossilOrigin-Name: fe99494d99df95f699ffab07b1e212800b5ff00d --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/test_func.c | 4 ++-- test/analyze9.test | 37 ++++++++++++++++++++++++++++--------- test/permutations.test | 2 ++ 5 files changed, 41 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 2f0577a5e2..e9fa983ce4 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests.\sNo\scode\schanges. -D 2013-08-16T14:23:02.198 +C Fixes\sfor\stest\scode\sthat\swas\snot\sworking\swith\sutf16\sdatabases.\sRun\sthe\sanalyze*.test\sscripts\sas\spart\sof\sthe\s'utf16'\spermutation\stest. +D 2013-08-16T14:48:23.663 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -243,7 +243,7 @@ F src/test_config.c 3d148e338b575bd937f7746824f36a9c6682d238 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f -F src/test_func.c 338a6e5ade3560ad36280881bbcf45f28d06cb68 +F src/test_func.c f8235719dff4bf9ffee04c55a190af8782ce9ab5 F src/test_hexio.c abfdecb6fa58c354623978efceb088ca18e379cd F src/test_init.c 3cbad7ce525aec925f8fda2192d576d47f0d478a F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61 @@ -308,7 +308,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test 0e25a960081391909659844ac87fdc26542630e4 +F test/analyze9.test 8a7880f29586581ee124fb787b722227ead47ee8 F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b @@ -716,7 +716,7 @@ F test/pagesize.test 1dd51367e752e742f58e861e65ed7390603827a0 F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54 -F test/permutations.test 461ef4ea10db02cd421dfe5f988eac3e99b5cd9a +F test/permutations.test c5e7ae8a18cb8a0ced38dbbc9e2463536c1de45b F test/pragma.test 5e7de6c32a5d764f09437d2025f07e4917b9e178 F test/pragma2.test 3a55f82b954242c642f8342b17dffc8b47472947 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9f85b6a52a0b9b1524daa6f24d85257e7f591e95 -R 236823b823655ed29e083b8eac1f75f3 +P 949127d53e0a907ed6f73c6828637d94b265468c +R c976c76cd7c81566939cdf0a48444b08 U dan -Z b297cf1029f061cc4748c85c1d9ef2e3 +Z b5e8345ccccb17f5909eea938b24d774 diff --git a/manifest.uuid b/manifest.uuid index 070ba78f13..fe107f0a22 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -949127d53e0a907ed6f73c6828637d94b265468c \ No newline at end of file +fe99494d99df95f699ffab07b1e212800b5ff00d \ No newline at end of file diff --git a/src/test_func.c b/src/test_func.c index cc4596d488..b250e33106 100644 --- a/src/test_func.c +++ b/src/test_func.c @@ -496,7 +496,7 @@ static void test_extract( memset(&mem, 0, sizeof(mem)); mem.db = db; - mem.enc = SQLITE_UTF8; + mem.enc = ENC(db); pHdr += sqlite3GetVarint(pHdr, &iSerialType); pBody += sqlite3VdbeSerialGet(pBody, (u32)iSerialType, &mem); sqlite3VdbeMemStoreType(&mem); @@ -545,7 +545,7 @@ static void test_decode( memset(&mem, 0, sizeof(mem)); mem.db = db; - mem.enc = SQLITE_UTF8; + mem.enc = ENC(db); pHdr += sqlite3GetVarint(pHdr, &iSerialType); pBody += sqlite3VdbeSerialGet(pBody, (u32)iSerialType, &mem); diff --git a/test/analyze9.test b/test/analyze9.test index 710f66793d..655929f909 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -52,7 +52,7 @@ do_execsql_test 1.1 { ANALYZE; } {} -do_execsql_test 1.3 { +do_execsql_test 1.2 { SELECT tbl,idx,nEq,nLt,nDLt,test_decode(sample) FROM sqlite_stat4; } { t1 i1 {1 1 1} {0 0 0} {0 0 0} {(0) (0) 1} @@ -62,14 +62,16 @@ do_execsql_test 1.3 { t1 i1 {1 1 1} {4 4 4} {4 4 4} {(4) (4) 5} } -do_execsql_test 1.2 { - SELECT tbl,idx,nEq,nLt,nDLt,s(sample) FROM sqlite_stat4; -} { - t1 i1 {1 1 1} {0 0 0} {0 0 0} ....(0)(0) - t1 i1 {1 1 1} {1 1 1} {1 1 1} ....(1)(1). - t1 i1 {1 1 1} {2 2 2} {2 2 2} ....(2)(2). - t1 i1 {1 1 1} {3 3 3} {3 3 3} ....(3)(3). - t1 i1 {1 1 1} {4 4 4} {4 4 4} ....(4)(4). +if {[permutation] != "utf16"} { + do_execsql_test 1.3 { + SELECT tbl,idx,nEq,nLt,nDLt,s(sample) FROM sqlite_stat4; + } { + t1 i1 {1 1 1} {0 0 0} {0 0 0} ....(0)(0) + t1 i1 {1 1 1} {1 1 1} {1 1 1} ....(1)(1). + t1 i1 {1 1 1} {2 2 2} {2 2 2} ....(2)(2). + t1 i1 {1 1 1} {3 3 3} {3 3 3} ....(3)(3). + t1 i1 {1 1 1} {4 4 4} {4 4 4} ....(4)(4). + } } @@ -364,6 +366,23 @@ do_execsql_test 7.5 { SELECT * FROM t1 WHERE a = 5; } {5 5} +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 8.1 { + CREATE TABLE t1(x TEXT); + CREATE INDEX i1 ON t1(x); + INSERT INTO t1 VALUES('1'); + INSERT INTO t1 VALUES('2'); + INSERT INTO t1 VALUES('3'); + INSERT INTO t1 VALUES('4'); + ANALYZE; +} + +breakpoint +do_execsql_test 8.2 { + SELECT * FROM t1 WHERE x = 3; +} {3} finish_test diff --git a/test/permutations.test b/test/permutations.test index f0494870ae..755565f1d7 100644 --- a/test/permutations.test +++ b/test/permutations.test @@ -500,6 +500,8 @@ test_suite "utf16" -description { pragma encoding = 'UTF-16' } -files { alter.test alter3.test + analyze.test analyze3.test analyze4.test analyze5.test analyze6.test + analyze7.test analyze8.test analyze9.test analyzeA.test auth.test bind.test blob.test capi2.test capi3.test collate1.test collate2.test collate3.test collate4.test collate5.test collate6.test conflict.test date.test delete.test expr.test fkey1.test func.test From 0e1f0029c0e6f85c88080370920ed64ed3e1bf56 Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 Aug 2013 14:49:00 +0000 Subject: [PATCH 36/67] Fix valueFromExpr() so that it returns SQLITE_NOMEM following an OOM when changing text encodings. Also fix some asserts to accommodate OOM errors. FossilOrigin-Name: dc1ccd09c443cebc4731dbe1a0dea84a9ccf6a9b --- manifest | 16 ++++++++-------- manifest.uuid | 2 +- src/vdbemem.c | 5 +++-- src/where.c | 9 ++++++--- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 2f0577a5e2..d3b84aa06c 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sextra\stests.\sNo\scode\schanges. -D 2013-08-16T14:23:02.198 +C Fix\svalueFromExpr()\sso\sthat\sit\sreturns\sSQLITE_NOMEM\sfollowing\san\sOOM\swhen\nchanging\stext\sencodings.\s\sAlso\sfix\ssome\sasserts\sto\saccommodate\sOOM\serrors. +D 2013-08-16T14:49:00.182 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,14 +283,14 @@ F src/vdbeInt.h e9b7c6b165a31a4715c5aa97223d20d265515231 F src/vdbeapi.c 4d13580bd058b39623e8fcfc233b7df4b8191e8b F src/vdbeaux.c a6ea36a9dc714e1128a0173249a0532ddcab0489 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 -F src/vdbemem.c c08dd81009fd6708dbb961d8ec7f82549765d680 +F src/vdbemem.c 374c2c80202db87032800653361a63681176e6aa F src/vdbesort.c 3937e06b2a0e354500e17dc206ef4c35770a5017 F src/vdbetrace.c e7ec40e1999ff3c6414424365d5941178966dcbc F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c d97b5cb215c2e3e5b172b84000d2f3ee67ca0e86 +F src/where.c 9be69c42a02376c46d44907b38edd70f8a2cea7a F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9f85b6a52a0b9b1524daa6f24d85257e7f591e95 -R 236823b823655ed29e083b8eac1f75f3 -U dan -Z b297cf1029f061cc4748c85c1d9ef2e3 +P 949127d53e0a907ed6f73c6828637d94b265468c +R 48e5653eb5a198286249664d90ff2dc2 +U drh +Z c981aacf279c866f9ae15fcea49df2b7 diff --git a/manifest.uuid b/manifest.uuid index 070ba78f13..b169310d7e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -949127d53e0a907ed6f73c6828637d94b265468c \ No newline at end of file +dc1ccd09c443cebc4731dbe1a0dea84a9ccf6a9b \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index b75e86b035..109db96ace 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1091,6 +1091,7 @@ int valueFromExpr( sqlite3_value *pVal = 0; int negInt = 1; const char *zNeg = ""; + int rc = SQLITE_OK; if( !pExpr ){ *ppVal = 0; @@ -1137,7 +1138,7 @@ int valueFromExpr( } if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str; if( enc!=SQLITE_UTF8 ){ - sqlite3VdbeChangeEncoding(pVal, enc); + rc = sqlite3VdbeChangeEncoding(pVal, enc); } }else if( op==TK_UMINUS ) { /* This branch happens for multiple negative signs. Ex: -(-5) */ @@ -1178,7 +1179,7 @@ int valueFromExpr( sqlite3VdbeMemStoreType(pVal); } *ppVal = pVal; - return SQLITE_OK; + return rc; no_mem: db->mallocFailed = 1; diff --git a/src/where.c b/src/where.c index 3b9d023938..5180d9e166 100644 --- a/src/where.c +++ b/src/where.c @@ -2440,14 +2440,17 @@ static void whereKeyStats( if( res==0 ){ /* If (res==0) is true, then sample $i must be equal to pRec */ assert( inSample ); - assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) ); + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) + || pParse->db->mallocFailed ); }else{ /* Otherwise, pRec must be smaller than sample $i and larger than ** sample ($i-1). */ assert( i==pIdx->nSample - || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 ); + || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 + || pParse->db->mallocFailed ); assert( i==0 - || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 ); + || sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 + || pParse->db->mallocFailed ); } #endif /* ifdef SQLITE_DEBUG */ From 202ca9075f9628ad3c6138d197e781e4ffc05e1a Mon Sep 17 00:00:00 2001 From: drh Date: Fri, 16 Aug 2013 17:18:02 +0000 Subject: [PATCH 37/67] In sqlite3Stat4ProbeSetValue() change a local variable name iVar to iBindVar to avoid confusion with iVal, and fix a place where the name was actually confused. FossilOrigin-Name: 91733bc4856034c10bc2ba0acdae1970b05d62b9 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/vdbemem.c | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index 6427c534ac..88a72b7ed9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sfork\sthat\sresulted\sfrom\sa\scheck-in\srace. -D 2013-08-16T14:51:32.595 +C In\ssqlite3Stat4ProbeSetValue()\schange\sa\slocal\svariable\sname\siVar\sto\siBindVar\nto\savoid\sconfusion\swith\siVal,\sand\sfix\sa\splace\swhere\sthe\sname\swas\sactually\nconfused. +D 2013-08-16T17:18:02.373 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -283,7 +283,7 @@ F src/vdbeInt.h e9b7c6b165a31a4715c5aa97223d20d265515231 F src/vdbeapi.c 4d13580bd058b39623e8fcfc233b7df4b8191e8b F src/vdbeaux.c a6ea36a9dc714e1128a0173249a0532ddcab0489 F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 -F src/vdbemem.c 374c2c80202db87032800653361a63681176e6aa +F src/vdbemem.c 466784a5626d9cf45ed28a187ef809471b8a5675 F src/vdbesort.c 3937e06b2a0e354500e17dc206ef4c35770a5017 F src/vdbetrace.c e7ec40e1999ff3c6414424365d5941178966dcbc F src/vtab.c 2e8b489db47e20ae36cd247932dc671c9ded0624 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P dc1ccd09c443cebc4731dbe1a0dea84a9ccf6a9b fe99494d99df95f699ffab07b1e212800b5ff00d -R d4423f6bfe7897e7f162dd09be03fa9b +P b7fe4f362bdf7b233a7b09eb9ce16d296165f82a +R cab7e60081f3cc3c7528ec9934a95702 U drh -Z 09d98cea9440e05f9750de825f016c53 +Z 9a6aa61e3c8ed44f20601f67504227f9 diff --git a/manifest.uuid b/manifest.uuid index 7e02239584..1c77deb1c2 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b7fe4f362bdf7b233a7b09eb9ce16d296165f82a \ No newline at end of file +91733bc4856034c10bc2ba0acdae1970b05d62b9 \ No newline at end of file diff --git a/src/vdbemem.c b/src/vdbemem.c index 109db96ace..20fe8b7165 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1322,12 +1322,12 @@ int sqlite3Stat4ProbeSetValue( || (pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) ){ Vdbe *v; - int iVar = pExpr->iColumn; - sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); - if( (v = pParse->pReprepare) ){ + int iBindVar = pExpr->iColumn; + sqlite3VdbeSetVarmask(pParse->pVdbe, iBindVar); + if( (v = pParse->pReprepare)!=0 ){ pVal = valueNew(pParse->db, &alloc); if( pVal ){ - rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iVal-1]); + rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iBindVar-1]); if( rc==SQLITE_OK ){ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } From 0e8194a9c7a5ea2bb75f37f7b027aac9da8d80a1 Mon Sep 17 00:00:00 2001 From: dan Date: Fri, 16 Aug 2013 17:46:49 +0000 Subject: [PATCH 38/67] Add a test for the problem fixed by [91733bc485]. FossilOrigin-Name: 5c591104813a7e55e59f4f557cbb3e500fa817c2 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/analyze9.test | 42 ++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 88a72b7ed9..790e35101b 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\ssqlite3Stat4ProbeSetValue()\schange\sa\slocal\svariable\sname\siVar\sto\siBindVar\nto\savoid\sconfusion\swith\siVal,\sand\sfix\sa\splace\swhere\sthe\sname\swas\sactually\nconfused. -D 2013-08-16T17:18:02.373 +C Add\sa\stest\sfor\sthe\sproblem\sfixed\sby\s[91733bc485]. +D 2013-08-16T17:46:49.564 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -308,7 +308,7 @@ F test/analyze5.test 765c4e284aa69ca172772aa940946f55629bc8c4 F test/analyze6.test 19151da2c4e918905d2081b74ac5c4d47fc850ab F test/analyze7.test bb1409afc9e8629e414387ef048b8e0e3e0bdc4f F test/analyze8.test 093d15c1c888eed5034304a98c992f7360130b88 -F test/analyze9.test 8a7880f29586581ee124fb787b722227ead47ee8 +F test/analyze9.test 3095a9ebfea4a2b1f9db60375320ae7f219595ba F test/analyzeA.test 1a5c40079894847976d983ca39c707aaa44b6944 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b7fe4f362bdf7b233a7b09eb9ce16d296165f82a -R cab7e60081f3cc3c7528ec9934a95702 -U drh -Z 9a6aa61e3c8ed44f20601f67504227f9 +P 91733bc4856034c10bc2ba0acdae1970b05d62b9 +R ae06ac9f2ea4ec06a2d42ca5809eb2a5 +U dan +Z 9e49d814bd14c401a388c6f763be5e56 diff --git a/manifest.uuid b/manifest.uuid index 1c77deb1c2..6d9c4c51ab 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -91733bc4856034c10bc2ba0acdae1970b05d62b9 \ No newline at end of file +5c591104813a7e55e59f4f557cbb3e500fa817c2 \ No newline at end of file diff --git a/test/analyze9.test b/test/analyze9.test index 655929f909..be52eb94b2 100644 --- a/test/analyze9.test +++ b/test/analyze9.test @@ -378,11 +378,49 @@ do_execsql_test 8.1 { INSERT INTO t1 VALUES('4'); ANALYZE; } - -breakpoint do_execsql_test 8.2 { SELECT * FROM t1 WHERE x = 3; } {3} +#------------------------------------------------------------------------- +# Check that the bug fixed by [91733bc485] really is fixed. +# +reset_db +do_execsql_test 9.1 { + CREATE TABLE t1(a, b, c, d, e); + CREATE INDEX i1 ON t1(a, b, c, d); + CREATE INDEX i2 ON t1(e); +} +do_test 9.2 { + execsql BEGIN; + for {set i 0} {$i < 100} {incr i} { + execsql "INSERT INTO t1 VALUES('x', 'y', 'z', $i, [expr $i/2])" + } + for {set i 0} {$i < 20} {incr i} { + execsql "INSERT INTO t1 VALUES('x', 'y', 'z', 101, $i)" + } + for {set i 102} {$i < 200} {incr i} { + execsql "INSERT INTO t1 VALUES('x', 'y', 'z', $i, [expr $i/2])" + } + execsql COMMIT + execsql ANALYZE +} {} + +do_eqp_test 9.3.1 { + SELECT * FROM t1 WHERE a='x' AND b='y' AND c='z' AND d=101 AND e=5; +} {/t1 USING INDEX i2/} +do_eqp_test 9.3.2 { + SELECT * FROM t1 WHERE a='x' AND b='y' AND c='z' AND d=99 AND e=5; +} {/t1 USING INDEX i1/} + +set value_d [expr 101] +do_eqp_test 9.4.1 { + SELECT * FROM t1 WHERE a='x' AND b='y' AND c='z' AND d=$value_d AND e=5 +} {/t1 USING INDEX i2/} +set value_d [expr 99] +do_eqp_test 9.4.2 { + SELECT * FROM t1 WHERE a='x' AND b='y' AND c='z' AND d=$value_d AND e=5 +} {/t1 USING INDEX i1/} + finish_test From 92707acfd07ef47d9147850e29026980befca20b Mon Sep 17 00:00:00 2001 From: drh Date: Sat, 17 Aug 2013 18:57:15 +0000 Subject: [PATCH 39/67] Adjustments to #ifdefs in analyze.c to all a clean compile with no extra code with both ENABLE_STAT3 and ENABLE_STAT4 and with neither. FossilOrigin-Name: f86b75b6c7290ee6ddb3636090b00e99fc68c45e --- manifest | 14 +++--- manifest.uuid | 2 +- src/analyze.c | 122 +++++++++++++++++++++++++++++--------------------- 3 files changed, 79 insertions(+), 59 deletions(-) diff --git a/manifest b/manifest index 790e35101b..36d9810593 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\sfor\sthe\sproblem\sfixed\sby\s[91733bc485]. -D 2013-08-16T17:46:49.564 +C Adjustments\sto\s#ifdefs\sin\sanalyze.c\sto\sall\sa\sclean\scompile\swith\sno\nextra\scode\swith\sboth\sENABLE_STAT3\sand\sENABLE_STAT4\sand\swith\sneither. +D 2013-08-17T18:57:15.775 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 30dd8962a6b95ed62793dc1e488ddec440c8a589 +F src/analyze.c 10a4f1cda87aec6c56760c66e5feaaf1cdc9a641 F src/attach.c 1816f5a9eea8d2010fc2b22b44f0f63eb3a62704 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 43b348822db3e4cef48b2ae5a445fbeb6c73a165 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 91733bc4856034c10bc2ba0acdae1970b05d62b9 -R ae06ac9f2ea4ec06a2d42ca5809eb2a5 -U dan -Z 9e49d814bd14c401a388c6f763be5e56 +P 5c591104813a7e55e59f4f557cbb3e500fa817c2 +R fed6486268a39dfc52589cfac215691e +U drh +Z ee5235686bd18f87bb4458ce74357891 diff --git a/manifest.uuid b/manifest.uuid index 6d9c4c51ab..cce34a49db 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5c591104813a7e55e59f4f557cbb3e500fa817c2 \ No newline at end of file +f86b75b6c7290ee6ddb3636090b00e99fc68c45e \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 421890034b..e68070a112 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -141,14 +141,17 @@ #include "sqliteInt.h" #if defined(SQLITE_ENABLE_STAT4) -# define IsStat4 1 -# define IsStat3 0 +# define IsStat4 1 +# define IsStat3 0 +# define SQLITE_ENABLE_STAT34 1 #elif defined(SQLITE_ENABLE_STAT3) -# define IsStat4 0 -# define IsStat3 1 +# define IsStat4 0 +# define IsStat3 1 +# define SQLITE_ENABLE_STAT34 1 #else -# define IsStat4 0 -# define IsStat3 0 +# define IsStat4 0 +# define IsStat3 0 +# undef SQLITE_ENABLE_STAT34 #endif /* @@ -366,6 +369,7 @@ static const FuncDef statInitFuncdef = { 0 /* pDestructor */ }; +#ifdef SQLITE_ENABLE_STAT34 /* ** Return true if pNew is to be preferred over pOld. */ @@ -487,6 +491,7 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ p->iMin = iMin; } } +#endif /* SQLITE_ENABLE_STAT34 */ /* ** Field iChng of the index being scanned has changed. So at this point @@ -495,30 +500,31 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ ** correct at this point. */ static void samplePushPrevious(Stat4Accum *p, int iChng){ - if( IsStat4 ){ - int i; +#ifdef SQLITE_ENABLE_STAT4 + int i; - /* Check if any samples from the aBest[] array should be pushed - ** into IndexSample.a[] at this point. */ - for(i=(p->nCol-2); i>=iChng; i--){ - Stat4Sample *pBest = &p->aBest[i]; - if( p->nSamplemxSample - || sampleIsBetter(pBest, &p->a[p->iMin]) - ){ - sampleInsert(p, pBest, i); - } - } - - /* Update the anEq[] fields of any samples already collected. */ - for(i=p->nSample-1; i>=0; i--){ - int j; - for(j=iChng; jnCol; j++){ - if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; - } + /* Check if any samples from the aBest[] array should be pushed + ** into IndexSample.a[] at this point. */ + for(i=(p->nCol-2); i>=iChng; i--){ + Stat4Sample *pBest = &p->aBest[i]; + if( p->nSamplemxSample + || sampleIsBetter(pBest, &p->a[p->iMin]) + ){ + sampleInsert(p, pBest, i); } } - if( IsStat3 && iChng==0 ){ + /* Update the anEq[] fields of any samples already collected. */ + for(i=p->nSample-1; i>=0; i--){ + int j; + for(j=iChng; jnCol; j++){ + if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + } + } +#endif + +#if defined(SQLITE_ENABLE_STAT3) && !defined(SQLITE_ENABLE_STAT4) + if( iChng==0 ){ tRowcnt nLt = p->current.anLt[0]; tRowcnt nEq = p->current.anEq[0]; @@ -534,6 +540,7 @@ static void samplePushPrevious(Stat4Accum *p, int iChng){ sampleInsert(p, &p->current, 0); } } +#endif } /* @@ -583,7 +590,8 @@ static void statPush( p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; } - if( IsStat4 ){ +#ifdef SQLITE_ENABLE_STAT4 + { tRowcnt nLt = p->current.anLt[p->nCol-1]; /* Check if this is to be a periodic sample. If so, add it. */ @@ -602,6 +610,7 @@ static void statPush( } } } +#endif } static const FuncDef statPushFuncdef = { 3, /* nArg */ @@ -624,15 +633,14 @@ static const FuncDef statPushFuncdef = { #define STAT_GET_NDLT 4 /* "ndlt" column of stat[34] entry */ /* -** Implementation of the stat3_get(P,N,...) SQL function. This routine is -** used to query the results. Content is returned for the Nth sqlite_stat3 -** row where N is between 0 and S-1 and S is the number of samples. The -** value returned depends on the number of arguments. +** Implementation of the stat_get(P,J) SQL function. This routine is +** used to query the results. Content is returned for parameter J +** which is one of the STAT_GET_xxxx values defined above. ** -** argc==2 result: rowid -** argc==3 result: nEq -** argc==4 result: nLt -** argc==5 result: nDLt +** If neither STAT3 nor STAT4 are enabled, then J is always +** STAT_GET_STAT1 and is hence omitted and this routine becomes +** a one-parameter function, stat_get(P), that always returns the +** stat1 table entry information. */ static void statGet( sqlite3_context *context, @@ -640,13 +648,19 @@ static void statGet( sqlite3_value **argv ){ Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); +#ifdef SQLITE_ENABLE_STAT34 + /* STAT3 and STAT4 have a parameter on this routine. */ int eCall = sqlite3_value_int(argv[1]); + assert( argc==2 ); assert( eCall==STAT_GET_STAT1 || eCall==STAT_GET_NEQ || eCall==STAT_GET_ROWID || eCall==STAT_GET_NLT || eCall==STAT_GET_NDLT ); - - if( eCall==STAT_GET_STAT1 ){ + if( eCall==STAT_GET_STAT1 ) +#else + assert( argc==1 ); +#endif + { /* Return the value to store in the "stat" column of the sqlite_stat1 ** table for this index. ** @@ -689,7 +703,9 @@ static void statGet( assert( z[0]=='\0' && z>zRet ); sqlite3_result_text(context, zRet, -1, sqlite3_free); - }else if( eCall==STAT_GET_ROWID ){ + } +#ifdef SQLITE_ENABLE_STAT34 + else if( eCall==STAT_GET_ROWID ){ if( p->iGet<0 ){ samplePushPrevious(p, 0); p->iGet = 0; @@ -730,6 +746,7 @@ static void statGet( } } } +#endif /* SQLITE_ENABLE_STAT34 */ } static const FuncDef statGetFuncdef = { 2, /* nArg */ @@ -747,10 +764,14 @@ static const FuncDef statGetFuncdef = { static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ assert( regOut!=regStat4 && regOut!=regStat4+1 ); +#ifdef SQLITE_ENABLE_STAT34 sqlite3VdbeAddOp2(v, OP_Integer, iParam, regStat4+1); +#else + assert( iParam==STAT_GET_STAT1 ); +#endif sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4, regOut); sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 2); + sqlite3VdbeChangeP5(v, 1 + IsStat3 + IsStat4); } /* @@ -1248,7 +1269,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** and its contents. */ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT34 if( pIdx->aSample ){ int j; for(j=0; jnSample; j++){ @@ -1264,11 +1285,10 @@ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ #else UNUSED_PARAMETER(db); UNUSED_PARAMETER(pIdx); -#endif +#endif /* SQLITE_ENABLE_STAT34 */ } -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) - +#ifdef SQLITE_ENABLE_STAT34 /* ** Populate the pIdx->aAvgEq[] array based on the samples currently ** stored in pIdx->aSample[]. @@ -1462,20 +1482,20 @@ static int loadStat4(sqlite3 *db, const char *zDb){ return rc; } -#endif /* SQLITE_ENABLE_STAT4 */ +#endif /* SQLITE_ENABLE_STAT34 */ /* -** Load the content of the sqlite_stat1 and sqlite_stat4 tables. The +** Load the content of the sqlite_stat1 and sqlite_stat3/4 tables. The ** contents of sqlite_stat1 are used to populate the Index.aiRowEst[] -** arrays. The contents of sqlite_stat4 are used to populate the +** arrays. The contents of sqlite_stat3/4 are used to populate the ** Index.aSample[] arrays. ** ** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR -** is returned. In this case, even if SQLITE_ENABLE_STAT4 was defined -** during compilation and the sqlite_stat4 table is present, no data is +** is returned. In this case, even if SQLITE_ENABLE_STAT3/4 was defined +** during compilation and the sqlite_stat3/4 table is present, no data is ** read from it. ** -** If SQLITE_ENABLE_STAT4 was defined during compilation and the +** If SQLITE_ENABLE_STAT3/4 was defined during compilation and the ** sqlite_stat4 table is not present in the database, SQLITE_ERROR is ** returned. However, in this case, data is read from the sqlite_stat1 ** table (if it is present) before returning. @@ -1498,7 +1518,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3DefaultRowEst(pIdx); -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT34 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; #endif @@ -1523,7 +1543,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat4 table. */ -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT34 if( rc==SQLITE_OK ){ int lookasideEnabled = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; From e98844f723bc932426691a685645bba5a811e088 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 24 Aug 2013 00:59:24 +0000 Subject: [PATCH 40/67] Unify the result of PRAGMA mmap_size when mmap support is disabled. FossilOrigin-Name: 032c31593d6f569842830cac6222362be68b2084 --- manifest | 19 +++++++++++-------- manifest.uuid | 2 +- src/os_unix.c | 22 ++++++++++++++-------- src/pragma.c | 8 +++++--- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 5f24bd8b5c..6bcd5436ed 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\s#ifdefs\sto\sget\sSQLITE_OMIT_AUTOMATIC_INDEX\sand\sSQLITE_OMIT_PAGER_PRAGMAS\nto\sbuild. -D 2013-08-23T17:33:46.203 +C Unify\sthe\sresult\sof\sPRAGMA\smmap_size\swhen\smmap\ssupport\sis\sdisabled. +D 2013-08-24T00:59:24.722 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -202,7 +202,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 9eafa5458cf2ff684ddccff82c9bb113c7cad847 +F src/os_unix.c e767bca6caa9db1eabc7e0657e425952d3272464 F src/os_win.c 1d84f2079d9b91f91a4b5dbfa5e08f1b1a0ed0ff F src/pager.c 2aa4444ffe86e9282d03bc349a4a5e49bd77c0e8 F src/pager.h f094af9f6ececfaa8a1e93876905a4f34233fb0c @@ -210,7 +210,7 @@ F src/parse.y 27c6b4138497d6f8360ba7847da6ed48033f957f F src/pcache.c f8043b433a57aba85384a531e3937a804432a346 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222 F src/pcache1.c a467393909a4ed7ca9de066d85ba5c5b04a5be63 -F src/pragma.c dc9ceab9b361e3fbd461ebaa7fb8003233e20725 +F src/pragma.c 3aa3d8c8623b7b71c5b1bfb72dcc31fb0c25665f F src/prepare.c fa6988589f39af8504a61731614cd4f6ae71554f F src/printf.c da9119eb31a187a4b99f60aa4a225141c0ebb74b F src/random.c 0b2dbc37fdfbfa6bd455b091dfcef5bdb32dba68 @@ -1105,7 +1105,10 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3ad1f998e5db044849b528c31c4a7e1f3d527320 -R 5a6ead2576f296902c1b2ccbdf985254 -U drh -Z 27173a06b09c0a2f08350ebdebfa00ec +P ed310201628cf79c0f57674ae7478ee6738b1c6e +R bd8622b0fc71103f701b6641f54bb274 +T *branch * mmapDisabled +T *sym-mmapDisabled * +T -sym-trunk * +U mistachkin +Z 80111f228705ea035bb71c4b1488f527 diff --git a/manifest.uuid b/manifest.uuid index a81384f06a..1b4ba748ad 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed310201628cf79c0f57674ae7478ee6738b1c6e \ No newline at end of file +032c31593d6f569842830cac6222362be68b2084 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 152bf9a113..339bae703b 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -218,11 +218,13 @@ struct unixFile { const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ +#if SQLITE_MAX_MMAP_SIZE>0 int nFetchOut; /* Number of outstanding xFetch refs */ sqlite3_int64 mmapSize; /* Usable size of mapping at pMapRegion */ sqlite3_int64 mmapSizeActual; /* Actual size of mapping at pMapRegion */ sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ void *pMapRegion; /* Memory mapped region */ +#endif #ifdef __QNXNTO__ int sectorSize; /* Device sector size */ int deviceCharacteristics; /* Precomputed device characteristics */ @@ -1871,8 +1873,10 @@ static int unixUnlock(sqlite3_file *id, int eFileLock){ return posixUnlock(id, eFileLock, 0); } +#if SQLITE_MAX_MMAP_SIZE>0 static int unixMapfile(unixFile *pFd, i64 nByte); static void unixUnmapfile(unixFile *pFd); +#endif /* ** This function performs the parts of the "close file" operation @@ -1886,7 +1890,9 @@ static void unixUnmapfile(unixFile *pFd); */ static int closeUnixFile(sqlite3_file *id){ unixFile *pFile = (unixFile*)id; +#if SQLITE_MAX_MMAP_SIZE>0 unixUnmapfile(pFile); +#endif if( pFile->h>=0 ){ robust_close(pFile, pFile->h, __LINE__); pFile->h = -1; @@ -3590,6 +3596,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ } #endif +#if SQLITE_MAX_MMAP_SIZE>0 /* If the file was just truncated to a size smaller than the currently ** mapped region, reduce the effective mapping size as well. SQLite will ** use read() and write() to access data beyond this point from now on. @@ -3597,6 +3604,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ if( nBytemmapSize ){ pFile->mmapSize = nByte; } +#endif return SQLITE_OK; } @@ -3686,6 +3694,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ } } +#if SQLITE_MAX_MMAP_SIZE>0 if( pFile->mmapSizeMax>0 && nByte>pFile->mmapSize ){ int rc; if( pFile->szChunk<=0 ){ @@ -3698,6 +3707,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ rc = unixMapfile(pFile, nByte); return rc; } +#endif return SQLITE_OK; } @@ -3766,6 +3776,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return SQLITE_OK; } +#if SQLITE_MAX_MMAP_SIZE>0 case SQLITE_FCNTL_MMAP_SIZE: { i64 newLimit = *(i64*)pArg; int rc = SQLITE_OK; @@ -3782,6 +3793,7 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ } return rc; } +#endif #ifdef SQLITE_DEBUG /* The pager calls this method to signal that it has done ** a rollback and that the database is therefore unchanged and @@ -4592,22 +4604,20 @@ static int unixShmUnmap( # define unixShmUnmap 0 #endif /* #ifndef SQLITE_OMIT_WAL */ +#if SQLITE_MAX_MMAP_SIZE>0 /* ** If it is currently memory mapped, unmap file pFd. */ static void unixUnmapfile(unixFile *pFd){ assert( pFd->nFetchOut==0 ); -#if SQLITE_MAX_MMAP_SIZE>0 if( pFd->pMapRegion ){ osMunmap(pFd->pMapRegion, pFd->mmapSizeActual); pFd->pMapRegion = 0; pFd->mmapSize = 0; pFd->mmapSizeActual = 0; } -#endif } -#if SQLITE_MAX_MMAP_SIZE>0 /* ** Return the system page size. */ @@ -4620,9 +4630,7 @@ static int unixGetPagesize(void){ return (int)sysconf(_SC_PAGESIZE); #endif } -#endif /* SQLITE_MAX_MMAP_SIZE>0 */ -#if SQLITE_MAX_MMAP_SIZE>0 /* ** Attempt to set the size of the memory mapping maintained by file ** descriptor pFd to nNew bytes. Any existing mapping is discarded. @@ -4707,7 +4715,6 @@ static void unixRemapfile( pFd->pMapRegion = (void *)pNew; pFd->mmapSize = pFd->mmapSizeActual = nNew; } -#endif /* ** Memory map or remap the file opened by file-descriptor pFd (if the file @@ -4726,7 +4733,6 @@ static void unixRemapfile( ** code otherwise. */ static int unixMapfile(unixFile *pFd, i64 nByte){ -#if SQLITE_MAX_MMAP_SIZE>0 i64 nMap = nByte; int rc; @@ -4752,10 +4758,10 @@ static int unixMapfile(unixFile *pFd, i64 nByte){ unixUnmapfile(pFd); } } -#endif return SQLITE_OK; } +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ /* ** If possible, return a pointer to a mapping of file fd starting at offset diff --git a/src/pragma.c b/src/pragma.c index ffaf69e9e4..d4cf597ff8 100644 --- a/src/pragma.c +++ b/src/pragma.c @@ -661,7 +661,7 @@ void sqlite3Pragma( Pager *pPager = sqlite3BtreePager(pDb->pBt); i64 iLimit = -2; if( zRight ){ - sqlite3Atoi64(zRight, &iLimit, 1000000, SQLITE_UTF8); + sqlite3Atoi64(zRight, &iLimit, sqlite3Strlen30(zRight), SQLITE_UTF8); if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); @@ -795,10 +795,11 @@ void sqlite3Pragma( */ if( sqlite3StrICmp(zLeft,"mmap_size")==0 ){ sqlite3_int64 sz; +#if SQLITE_MAX_MMAP_SIZE>0 assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( zRight ){ int ii; - sqlite3Atoi64(zRight, &sz, 1000, SQLITE_UTF8); + sqlite3Atoi64(zRight, &sz, sqlite3Strlen30(zRight), SQLITE_UTF8); if( sz<0 ) sz = sqlite3GlobalConfig.szMmap; if( pId2->n==0 ) db->szMmap = sz; for(ii=db->nDb-1; ii>=0; ii--){ @@ -809,8 +810,9 @@ void sqlite3Pragma( } sz = -1; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_MMAP_SIZE, &sz); -#if SQLITE_MAX_MMAP_SIZE==0 +#else sz = 0; + rc = SQLITE_OK; #endif if( rc==SQLITE_OK ){ returnSingleInt(pParse, "mmap_size", sz); From b5ca3cbcd3fe39a55f6b22ddea3c026c355d3102 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 24 Aug 2013 01:12:03 +0000 Subject: [PATCH 41/67] Fix a couple compilation issues on Unix. FossilOrigin-Name: 25b029d8f32440a94ef8af45153423f6702d7431 --- manifest | 15 ++++++--------- manifest.uuid | 2 +- src/os_unix.c | 4 ++++ 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 6bcd5436ed..50cfe34e75 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Unify\sthe\sresult\sof\sPRAGMA\smmap_size\swhen\smmap\ssupport\sis\sdisabled. -D 2013-08-24T00:59:24.722 +C Fix\sa\scouple\scompilation\sissues\son\sUnix. +D 2013-08-24T01:12:03.598 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -202,7 +202,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c e767bca6caa9db1eabc7e0657e425952d3272464 +F src/os_unix.c 44a2b26acd3f3f3a0b5d8495af945cec70a9e9df F src/os_win.c 1d84f2079d9b91f91a4b5dbfa5e08f1b1a0ed0ff F src/pager.c 2aa4444ffe86e9282d03bc349a4a5e49bd77c0e8 F src/pager.h f094af9f6ececfaa8a1e93876905a4f34233fb0c @@ -1105,10 +1105,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ed310201628cf79c0f57674ae7478ee6738b1c6e -R bd8622b0fc71103f701b6641f54bb274 -T *branch * mmapDisabled -T *sym-mmapDisabled * -T -sym-trunk * +P 032c31593d6f569842830cac6222362be68b2084 +R e9c8a1891e536319ba2e72bdf41d1b7c U mistachkin -Z 80111f228705ea035bb71c4b1488f527 +Z cf772d6485c0eea6b9958d111508c114 diff --git a/manifest.uuid b/manifest.uuid index 1b4ba748ad..3992a4377e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -032c31593d6f569842830cac6222362be68b2084 \ No newline at end of file +25b029d8f32440a94ef8af45153423f6702d7431 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 339bae703b..0ae88517e4 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -4810,6 +4810,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ UNUSED_PARAMETER(iOff); +#if SQLITE_MAX_MMAP_SIZE>0 /* If p==0 (unmap the entire file) then there must be no outstanding ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), ** then there must be at least one outstanding. */ @@ -4825,6 +4826,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ } assert( pFd->nFetchOut>=0 ); +#endif return SQLITE_OK; } @@ -5156,7 +5158,9 @@ static int fillInUnixFile( pNew->pVfs = pVfs; pNew->zPath = zFilename; pNew->ctrlFlags = (u8)ctrlFlags; +#if SQLITE_MAX_MMAP_SIZE>0 pNew->mmapSizeMax = sqlite3GlobalConfig.szMmap; +#endif if( sqlite3_uri_boolean(((ctrlFlags & UNIXFILE_URI) ? zFilename : 0), "psow", SQLITE_POWERSAFE_OVERWRITE) ){ pNew->ctrlFlags |= UNIXFILE_PSOW; From e49d98fb0a5f0e2987d128110d08c833babd6606 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Sat, 24 Aug 2013 23:55:01 +0000 Subject: [PATCH 42/67] Several modifications to the use of the MAX_PATH macro on Windows to improve consistency. FossilOrigin-Name: bda4c47df8e80b4cc9e8aac8fd74482869f96107 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 35 ++++++++++++++++++++++------------- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/manifest b/manifest index 50cfe34e75..99343dbe57 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\scompilation\sissues\son\sUnix. -D 2013-08-24T01:12:03.598 +C Several\smodifications\sto\sthe\suse\sof\sthe\sMAX_PATH\smacro\son\sWindows\sto\simprove\sconsistency. +D 2013-08-24T23:55:01.356 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -203,7 +203,7 @@ F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c 44a2b26acd3f3f3a0b5d8495af945cec70a9e9df -F src/os_win.c 1d84f2079d9b91f91a4b5dbfa5e08f1b1a0ed0ff +F src/os_win.c 339cdb887654f805a3e7225dd67f210cd5c831d3 F src/pager.c 2aa4444ffe86e9282d03bc349a4a5e49bd77c0e8 F src/pager.h f094af9f6ececfaa8a1e93876905a4f34233fb0c F src/parse.y 27c6b4138497d6f8360ba7847da6ed48033f957f @@ -1105,7 +1105,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 032c31593d6f569842830cac6222362be68b2084 -R e9c8a1891e536319ba2e72bdf41d1b7c +P 25b029d8f32440a94ef8af45153423f6702d7431 +R 43a5201efd5c79e69d4099d70d2d8790 U mistachkin -Z cf772d6485c0eea6b9958d111508c114 +Z 68ffb990c84dd84cc8ba4e540d45c8d3 diff --git a/manifest.uuid b/manifest.uuid index 3992a4377e..ad42030f96 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -25b029d8f32440a94ef8af45153423f6702d7431 \ No newline at end of file +bda4c47df8e80b4cc9e8aac8fd74482869f96107 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 10c4316448..0f27ac43f2 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -50,6 +50,24 @@ # define SQLITE_WIN32_HAS_WIDE #endif +/* +** Maximum pathname length (in bytes) for Win32. The MAX_PATH macro is in +** characters, so we allocate 3 bytes per character assuming worst-case of +** 3-bytes-per-character for UTF8. +*/ +#ifndef SQLITE_WIN32_MAX_PATH +# define SQLITE_WIN32_MAX_PATH (MAX_PATH*3) +#endif + +/* +** Maximum error message length (in bytes) for WinRT. The MAX_PATH macro is +** in characters, so we allocate 3 bytes per character assuming worst-case of +** 3-bytes-per-character for UTF8. +*/ +#ifndef SQLITE_WIN32_MAX_ERRMSG +# define SQLITE_WIN32_MAX_ERRMSG (MAX_PATH*3) +#endif + /* ** Do we need to manually define the Win32 file mapping APIs for use with WAL ** mode (e.g. these APIs are available in the Windows CE SDK; however, they @@ -1463,14 +1481,14 @@ static int getLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ if( isNT() ){ #if SQLITE_OS_WINRT - WCHAR zTempWide[MAX_PATH+1]; /* NOTE: Somewhat arbitrary. */ + WCHAR zTempWide[SQLITE_WIN32_MAX_ERRMSG+1]; /* NOTE: Somewhat arbitrary. */ dwLen = osFormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, lastErrno, 0, zTempWide, - MAX_PATH, + SQLITE_WIN32_MAX_ERRMSG, 0); #else LPWSTR zTempWide = NULL; @@ -3868,15 +3886,6 @@ static void *convertUtf8Filename(const char *zFilename){ return zConverted; } -/* -** Maximum pathname length (in bytes) for windows. The MAX_PATH macro is -** in characters, so we allocate 3 bytes per character assuming worst-case -** 3-bytes-per-character UTF8. -*/ -#ifndef SQLITE_WIN32_MAX_PATH -# define SQLITE_WIN32_MAX_PATH (MAX_PATH*3) -#endif - /* ** Create a temporary file name in zBuf. zBuf must be big enough to ** hold at pVfs->mxPathname characters. @@ -3903,8 +3912,8 @@ static int getTempname(int nBuf, char *zBuf){ #if !SQLITE_OS_WINRT else if( isNT() ){ char *zMulti; - WCHAR zWidePath[MAX_PATH]; - if( osGetTempPathW(MAX_PATH-30, zWidePath)==0 ){ + WCHAR zWidePath[SQLITE_WIN32_MAX_PATH]; + if( osGetTempPathW(SQLITE_WIN32_MAX_PATH-30, zWidePath)==0 ){ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_GETTEMPPATH\n")); return SQLITE_IOERR_GETTEMPPATH; } From f68060839c5e10a5a75180510b0a10ef85e7487e Mon Sep 17 00:00:00 2001 From: drh Date: Mon, 26 Aug 2013 04:50:08 +0000 Subject: [PATCH 43/67] Version 3.8.0 FossilOrigin-Name: f64cd21e2e23ed7cff48f7dafa5e76adde9321c2 --- manifest | 11 +++++++---- manifest.uuid | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/manifest b/manifest index 5f24bd8b5c..c25ddb4000 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\s#ifdefs\sto\sget\sSQLITE_OMIT_AUTOMATIC_INDEX\sand\sSQLITE_OMIT_PAGER_PRAGMAS\nto\sbuild. -D 2013-08-23T17:33:46.203 +C Version\s3.8.0 +D 2013-08-26T04:50:08.905 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1105,7 +1105,10 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3ad1f998e5db044849b528c31c4a7e1f3d527320 +P ed310201628cf79c0f57674ae7478ee6738b1c6e R 5a6ead2576f296902c1b2ccbdf985254 +T +bgcolor * #d0c0ff +T +sym-release * +T +sym-version-3.8.0 * U drh -Z 27173a06b09c0a2f08350ebdebfa00ec +Z 16184cb835825e1de42c653d441df656 diff --git a/manifest.uuid b/manifest.uuid index a81384f06a..83c0ca7e6f 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ed310201628cf79c0f57674ae7478ee6738b1c6e \ No newline at end of file +f64cd21e2e23ed7cff48f7dafa5e76adde9321c2 \ No newline at end of file From 4dd5144366484feb8a108739751f9ed8d9f332d3 Mon Sep 17 00:00:00 2001 From: dan Date: Mon, 26 Aug 2013 14:30:25 +0000 Subject: [PATCH 44/67] Fix for builds with both SQLITE_OMIT_WAL and SQLITE_MAX_MMAP_SIZE=0 defined. FossilOrigin-Name: edd5dbdc3239fc88799b822941603fcc828ecbb6 --- manifest | 17 +++++++---------- manifest.uuid | 2 +- src/os_unix.c | 2 ++ 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index c25ddb4000..183ea44fcb 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Version\s3.8.0 -D 2013-08-26T04:50:08.905 +C Fix\sfor\sbuilds\swith\sboth\sSQLITE_OMIT_WAL\sand\sSQLITE_MAX_MMAP_SIZE=0\sdefined. +D 2013-08-26T14:30:25.813 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -202,7 +202,7 @@ F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 -F src/os_unix.c 9eafa5458cf2ff684ddccff82c9bb113c7cad847 +F src/os_unix.c 6005be1c24797520cd5671463e1c735d197e6e83 F src/os_win.c 1d84f2079d9b91f91a4b5dbfa5e08f1b1a0ed0ff F src/pager.c 2aa4444ffe86e9282d03bc349a4a5e49bd77c0e8 F src/pager.h f094af9f6ececfaa8a1e93876905a4f34233fb0c @@ -1105,10 +1105,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ed310201628cf79c0f57674ae7478ee6738b1c6e -R 5a6ead2576f296902c1b2ccbdf985254 -T +bgcolor * #d0c0ff -T +sym-release * -T +sym-version-3.8.0 * -U drh -Z 16184cb835825e1de42c653d441df656 +P f64cd21e2e23ed7cff48f7dafa5e76adde9321c2 +R 2c7ed4c326daab71e6477ad6fcbe3595 +U dan +Z 1fc299d2108dcbca170e4fd8db67c42b diff --git a/manifest.uuid b/manifest.uuid index 83c0ca7e6f..a436785de3 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -f64cd21e2e23ed7cff48f7dafa5e76adde9321c2 \ No newline at end of file +edd5dbdc3239fc88799b822941603fcc828ecbb6 \ No newline at end of file diff --git a/src/os_unix.c b/src/os_unix.c index 152bf9a113..c5152e5a54 100644 --- a/src/os_unix.c +++ b/src/os_unix.c @@ -449,6 +449,7 @@ static struct unix_syscall { { "fchown", (sqlite3_syscall_ptr)posixFchown, 0 }, #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) +#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, #define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[21].pCurrent) @@ -461,6 +462,7 @@ static struct unix_syscall { { "mremap", (sqlite3_syscall_ptr)0, 0 }, #endif #define osMremap ((void*(*)(void*,size_t,size_t,int,...))aSyscall[23].pCurrent) +#endif }; /* End of the overrideable system calls */ From 31706a2d940459e1de326942fcf2080d33cde398 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 26 Aug 2013 20:45:50 +0000 Subject: [PATCH 45/67] Change MAX_PATH related defines to use character lengths where WCHARs are used. FossilOrigin-Name: 0a497083e915520c0807cb6611264b1a35ff62b7 --- manifest | 12 +++++------ manifest.uuid | 2 +- src/os_win.c | 56 ++++++++++++++++++++++++++++----------------------- 3 files changed, 38 insertions(+), 32 deletions(-) diff --git a/manifest b/manifest index e9f6e2333f..43a3a04a88 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\supdates\sfrom\strunk. -D 2013-08-26T19:36:18.286 +C Change\sMAX_PATH\srelated\sdefines\sto\suse\scharacter\slengths\swhere\sWCHARs\sare\sused. +D 2013-08-26T20:45:50.504 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -203,7 +203,7 @@ F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c c27a14a05061e4e690bd3949dc0246bda35e399d -F src/os_win.c 339cdb887654f805a3e7225dd67f210cd5c831d3 +F src/os_win.c c05cd62a7af20004414a425c0f798cf5db36ba9f F src/pager.c 2aa4444ffe86e9282d03bc349a4a5e49bd77c0e8 F src/pager.h f094af9f6ececfaa8a1e93876905a4f34233fb0c F src/parse.y 27c6b4138497d6f8360ba7847da6ed48033f957f @@ -1105,7 +1105,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P bda4c47df8e80b4cc9e8aac8fd74482869f96107 edd5dbdc3239fc88799b822941603fcc828ecbb6 -R f8b228db3f91433cfd67191a80e72ad7 +P 9d6860098f96efc7ea60e2d6116fb7d0e2685a55 +R 299a595b50bdbcb5444938600e71cf89 U mistachkin -Z d72989ebb46558e2aa62256a6f0f9391 +Z 552a2531f97ec783e6ad2f14086120a6 diff --git a/manifest.uuid b/manifest.uuid index ed4f6aa8e8..24daa7a0ef 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d6860098f96efc7ea60e2d6116fb7d0e2685a55 \ No newline at end of file +0a497083e915520c0807cb6611264b1a35ff62b7 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 0f27ac43f2..7bf8828dbc 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -50,22 +50,28 @@ # define SQLITE_WIN32_HAS_WIDE #endif +/* +** Maximum pathname length (in chars) for Win32. This should normally be +** MAX_PATH. +*/ +#ifndef SQLITE_WIN32_MAX_PATH_CHARS +# define SQLITE_WIN32_MAX_PATH_CHARS (MAX_PATH) +#endif + /* ** Maximum pathname length (in bytes) for Win32. The MAX_PATH macro is in ** characters, so we allocate 3 bytes per character assuming worst-case of ** 3-bytes-per-character for UTF8. */ -#ifndef SQLITE_WIN32_MAX_PATH -# define SQLITE_WIN32_MAX_PATH (MAX_PATH*3) +#ifndef SQLITE_WIN32_MAX_PATH_BYTES +# define SQLITE_WIN32_MAX_PATH_BYTES (SQLITE_WIN32_MAX_PATH_CHARS*3) #endif /* -** Maximum error message length (in bytes) for WinRT. The MAX_PATH macro is -** in characters, so we allocate 3 bytes per character assuming worst-case of -** 3-bytes-per-character for UTF8. +** Maximum error message length (in chars) for WinRT. */ -#ifndef SQLITE_WIN32_MAX_ERRMSG -# define SQLITE_WIN32_MAX_ERRMSG (MAX_PATH*3) +#ifndef SQLITE_WIN32_MAX_ERRMSG_CHARS +# define SQLITE_WIN32_MAX_ERRMSG_CHARS (1024) #endif /* @@ -1481,14 +1487,14 @@ static int getLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ if( isNT() ){ #if SQLITE_OS_WINRT - WCHAR zTempWide[SQLITE_WIN32_MAX_ERRMSG+1]; /* NOTE: Somewhat arbitrary. */ + WCHAR zTempWide[SQLITE_WIN32_MAX_ERRMSG_CHARS+1]; dwLen = osFormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL, lastErrno, 0, zTempWide, - SQLITE_WIN32_MAX_ERRMSG, + SQLITE_WIN32_MAX_ERRMSG_CHARS, 0); #else LPWSTR zTempWide = NULL; @@ -3897,7 +3903,7 @@ static int getTempname(int nBuf, char *zBuf){ "0123456789"; size_t i, j; int nTempPath; - char zTempPath[SQLITE_WIN32_MAX_PATH+2]; + char zTempPath[SQLITE_WIN32_MAX_PATH_BYTES+2]; /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -3906,20 +3912,20 @@ static int getTempname(int nBuf, char *zBuf){ SimulateIOError( return SQLITE_IOERR ); if( sqlite3_temp_directory ){ - sqlite3_snprintf(SQLITE_WIN32_MAX_PATH-30, zTempPath, "%s", + sqlite3_snprintf(SQLITE_WIN32_MAX_PATH_BYTES-30, zTempPath, "%s", sqlite3_temp_directory); } #if !SQLITE_OS_WINRT else if( isNT() ){ char *zMulti; - WCHAR zWidePath[SQLITE_WIN32_MAX_PATH]; - if( osGetTempPathW(SQLITE_WIN32_MAX_PATH-30, zWidePath)==0 ){ + WCHAR zWidePath[SQLITE_WIN32_MAX_PATH_CHARS]; + if( osGetTempPathW(SQLITE_WIN32_MAX_PATH_CHARS-30, zWidePath)==0 ){ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_GETTEMPPATH\n")); return SQLITE_IOERR_GETTEMPPATH; } zMulti = unicodeToUtf8(zWidePath); if( zMulti ){ - sqlite3_snprintf(SQLITE_WIN32_MAX_PATH-30, zTempPath, "%s", zMulti); + sqlite3_snprintf(SQLITE_WIN32_MAX_PATH_BYTES-30, zTempPath, "%s", zMulti); sqlite3_free(zMulti); }else{ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); @@ -3929,14 +3935,14 @@ static int getTempname(int nBuf, char *zBuf){ #ifdef SQLITE_WIN32_HAS_ANSI else{ char *zUtf8; - char zMbcsPath[SQLITE_WIN32_MAX_PATH]; - if( osGetTempPathA(SQLITE_WIN32_MAX_PATH-30, zMbcsPath)==0 ){ + char zMbcsPath[SQLITE_WIN32_MAX_PATH_BYTES]; + if( osGetTempPathA(SQLITE_WIN32_MAX_PATH_BYTES-30, zMbcsPath)==0 ){ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_GETTEMPPATH\n")); return SQLITE_IOERR_GETTEMPPATH; } zUtf8 = sqlite3_win32_mbcs_to_utf8(zMbcsPath); if( zUtf8 ){ - sqlite3_snprintf(SQLITE_WIN32_MAX_PATH-30, zTempPath, "%s", zUtf8); + sqlite3_snprintf(SQLITE_WIN32_MAX_PATH_BYTES-30, zTempPath, "%s", zUtf8); sqlite3_free(zUtf8); }else{ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); @@ -3949,7 +3955,7 @@ static int getTempname(int nBuf, char *zBuf){ ** Compiled without ANSI support and the current operating system ** is not Windows NT; therefore, just zero the temporary buffer. */ - memset(zTempPath, 0, SQLITE_WIN32_MAX_PATH+2); + memset(zTempPath, 0, SQLITE_WIN32_MAX_PATH_BYTES+2); } #endif /* SQLITE_WIN32_HAS_ANSI */ #else @@ -3958,7 +3964,7 @@ static int getTempname(int nBuf, char *zBuf){ ** Compiled for WinRT and the sqlite3_temp_directory is not set; ** therefore, just zero the temporary buffer. */ - memset(zTempPath, 0, SQLITE_WIN32_MAX_PATH+2); + memset(zTempPath, 0, SQLITE_WIN32_MAX_PATH_BYTES+2); } #endif /* !SQLITE_OS_WINRT */ @@ -4046,7 +4052,7 @@ static int winOpen( /* If argument zPath is a NULL pointer, this function is required to open ** a temporary file. Use this buffer to store the file name in. */ - char zTmpname[SQLITE_WIN32_MAX_PATH+2]; /* Buffer used to create temp filename */ + char zTmpname[SQLITE_WIN32_MAX_PATH_BYTES+2]; /* Buffer for temp filename */ int rc = SQLITE_OK; /* Function Return Code */ #if !defined(NDEBUG) || SQLITE_OS_WINCE @@ -4112,7 +4118,7 @@ static int winOpen( */ if( !zUtf8Name ){ assert(isDelete && !isOpenJournal); - rc = getTempname(SQLITE_WIN32_MAX_PATH+2, zTmpname); + rc = getTempname(SQLITE_WIN32_MAX_PATH_BYTES+2, zTmpname); if( rc!=SQLITE_OK ){ OSTRACE(("OPEN name=%s, rc=%s", zUtf8Name, sqlite3ErrName(rc))); return rc; @@ -4543,7 +4549,7 @@ static int winFullPathname( #if defined(__CYGWIN__) SimulateIOError( return SQLITE_ERROR ); UNUSED_PARAMETER(nFull); - assert( pVfs->mxPathname>=SQLITE_WIN32_MAX_PATH ); + assert( pVfs->mxPathname>=SQLITE_WIN32_MAX_PATH_BYTES ); assert( nFull>=pVfs->mxPathname ); if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ /* @@ -4552,9 +4558,9 @@ static int winFullPathname( ** for converting the relative path name to an absolute ** one by prepending the data directory and a slash. */ - char zOut[SQLITE_WIN32_MAX_PATH+1]; + char zOut[SQLITE_WIN32_MAX_PATH_BYTES+1]; if( cygwin_conv_path(CCP_POSIX_TO_WIN_A|CCP_RELATIVE, zRelative, zOut, - SQLITE_WIN32_MAX_PATH+1)<0 ){ + SQLITE_WIN32_MAX_PATH_BYTES+1)<0 ){ winLogError(SQLITE_CANTOPEN_FULLPATH, (DWORD)errno, "cygwin_conv_path", zRelative); return SQLITE_CANTOPEN_FULLPATH; @@ -4908,7 +4914,7 @@ int sqlite3_os_init(void){ static sqlite3_vfs winVfs = { 3, /* iVersion */ sizeof(winFile), /* szOsFile */ - SQLITE_WIN32_MAX_PATH, /* mxPathname */ + SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ 0, /* pNext */ "win32", /* zName */ 0, /* pAppData */ From 4496a2329a0fd2f922e6f80fa7081294bd14a811 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Mon, 26 Aug 2013 23:18:57 +0000 Subject: [PATCH 46/67] Preparation for further Windows path name handling changes. FossilOrigin-Name: ec99224b0c9cf1ccd64b3dc93252556d888e93a3 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 23 ++++++++++++++++++++--- 3 files changed, 27 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 43a3a04a88..36f923a1e8 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Change\sMAX_PATH\srelated\sdefines\sto\suse\scharacter\slengths\swhere\sWCHARs\sare\sused. -D 2013-08-26T20:45:50.504 +C Preparation\sfor\sfurther\sWindows\spath\sname\shandling\schanges. +D 2013-08-26T23:18:57.140 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -203,7 +203,7 @@ F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c c27a14a05061e4e690bd3949dc0246bda35e399d -F src/os_win.c c05cd62a7af20004414a425c0f798cf5db36ba9f +F src/os_win.c 52bbfe7f96fdb61fa71ba6f96eac43bcdc5c29a8 F src/pager.c 2aa4444ffe86e9282d03bc349a4a5e49bd77c0e8 F src/pager.h f094af9f6ececfaa8a1e93876905a4f34233fb0c F src/parse.y 27c6b4138497d6f8360ba7847da6ed48033f957f @@ -1105,7 +1105,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9d6860098f96efc7ea60e2d6116fb7d0e2685a55 -R 299a595b50bdbcb5444938600e71cf89 +P 0a497083e915520c0807cb6611264b1a35ff62b7 +R 281d5c8316076afa3106297ea64f6c5c U mistachkin -Z 552a2531f97ec783e6ad2f14086120a6 +Z adbcf8e3a91ca172565c0b57ab07dba0 diff --git a/manifest.uuid b/manifest.uuid index 24daa7a0ef..82de3f15c0 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -0a497083e915520c0807cb6611264b1a35ff62b7 \ No newline at end of file +ec99224b0c9cf1ccd64b3dc93252556d888e93a3 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index 7bf8828dbc..e07883a611 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -30,7 +30,7 @@ ** available in Windows platforms based on the NT kernel. */ #if !SQLITE_OS_WINNT && !defined(SQLITE_OMIT_WAL) -# error "WAL mode requires support from the Windows NT kernel, compile\ +# error "WAL mode requires support from the Windows NT kernel, compile\ with SQLITE_OMIT_WAL." #endif @@ -58,13 +58,30 @@ # define SQLITE_WIN32_MAX_PATH_CHARS (MAX_PATH) #endif +/* +** Maximum pathname length (in chars) for WinNT. This should normally be +** 32767. +*/ +#ifndef SQLITE_WINNT_MAX_PATH_CHARS +# define SQLITE_WINNT_MAX_PATH_CHARS (32767) +#endif + /* ** Maximum pathname length (in bytes) for Win32. The MAX_PATH macro is in ** characters, so we allocate 3 bytes per character assuming worst-case of -** 3-bytes-per-character for UTF8. +** 4-bytes-per-character for UTF8. */ #ifndef SQLITE_WIN32_MAX_PATH_BYTES -# define SQLITE_WIN32_MAX_PATH_BYTES (SQLITE_WIN32_MAX_PATH_CHARS*3) +# define SQLITE_WIN32_MAX_PATH_BYTES (SQLITE_WIN32_MAX_PATH_CHARS*4) +#endif + +/* +** Maximum pathname length (in bytes) for WinNT. This should normally be +** 32767 * sizeof(WCHAR). +*/ +#ifndef SQLITE_WINNT_MAX_PATH_BYTES +# define SQLITE_WINNT_MAX_PATH_BYTES \ + (sizeof(WCHAR) * SQLITE_WINNT_MAX_PATH_CHARS) #endif /* From 59b08dd449760338d04d19f6f34847e9a92b1bd8 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 27 Aug 2013 14:14:14 +0000 Subject: [PATCH 47/67] In the ANALYZE command implementation make statInit() a 2-value function since the 3rd parameter was always the same constant. FossilOrigin-Name: 959bb5acdc3b4e2b481e3c38f20867131bfc9dbc --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 13 ++++++------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/manifest b/manifest index 41b9caf087..c091ea3ce5 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Merge\sthe\sSTAT4\scapability\sinto\strunk. -D 2013-08-26T23:18:06.972 +C In\sthe\sANALYZE\scommand\simplementation\smake\sstatInit()\sa\s2-value\sfunction\nsince\sthe\s3rd\sparameter\swas\salways\sthe\ssame\sconstant. +D 2013-08-27T14:14:14.669 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c 10a4f1cda87aec6c56760c66e5feaaf1cdc9a641 +F src/analyze.c ce2a3385ac05f61e07d384b1c7e8c52e6ef1a3a6 F src/attach.c fea00cab11c854646a27641a263f5876569a51f9 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 2f1987981139bd2f6d8c728d64bf09fb387443c3 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P edd5dbdc3239fc88799b822941603fcc828ecbb6 f86b75b6c7290ee6ddb3636090b00e99fc68c45e -R f565ac94f1ba86f7abdb8be442158ff1 +P a32af0abe5fa6d570604fa3534e8230d5b6042fc +R 5b1a9e2b24a504e796d0e448f0b689ac U drh -Z 3380b8636b10d59476656dd5f78e188d +Z 459d7705f4f327bb53f4cbbd349b98f8 diff --git a/manifest.uuid b/manifest.uuid index 877123ce44..08b8f406e7 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -a32af0abe5fa6d570604fa3534e8230d5b6042fc \ No newline at end of file +959bb5acdc3b4e2b481e3c38f20867131bfc9dbc \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index e68070a112..d2fd3682ae 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -285,9 +285,9 @@ struct Stat4Accum { }; /* -** Implementation of the stat_init(C,N,S) SQL function. The three parameters -** are the number of rows in the table or index (C), the number of columns -** in the index (N) and the number of samples to accumulate (S). +** Implementation of the stat_init(C,N) SQL function. The two parameters +** are the number of rows in the table or index (C) and the number of columns +** in the index (N). ** ** This routine allocates the Stat4Accum object in heap memory. The return ** value is a pointer to the the Stat4Accum object encoded as a blob (i.e. @@ -309,7 +309,7 @@ static void statInit( /* Decode the three function arguments */ UNUSED_PARAMETER(argc); nRow = (tRowcnt)sqlite3_value_int64(argv[0]); - mxSample = sqlite3_value_int(argv[2]); + mxSample = SQLITE_STAT4_SAMPLES; nCol = sqlite3_value_int(argv[1]); assert( nCol>1 ); /* >1 because it includes the rowid column */ @@ -356,7 +356,7 @@ static void statInit( sqlite3_result_blob(context, p, sizeof(p), sqlite3_free); } static const FuncDef statInitFuncdef = { - 3, /* nArg */ + 2, /* nArg */ SQLITE_UTF8, /* iPrefEnc */ 0, /* flags */ 0, /* pUserData */ @@ -911,10 +911,9 @@ static void analyzeOneTable( */ sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+1); sqlite3VdbeAddOp2(v, OP_Integer, nCol+1, regStat4+2); - sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_STAT4_SAMPLES, regStat4+3); sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 3); + sqlite3VdbeChangeP5(v, 2); /* Implementation of the following: ** From 0dd8b87955819cfaef99603f17290fb289d9385e Mon Sep 17 00:00:00 2001 From: dan Date: Tue, 27 Aug 2013 15:41:09 +0000 Subject: [PATCH 48/67] Update sqlite3.pc.in to use @PACKAGE_VERSION@ instead of @RELEASE@. FossilOrigin-Name: 2460dfd8825d251e622e866e8dc1c0bf7fe7ec9f --- manifest | 14 +++++++------- manifest.uuid | 2 +- sqlite3.pc.in | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index c091ea3ce5..bf7e78d21a 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\sANALYZE\scommand\simplementation\smake\sstatInit()\sa\s2-value\sfunction\nsince\sthe\s3rd\sparameter\swas\salways\sthe\ssame\sconstant. -D 2013-08-27T14:14:14.669 +C Update\ssqlite3.pc.in\sto\suse\s@PACKAGE_VERSION@\sinstead\sof\s@RELEASE@. +D 2013-08-27T15:41:09.212 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -155,7 +155,7 @@ F mptest/multiwrite01.test 499ad0310da8dff8e8f98d2e272fc2a8aa741b2e F spec.template 86a4a43b99ebb3e75e6b9a735d5fd293a24e90ca F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc -F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad +F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 F src/analyze.c ce2a3385ac05f61e07d384b1c7e8c52e6ef1a3a6 F src/attach.c fea00cab11c854646a27641a263f5876569a51f9 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P a32af0abe5fa6d570604fa3534e8230d5b6042fc -R 5b1a9e2b24a504e796d0e448f0b689ac -U drh -Z 459d7705f4f327bb53f4cbbd349b98f8 +P 959bb5acdc3b4e2b481e3c38f20867131bfc9dbc +R 2e3b4a437bff68926c962c9ab6977363 +U dan +Z 665cf10c770598d3c963d389b4e25d18 diff --git a/manifest.uuid b/manifest.uuid index 08b8f406e7..c8cd2432dc 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -959bb5acdc3b4e2b481e3c38f20867131bfc9dbc \ No newline at end of file +2460dfd8825d251e622e866e8dc1c0bf7fe7ec9f \ No newline at end of file diff --git a/sqlite3.pc.in b/sqlite3.pc.in index c8d0aa9d5d..3799671e61 100644 --- a/sqlite3.pc.in +++ b/sqlite3.pc.in @@ -7,7 +7,7 @@ includedir=@includedir@ Name: SQLite Description: SQL database engine -Version: @RELEASE@ +Version: @PACKAGE_VERSION@ Libs: -L${libdir} -lsqlite3 Libs.private: @LIBS@ Cflags: -I${includedir} From 9fecc546cb32bf293a5f91ba9c472115577b85a5 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 27 Aug 2013 20:16:48 +0000 Subject: [PATCH 49/67] Reduce the amount of code run and memory used for ANALYZE in the common case where neither STAT3 and STAT4 are enabled. FossilOrigin-Name: 9d1424c91a21ed740aca53e437b8f7c1f0823c03 --- manifest | 16 ++-- manifest.uuid | 2 +- src/analyze.c | 238 +++++++++++++++++++++++++++--------------------- src/sqliteInt.h | 4 +- 4 files changed, 148 insertions(+), 112 deletions(-) diff --git a/manifest b/manifest index bf7e78d21a..c679578fc0 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Update\ssqlite3.pc.in\sto\suse\s@PACKAGE_VERSION@\sinstead\sof\s@RELEASE@. -D 2013-08-27T15:41:09.212 +C Reduce\sthe\samount\sof\scode\srun\sand\smemory\sused\sfor\sANALYZE\sin\sthe\scommon\scase\nwhere\sneither\sSTAT3\sand\sSTAT4\sare\senabled. +D 2013-08-27T20:16:48.420 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c ce2a3385ac05f61e07d384b1c7e8c52e6ef1a3a6 +F src/analyze.c f1dbc79cbe97f960b20dfc3d24143dff988db41d F src/attach.c fea00cab11c854646a27641a263f5876569a51f9 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 2f1987981139bd2f6d8c728d64bf09fb387443c3 @@ -221,7 +221,7 @@ F src/shell.c 00a23311466829d9b77f0be4f7cedee9328279db F src/sqlite.h.in bd1451ba1ab681022a53bccc3c39580ba094a3ff F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h f3636620ad376f65bdbbf5e183f0e4309cb7526e +F src/sqliteInt.h 97b1005b812048469c80ec9394a77e0ad83ea9c0 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 959bb5acdc3b4e2b481e3c38f20867131bfc9dbc -R 2e3b4a437bff68926c962c9ab6977363 -U dan -Z 665cf10c770598d3c963d389b4e25d18 +P 2460dfd8825d251e622e866e8dc1c0bf7fe7ec9f +R d4ca9812963b4361724d44342b635223 +U drh +Z 6faf42c735288a2cfa68556541b907cb diff --git a/manifest.uuid b/manifest.uuid index c8cd2432dc..94e1055f64 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -2460dfd8825d251e622e866e8dc1c0bf7fe7ec9f \ No newline at end of file +9d1424c91a21ed740aca53e437b8f7c1f0823c03 \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index d2fd3682ae..9ea16b91e9 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1,5 +1,5 @@ /* -** 2005 July 8 +** 2005-07-08 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -31,7 +31,9 @@ ** SQLITE_ENABLE_STAT3 defined. The functionality of sqlite_stat3 ** is a superset of sqlite_stat2. The sqlite_stat4 is an enhanced ** version of sqlite_stat3 and is only available when compiled with -** SQLITE_ENABLE_STAT4 and in SQLite versions 3.8.0 and later. +** SQLITE_ENABLE_STAT4 and in SQLite versions 3.8.0 and later. It is +** not possible to enable both STAT3 and STAT4 at the same time. If they +** are both enabled, then STAT4 is precedence. ** ** For most applications, sqlite_stat1 provides all the statisics required ** for the query planner to make good choices. @@ -152,23 +154,23 @@ # define IsStat4 0 # define IsStat3 0 # undef SQLITE_ENABLE_STAT34 +# undef SQLITE_STAT4_SAMPLES +# define SQLITE_STAT4_SAMPLES 1 #endif +#define IsStat34 (IsStat3+IsStat4) /* 1 for STAT3 or STAT4. 0 otherwise */ /* -** This routine generates code that opens the sqlite_stat1 table for -** writing with cursor iStatCur. If the library was built with the -** SQLITE_ENABLE_STAT4 macro defined, then the sqlite_stat4 table is -** opened for writing using cursor (iStatCur+1) +** This routine generates code that opens the sqlite_statN tables. +** The sqlite_stat1 table is always relevant. sqlite_stat2 is now +** obsolete. sqlite_stat3 and sqlite_stat4 are only opened when +** appropriate compile-time options are provided. ** -** If the sqlite_stat1 tables does not previously exist, it is created. -** Similarly, if the sqlite_stat4 table does not exist and the library -** is compiled with SQLITE_ENABLE_STAT4 defined, it is created. +** If the sqlite_statN tables do not previously exist, it is created. ** ** Argument zWhere may be a pointer to a buffer containing a table name, ** or it may be a NULL pointer. If it is not NULL, then all entries in -** the sqlite_stat1 and (if applicable) sqlite_stat4 tables associated -** with the named table are deleted. If zWhere==0, then code is generated -** to delete all stat table entries. +** the sqlite_statN tables associated with the named table are deleted. +** If zWhere==0, then code is generated to delete all stat table entries. */ static void openStatTable( Parse *pParse, /* Parsing context */ @@ -188,16 +190,18 @@ static void openStatTable( #elif defined(SQLITE_ENABLE_STAT3) { "sqlite_stat3", "tbl,idx,neq,nlt,ndlt,sample" }, { "sqlite_stat4", 0 }, +#else + { "sqlite_stat3", 0 }, + { "sqlite_stat4", 0 }, #endif }; - - int aRoot[] = {0, 0}; - u8 aCreateTbl[] = {0, 0}; - int i; sqlite3 *db = pParse->db; Db *pDb; Vdbe *v = sqlite3GetVdbe(pParse); + int aRoot[ArraySize(aTable)]; + u8 aCreateTbl[ArraySize(aTable)]; + if( v==0 ) return; assert( sqlite3BtreeHoldsAllMutexes(db) ); assert( sqlite3VdbeDb(v)==db ); @@ -211,7 +215,7 @@ static void openStatTable( Table *pStat; if( (pStat = sqlite3FindTable(db, zTab, pDb->zName))==0 ){ if( aTable[i].zCols ){ - /* The sqlite_stat[12] table does not exist. Create it. Note that a + /* The sqlite_statN table does not exist. Create it. Note that a ** side-effect of the CREATE TABLE statement is to leave the rootpage ** of the new table in register pParse->regRoot. This is important ** because the OpenWrite opcode below will be needing it. */ @@ -226,6 +230,7 @@ static void openStatTable( ** associated with the table zWhere. If zWhere is NULL, delete the ** entire contents of the table. */ aRoot[i] = pStat->tnum; + aCreateTbl[i] = 0; sqlite3TableLock(pParse, iDb, aRoot[i], 1, zTab); if( zWhere ){ sqlite3NestedParse(pParse, @@ -239,11 +244,14 @@ static void openStatTable( } /* Open the sqlite_stat[134] tables for writing. */ - for(i=0; iaSample[] */ +#ifdef SQLITE_ENABLE_STAT34 + int mxSample = SQLITE_STAT4_SAMPLES; +#endif /* Decode the three function arguments */ UNUSED_PARAMETER(argc); - nRow = (tRowcnt)sqlite3_value_int64(argv[0]); - mxSample = SQLITE_STAT4_SAMPLES; - nCol = sqlite3_value_int(argv[1]); + nCol = sqlite3_value_int(argv[0]); assert( nCol>1 ); /* >1 because it includes the rowid column */ + nColUp = sizeof(tRowcnt)<8 ? (nCol+1)&~1 : nCol; /* Allocate the space required for the Stat4Accum object */ n = sizeof(*p) - + sizeof(tRowcnt)*nCol /* Stat4Accum.anEq */ - + sizeof(tRowcnt)*nCol /* Stat4Accum.anLt */ - + sizeof(tRowcnt)*nCol /* Stat4Accum.anDLt */ - + sizeof(Stat4Sample)*(nCol+mxSample) /* Stat4Accum.aBest[], a[] */ - + sizeof(tRowcnt)*3*nCol*(nCol+mxSample); + + sizeof(tRowcnt)*nColUp /* Stat4Accum.anEq */ + + sizeof(tRowcnt)*nColUp /* Stat4Accum.anDLt */ +#ifdef SQLITE_ENABLE_STAT34 + + sizeof(tRowcnt)*nColUp /* Stat4Accum.anLt */ + + sizeof(Stat4Sample)*(nCol+mxSample) /* Stat4Accum.aBest[], a[] */ + + sizeof(tRowcnt)*3*nColUp*(nCol+mxSample) +#endif + ; p = sqlite3MallocZero(n); if( p==0 ){ sqlite3_result_error_nomem(context); return; } - p->nRow = nRow; + p->nRow = 0; p->nCol = nCol; - p->mxSample = mxSample; - p->nPSample = p->nRow/(mxSample/3+1) + 1; - p->iGet = -1; - p->current.anDLt = (tRowcnt*)&p[1]; - p->current.anEq = &p->current.anDLt[nCol]; - p->current.anLt = &p->current.anEq[nCol]; - sqlite3_randomness(sizeof(p->iPrn), &p->iPrn); + p->current.anEq = &p->current.anDLt[nColUp]; - /* Set up the Stat4Accum.a[] and aBest[] arrays */ - p->a = (struct Stat4Sample*)&p->current.anLt[nCol]; - p->aBest = &p->a[mxSample]; - pSpace = (u8*)(&p->a[mxSample+nCol]); - for(i=0; i<(mxSample+nCol); i++){ - p->a[i].anEq = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nCol); - p->a[i].anLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nCol); - p->a[i].anDLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nCol); - } - assert( (pSpace - (u8*)p)==n ); +#ifdef SQLITE_ENABLE_STAT34 + { + u8 *pSpace; /* Allocated space not yet assigned */ + int i; /* Used to iterate through p->aSample[] */ - for(i=0; iaBest[i].iCol = i; + p->iGet = -1; + p->mxSample = mxSample; + p->nPSample = sqlite3_value_int64(argv[1])/(mxSample/3+1) + 1; + p->current.anLt = &p->current.anEq[nColUp]; + sqlite3_randomness(sizeof(p->iPrn), &p->iPrn); + + /* Set up the Stat4Accum.a[] and aBest[] arrays */ + p->a = (struct Stat4Sample*)&p->current.anLt[nColUp]; + p->aBest = &p->a[mxSample]; + pSpace = (u8*)(&p->a[mxSample+nCol]); + for(i=0; i<(mxSample+nCol); i++){ + p->a[i].anEq = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + p->a[i].anLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + p->a[i].anDLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); + } + assert( (pSpace - (u8*)p)==n ); + + for(i=0; iaBest[i].iCol = i; + } } +#endif /* Return a pointer to the allocated object to the caller */ sqlite3_result_blob(context, p, sizeof(p), sqlite3_free); } static const FuncDef statInitFuncdef = { - 2, /* nArg */ + 1+IsStat34, /* nArg */ SQLITE_UTF8, /* iPrefEnc */ 0, /* flags */ 0, /* pUserData */ @@ -544,11 +563,16 @@ static void samplePushPrevious(Stat4Accum *p, int iChng){ } /* -** Implementation of the stat_push SQL function. +** Implementation of the stat_push SQL function: stat_push(P,R,C) +** Arguments: ** -** stat_push(P,R,C) +** P Pointer to the Stat4Accum object created by stat_init() +** C Index of left-most column to differ from previous row +** R Rowid for the current row ** -** The return value is always NULL. +** The SQL function always returns NULL. +** +** The R parameter is only used for STAT3 and STAT4. */ static void statPush( sqlite3_context *context, @@ -559,15 +583,17 @@ static void statPush( /* The three function arguments */ Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); - i64 rowid = sqlite3_value_int64(argv[1]); - int iChng = sqlite3_value_int(argv[2]); + int iChng = sqlite3_value_int(argv[1]); assert( p->nCol>1 ); /* Includes rowid field */ assert( iChngnCol ); - /* p->current.anEq[0] is false the first time this function is called. */ - if( p->current.anEq[0] ){ - + if( p->nRow==0 ){ + /* anEq[0] is only zero for the very first call to this function. Do + ** appropriate initialization */ + for(i=0; inCol; i++) p->current.anEq[i] = 1; + }else{ + /* Second and subsequent calls get processed here */ samplePushPrevious(p, iChng); /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply @@ -577,18 +603,17 @@ static void statPush( } for(i=iChng; inCol; i++){ p->current.anDLt[i]++; +#ifdef SQLITE_ENABLE_STAT34 p->current.anLt[i] += p->current.anEq[i]; +#endif p->current.anEq[i] = 1; } - - }else{ - for(i=0; inCol; i++) p->current.anEq[i] = 1; - } - - if( IsStat4 || IsStat3 ){ - p->current.iRowid = rowid; - p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; } + p->nRow++; +#ifdef SQLITE_ENABLE_STAT34 + p->current.iRowid = sqlite3_value_int64(argv[2]); + p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; +#endif #ifdef SQLITE_ENABLE_STAT4 { @@ -613,7 +638,7 @@ static void statPush( #endif } static const FuncDef statPushFuncdef = { - 3, /* nArg */ + 2+IsStat34, /* nArg */ SQLITE_UTF8, /* iPrefEnc */ 0, /* flags */ 0, /* pUserData */ @@ -749,7 +774,7 @@ static void statGet( #endif /* SQLITE_ENABLE_STAT34 */ } static const FuncDef statGetFuncdef = { - 2, /* nArg */ + 1+IsStat34, /* nArg */ SQLITE_UTF8, /* iPrefEnc */ 0, /* flags */ 0, /* pUserData */ @@ -771,7 +796,7 @@ static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ #endif sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4, regOut); sqlite3VdbeChangeP4(v, -1, (char*)&statGetFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 1 + IsStat3 + IsStat4); + sqlite3VdbeChangeP5(v, 1 + IsStat34); } /* @@ -797,8 +822,10 @@ static void analyzeOneTable( u8 needTableCnt = 1; /* True to count the table */ int regNewRowid = iMem++; /* Rowid for the inserted record */ int regStat4 = iMem++; /* Register to hold Stat4Accum object */ - int regRowid = iMem++; /* Rowid argument passed to stat_push() */ int regChng = iMem++; /* Index of changed index field */ +#ifdef SQLITE_ENABLE_STAT34 + int regRowid = iMem++; /* Rowid argument passed to stat_push() */ +#endif int regTemp = iMem++; /* Temporary use register */ int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ @@ -884,7 +911,7 @@ static void analyzeOneTable( ** ** chng_addr_N: ** regRowid = idx(rowid) - ** stat_push(P, regRowid, regChng) + ** stat_push(P, regChng, regRowid) ** Next csr ** if !eof(csr) goto next_row; ** @@ -905,15 +932,18 @@ static void analyzeOneTable( /* Invoke the stat_init() function. The arguments are: ** - ** * the number of rows in the index, - ** * the number of columns in the index including the rowid, - ** * the recommended number of samples for the stat3/stat4 table. + ** (1) the number of columns in the index including the rowid, + ** (2) the number of rows in the index, + ** + ** The second argument is only used for STAT3 and STAT4 */ - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+1); - sqlite3VdbeAddOp2(v, OP_Integer, nCol+1, regStat4+2); +#ifdef SQLITE_ENABLE_STAT34 + sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+2); +#endif + sqlite3VdbeAddOp2(v, OP_Integer, nCol+1, regStat4+1); sqlite3VdbeAddOp3(v, OP_Function, 0, regStat4+1, regStat4); sqlite3VdbeChangeP4(v, -1, (char*)&statInitFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 2); + sqlite3VdbeChangeP5(v, 1+IsStat34); /* Implementation of the following: ** @@ -964,17 +994,20 @@ static void analyzeOneTable( /* ** chng_addr_N: - ** regRowid = idx(rowid) - ** stat_push(P, regRowid, regChng) + ** regRowid = idx(rowid) // STAT34 only + ** stat_push(P, regChng, regRowid) // 3rd parameter STAT34 only ** Next csr ** if !eof(csr) goto next_row; */ sqlite3VdbeJumpHere(v, aGotoChng[nCol]); +#ifdef SQLITE_ENABLE_STAT34 sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid); + assert( regRowid==(regStat4+2) ); +#endif + assert( regChng==(regStat4+1) ); sqlite3VdbeAddOp3(v, OP_Function, 1, regStat4, regTemp); sqlite3VdbeChangeP4(v, -1, (char*)&statPushFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 3); - assert( regRowid==(regStat4+1) && regChng==(regStat4+2) ); + sqlite3VdbeChangeP5(v, 2+IsStat34); sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); /* Add the entry to the stat1 table. */ @@ -985,7 +1018,8 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); /* Add the entries to the stat3 or stat4 table. */ - if( IsStat3 || IsStat4 ){ +#ifdef SQLITE_ENABLE_STAT34 + { int regEq = regStat1; int regLt = regStat1+1; int regDLt = regStat1+2; @@ -1004,25 +1038,25 @@ static void analyzeOneTable( callStatGet(v, regStat4, STAT_GET_NLT, regLt); callStatGet(v, regStat4, STAT_GET_NDLT, regDLt); sqlite3VdbeAddOp3(v, OP_NotExists, iTabCur, addrNext, regSampleRowid); - if( IsStat3 ){ - int iCol = pIdx->aiColumn[0]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regSample); - }else{ - for(i=0; iaiColumn[i]; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); - } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol+1, regSample); +#ifdef SQLITE_ENABLE_STAT3 + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, + pIdx->aiColumn[0], regSample); +#else + for(i=0; iaiColumn[i]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, iCol, regCol+i); } - + sqlite3VdbeAddOp3(v, OP_MakeRecord, regCol, nCol+1, regSample); +#endif sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 6, regTemp, "bbbbbb", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid); sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regTemp, regNewRowid); sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNext); sqlite3VdbeJumpHere(v, addrIsNull); } +#endif /* SQLITE_ENABLE_STAT34 */ - /* Jump here if the index is empty */ + /* End of analysis */ sqlite3VdbeJumpHere(v, addrRewind); sqlite3DbFree(db, aGotoChng); } diff --git a/src/sqliteInt.h b/src/sqliteInt.h index d705bd7500..47d085ca93 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -3056,7 +3056,6 @@ extern int sqlite3PendingByte; void sqlite3RootPageMoved(sqlite3*, int, int, int); void sqlite3Reindex(Parse*, Token*, Token*); void sqlite3AlterFunctions(void); -void sqlite3AnalyzeFunctions(void); void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); int sqlite3GetToken(const unsigned char *, int *); void sqlite3NestedParse(Parse*, const char*, ...); @@ -3107,8 +3106,11 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int); void sqlite3BackupRestart(sqlite3_backup *); void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); +#if defined(SQLITE_ENABLE_STAT3) || defined(SQLITE_ENABLE_STAT4) +void sqlite3AnalyzeFunctions(void); int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*); void sqlite3Stat4ProbeFree(UnpackedRecord*); +#endif /* ** The interface to the LEMON-generated parser From 1435a9a12604d35e7560598c76793dc3a82634d2 Mon Sep 17 00:00:00 2001 From: drh Date: Tue, 27 Aug 2013 23:15:44 +0000 Subject: [PATCH 50/67] Adjust ANALYZE for improved test coverage. Use the SQLITE_ENABLE_STAT3_OR_STAT4 macro (created in sqliteInt.h) to conditionally include code, instead of a boolean specifying both macros separately. FossilOrigin-Name: 67a9a392edd62ef5a5a1ad3010b20a39b285793d --- manifest | 18 ++++++------- manifest.uuid | 2 +- src/analyze.c | 71 +++++++++++++++++++++++-------------------------- src/sqliteInt.h | 18 +++++++++++-- src/vdbemem.c | 10 ++++--- src/where.c | 30 ++++++++++----------- 6 files changed, 81 insertions(+), 68 deletions(-) diff --git a/manifest b/manifest index c679578fc0..8af672c496 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Reduce\sthe\samount\sof\scode\srun\sand\smemory\sused\sfor\sANALYZE\sin\sthe\scommon\scase\nwhere\sneither\sSTAT3\sand\sSTAT4\sare\senabled. -D 2013-08-27T20:16:48.420 +C Adjust\sANALYZE\sfor\simproved\stest\scoverage.\s\sUse\sthe\sSQLITE_ENABLE_STAT3_OR_STAT4\nmacro\s(created\sin\ssqliteInt.h)\sto\sconditionally\sinclude\scode,\sinstead\sof\s\na\sboolean\sspecifying\sboth\smacros\sseparately. +D 2013-08-27T23:15:44.027 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c f1dbc79cbe97f960b20dfc3d24143dff988db41d +F src/analyze.c fa1a9eeeebdeceddf0b0ed141656dea6d4458699 F src/attach.c fea00cab11c854646a27641a263f5876569a51f9 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 2f1987981139bd2f6d8c728d64bf09fb387443c3 @@ -221,7 +221,7 @@ F src/shell.c 00a23311466829d9b77f0be4f7cedee9328279db F src/sqlite.h.in bd1451ba1ab681022a53bccc3c39580ba094a3ff F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc -F src/sqliteInt.h 97b1005b812048469c80ec9394a77e0ad83ea9c0 +F src/sqliteInt.h 600086a5082e2291b0aeeefcfbb546f2bbda67b2 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e @@ -283,14 +283,14 @@ F src/vdbeInt.h cbe71b8b36d8b3bba5709cc3f436c7e3b47b7b08 F src/vdbeapi.c 96b24b946cf21894f63d9393e821baa2f0a80979 F src/vdbeaux.c c7fe2695e256dbf254113c4fe90d3ec9aabe3bbe F src/vdbeblob.c 5dc79627775bd9a9b494dd956e26297946417d69 -F src/vdbemem.c 1bec36bf054521df810b2f9cc5078cba929ff10d +F src/vdbemem.c 4511e1d2304a7d7916d14be20080036331740fcf F src/vdbesort.c 3937e06b2a0e354500e17dc206ef4c35770a5017 F src/vdbetrace.c e7ec40e1999ff3c6414424365d5941178966dcbc F src/vtab.c 165ce0e797c2cd23badb104c9f2ae9042d6d942c F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 18cd15160c089dd7854febeaf5a9c065fce6a95a +F src/where.c 289bfa239207945e27d4689a22e838aea4afcc49 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 2460dfd8825d251e622e866e8dc1c0bf7fe7ec9f -R d4ca9812963b4361724d44342b635223 +P 9d1424c91a21ed740aca53e437b8f7c1f0823c03 +R 89270b1d529ed825a8ce16c2a3d7caa2 U drh -Z 6faf42c735288a2cfa68556541b907cb +Z 2d085e9043e9441bdd89e3f37cf9f342 diff --git a/manifest.uuid b/manifest.uuid index 94e1055f64..fdb9809352 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9d1424c91a21ed740aca53e437b8f7c1f0823c03 \ No newline at end of file +67a9a392edd62ef5a5a1ad3010b20a39b285793d \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index 9ea16b91e9..dae0f3aa0f 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -145,15 +145,12 @@ #if defined(SQLITE_ENABLE_STAT4) # define IsStat4 1 # define IsStat3 0 -# define SQLITE_ENABLE_STAT34 1 #elif defined(SQLITE_ENABLE_STAT3) # define IsStat4 0 # define IsStat3 1 -# define SQLITE_ENABLE_STAT34 1 #else # define IsStat4 0 # define IsStat3 0 -# undef SQLITE_ENABLE_STAT34 # undef SQLITE_STAT4_SAMPLES # define SQLITE_STAT4_SAMPLES 1 #endif @@ -234,7 +231,8 @@ static void openStatTable( sqlite3TableLock(pParse, iDb, aRoot[i], 1, zTab); if( zWhere ){ sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE %s=%Q", pDb->zName, zTab, zWhereType, zWhere + "DELETE FROM %Q.%s WHERE %s=%Q", + pDb->zName, zTab, zWhereType, zWhere ); }else{ /* The sqlite_stat[134] table already exists. Delete all rows. */ @@ -244,14 +242,11 @@ static void openStatTable( } /* Open the sqlite_stat[134] tables for writing. */ - for(i=0; icurrent.anDLt = (tRowcnt*)&p[1]; p->current.anEq = &p->current.anDLt[nColUp]; -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 { u8 *pSpace; /* Allocated space not yet assigned */ int i; /* Used to iterate through p->aSample[] */ @@ -388,7 +383,7 @@ static const FuncDef statInitFuncdef = { 0 /* pDestructor */ }; -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** Return true if pNew is to be preferred over pOld. */ @@ -510,7 +505,7 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ p->iMin = iMin; } } -#endif /* SQLITE_ENABLE_STAT34 */ +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* ** Field iChng of the index being scanned has changed. So at this point @@ -603,14 +598,14 @@ static void statPush( } for(i=iChng; inCol; i++){ p->current.anDLt[i]++; -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 p->current.anLt[i] += p->current.anEq[i]; #endif p->current.anEq[i] = 1; } } p->nRow++; -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 p->current.iRowid = sqlite3_value_int64(argv[2]); p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; #endif @@ -673,7 +668,7 @@ static void statGet( sqlite3_value **argv ){ Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* STAT3 and STAT4 have a parameter on this routine. */ int eCall = sqlite3_value_int(argv[1]); assert( argc==2 ); @@ -729,7 +724,7 @@ static void statGet( sqlite3_result_text(context, zRet, -1, sqlite3_free); } -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 else if( eCall==STAT_GET_ROWID ){ if( p->iGet<0 ){ samplePushPrevious(p, 0); @@ -771,7 +766,7 @@ static void statGet( } } } -#endif /* SQLITE_ENABLE_STAT34 */ +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ } static const FuncDef statGetFuncdef = { 1+IsStat34, /* nArg */ @@ -789,7 +784,7 @@ static const FuncDef statGetFuncdef = { static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ assert( regOut!=regStat4 && regOut!=regStat4+1 ); -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3VdbeAddOp2(v, OP_Integer, iParam, regStat4+1); #else assert( iParam==STAT_GET_STAT1 ); @@ -823,7 +818,7 @@ static void analyzeOneTable( int regNewRowid = iMem++; /* Rowid for the inserted record */ int regStat4 = iMem++; /* Register to hold Stat4Accum object */ int regChng = iMem++; /* Index of changed index field */ -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int regRowid = iMem++; /* Rowid argument passed to stat_push() */ #endif int regTemp = iMem++; /* Temporary use register */ @@ -937,7 +932,7 @@ static void analyzeOneTable( ** ** The second argument is only used for STAT3 and STAT4 */ -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+2); #endif sqlite3VdbeAddOp2(v, OP_Integer, nCol+1, regStat4+1); @@ -1000,7 +995,7 @@ static void analyzeOneTable( ** if !eof(csr) goto next_row; */ sqlite3VdbeJumpHere(v, aGotoChng[nCol]); -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid); assert( regRowid==(regStat4+2) ); #endif @@ -1018,7 +1013,7 @@ static void analyzeOneTable( sqlite3VdbeChangeP5(v, OPFLAG_APPEND); /* Add the entries to the stat3 or stat4 table. */ -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 { int regEq = regStat1; int regLt = regStat1+1; @@ -1054,7 +1049,7 @@ static void analyzeOneTable( sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNext); sqlite3VdbeJumpHere(v, addrIsNull); } -#endif /* SQLITE_ENABLE_STAT34 */ +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* End of analysis */ sqlite3VdbeJumpHere(v, addrRewind); @@ -1235,8 +1230,8 @@ static void decodeIntArray( tRowcnt v; assert( pbUnordered==0 || *pbUnordered==0 ); - - if( z==0 ) z = ""; + + if( NEVER(z==0) ) z = ""; for(i=0; *z && i='0' && c<='9' ){ @@ -1287,7 +1282,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pIndex ){ int bUnordered = 0; - decodeIntArray((char*)z, pIndex->nColumn+1, pIndex->aiRowEst, &bUnordered); + decodeIntArray((char*)z, pIndex->nColumn+1, pIndex->aiRowEst,&bUnordered); if( pIndex->pPartIdxWhere==0 ) pTable->nRowEst = pIndex->aiRowEst[0]; pIndex->bUnordered = bUnordered; }else{ @@ -1302,7 +1297,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** and its contents. */ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( pIdx->aSample ){ int j; for(j=0; jnSample; j++){ @@ -1318,10 +1313,10 @@ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ #else UNUSED_PARAMETER(db); UNUSED_PARAMETER(pIdx); -#endif /* SQLITE_ENABLE_STAT34 */ +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ } -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** Populate the pIdx->aAvgEq[] array based on the samples currently ** stored in pIdx->aSample[]. @@ -1515,7 +1510,7 @@ static int loadStat4(sqlite3 *db, const char *zDb){ return rc; } -#endif /* SQLITE_ENABLE_STAT34 */ +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* ** Load the content of the sqlite_stat1 and sqlite_stat3/4 tables. The @@ -1551,7 +1546,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3DefaultRowEst(pIdx); -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; #endif @@ -1576,7 +1571,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load the statistics from the sqlite_stat4 table. */ -#ifdef SQLITE_ENABLE_STAT34 +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( rc==SQLITE_OK ){ int lookasideEnabled = db->lookaside.bEnabled; db->lookaside.bEnabled = 0; diff --git a/src/sqliteInt.h b/src/sqliteInt.h index 47d085ca93..49264238b8 100644 --- a/src/sqliteInt.h +++ b/src/sqliteInt.h @@ -568,6 +568,20 @@ extern const int sqlite3one; # define SQLITE_DEFAULT_MMAP_SIZE SQLITE_MAX_MMAP_SIZE #endif +/* +** Only one of SQLITE_ENABLE_STAT3 or SQLITE_ENABLE_STAT4 can be defined. +** Priority is given to SQLITE_ENABLE_STAT4. If either are defined, also +** define SQLITE_ENABLE_STAT3_OR_STAT4 +*/ +#ifdef SQLITE_ENABLE_STAT4 +# undef SQLITE_ENABLE_STAT3 +# define SQLITE_ENABLE_STAT3_OR_STAT4 1 +#elif SQLITE_ENABLE_STAT3 +# define SQLITE_ENABLE_STAT3_OR_STAT4 1 +#elif SQLITE_ENABLE_STAT3_OR_STAT4 +# undef SQLITE_ENABLE_STAT3_OR_STAT4 +#endif + /* ** An instance of the following structure is used to store the busy-handler ** callback for a given sqlite handle. @@ -1550,7 +1564,7 @@ struct Index { unsigned autoIndex:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ unsigned bUnordered:1; /* Use this index for == or IN queries only */ unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nSample; /* Number of elements in aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ @@ -3106,7 +3120,7 @@ Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int); void sqlite3BackupRestart(sqlite3_backup *); void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); -#if defined(SQLITE_ENABLE_STAT3) || defined(SQLITE_ENABLE_STAT4) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 void sqlite3AnalyzeFunctions(void); int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*); void sqlite3Stat4ProbeFree(UnpackedRecord*); diff --git a/src/vdbemem.c b/src/vdbemem.c index 99379388a6..e6ad291532 100644 --- a/src/vdbemem.c +++ b/src/vdbemem.c @@ -1023,7 +1023,7 @@ struct ValueNewStat4Ctx { ** an sqlite3_value within the UnpackedRecord.a[] array. */ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( p ){ UnpackedRecord *pRec = p->ppRec[0]; @@ -1098,7 +1098,7 @@ int valueFromExpr( ** The ifdef here is to enable us to achieve 100% branch test coverage even ** when SQLITE_ENABLE_STAT4 is omitted. */ -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( op==TK_REGISTER ) op = pExpr->op2; #else if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; @@ -1180,7 +1180,11 @@ no_mem: db->mallocFailed = 1; sqlite3DbFree(db, zVal); assert( *ppVal==0 ); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( pCtx==0 ) sqlite3ValueFree(pVal); +#else + assert( pCtx==0 ); sqlite3ValueFree(pVal); +#endif return SQLITE_NOMEM; } @@ -1204,7 +1208,7 @@ int sqlite3ValueFromExpr( return valueFromExpr(db, pExpr, enc, affinity, ppVal, 0); } -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** The implementation of the sqlite_record() function. This function accepts ** a single argument of any type. The return value is a formatted database diff --git a/src/where.c b/src/where.c index 1a53c75343..dc944612a3 100644 --- a/src/where.c +++ b/src/where.c @@ -286,7 +286,7 @@ struct WhereTerm { #define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */ #define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */ #define TERM_OR_OK 0x40 /* Used during OR-clause processing */ -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 # define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */ #else # define TERM_VNULL 0x00 /* Disabled if not using stat3 */ @@ -392,7 +392,7 @@ struct WhereLoopBuilder { ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 UnpackedRecord *pRec; /* Probe for stat4 (if required) */ int nRecValid; /* Number of valid fields currently in pRec */ #endif @@ -1790,7 +1790,7 @@ static void exprAnalyze( } #endif /* SQLITE_OMIT_VIRTUALTABLE */ -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* When sqlite_stat3 histogram data is available an operator of the ** form "x IS NOT NULL" can sometimes be evaluated more efficiently ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a @@ -1830,7 +1830,7 @@ static void exprAnalyze( pNewTerm->prereqAll = pTerm->prereqAll; } } -#endif /* SQLITE_ENABLE_STAT */ +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* Prevent ON clause terms of a LEFT JOIN from being used to drive ** an index for tables to the left of the join. @@ -2398,7 +2398,7 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** Estimate the location of a particular key among all keys in an ** index. Store the results in aStat as follows: @@ -2485,7 +2485,7 @@ static void whereKeyStats( aStat[0] = iLower + iGap; } } -#endif /* SQLITE_ENABLE_STAT4 */ +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* ** This function is used to estimate the number of rows that will be visited @@ -2537,7 +2537,7 @@ static int whereRangeScanEst( int rc = SQLITE_OK; int nOut = (int)*pnOut; -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 Index *p = pBuilder->pNew->u.btree.pIndex; int nEq = pBuilder->pNew->u.btree.nEq; @@ -2645,7 +2645,7 @@ static int whereRangeScanEst( return rc; } -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** Estimate the number of rows that will be returned based on ** an equality constraint x=VALUE and where that VALUE occurs in @@ -2709,9 +2709,9 @@ static int whereEqualScanEst( return rc; } -#endif /* defined(SQLITE_ENABLE_STAT4) */ +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 /* ** Estimate the number of rows that will be returned based on ** an IN constraint where the right-hand side of the IN operator @@ -2757,7 +2757,7 @@ static int whereInScanEst( assert( pBuilder->nRecValid==nRecValid ); return rc; } -#endif /* defined(SQLITE_ENABLE_STAT4) */ +#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ /* ** Disable a term in the WHERE clause. Except, do not disable the term @@ -4302,7 +4302,7 @@ static int whereLoopAddBtreeIndex( rLogSize = estLog(whereCost(pProbe->aiRowEst[0])); for(; rc==SQLITE_OK && pTerm!=0; pTerm = whereScanNext(&scan)){ int nIn = 0; -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nRecValid = pBuilder->nRecValid; assert( pNew->nOut==saved_nOut ); if( (pTerm->wtFlags & TERM_VNULL)!=0 && pSrc->pTab->aCol[iCol].notNull ){ @@ -4372,7 +4372,7 @@ static int whereLoopAddBtreeIndex( assert( pNew->nOut==saved_nOut ); whereRangeScanEst(pParse, pBuilder, pBtm, pTop, &pNew->nOut); } -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 if( nInMul==0 && pProbe->nSample && pNew->u.btree.nEq<=pProbe->nSampleCol @@ -4410,7 +4410,7 @@ static int whereLoopAddBtreeIndex( whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } pNew->nOut = saved_nOut; -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 pBuilder->nRecValid = nRecValid; #endif } @@ -4644,7 +4644,7 @@ static int whereLoopAddBtree( } rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); -#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_ENABLE_STAT3) +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); pBuilder->nRecValid = 0; pBuilder->pRec = 0; From f64188910dbc4a7152827b3dd532b3360fcb4512 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 28 Aug 2013 01:54:12 +0000 Subject: [PATCH 51/67] Fix several harmless compiler warnings. Fix a couple compiler issues with the shell. FossilOrigin-Name: 8917e9f9a0802cbfb6f33e2ab1c2f98e4df5babd --- manifest | 22 +++++++++++----------- manifest.uuid | 2 +- src/expr.c | 2 +- src/mem2.c | 14 +++++++------- src/mutex_w32.c | 2 +- src/shell.c | 8 ++++---- src/test_malloc.c | 2 +- src/where.c | 2 +- 8 files changed, 27 insertions(+), 27 deletions(-) diff --git a/manifest b/manifest index 36f923a1e8..4916545b93 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Preparation\sfor\sfurther\sWindows\spath\sname\shandling\schanges. -D 2013-08-26T23:18:57.140 +C Fix\sseveral\sharmless\scompiler\swarnings.\s\sFix\sa\scouple\scompiler\sissues\swith\sthe\sshell. +D 2013-08-28T01:54:12.702 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -172,7 +172,7 @@ F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac F src/ctime.c 4262c227bc91cecc61ae37ed3a40f08069cfa267 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4 F src/delete.c 2317c814866d9aa71fea16b3faf4fdd4d6a49b94 -F src/expr.c 6bab61c757c4c4c94a92e98d507025d5d18afd3c +F src/expr.c 4d89bd03a04fcdb5ff71d86b4e0cc7d3230797b8 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb F src/fkey.c 914a6bbd987d857c41ac9d244efa6641f36faadb F src/func.c d7be77897c91d5b9887beb372f1e6deb515c92af @@ -189,7 +189,7 @@ F src/main.c dc02c7c118d58049e0cccc6c9c47c0a4d3e2bcda F src/malloc.c fe085aa851b666b7c375c1ff957643dc20a04bf6 F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645 F src/mem1.c 437c7c4af964895d4650f29881df63535caaa1fa -F src/mem2.c e307323e86b5da1853d7111b68fd6b84ad6f09cf +F src/mem2.c e243acb034f91e75e4c06aed5d3d960025c62d1f F src/mem3.c 61c9d47b792908c532ca3a62b999cf21795c6534 F src/mem5.c 0025308a93838022bd5696cf9627ff4e40b19918 F src/memjournal.c 0683aac6cab6ec2b5374c0db37c0deb2436a3785 @@ -197,7 +197,7 @@ F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc -F src/mutex_w32.c 32a9b3841e2d757355f0012b860b1bc5e01eafa0 +F src/mutex_w32.c db4e14c30da72ffc8ead548469bc786c7a13145a F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f @@ -217,7 +217,7 @@ F src/random.c 0b2dbc37fdfbfa6bd455b091dfcef5bdb32dba68 F src/resolve.c 9d53899cc6e1f4ec0b4632d07e97d57827bf63b9 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 8b148eb851f384412aea57091659d14b369918ca -F src/shell.c 1c317a4c96d61d8d9fdad9fd1811d9b10b8c7f57 +F src/shell.c 250e4770afe136954088f0edcbc8ba76bcdb0e2e F src/sqlite.h.in bd1451ba1ab681022a53bccc3c39580ba094a3ff F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -250,7 +250,7 @@ F src/test_intarray.c 87847c71c3c36889c0bcc9c4baf9d31881665d61 F src/test_intarray.h 2ece66438cfd177b78d1bfda7a4180cd3a10844d F src/test_journal.c f5c0a05b7b3d5930db769b5ee6c3766dc2221a64 F src/test_loadext.c df586c27176e3c2cb2e099c78da67bf14379a56e -F src/test_malloc.c a105801222c0514f8befa2a1712a95505cce099a +F src/test_malloc.c eba4e1c5847cc98e7edc98f62265cd2abafba7a6 F src/test_multiplex.c 5d691eeb6cb6aa7888da28eba5e62a9a857d3c0f F src/test_multiplex.h 110a8c4d356e0aa464ca8730375608a9a0b61ae1 F src/test_mutex.c 293042d623ebba969160f471a82aa1551626454f @@ -290,7 +290,7 @@ F src/vtab.c 165ce0e797c2cd23badb104c9f2ae9042d6d942c F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 6e718c39d6b2964f15f6c96ce5938b4652e3538e +F src/where.c ee148dd577aaef8a4b5923f6046e3074a1ddf130 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1105,7 +1105,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 0a497083e915520c0807cb6611264b1a35ff62b7 -R 281d5c8316076afa3106297ea64f6c5c +P ec99224b0c9cf1ccd64b3dc93252556d888e93a3 +R 83bbd06201b37fcfd4ba93e170854819 U mistachkin -Z adbcf8e3a91ca172565c0b57ab07dba0 +Z 7a5beb9b27a8339428285f5cf59d377b diff --git a/manifest.uuid b/manifest.uuid index 82de3f15c0..58a03672d6 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -ec99224b0c9cf1ccd64b3dc93252556d888e93a3 \ No newline at end of file +8917e9f9a0802cbfb6f33e2ab1c2f98e4df5babd \ No newline at end of file diff --git a/src/expr.c b/src/expr.c index dd4f5de15f..aa6f275438 100644 --- a/src/expr.c +++ b/src/expr.c @@ -1278,7 +1278,7 @@ int sqlite3ExprIsInteger(Expr *p, int *pValue){ case TK_UMINUS: { int v; if( sqlite3ExprIsInteger(p->pLeft, &v) ){ - assert( v!=-2147483648 ); + assert( v!=(-2147483647-1) ); *pValue = -v; rc = 1; } diff --git a/src/mem2.c b/src/mem2.c index 26448ea8aa..d461dffab1 100644 --- a/src/mem2.c +++ b/src/mem2.c @@ -179,7 +179,7 @@ static int sqlite3MemSize(void *p){ return 0; } pHdr = sqlite3MemsysGetHeader(p); - return pHdr->iSize; + return (int)pHdr->iSize; } /* @@ -221,7 +221,7 @@ static void randomFill(char *pBuf, int nByte){ x = SQLITE_PTR_TO_INT(pBuf); y = nByte | 1; while( nByte >= 4 ){ - x = (x>>1) ^ (-(x&1) & 0xd0000001); + x = (x>>1) ^ (-(int)(x&1) & 0xd0000001); y = y*1103515245 + 12345; r = x ^ y; *(int*)pBuf = r; @@ -229,7 +229,7 @@ static void randomFill(char *pBuf, int nByte){ nByte -= 4; } while( nByte-- > 0 ){ - x = (x>>1) ^ (-(x&1) & 0xd0000001); + x = (x>>1) ^ (-(int)(x&1) & 0xd0000001); y = y*1103515245 + 12345; r = x ^ y; *(pBuf++) = r & 0xff; @@ -324,9 +324,9 @@ static void sqlite3MemFree(void *pPrior){ } z = (char*)pBt; z -= pHdr->nTitle; - adjustStats(pHdr->iSize, -1); + adjustStats((int)pHdr->iSize, -1); randomFill(z, sizeof(void*)*pHdr->nBacktraceSlots + sizeof(*pHdr) + - pHdr->iSize + sizeof(int) + pHdr->nTitle); + (int)pHdr->iSize + sizeof(int) + pHdr->nTitle); free(z); sqlite3_mutex_leave(mem.mutex); } @@ -350,7 +350,7 @@ static void *sqlite3MemRealloc(void *pPrior, int nByte){ if( pNew ){ memcpy(pNew, pPrior, nByteiSize ? nByte : pOldHdr->iSize); if( nByte>pOldHdr->iSize ){ - randomFill(&((char*)pNew)[pOldHdr->iSize], nByte - pOldHdr->iSize); + randomFill(&((char*)pNew)[pOldHdr->iSize], nByte - (int)pOldHdr->iSize); } sqlite3MemFree(pPrior); } @@ -465,7 +465,7 @@ void sqlite3MemdebugSync(){ for(pHdr=mem.pFirst; pHdr; pHdr=pHdr->pNext){ void **pBt = (void**)pHdr; pBt -= pHdr->nBacktraceSlots; - mem.xBacktrace(pHdr->iSize, pHdr->nBacktrace-1, &pBt[1]); + mem.xBacktrace((int)pHdr->iSize, pHdr->nBacktrace-1, &pBt[1]); } } diff --git a/src/mutex_w32.c b/src/mutex_w32.c index 27d10af5bd..f89a611b63 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -107,7 +107,7 @@ static int winMutex_isInit = 0; ** processing, the "interlocked" magic is probably not ** strictly necessary. */ -static long winMutex_lock = 0; +static LONG winMutex_lock = 0; void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ diff --git a/src/shell.c b/src/shell.c index 64c17ed054..093b58bc2a 100644 --- a/src/shell.c +++ b/src/shell.c @@ -71,12 +71,12 @@ /* Make sure isatty() has a prototype. */ extern int isatty(int); -#endif /* popen and pclose are not C89 functions and so are sometimes omitted from ** the header */ -FILE *popen(const char*,const char*); -int pclose(FILE*); +extern FILE *popen(const char*,const char*); +extern int pclose(FILE*); +#endif #if defined(_WIN32_WCE) /* Windows CE (arm-wince-mingw32ce-gcc) does not provide isatty() @@ -554,7 +554,7 @@ static void output_c_string(FILE *out, const char *z){ }else if( c=='\r' ){ fputc('\\', out); fputc('r', out); - }else if( !isprint(c) ){ + }else if( !isprint(c&0xff) ){ fprintf(out, "\\%03o", c&0xff); }else{ fputc(c, out); diff --git a/src/test_malloc.c b/src/test_malloc.c index 2e31f0833d..f513e24bf4 100644 --- a/src/test_malloc.c +++ b/src/test_malloc.c @@ -749,7 +749,7 @@ static void test_memdebug_callback(int nByte, int nFrame, void **aFrame){ int isNew; int aKey[MALLOC_LOG_KEYINTS]; - int nKey = sizeof(int)*MALLOC_LOG_KEYINTS; + unsigned int nKey = sizeof(int)*MALLOC_LOG_KEYINTS; memset(aKey, 0, nKey); if( (sizeof(void*)*nFrame)=nEq ); + assert( zAff==0 || (int)strlen(zAff)>=nEq ); for(j=0; jaLTerm[j]; From 533fb6de5bfb86930e58718200b7b8eea34764dc Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 28 Aug 2013 02:26:48 +0000 Subject: [PATCH 52/67] Remove hard-coding of the directory separator in the Win32 VFS. Fixes to OSTRACE macros. FossilOrigin-Name: fc98092f4bd42d64059024f09547904c1d70a517 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/os_win.c | 51 ++++++++++++++++++++++++++++++++++++--------------- 3 files changed, 43 insertions(+), 22 deletions(-) diff --git a/manifest b/manifest index 4916545b93..c3dde0dc4f 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sseveral\sharmless\scompiler\swarnings.\s\sFix\sa\scouple\scompiler\sissues\swith\sthe\sshell. -D 2013-08-28T01:54:12.702 +C Remove\shard-coding\sof\sthe\sdirectory\sseparator\sin\sthe\sWin32\sVFS.\s\sFixes\sto\sOSTRACE\smacros. +D 2013-08-28T02:26:48.559 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -203,7 +203,7 @@ F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c c27a14a05061e4e690bd3949dc0246bda35e399d -F src/os_win.c 52bbfe7f96fdb61fa71ba6f96eac43bcdc5c29a8 +F src/os_win.c 7f8cabe0ae13bf6303543197c349bd13a1aaaa0f F src/pager.c 2aa4444ffe86e9282d03bc349a4a5e49bd77c0e8 F src/pager.h f094af9f6ececfaa8a1e93876905a4f34233fb0c F src/parse.y 27c6b4138497d6f8360ba7847da6ed48033f957f @@ -1105,7 +1105,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P ec99224b0c9cf1ccd64b3dc93252556d888e93a3 -R 83bbd06201b37fcfd4ba93e170854819 +P 8917e9f9a0802cbfb6f33e2ab1c2f98e4df5babd +R 596c11dfb76cd23db3586c983e12df43 U mistachkin -Z 7a5beb9b27a8339428285f5cf59d377b +Z 1567302589a948f8e7270192d6781870 diff --git a/manifest.uuid b/manifest.uuid index 58a03672d6..3b379e9316 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -8917e9f9a0802cbfb6f33e2ab1c2f98e4df5babd \ No newline at end of file +fc98092f4bd42d64059024f09547904c1d70a517 \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index e07883a611..c9dd187ff8 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -91,6 +91,25 @@ # define SQLITE_WIN32_MAX_ERRMSG_CHARS (1024) #endif +/* +** Returns non-zero if the character should be treated as a directory +** separator. +*/ +#ifndef winIsDirSep +# define winIsDirSep(a) (((a) == '/') || ((a) == '\\')) +#endif + +/* +** Returns the string that should be used as the directory separator. +*/ +#ifndef winGetDirDep +# ifdef __CYGWIN__ +# define winGetDirDep() "/" +# else +# define winGetDirDep() "\\" +# endif +#endif + /* ** Do we need to manually define the Win32 file mapping APIs for use with WAL ** mode (e.g. these APIs are available in the Windows CE SDK; however, they @@ -2456,6 +2475,7 @@ static int winSync(sqlite3_file *id, int flags){ ** no-op */ #ifdef SQLITE_NO_SYNC + OSTRACE(("SYNC-NOP file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; #else rc = osFlushFileBuffers(pFile->h); @@ -2749,10 +2769,10 @@ static int winLock(sqlite3_file *id, int locktype){ if( res ){ rc = SQLITE_OK; }else{ - OSTRACE(("LOCK-FAIL file=%p, wanted=%d, got=%d\n", - pFile->h, locktype, newLocktype)); pFile->lastErrno = lastErrno; rc = SQLITE_BUSY; + OSTRACE(("LOCK-FAIL file=%p, wanted=%d, got=%d\n", + pFile->h, locktype, newLocktype)); } pFile->locktype = (u8)newLocktype; OSTRACE(("LOCK file=%p, lock=%d, rc=%s\n", @@ -3995,12 +4015,12 @@ static int getTempname(int nBuf, char *zBuf){ return SQLITE_ERROR; } - for(i=nTempPath; i>0 && zTempPath[i-1]=='\\'; i--){} + for(i=nTempPath; i>0 && winIsDirSep(zTempPath[i-1]); i--){} zTempPath[i] = 0; sqlite3_snprintf(nBuf-18, zBuf, (nTempPath > 0) ? - "%s\\"SQLITE_TEMP_FILE_PREFIX : SQLITE_TEMP_FILE_PREFIX, - zTempPath); + "%s%s" SQLITE_TEMP_FILE_PREFIX : SQLITE_TEMP_FILE_PREFIX, + zTempPath, winGetDirDep()); j = sqlite3Strlen30(zBuf); sqlite3_randomness(15, &zBuf[j]); for(i=0; i<15; i++, j++){ @@ -4124,7 +4144,7 @@ static int winOpen( pFile->h = INVALID_HANDLE_VALUE; #if SQLITE_OS_WINRT - if( !sqlite3_temp_directory ){ + if( !zUtf8Name && !sqlite3_temp_directory ){ sqlite3_log(SQLITE_ERROR, "sqlite3_temp_directory variable should be set for WinRT"); } @@ -4134,7 +4154,7 @@ static int winOpen( ** temporary file name to use */ if( !zUtf8Name ){ - assert(isDelete && !isOpenJournal); + assert( isDelete && !isOpenJournal ); rc = getTempname(SQLITE_WIN32_MAX_PATH_BYTES+2, zTmpname); if( rc!=SQLITE_OK ){ OSTRACE(("OPEN name=%s, rc=%s", zUtf8Name, sqlite3ErrName(rc))); @@ -4148,7 +4168,7 @@ static int winOpen( ** sqlite3_uri_parameter(). */ assert( (eType!=SQLITE_OPEN_MAIN_DB) || (flags & SQLITE_OPEN_URI) || - zUtf8Name[strlen(zUtf8Name)+1]==0 ); + zUtf8Name[sqlite3Strlen30(zUtf8Name)+1]==0 ); /* Convert the filename to the system encoding. */ zConverted = convertUtf8Filename(zUtf8Name); @@ -4349,6 +4369,7 @@ static int winDelete( zConverted = convertUtf8Filename(zFilename); if( zConverted==0 ){ + OSTRACE(("DELETE name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); return SQLITE_IOERR_NOMEM; } if( isNT() ){ @@ -4530,7 +4551,7 @@ static BOOL winIsVerbatimPathname( ** the final two cases; therefore, we return the safer return value of TRUE ** so that callers of this function will simply use it verbatim. */ - if ( zPathname[0]=='/' || zPathname[0]=='\\' ){ + if ( winIsDirSep(zPathname[0]) ){ return TRUE; } @@ -4582,8 +4603,8 @@ static int winFullPathname( zRelative); return SQLITE_CANTOPEN_FULLPATH; } - sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s\\%s", - sqlite3_data_directory, zOut); + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%s%s", + sqlite3_data_directory, winGetDirDep(), zOut); }else{ if( cygwin_conv_path(CCP_POSIX_TO_WIN_A, zRelative, zFull, nFull)<0 ){ winLogError(SQLITE_CANTOPEN_FULLPATH, (DWORD)errno, "cygwin_conv_path", @@ -4605,8 +4626,8 @@ static int winFullPathname( ** for converting the relative path name to an absolute ** one by prepending the data directory and a backslash. */ - sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s\\%s", - sqlite3_data_directory, zRelative); + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%s%s", + sqlite3_data_directory, winGetDirDep(), zRelative); }else{ sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s", zRelative); } @@ -4638,8 +4659,8 @@ static int winFullPathname( ** for converting the relative path name to an absolute ** one by prepending the data directory and a backslash. */ - sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s\\%s", - sqlite3_data_directory, zRelative); + sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%s%s", + sqlite3_data_directory, winGetDirDep(), zRelative); return SQLITE_OK; } zConverted = convertUtf8Filename(zRelative); From b324bc79e9e6971eba45bc1a861320fb4ed81d85 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 28 Aug 2013 02:37:29 +0000 Subject: [PATCH 53/67] Make names of private functions in the Win32 VFS consistent. Fix comment typo in Win32 mutex implementation. FossilOrigin-Name: c3b82c5bf97cfb35544c5b1fbcdf7b9c4827d1cf --- manifest | 14 ++-- manifest.uuid | 2 +- src/mutex_w32.c | 2 +- src/os_win.c | 168 ++++++++++++++++++++++++------------------------ 4 files changed, 93 insertions(+), 93 deletions(-) diff --git a/manifest b/manifest index c3dde0dc4f..dc3a924ab1 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Remove\shard-coding\sof\sthe\sdirectory\sseparator\sin\sthe\sWin32\sVFS.\s\sFixes\sto\sOSTRACE\smacros. -D 2013-08-28T02:26:48.559 +C Make\snames\sof\sprivate\sfunctions\sin\sthe\sWin32\sVFS\sconsistent.\s\sFix\scomment\stypo\sin\sWin32\smutex\simplementation. +D 2013-08-28T02:37:29.527 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -197,13 +197,13 @@ F src/mutex.c d3b66a569368015e0fcb1ac15f81c119f504d3bc F src/mutex.h 5bc526e19dccc412b7ff04642f6fdad3fdfdabea F src/mutex_noop.c 7682796b7d8d39bf1c138248858efcd10c9e1553 F src/mutex_unix.c c3a4e00f96ba068a8dbef34084465979aaf369cc -F src/mutex_w32.c db4e14c30da72ffc8ead548469bc786c7a13145a +F src/mutex_w32.c 6108c88e1cb38d8fbb3534b170793815cbedbf97 F src/notify.c 976dd0f6171d4588e89e874fcc765e92914b6d30 F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c c27a14a05061e4e690bd3949dc0246bda35e399d -F src/os_win.c 7f8cabe0ae13bf6303543197c349bd13a1aaaa0f +F src/os_win.c 0b9ab09dd3ccf8933f888c542a9b11b0bfa62659 F src/pager.c 2aa4444ffe86e9282d03bc349a4a5e49bd77c0e8 F src/pager.h f094af9f6ececfaa8a1e93876905a4f34233fb0c F src/parse.y 27c6b4138497d6f8360ba7847da6ed48033f957f @@ -1105,7 +1105,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 8917e9f9a0802cbfb6f33e2ab1c2f98e4df5babd -R 596c11dfb76cd23db3586c983e12df43 +P fc98092f4bd42d64059024f09547904c1d70a517 +R 1438e04cfc27e16b0dae733cf906e3e9 U mistachkin -Z 1567302589a948f8e7270192d6781870 +Z 1716564dd5627ab9e624a1f87aa67863 diff --git a/manifest.uuid b/manifest.uuid index 3b379e9316..38eda44345 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -fc98092f4bd42d64059024f09547904c1d70a517 \ No newline at end of file +c3b82c5bf97cfb35544c5b1fbcdf7b9c4827d1cf \ No newline at end of file diff --git a/src/mutex_w32.c b/src/mutex_w32.c index f89a611b63..4b88c17452 100644 --- a/src/mutex_w32.c +++ b/src/mutex_w32.c @@ -69,7 +69,7 @@ struct sqlite3_mutex { } return osType==2; } -#endif /* SQLITE_OS_WINCE */ +#endif /* SQLITE_OS_WINCE || SQLITE_OS_WINRT */ #endif #ifdef SQLITE_DEBUG diff --git a/src/os_win.c b/src/os_win.c index c9dd187ff8..ff5575ddb5 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -1117,11 +1117,11 @@ void sqlite3_win32_sleep(DWORD milliseconds){ ** the LockFileEx() API. */ #if SQLITE_OS_WINCE || SQLITE_OS_WINRT -# define isNT() (1) +# define osIsNT() (1) #elif !defined(SQLITE_WIN32_HAS_WIDE) -# define isNT() (0) +# define osIsNT() (0) #else - static int isNT(void){ + static int osIsNT(void){ if( sqlite3_os_type==0 ){ OSVERSIONINFOA sInfo; sInfo.dwOSVersionInfoSize = sizeof(sInfo); @@ -1332,7 +1332,7 @@ void sqlite3MemSetDefault(void){ ** ** Space to hold the returned string is obtained from malloc. */ -static LPWSTR utf8ToUnicode(const char *zFilename){ +static LPWSTR winUtf8ToUnicode(const char *zFilename){ int nChar; LPWSTR zWideFilename; @@ -1357,7 +1357,7 @@ static LPWSTR utf8ToUnicode(const char *zFilename){ ** Convert Microsoft Unicode to UTF-8. Space to hold the returned string is ** obtained from sqlite3_malloc(). */ -static char *unicodeToUtf8(LPCWSTR zWideFilename){ +static char *winUnicodeToUtf8(LPCWSTR zWideFilename){ int nByte; char *zFilename; @@ -1385,7 +1385,7 @@ static char *unicodeToUtf8(LPCWSTR zWideFilename){ ** Space to hold the returned string is obtained ** from sqlite3_malloc. */ -static LPWSTR mbcsToUnicode(const char *zFilename){ +static LPWSTR winMbcsToUnicode(const char *zFilename){ int nByte; LPWSTR zMbcsFilename; int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; @@ -1415,7 +1415,7 @@ static LPWSTR mbcsToUnicode(const char *zFilename){ ** Space to hold the returned string is obtained from ** sqlite3_malloc(). */ -static char *unicodeToMbcs(LPCWSTR zWideFilename){ +static char *winUnicodeToMbcs(LPCWSTR zWideFilename){ int nByte; char *zFilename; int codepage = osAreFileApisANSI() ? CP_ACP : CP_OEMCP; @@ -1445,11 +1445,11 @@ char *sqlite3_win32_mbcs_to_utf8(const char *zFilename){ char *zFilenameUtf8; LPWSTR zTmpWide; - zTmpWide = mbcsToUnicode(zFilename); + zTmpWide = winMbcsToUnicode(zFilename); if( zTmpWide==0 ){ return 0; } - zFilenameUtf8 = unicodeToUtf8(zTmpWide); + zFilenameUtf8 = winUnicodeToUtf8(zTmpWide); sqlite3_free(zTmpWide); return zFilenameUtf8; } @@ -1462,11 +1462,11 @@ char *sqlite3_win32_utf8_to_mbcs(const char *zFilename){ char *zFilenameMbcs; LPWSTR zTmpWide; - zTmpWide = utf8ToUnicode(zFilename); + zTmpWide = winUtf8ToUnicode(zFilename); if( zTmpWide==0 ){ return 0; } - zFilenameMbcs = unicodeToMbcs(zTmpWide); + zFilenameMbcs = winUnicodeToMbcs(zTmpWide); sqlite3_free(zTmpWide); return zFilenameMbcs; } @@ -1496,7 +1496,7 @@ int sqlite3_win32_set_directory(DWORD type, LPCWSTR zValue){ if( ppDirectory ){ char *zValueUtf8 = 0; if( zValue && zValue[0] ){ - zValueUtf8 = unicodeToUtf8(zValue); + zValueUtf8 = winUnicodeToUtf8(zValue); if ( zValueUtf8==0 ){ return SQLITE_NOMEM; } @@ -1509,11 +1509,11 @@ int sqlite3_win32_set_directory(DWORD type, LPCWSTR zValue){ } /* -** The return value of getLastErrorMsg +** The return value of winGetLastErrorMsg ** is zero if the error message fits in the buffer, or non-zero ** otherwise (if the message was truncated). */ -static int getLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ +static int winGetLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ /* FormatMessage returns 0 on failure. Otherwise it ** returns the number of TCHARs written to the output ** buffer, excluding the terminating null char. @@ -1521,7 +1521,7 @@ static int getLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ DWORD dwLen = 0; char *zOut = 0; - if( isNT() ){ + if( osIsNT() ){ #if SQLITE_OS_WINRT WCHAR zTempWide[SQLITE_WIN32_MAX_ERRMSG_CHARS+1]; dwLen = osFormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | @@ -1547,7 +1547,7 @@ static int getLastErrorMsg(DWORD lastErrno, int nBuf, char *zBuf){ if( dwLen > 0 ){ /* allocate a buffer and convert to UTF8 */ sqlite3BeginBenignMalloc(); - zOut = unicodeToUtf8(zTempWide); + zOut = winUnicodeToUtf8(zTempWide); sqlite3EndBenignMalloc(); #if !SQLITE_OS_WINRT /* free the system buffer allocated by FormatMessage */ @@ -1615,7 +1615,7 @@ static int winLogErrorAtLine( int i; /* Loop counter */ zMsg[0] = 0; - getLastErrorMsg(lastErrno, sizeof(zMsg), zMsg); + winGetLastErrorMsg(lastErrno, sizeof(zMsg), zMsg); assert( errcode!=SQLITE_OK ); if( zPath==0 ) zPath = ""; for(i=0; zMsg[i] && zMsg[i]!='\r' && zMsg[i]!='\n'; i++){} @@ -1640,17 +1640,17 @@ static int winLogErrorAtLine( #ifndef SQLITE_WIN32_IOERR_RETRY_DELAY # define SQLITE_WIN32_IOERR_RETRY_DELAY 25 #endif -static int win32IoerrRetry = SQLITE_WIN32_IOERR_RETRY; -static int win32IoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; +static int winIoerrRetry = SQLITE_WIN32_IOERR_RETRY; +static int winIoerrRetryDelay = SQLITE_WIN32_IOERR_RETRY_DELAY; /* ** If a ReadFile() or WriteFile() error occurs, invoke this routine ** to see if it should be retried. Return TRUE to retry. Return FALSE ** to give up with an error. */ -static int retryIoerr(int *pnRetry, DWORD *pError){ +static int winRetryIoerr(int *pnRetry, DWORD *pError){ DWORD e = osGetLastError(); - if( *pnRetry>=win32IoerrRetry ){ + if( *pnRetry>=winIoerrRetry ){ if( pError ){ *pError = e; } @@ -1659,7 +1659,7 @@ static int retryIoerr(int *pnRetry, DWORD *pError){ if( e==ERROR_ACCESS_DENIED || e==ERROR_LOCK_VIOLATION || e==ERROR_SHARING_VIOLATION ){ - sqlite3_win32_sleep(win32IoerrRetryDelay*(1+*pnRetry)); + sqlite3_win32_sleep(winIoerrRetryDelay*(1+*pnRetry)); ++*pnRetry; return 1; } @@ -1672,11 +1672,11 @@ static int retryIoerr(int *pnRetry, DWORD *pError){ /* ** Log a I/O error retry episode. */ -static void logIoerr(int nRetry){ +static void winLogIoerr(int nRetry){ if( nRetry ){ sqlite3_log(SQLITE_IOERR, "delayed %dms for lock/sharing conflict", - win32IoerrRetryDelay*nRetry*(nRetry+1)/2 + winIoerrRetryDelay*nRetry*(nRetry+1)/2 ); } } @@ -1741,7 +1741,7 @@ static int winceCreateLock(const char *zFilename, winFile *pFile){ BOOL bLogged = FALSE; BOOL bInit = TRUE; - zName = utf8ToUnicode(zFilename); + zName = winUtf8ToUnicode(zFilename); if( zName==0 ){ /* out of memory */ return SQLITE_IOERR_NOMEM; @@ -2014,7 +2014,7 @@ static BOOL winLockFile( return winceLockFile(phFile, offsetLow, offsetHigh, numBytesLow, numBytesHigh); #else - if( isNT() ){ + if( osIsNT() ){ OVERLAPPED ovlp; memset(&ovlp, 0, sizeof(OVERLAPPED)); ovlp.Offset = offsetLow; @@ -2045,7 +2045,7 @@ static BOOL winUnlockFile( return winceUnlockFile(phFile, offsetLow, offsetHigh, numBytesLow, numBytesHigh); #else - if( isNT() ){ + if( osIsNT() ){ OVERLAPPED ovlp; memset(&ovlp, 0, sizeof(OVERLAPPED)); ovlp.Offset = offsetLow; @@ -2075,7 +2075,7 @@ static BOOL winUnlockFile( ** argument to offset iOffset within the file. If successful, return 0. ** Otherwise, set pFile->lastErrno and return non-zero. */ -static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ +static int winSeekFile(winFile *pFile, sqlite3_int64 iOffset){ #if !SQLITE_OS_WINRT LONG upperBits; /* Most sig. 32 bits of new offset */ LONG lowerBits; /* Least sig. 32 bits of new offset */ @@ -2100,7 +2100,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ && ((lastErrno = osGetLastError())!=NO_ERROR)) ){ pFile->lastErrno = lastErrno; winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, - "seekWinFile", pFile->zPath); + "winSeekFile", pFile->zPath); OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); return 1; } @@ -2121,7 +2121,7 @@ static int seekWinFile(winFile *pFile, sqlite3_int64 iOffset){ if(!bRet){ pFile->lastErrno = osGetLastError(); winLogError(SQLITE_IOERR_SEEK, pFile->lastErrno, - "seekWinFile", pFile->zPath); + "winSeekFile", pFile->zPath); OSTRACE(("SEEK file=%p, rc=SQLITE_IOERR_SEEK\n", pFile->h)); return 1; } @@ -2236,7 +2236,7 @@ static int winRead( #endif #if SQLITE_OS_WINCE - if( seekWinFile(pFile, offset) ){ + if( winSeekFile(pFile, offset) ){ OSTRACE(("READ file=%p, rc=SQLITE_FULL\n", pFile->h)); return SQLITE_FULL; } @@ -2249,13 +2249,13 @@ static int winRead( osGetLastError()!=ERROR_HANDLE_EOF ){ #endif DWORD lastErrno; - if( retryIoerr(&nRetry, &lastErrno) ) continue; + if( winRetryIoerr(&nRetry, &lastErrno) ) continue; pFile->lastErrno = lastErrno; OSTRACE(("READ file=%p, rc=SQLITE_IOERR_READ\n", pFile->h)); return winLogError(SQLITE_IOERR_READ, pFile->lastErrno, "winRead", pFile->zPath); } - logIoerr(nRetry); + winLogIoerr(nRetry); if( nRead<(DWORD)amt ){ /* Unread parts of the buffer must be zero-filled */ memset(&((char*)pBuf)[nRead], 0, amt-nRead); @@ -2308,7 +2308,7 @@ static int winWrite( #endif #if SQLITE_OS_WINCE - rc = seekWinFile(pFile, offset); + rc = winSeekFile(pFile, offset); if( rc==0 ){ #else { @@ -2333,7 +2333,7 @@ static int winWrite( #else if( !osWriteFile(pFile->h, aRem, nRem, &nWrite, &overlapped) ){ #endif - if( retryIoerr(&nRetry, &lastErrno) ) continue; + if( winRetryIoerr(&nRetry, &lastErrno) ) continue; break; } assert( nWrite==0 || nWrite<=(DWORD)nRem ); @@ -2365,7 +2365,7 @@ static int winWrite( return winLogError(SQLITE_IOERR_WRITE, pFile->lastErrno, "winWrite", pFile->zPath); }else{ - logIoerr(nRetry); + winLogIoerr(nRetry); } OSTRACE(("WRITE file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; @@ -2394,7 +2394,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ } /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ - if( seekWinFile(pFile, nByte) ){ + if( winSeekFile(pFile, nByte) ){ rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, "winTruncate1", pFile->zPath); }else if( 0==osSetEndOfFile(pFile->h) && @@ -2573,10 +2573,10 @@ static int winFileSize(sqlite3_file *id, sqlite3_int64 *pSize){ ** Different API routines are called depending on whether or not this ** is Win9x or WinNT. */ -static int getReadLock(winFile *pFile){ +static int winGetReadLock(winFile *pFile){ int res; OSTRACE(("READ-LOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); - if( isNT() ){ + if( osIsNT() ){ #if SQLITE_OS_WINCE /* ** NOTE: Windows CE is handled differently here due its lack of the Win32 @@ -2608,11 +2608,11 @@ static int getReadLock(winFile *pFile){ /* ** Undo a readlock */ -static int unlockReadLock(winFile *pFile){ +static int winUnlockReadLock(winFile *pFile){ int res; DWORD lastErrno; OSTRACE(("READ-UNLOCK file=%p, lock=%d\n", pFile->h, pFile->locktype)); - if( isNT() ){ + if( osIsNT() ){ res = winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); } #ifdef SQLITE_WIN32_HAS_ANSI @@ -2623,7 +2623,7 @@ static int unlockReadLock(winFile *pFile){ if( res==0 && ((lastErrno = osGetLastError())!=ERROR_NOT_LOCKED) ){ pFile->lastErrno = lastErrno; winLogError(SQLITE_IOERR_UNLOCK, pFile->lastErrno, - "unlockReadLock", pFile->zPath); + "winUnlockReadLock", pFile->zPath); } OSTRACE(("READ-UNLOCK file=%p, rc=%s\n", pFile->h, sqlite3ErrName(res))); return res; @@ -2714,7 +2714,7 @@ static int winLock(sqlite3_file *id, int locktype){ */ if( locktype==SHARED_LOCK && res ){ assert( pFile->locktype==NO_LOCK ); - res = getReadLock(pFile); + res = winGetReadLock(pFile); if( res ){ newLocktype = SHARED_LOCK; }else{ @@ -2745,14 +2745,14 @@ static int winLock(sqlite3_file *id, int locktype){ */ if( locktype==EXCLUSIVE_LOCK && res ){ assert( pFile->locktype>=SHARED_LOCK ); - res = unlockReadLock(pFile); + res = winUnlockReadLock(pFile); res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, SHARED_SIZE, 0); if( res ){ newLocktype = EXCLUSIVE_LOCK; }else{ lastErrno = osGetLastError(); - getReadLock(pFile); + winGetReadLock(pFile); } } @@ -2832,7 +2832,7 @@ static int winUnlock(sqlite3_file *id, int locktype){ type = pFile->locktype; if( type>=EXCLUSIVE_LOCK ){ winUnlockFile(&pFile->h, SHARED_FIRST, 0, SHARED_SIZE, 0); - if( locktype==SHARED_LOCK && !getReadLock(pFile) ){ + if( locktype==SHARED_LOCK && !winGetReadLock(pFile) ){ /* This should never happen. We should always be able to ** reacquire the read lock */ rc = winLogError(SQLITE_IOERR_UNLOCK, osGetLastError(), @@ -2843,7 +2843,7 @@ static int winUnlock(sqlite3_file *id, int locktype){ winUnlockFile(&pFile->h, RESERVED_BYTE, 0, 1, 0); } if( locktype==NO_LOCK && type>=SHARED_LOCK ){ - unlockReadLock(pFile); + winUnlockReadLock(pFile); } if( type>=PENDING_LOCK ){ winUnlockFile(&pFile->h, PENDING_BYTE, 0, 1, 0); @@ -2871,7 +2871,7 @@ static void winModeBit(winFile *pFile, unsigned char mask, int *pArg){ } /* Forward declaration */ -static int getTempname(int nBuf, char *zBuf); +static int winGetTempname(int nBuf, char *zBuf); #if SQLITE_MAX_MMAP_SIZE>0 static int winMapfile(winFile*, sqlite3_int64); #endif @@ -2934,14 +2934,14 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ case SQLITE_FCNTL_WIN32_AV_RETRY: { int *a = (int*)pArg; if( a[0]>0 ){ - win32IoerrRetry = a[0]; + winIoerrRetry = a[0]; }else{ - a[0] = win32IoerrRetry; + a[0] = winIoerrRetry; } if( a[1]>0 ){ - win32IoerrRetryDelay = a[1]; + winIoerrRetryDelay = a[1]; }else{ - a[1] = win32IoerrRetryDelay; + a[1] = winIoerrRetryDelay; } OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; @@ -2949,7 +2949,7 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ case SQLITE_FCNTL_TEMPFILENAME: { char *zTFile = sqlite3MallocZero( pFile->pVfs->mxPathname ); if( zTFile ){ - getTempname(pFile->pVfs->mxPathname, zTFile); + winGetTempname(pFile->pVfs->mxPathname, zTFile); *(char**)pArg = zTFile; } OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); @@ -3915,10 +3915,10 @@ static const sqlite3_io_methods winIoMethod = { ** is obtained from malloc and must be freed by the calling ** function. */ -static void *convertUtf8Filename(const char *zFilename){ +static void *winConvertUtf8Filename(const char *zFilename){ void *zConverted = 0; - if( isNT() ){ - zConverted = utf8ToUnicode(zFilename); + if( osIsNT() ){ + zConverted = winUtf8ToUnicode(zFilename); } #ifdef SQLITE_WIN32_HAS_ANSI else{ @@ -3933,7 +3933,7 @@ static void *convertUtf8Filename(const char *zFilename){ ** Create a temporary file name in zBuf. zBuf must be big enough to ** hold at pVfs->mxPathname characters. */ -static int getTempname(int nBuf, char *zBuf){ +static int winGetTempname(int nBuf, char *zBuf){ static char zChars[] = "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" @@ -3953,14 +3953,14 @@ static int getTempname(int nBuf, char *zBuf){ sqlite3_temp_directory); } #if !SQLITE_OS_WINRT - else if( isNT() ){ + else if( osIsNT() ){ char *zMulti; WCHAR zWidePath[SQLITE_WIN32_MAX_PATH_CHARS]; if( osGetTempPathW(SQLITE_WIN32_MAX_PATH_CHARS-30, zWidePath)==0 ){ OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_GETTEMPPATH\n")); return SQLITE_IOERR_GETTEMPPATH; } - zMulti = unicodeToUtf8(zWidePath); + zMulti = winUnicodeToUtf8(zWidePath); if( zMulti ){ sqlite3_snprintf(SQLITE_WIN32_MAX_PATH_BYTES-30, zTempPath, "%s", zMulti); sqlite3_free(zMulti); @@ -4043,13 +4043,13 @@ static int winIsDir(const void *zConverted){ int rc = 0; DWORD lastErrno; - if( isNT() ){ + if( osIsNT() ){ int cnt = 0; WIN32_FILE_ATTRIBUTE_DATA sAttrData; memset(&sAttrData, 0, sizeof(sAttrData)); while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, GetFileExInfoStandard, - &sAttrData)) && retryIoerr(&cnt, &lastErrno) ){} + &sAttrData)) && winRetryIoerr(&cnt, &lastErrno) ){} if( !rc ){ return 0; /* Invalid name? */ } @@ -4155,7 +4155,7 @@ static int winOpen( */ if( !zUtf8Name ){ assert( isDelete && !isOpenJournal ); - rc = getTempname(SQLITE_WIN32_MAX_PATH_BYTES+2, zTmpname); + rc = winGetTempname(SQLITE_WIN32_MAX_PATH_BYTES+2, zTmpname); if( rc!=SQLITE_OK ){ OSTRACE(("OPEN name=%s, rc=%s", zUtf8Name, sqlite3ErrName(rc))); return rc; @@ -4171,7 +4171,7 @@ static int winOpen( zUtf8Name[sqlite3Strlen30(zUtf8Name)+1]==0 ); /* Convert the filename to the system encoding. */ - zConverted = convertUtf8Filename(zUtf8Name); + zConverted = winConvertUtf8Filename(zUtf8Name); if( zConverted==0 ){ OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); return SQLITE_IOERR_NOMEM; @@ -4225,7 +4225,7 @@ static int winOpen( dwFlagsAndAttributes |= FILE_FLAG_RANDOM_ACCESS; #endif - if( isNT() ){ + if( osIsNT() ){ #if SQLITE_OS_WINRT CREATEFILE2_EXTENDED_PARAMETERS extendedParameters; extendedParameters.dwSize = sizeof(CREATEFILE2_EXTENDED_PARAMETERS); @@ -4240,7 +4240,7 @@ static int winOpen( dwShareMode, dwCreationDisposition, &extendedParameters))==INVALID_HANDLE_VALUE && - retryIoerr(&cnt, &lastErrno) ){ + winRetryIoerr(&cnt, &lastErrno) ){ /* Noop */ } #else @@ -4250,7 +4250,7 @@ static int winOpen( dwCreationDisposition, dwFlagsAndAttributes, NULL))==INVALID_HANDLE_VALUE && - retryIoerr(&cnt, &lastErrno) ){ + winRetryIoerr(&cnt, &lastErrno) ){ /* Noop */ } #endif @@ -4263,12 +4263,12 @@ static int winOpen( dwCreationDisposition, dwFlagsAndAttributes, NULL))==INVALID_HANDLE_VALUE && - retryIoerr(&cnt, &lastErrno) ){ + winRetryIoerr(&cnt, &lastErrno) ){ /* Noop */ } } #endif - logIoerr(cnt); + winLogIoerr(cnt); OSTRACE(("OPEN file=%p, name=%s, access=%lx, rc=%s\n", h, zUtf8Name, dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); @@ -4367,12 +4367,12 @@ static int winDelete( SimulateIOError(return SQLITE_IOERR_DELETE); OSTRACE(("DELETE name=%s, syncDir=%d\n", zFilename, syncDir)); - zConverted = convertUtf8Filename(zFilename); + zConverted = winConvertUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("DELETE name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); return SQLITE_IOERR_NOMEM; } - if( isNT() ){ + if( osIsNT() ){ do { #if SQLITE_OS_WINRT WIN32_FILE_ATTRIBUTE_DATA sAttrData; @@ -4411,7 +4411,7 @@ static int winDelete( rc = SQLITE_OK; /* Deleted OK. */ break; } - if ( !retryIoerr(&cnt, &lastErrno) ){ + if ( !winRetryIoerr(&cnt, &lastErrno) ){ rc = SQLITE_ERROR; /* No more retries. */ break; } @@ -4439,7 +4439,7 @@ static int winDelete( rc = SQLITE_OK; /* Deleted OK. */ break; } - if ( !retryIoerr(&cnt, &lastErrno) ){ + if ( !winRetryIoerr(&cnt, &lastErrno) ){ rc = SQLITE_ERROR; /* No more retries. */ break; } @@ -4450,7 +4450,7 @@ static int winDelete( rc = winLogError(SQLITE_IOERR_DELETE, lastErrno, "winDelete", zFilename); }else{ - logIoerr(cnt); + winLogIoerr(cnt); } sqlite3_free(zConverted); OSTRACE(("DELETE name=%s, rc=%s\n", zFilename, sqlite3ErrName(rc))); @@ -4476,18 +4476,18 @@ static int winAccess( OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", zFilename, flags, pResOut)); - zConverted = convertUtf8Filename(zFilename); + zConverted = winConvertUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); return SQLITE_IOERR_NOMEM; } - if( isNT() ){ + if( osIsNT() ){ int cnt = 0; WIN32_FILE_ATTRIBUTE_DATA sAttrData; memset(&sAttrData, 0, sizeof(sAttrData)); while( !(rc = osGetFileAttributesExW((LPCWSTR)zConverted, GetFileExInfoStandard, - &sAttrData)) && retryIoerr(&cnt, &lastErrno) ){} + &sAttrData)) && winRetryIoerr(&cnt, &lastErrno) ){} if( rc ){ /* For an SQLITE_ACCESS_EXISTS query, treat a zero-length file ** as if it does not exist. @@ -4500,7 +4500,7 @@ static int winAccess( attr = sAttrData.dwFileAttributes; } }else{ - logIoerr(cnt); + winLogIoerr(cnt); if( lastErrno!=ERROR_FILE_NOT_FOUND && lastErrno!=ERROR_PATH_NOT_FOUND ){ winLogError(SQLITE_IOERR_ACCESS, lastErrno, "winAccess", zFilename); sqlite3_free(zConverted); @@ -4663,11 +4663,11 @@ static int winFullPathname( sqlite3_data_directory, winGetDirDep(), zRelative); return SQLITE_OK; } - zConverted = convertUtf8Filename(zRelative); + zConverted = winConvertUtf8Filename(zRelative); if( zConverted==0 ){ return SQLITE_IOERR_NOMEM; } - if( isNT() ){ + if( osIsNT() ){ LPWSTR zTemp; nByte = osGetFullPathNameW((LPCWSTR)zConverted, 0, 0, 0); if( nByte==0 ){ @@ -4691,7 +4691,7 @@ static int winFullPathname( return SQLITE_CANTOPEN_FULLPATH; } sqlite3_free(zConverted); - zOut = unicodeToUtf8(zTemp); + zOut = winUnicodeToUtf8(zTemp); sqlite3_free(zTemp); } #ifdef SQLITE_WIN32_HAS_ANSI @@ -4744,12 +4744,12 @@ static int winFullPathname( */ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ HANDLE h; - void *zConverted = convertUtf8Filename(zFilename); + void *zConverted = winConvertUtf8Filename(zFilename); UNUSED_PARAMETER(pVfs); if( zConverted==0 ){ return 0; } - if( isNT() ){ + if( osIsNT() ){ #if SQLITE_OS_WINRT h = osLoadPackagedLibrary((LPCWSTR)zConverted, 0); #else @@ -4766,7 +4766,7 @@ static void *winDlOpen(sqlite3_vfs *pVfs, const char *zFilename){ } static void winDlError(sqlite3_vfs *pVfs, int nBuf, char *zBufOut){ UNUSED_PARAMETER(pVfs); - getLastErrorMsg(osGetLastError(), nBuf, zBufOut); + winGetLastErrorMsg(osGetLastError(), nBuf, zBufOut); } static void (*winDlSym(sqlite3_vfs *pVfs,void *pH,const char *zSym))(void){ UNUSED_PARAMETER(pVfs); @@ -4942,7 +4942,7 @@ static int winCurrentTime(sqlite3_vfs *pVfs, double *prNow){ */ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ UNUSED_PARAMETER(pVfs); - return getLastErrorMsg(osGetLastError(), nBuf, zBuf); + return winGetLastErrorMsg(osGetLastError(), nBuf, zBuf); } /* From 3741827ea16a519258785edb2bd3ea8ea3a237b1 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 28 Aug 2013 05:49:39 +0000 Subject: [PATCH 54/67] Support database file names longer than 260 characters using the new 'win32-longpath' VFS variant. FossilOrigin-Name: 37e85e444cde18f061049437980b965d4485f43c --- manifest | 15 ++-- manifest.uuid | 2 +- src/os_win.c | 164 ++++++++++++++++++++++++++++------------ src/test1.c | 144 +++++++++++++++++++++++++++++++++++ test/win32longpath.test | 160 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 429 insertions(+), 56 deletions(-) create mode 100644 test/win32longpath.test diff --git a/manifest b/manifest index dc3a924ab1..c81c0d5c3e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Make\snames\sof\sprivate\sfunctions\sin\sthe\sWin32\sVFS\sconsistent.\s\sFix\scomment\stypo\sin\sWin32\smutex\simplementation. -D 2013-08-28T02:37:29.527 +C Support\sdatabase\sfile\snames\slonger\sthan\s260\scharacters\susing\sthe\snew\s'win32-longpath'\sVFS\svariant. +D 2013-08-28T05:49:39.789 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -203,7 +203,7 @@ F src/os.c b4ad71336fd96f97776f75587cd9e8218288f5be F src/os.h 4a46270a64e9193af4a0aaa3bc2c66dc07c29b3f F src/os_common.h 92815ed65f805560b66166e3583470ff94478f04 F src/os_unix.c c27a14a05061e4e690bd3949dc0246bda35e399d -F src/os_win.c 0b9ab09dd3ccf8933f888c542a9b11b0bfa62659 +F src/os_win.c 26d752736dff0c7e4e384ab65b353cce1e7e19c5 F src/pager.c 2aa4444ffe86e9282d03bc349a4a5e49bd77c0e8 F src/pager.h f094af9f6ececfaa8a1e93876905a4f34233fb0c F src/parse.y 27c6b4138497d6f8360ba7847da6ed48033f957f @@ -226,7 +226,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 659dad8ef30b54831306a244b43e37af4725a444 -F src/test1.c 870fc648a48cb6d6808393174f7ebe82b8c840fa +F src/test1.c 031d00dbf1b48378e6d47af1a0f728c767320b96 F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -1057,6 +1057,7 @@ F test/whereF.test 136a7301512d72a08a272806c8767066311b7bc1 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 +F test/win32longpath.test 3018b1199270cb91194fc8cd8413912b6950577e F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test 209d7ed441f44cc5299e4ebffbef06fd5aabfefd F tool/build-all-msvc.bat c55f64ca200308fb5fa5c1ee751ea95a13977b5a x @@ -1105,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P fc98092f4bd42d64059024f09547904c1d70a517 -R 1438e04cfc27e16b0dae733cf906e3e9 +P c3b82c5bf97cfb35544c5b1fbcdf7b9c4827d1cf +R e20bd90758596867c642a3a14073401d U mistachkin -Z 1716564dd5627ab9e624a1f87aa67863 +Z e8ceaa194e29622b0b6b8c77bb0db1ea diff --git a/manifest.uuid b/manifest.uuid index 38eda44345..6cdcbf3836 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c3b82c5bf97cfb35544c5b1fbcdf7b9c4827d1cf \ No newline at end of file +37e85e444cde18f061049437980b965d4485f43c \ No newline at end of file diff --git a/src/os_win.c b/src/os_win.c index ff5575ddb5..89e64aa431 100644 --- a/src/os_win.c +++ b/src/os_win.c @@ -2871,7 +2871,7 @@ static void winModeBit(winFile *pFile, unsigned char mask, int *pArg){ } /* Forward declaration */ -static int winGetTempname(int nBuf, char *zBuf); +static int winGetTempname(sqlite3_vfs *, char **); #if SQLITE_MAX_MMAP_SIZE>0 static int winMapfile(winFile*, sqlite3_int64); #endif @@ -2947,13 +2947,13 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ return SQLITE_OK; } case SQLITE_FCNTL_TEMPFILENAME: { - char *zTFile = sqlite3MallocZero( pFile->pVfs->mxPathname ); - if( zTFile ){ - winGetTempname(pFile->pVfs->mxPathname, zTFile); + char *zTFile = 0; + int rc = winGetTempname(pFile->pVfs, &zTFile); + if( rc==SQLITE_OK ){ *(char**)pArg = zTFile; } - OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); - return SQLITE_OK; + OSTRACE(("FCNTL file=%p, rc=%d\n", pFile->h, rc)); + return rc; } #if SQLITE_MAX_MMAP_SIZE>0 case SQLITE_FCNTL_MMAP_SIZE: { @@ -3930,17 +3930,29 @@ static void *winConvertUtf8Filename(const char *zFilename){ } /* -** Create a temporary file name in zBuf. zBuf must be big enough to -** hold at pVfs->mxPathname characters. +** This function returns non-zero if the specified UTF-8 string buffer +** ends with a directory separator character. */ -static int winGetTempname(int nBuf, char *zBuf){ +static int winEndsInDirSep(char *zBuf){ + if( zBuf ){ + int nLen = sqlite3Strlen30(zBuf); + return nLen>0 && winIsDirSep(zBuf[nLen-1]); + } + return 0; +} + +/* +** Create a temporary file name and store the resulting pointer into pzBuf. +** The pointer returned in pzBuf must be freed via sqlite3_free(). +*/ +static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ static char zChars[] = "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789"; size_t i, j; - int nTempPath; - char zTempPath[SQLITE_WIN32_MAX_PATH_BYTES+2]; + int nBuf, nLen; + char *zBuf; /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this @@ -3948,23 +3960,49 @@ static int winGetTempname(int nBuf, char *zBuf){ */ SimulateIOError( return SQLITE_IOERR ); + /* Allocate a temporary buffer to store the fully qualified file + ** name for the temporary file. If this fails, we cannot continue. + */ + nBuf = pVfs->mxPathname; + zBuf = sqlite3MallocZero( nBuf+2 ); + if( !zBuf ){ + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); + return SQLITE_IOERR_NOMEM; + } + + /* Figure out the effective temporary directory. First, check if one + ** has been explicitly set by the application; otherwise, use the one + ** configured by the operating system. + */ + assert( nBuf>30 ); if( sqlite3_temp_directory ){ - sqlite3_snprintf(SQLITE_WIN32_MAX_PATH_BYTES-30, zTempPath, "%s", - sqlite3_temp_directory); + sqlite3_snprintf(nBuf-30, zBuf, "%s%s", sqlite3_temp_directory, + winEndsInDirSep(sqlite3_temp_directory) ? "" : + winGetDirDep()); } #if !SQLITE_OS_WINRT else if( osIsNT() ){ char *zMulti; - WCHAR zWidePath[SQLITE_WIN32_MAX_PATH_CHARS]; - if( osGetTempPathW(SQLITE_WIN32_MAX_PATH_CHARS-30, zWidePath)==0 ){ + LPWSTR zWidePath = sqlite3MallocZero( nBuf*sizeof(WCHAR) ); + if( !zWidePath ){ + sqlite3_free(zBuf); + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); + return SQLITE_IOERR_NOMEM; + } + if( osGetTempPathW(nBuf, zWidePath)==0 ){ + sqlite3_free(zWidePath); + sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_GETTEMPPATH\n")); return SQLITE_IOERR_GETTEMPPATH; } zMulti = winUnicodeToUtf8(zWidePath); if( zMulti ){ - sqlite3_snprintf(SQLITE_WIN32_MAX_PATH_BYTES-30, zTempPath, "%s", zMulti); + sqlite3_snprintf(nBuf-30, zBuf, "%s", zMulti); sqlite3_free(zMulti); + sqlite3_free(zWidePath); }else{ + sqlite3_free(zWidePath); + sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); return SQLITE_IOERR_NOMEM; } @@ -3972,55 +4010,43 @@ static int winGetTempname(int nBuf, char *zBuf){ #ifdef SQLITE_WIN32_HAS_ANSI else{ char *zUtf8; - char zMbcsPath[SQLITE_WIN32_MAX_PATH_BYTES]; - if( osGetTempPathA(SQLITE_WIN32_MAX_PATH_BYTES-30, zMbcsPath)==0 ){ + char *zMbcsPath = sqlite3MallocZero( nBuf ); + if( !zMbcsPath ){ + sqlite3_free(zBuf); + OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); + return SQLITE_IOERR_NOMEM; + } + if( osGetTempPathA(nBuf, zMbcsPath)==0 ){ + sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_GETTEMPPATH\n")); return SQLITE_IOERR_GETTEMPPATH; } zUtf8 = sqlite3_win32_mbcs_to_utf8(zMbcsPath); if( zUtf8 ){ - sqlite3_snprintf(SQLITE_WIN32_MAX_PATH_BYTES-30, zTempPath, "%s", zUtf8); + sqlite3_snprintf(nBuf-30, zBuf, "%s", zUtf8); sqlite3_free(zUtf8); }else{ + sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_IOERR_NOMEM\n")); return SQLITE_IOERR_NOMEM; } } -#else - else{ - /* - ** Compiled without ANSI support and the current operating system - ** is not Windows NT; therefore, just zero the temporary buffer. - */ - memset(zTempPath, 0, SQLITE_WIN32_MAX_PATH_BYTES+2); - } #endif /* SQLITE_WIN32_HAS_ANSI */ -#else - else{ - /* - ** Compiled for WinRT and the sqlite3_temp_directory is not set; - ** therefore, just zero the temporary buffer. - */ - memset(zTempPath, 0, SQLITE_WIN32_MAX_PATH_BYTES+2); - } #endif /* !SQLITE_OS_WINRT */ /* Check that the output buffer is large enough for the temporary file ** name. If it is not, return SQLITE_ERROR. */ - nTempPath = sqlite3Strlen30(zTempPath); + nLen = sqlite3Strlen30(zBuf); - if( (nTempPath + sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX) + 18) >= nBuf ){ + if( (nLen + sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX) + 18) >= nBuf ){ + sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); return SQLITE_ERROR; } - for(i=nTempPath; i>0 && winIsDirSep(zTempPath[i-1]); i--){} - zTempPath[i] = 0; + sqlite3_snprintf(nBuf-18-nLen, zBuf+nLen, SQLITE_TEMP_FILE_PREFIX); - sqlite3_snprintf(nBuf-18, zBuf, (nTempPath > 0) ? - "%s%s" SQLITE_TEMP_FILE_PREFIX : SQLITE_TEMP_FILE_PREFIX, - zTempPath, winGetDirDep()); j = sqlite3Strlen30(zBuf); sqlite3_randomness(15, &zBuf[j]); for(i=0; i<15; i++, j++){ @@ -4028,6 +4054,7 @@ static int winGetTempname(int nBuf, char *zBuf){ } zBuf[j] = 0; zBuf[j+1] = 0; + *pzBuf = zBuf; OSTRACE(("TEMP-FILENAME name=%s, rc=SQLITE_OK\n", zBuf)); return SQLITE_OK; @@ -4066,7 +4093,7 @@ static int winIsDir(const void *zConverted){ ** Open a file. */ static int winOpen( - sqlite3_vfs *pVfs, /* Not used */ + sqlite3_vfs *pVfs, /* Used to get maximum path name length */ const char *zName, /* Name of the file (UTF-8) */ sqlite3_file *id, /* Write the SQLite file handle here */ int flags, /* Open mode flags */ @@ -4089,7 +4116,7 @@ static int winOpen( /* If argument zPath is a NULL pointer, this function is required to open ** a temporary file. Use this buffer to store the file name in. */ - char zTmpname[SQLITE_WIN32_MAX_PATH_BYTES+2]; /* Buffer for temp filename */ + char *zTmpname = 0; /* For temporary filename, if necessary. */ int rc = SQLITE_OK; /* Function Return Code */ #if !defined(NDEBUG) || SQLITE_OS_WINCE @@ -4155,7 +4182,7 @@ static int winOpen( */ if( !zUtf8Name ){ assert( isDelete && !isOpenJournal ); - rc = winGetTempname(SQLITE_WIN32_MAX_PATH_BYTES+2, zTmpname); + rc = winGetTempname(pVfs, &zTmpname); if( rc!=SQLITE_OK ){ OSTRACE(("OPEN name=%s, rc=%s", zUtf8Name, sqlite3ErrName(rc))); return rc; @@ -4173,12 +4200,14 @@ static int winOpen( /* Convert the filename to the system encoding. */ zConverted = winConvertUtf8Filename(zUtf8Name); if( zConverted==0 ){ + sqlite3_free(zTmpname); OSTRACE(("OPEN name=%s, rc=SQLITE_IOERR_NOMEM", zUtf8Name)); return SQLITE_IOERR_NOMEM; } if( winIsDir(zConverted) ){ sqlite3_free(zConverted); + sqlite3_free(zTmpname); OSTRACE(("OPEN name=%s, rc=SQLITE_CANTOPEN_ISDIR", zUtf8Name)); return SQLITE_CANTOPEN_ISDIR; } @@ -4277,6 +4306,7 @@ static int winOpen( pFile->lastErrno = lastErrno; winLogError(SQLITE_CANTOPEN, pFile->lastErrno, "winOpen", zUtf8Name); sqlite3_free(zConverted); + sqlite3_free(zTmpname); if( isReadWrite && !isExclusive ){ return winOpen(pVfs, zName, id, ((flags|SQLITE_OPEN_READONLY) & @@ -4305,6 +4335,7 @@ static int winOpen( ){ osCloseHandle(h); sqlite3_free(zConverted); + sqlite3_free(zTmpname); OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); return rc; } @@ -4314,6 +4345,7 @@ static int winOpen( #endif { sqlite3_free(zConverted); + sqlite3_free(zTmpname); } pFile->pMethod = &winIoMethod; @@ -4587,7 +4619,6 @@ static int winFullPathname( #if defined(__CYGWIN__) SimulateIOError( return SQLITE_ERROR ); UNUSED_PARAMETER(nFull); - assert( pVfs->mxPathname>=SQLITE_WIN32_MAX_PATH_BYTES ); assert( nFull>=pVfs->mxPathname ); if ( sqlite3_data_directory && !winIsVerbatimPathname(zRelative) ){ /* @@ -4596,15 +4627,21 @@ static int winFullPathname( ** for converting the relative path name to an absolute ** one by prepending the data directory and a slash. */ - char zOut[SQLITE_WIN32_MAX_PATH_BYTES+1]; + char *zOut = sqlite3MallocZero( pVfs->mxPathname+1 ); + if( !zOut ){ + winLogError(SQLITE_IOERR_NOMEM, 0, "winFullPathname", zRelative); + return SQLITE_IOERR_NOMEM; + } if( cygwin_conv_path(CCP_POSIX_TO_WIN_A|CCP_RELATIVE, zRelative, zOut, - SQLITE_WIN32_MAX_PATH_BYTES+1)<0 ){ + pVfs->mxPathname+1)<0 ){ winLogError(SQLITE_CANTOPEN_FULLPATH, (DWORD)errno, "cygwin_conv_path", zRelative); + sqlite3_free(zOut); return SQLITE_CANTOPEN_FULLPATH; } sqlite3_snprintf(MIN(nFull, pVfs->mxPathname), zFull, "%s%s%s", sqlite3_data_directory, winGetDirDep(), zOut); + sqlite3_free(zOut); }else{ if( cygwin_conv_path(CCP_POSIX_TO_WIN_A, zRelative, zFull, nFull)<0 ){ winLogError(SQLITE_CANTOPEN_FULLPATH, (DWORD)errno, "cygwin_conv_path", @@ -4973,6 +5010,32 @@ int sqlite3_os_init(void){ winGetSystemCall, /* xGetSystemCall */ winNextSystemCall, /* xNextSystemCall */ }; +#if defined(SQLITE_WIN32_HAS_WIDE) + static sqlite3_vfs winLongPathVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-longpath", /* zName */ + 0, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#endif /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ @@ -4989,6 +5052,11 @@ int sqlite3_os_init(void){ assert( winSysInfo.dwPageSize>0 ); sqlite3_vfs_register(&winVfs, 1); + +#if defined(SQLITE_WIN32_HAS_WIDE) + sqlite3_vfs_register(&winLongPathVfs, 0); +#endif + return SQLITE_OK; } diff --git a/src/test1.c b/src/test1.c index 9c38b11a6d..6727d48899 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5934,6 +5934,145 @@ static int win32_file_lock( CloseHandle(ev); return TCL_OK; } + +/* +** exists_win32_path PATH +** +** Returns non-zero if the specified path exists, whose fully qualified name +** may exceed 248 characters if it is prefixed with "\\?\". +*/ +static int win32_exists_path( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "PATH"); + return TCL_ERROR; + } + Tcl_SetObjResult(interp, Tcl_NewBooleanObj( + GetFileAttributesW( Tcl_GetUnicode(objv[1]))!=INVALID_FILE_ATTRIBUTES )); + return TCL_OK; +} + +/* +** find_win32_file PATTERN +** +** Returns a list of entries in a directory that match the specified pattern, +** whose fully qualified name may exceed 248 characters if it is prefixed with +** "\\?\". +*/ +static int win32_find_file( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + HANDLE hFindFile = INVALID_HANDLE_VALUE; + WIN32_FIND_DATAW findData; + Tcl_Obj *listObj; + DWORD lastErrno; + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "PATTERN"); + return TCL_ERROR; + } + hFindFile = FindFirstFileW(Tcl_GetUnicode(objv[1]), &findData); + if( hFindFile==INVALID_HANDLE_VALUE ){ + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError())); + return TCL_ERROR; + } + listObj = Tcl_NewObj(); + Tcl_IncrRefCount(listObj); + do { + Tcl_ListObjAppendElement(interp, listObj, Tcl_NewUnicodeObj( + findData.cFileName, -1)); + Tcl_ListObjAppendElement(interp, listObj, Tcl_NewWideIntObj( + findData.dwFileAttributes)); + } while( FindNextFileW(hFindFile, &findData) ); + lastErrno = GetLastError(); + if( lastErrno!=NO_ERROR && lastErrno!=ERROR_NO_MORE_FILES ){ + FindClose(hFindFile); + Tcl_DecrRefCount(listObj); + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError())); + return TCL_ERROR; + } + FindClose(hFindFile); + Tcl_SetObjResult(interp, listObj); + return TCL_OK; +} + +/* +** delete_win32_file FILENAME +** +** Deletes the specified file, whose fully qualified name may exceed 248 +** characters if it is prefixed with "\\?\". +*/ +static int win32_delete_file( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "FILENAME"); + return TCL_ERROR; + } + if( !DeleteFileW(Tcl_GetUnicode(objv[1])) ){ + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError())); + return TCL_ERROR; + } + Tcl_ResetResult(interp); + return TCL_OK; +} + +/* +** make_win32_dir DIRECTORY +** +** Creates the specified directory, whose fully qualified name may exceed 248 +** characters if it is prefixed with "\\?\". +*/ +static int win32_mkdir( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DIRECTORY"); + return TCL_ERROR; + } + if( !CreateDirectoryW(Tcl_GetUnicode(objv[1]), NULL) ){ + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError())); + return TCL_ERROR; + } + Tcl_ResetResult(interp); + return TCL_OK; +} + +/* +** remove_win32_dir DIRECTORY +** +** Removes the specified directory, whose fully qualified name may exceed 248 +** characters if it is prefixed with "\\?\". +*/ +static int win32_rmdir( + void *clientData, + Tcl_Interp *interp, + int objc, + Tcl_Obj *CONST objv[] +){ + if( objc!=2 ){ + Tcl_WrongNumArgs(interp, 1, objv, "DIRECTORY"); + return TCL_ERROR; + } + if( !RemoveDirectoryW(Tcl_GetUnicode(objv[1])) ){ + Tcl_SetObjResult(interp, Tcl_NewWideIntObj(GetLastError())); + return TCL_ERROR; + } + Tcl_ResetResult(interp); + return TCL_OK; +} #endif @@ -6193,6 +6332,11 @@ int Sqlitetest1_Init(Tcl_Interp *interp){ { "optimization_control", optimization_control,0}, #if SQLITE_OS_WIN { "lock_win32_file", win32_file_lock, 0 }, + { "exists_win32_path", win32_exists_path, 0 }, + { "find_win32_file", win32_find_file, 0 }, + { "delete_win32_file", win32_delete_file, 0 }, + { "make_win32_dir", win32_mkdir, 0 }, + { "remove_win32_dir", win32_rmdir, 0 }, #endif { "tcl_objproc", runAsObjProc, 0 }, diff --git a/test/win32longpath.test b/test/win32longpath.test new file mode 100644 index 0000000000..bd6eb709f9 --- /dev/null +++ b/test/win32longpath.test @@ -0,0 +1,160 @@ +# 2013 August 27 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. The +# focus of this script is testing the file name handling provided +# by the "win32-longpath" VFS. +# + +if {$tcl_platform(platform)!="windows"} return + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +set testprefix win32longpath + +proc do_remove_win32_dir {args} { + set nRetry [getFileRetries] ;# Maximum number of retries. + set nDelay [getFileRetryDelay] ;# Delay in ms before retrying. + + foreach dirName $args { + # On windows, sometimes even a [remove_win32_dir] can fail just after + # a directory is emptied. The cause is usually "tag-alongs" - programs + # like anti-virus software, automatic backup tools and various explorer + # extensions that keep a file open a little longer than we expect, + # causing the delete to fail. + # + # The solution is to wait a short amount of time before retrying the + # removal. + # + if {$nRetry > 0} { + for {set i 0} {$i < $nRetry} {incr i} { + set rc [catch { + remove_win32_dir $dirName + } msg] + if {$rc == 0} break + if {$nDelay > 0} { after $nDelay } + } + if {$rc} { error $msg } + } else { + remove_win32_dir $dirName + } + } +} + +proc do_delete_win32_file {args} { + set nRetry [getFileRetries] ;# Maximum number of retries. + set nDelay [getFileRetryDelay] ;# Delay in ms before retrying. + + foreach fileName $args { + # On windows, sometimes even a [delete_win32_file] can fail just after + # a file is closed. The cause is usually "tag-alongs" - programs like + # anti-virus software, automatic backup tools and various explorer + # extensions that keep a file open a little longer than we expect, + # causing the delete to fail. + # + # The solution is to wait a short amount of time before retrying the + # delete. + # + if {$nRetry > 0} { + for {set i 0} {$i < $nRetry} {incr i} { + set rc [catch { + delete_win32_file $fileName + } msg] + if {$rc == 0} break + if {$nDelay > 0} { after $nDelay } + } + if {$rc} { error $msg } + } else { + delete_win32_file $fileName + } + } +} + +db close +set path [file nativename [get_pwd]] +sqlite3 db [file join $path test.db] -vfs win32-longpath + +do_test 1.1 { + db eval { + BEGIN EXCLUSIVE; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(1); + INSERT INTO t1 VALUES(2); + INSERT INTO t1 VALUES(3); + INSERT INTO t1 VALUES(4); + SELECT x FROM t1 ORDER BY x; + COMMIT; + } +} {1 2 3 4} + +set longPath(1) \\\\?\\$path\\[pid] +make_win32_dir $longPath(1) + +set longPath(2) $longPath(1)\\[string repeat X 255] +make_win32_dir $longPath(2) + +set longPath(3) $longPath(2)\\[string repeat Y 255] +make_win32_dir $longPath(3) + +set fileName $longPath(3)\\test.db + +do_test 1.2 { + list [catch {sqlite3 db2 [string range $fileName 4 end]} msg] $msg +} {1 {unable to open database file}} + +sqlite3 db3 $fileName -vfs win32-longpath + +do_test 1.3 { + db3 eval { + PRAGMA journal_mode = WAL; + } +} {wal} + +do_test 1.4 { + db3 eval { + BEGIN EXCLUSIVE; + CREATE TABLE t1(x); + INSERT INTO t1 VALUES(5); + INSERT INTO t1 VALUES(6); + INSERT INTO t1 VALUES(7); + INSERT INTO t1 VALUES(8); + SELECT x FROM t1 ORDER BY x; + COMMIT; + } +} {5 6 7 8} + +db3 close +# puts " Database exists \{[exists_win32_path $fileName]\}" + +sqlite3 db3 $fileName -vfs win32-longpath + +do_test 1.5 { + db3 eval { + BEGIN EXCLUSIVE; + INSERT INTO t1 VALUES(9); + INSERT INTO t1 VALUES(10); + INSERT INTO t1 VALUES(11); + INSERT INTO t1 VALUES(12); + SELECT x FROM t1 ORDER BY x; + COMMIT; + } +} {5 6 7 8 9 10 11 12} + +db3 close +# puts " Database exists \{[exists_win32_path $fileName]\}" + +do_delete_win32_file $fileName +# puts " Files remaining \{[find_win32_file $longPath(3)\\*]\}" + +do_remove_win32_dir $longPath(3) +do_remove_win32_dir $longPath(2) +do_remove_win32_dir $longPath(1) + +finish_test From 70e9ffa3bb7f75eb175b44316a755d16547a40f4 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 28 Aug 2013 07:42:43 +0000 Subject: [PATCH 55/67] In the win32longpath test, move the journal mode change down where it was intended to be. FossilOrigin-Name: 5cead293bbbcddd7d0976db41133e40704efcc99 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/win32longpath.test | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/manifest b/manifest index c81c0d5c3e..4529c41c94 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Support\sdatabase\sfile\snames\slonger\sthan\s260\scharacters\susing\sthe\snew\s'win32-longpath'\sVFS\svariant. -D 2013-08-28T05:49:39.789 +C In\sthe\swin32longpath\stest,\smove\sthe\sjournal\smode\schange\sdown\swhere\sit\swas\sintended\sto\sbe. +D 2013-08-28T07:42:43.637 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1057,7 +1057,7 @@ F test/whereF.test 136a7301512d72a08a272806c8767066311b7bc1 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 -F test/win32longpath.test 3018b1199270cb91194fc8cd8413912b6950577e +F test/win32longpath.test f888106783fc26515f393c8848c94cd6166addbb F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test 209d7ed441f44cc5299e4ebffbef06fd5aabfefd F tool/build-all-msvc.bat c55f64ca200308fb5fa5c1ee751ea95a13977b5a x @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P c3b82c5bf97cfb35544c5b1fbcdf7b9c4827d1cf -R e20bd90758596867c642a3a14073401d +P 37e85e444cde18f061049437980b965d4485f43c +R 56dde498dcecb04c21e3755845c4691c U mistachkin -Z e8ceaa194e29622b0b6b8c77bb0db1ea +Z 6c6b47fa5ba3b59f97ecaf4251b21c76 diff --git a/manifest.uuid b/manifest.uuid index 6cdcbf3836..102f430b4e 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -37e85e444cde18f061049437980b965d4485f43c \ No newline at end of file +5cead293bbbcddd7d0976db41133e40704efcc99 \ No newline at end of file diff --git a/test/win32longpath.test b/test/win32longpath.test index bd6eb709f9..45f8825858 100644 --- a/test/win32longpath.test +++ b/test/win32longpath.test @@ -112,12 +112,6 @@ do_test 1.2 { sqlite3 db3 $fileName -vfs win32-longpath do_test 1.3 { - db3 eval { - PRAGMA journal_mode = WAL; - } -} {wal} - -do_test 1.4 { db3 eval { BEGIN EXCLUSIVE; CREATE TABLE t1(x); @@ -135,6 +129,12 @@ db3 close sqlite3 db3 $fileName -vfs win32-longpath +do_test 1.4 { + db3 eval { + PRAGMA journal_mode = WAL; + } +} {wal} + do_test 1.5 { db3 eval { BEGIN EXCLUSIVE; From 6d57bbfe255584da2387558c6a1b2366997c63b6 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Aug 2013 11:43:49 +0000 Subject: [PATCH 56/67] Label a certain branch as NEVER() only for non-STAT3/4 builds. FossilOrigin-Name: b5ccf6e995ac2fe010f49d46d40b487ddbd28e5a --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/analyze.c | 4 ++++ 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index 8af672c496..aacd2e3508 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Adjust\sANALYZE\sfor\simproved\stest\scoverage.\s\sUse\sthe\sSQLITE_ENABLE_STAT3_OR_STAT4\nmacro\s(created\sin\ssqliteInt.h)\sto\sconditionally\sinclude\scode,\sinstead\sof\s\na\sboolean\sspecifying\sboth\smacros\sseparately. -D 2013-08-27T23:15:44.027 +C Label\sa\scertain\sbranch\sas\sNEVER()\sonly\sfor\snon-STAT3/4\sbuilds. +D 2013-08-28T11:43:49.105 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -157,7 +157,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc F sqlite3.pc.in 48fed132e7cb71ab676105d2a4dc77127d8c1f3a F src/alter.c 2af0330bb1b601af7a7789bf7229675fd772a083 -F src/analyze.c fa1a9eeeebdeceddf0b0ed141656dea6d4458699 +F src/analyze.c 7f869a251f3d58d3b95883eeae425d4ee17a66bd F src/attach.c fea00cab11c854646a27641a263f5876569a51f9 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34 F src/backup.c 2f1987981139bd2f6d8c728d64bf09fb387443c3 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9d1424c91a21ed740aca53e437b8f7c1f0823c03 -R 89270b1d529ed825a8ce16c2a3d7caa2 +P 67a9a392edd62ef5a5a1ad3010b20a39b285793d +R b7a40741ef64fba945b7b5951e4f4f5c U drh -Z 2d085e9043e9441bdd89e3f37cf9f342 +Z 13881c9df2b2879174eba7e7aa354c7f diff --git a/manifest.uuid b/manifest.uuid index fdb9809352..36eaa85253 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -67a9a392edd62ef5a5a1ad3010b20a39b285793d \ No newline at end of file +b5ccf6e995ac2fe010f49d46d40b487ddbd28e5a \ No newline at end of file diff --git a/src/analyze.c b/src/analyze.c index dae0f3aa0f..46ebac4c3a 100644 --- a/src/analyze.c +++ b/src/analyze.c @@ -1231,7 +1231,11 @@ static void decodeIntArray( assert( pbUnordered==0 || *pbUnordered==0 ); +#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( z==0 ) z = ""; +#else if( NEVER(z==0) ) z = ""; +#endif for(i=0; *z && i='0' && c<='9' ){ From f7f8de5fb20985dcd50d37e31673541a4971df35 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Aug 2013 11:57:52 +0000 Subject: [PATCH 57/67] Add a test case for empty quoted field in CSV import in the shell. FossilOrigin-Name: 3df56e849164811117f9ac8ee2aee1eee188bb96 --- manifest | 12 ++++++------ manifest.uuid | 2 +- test/shell5.test | 26 ++++++++++++++++++++++++++ 3 files changed, 33 insertions(+), 7 deletions(-) diff --git a/manifest b/manifest index aacd2e3508..8f40323e8e 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Label\sa\scertain\sbranch\sas\sNEVER()\sonly\sfor\snon-STAT3/4\sbuilds. -D 2013-08-28T11:43:49.105 +C Add\sa\stest\scase\sfor\sempty\squoted\sfield\sin\sCSV\simport\sin\sthe\sshell. +D 2013-08-28T11:57:52.201 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -784,7 +784,7 @@ F test/shell1.test 928547277d385038c696428e9d791cbbad098974 F test/shell2.test 037d6ad16e873354195d30bb2dc4b5321788154a F test/shell3.test 9196c42772d575685e722c92b4b39053c6ebba59 F test/shell4.test aa4eef8118b412d1a01477a53426ece169ea86a9 -F test/shell5.test 946e620a41b64f90d45dcf91c36e8d408d435cfd +F test/shell5.test 174a2a3df73ae5caf2cf74d5232fd869a841ecec F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 67a9a392edd62ef5a5a1ad3010b20a39b285793d -R b7a40741ef64fba945b7b5951e4f4f5c +P b5ccf6e995ac2fe010f49d46d40b487ddbd28e5a +R df6774582c305a98ada1f84d02d89329 U drh -Z 13881c9df2b2879174eba7e7aa354c7f +Z e58789328f00ee55080b8aec43c720b4 diff --git a/manifest.uuid b/manifest.uuid index 36eaa85253..bfb4c5f384 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5ccf6e995ac2fe010f49d46d40b487ddbd28e5a \ No newline at end of file +3df56e849164811117f9ac8ee2aee1eee188bb96 \ No newline at end of file diff --git a/test/shell5.test b/test/shell5.test index 6256b281bb..d881b96d14 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -242,4 +242,30 @@ SELECT * FROM t1;} 5,"this is 5"}} } +# Import columns containing quoted strings +do_test shell5-1.9 { + set out [open shell5.csv w] + puts $out {1,"",11} + puts $out {2,"x",22} + puts $out {3,"""",33} + puts $out {4,"hello",44} + puts $out {5,55,""} + puts $out {6,66,"x"} + puts $out {7,77,""""} + puts $out {8,88,"hello"} + puts $out {"",9,99} + puts $out {"x",10,110} + puts $out {"""",11,121} + puts $out {"hello",12,132} + close $out + file delete -force test.db + catchcmd test.db {.mode csv + CREATE TABLE t1(a,b,c); +.import shell5.csv t1 + } + sqlite3 db test.db + db eval {SELECT *, '|' FROM t1 ORDER BY rowid} +} {1 {} 11 | 2 x 22 | 3 {"} 33 | 4 hello 44 | 5 55 {} | 6 66 x | 7 77 {"} | 8 88 hello | {} 9 99 | x 10 110 | {"} 11 121 | hello 12 132 |} +db close + finish_test From 868ccf26797fa5ce73a4145b66696577d538bbdf Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Aug 2013 13:33:40 +0000 Subject: [PATCH 58/67] Fix an off-by-one error that causes a quoted empty string at the end of a CRNL-terminated line of CSV input to be misread by the shell. FossilOrigin-Name: b5617e4fdadc4cded93c985008e90982dd48bc3b --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/shell.c | 2 +- test/shell5.test | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index 8f40323e8e..8a6b4333e9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sa\stest\scase\sfor\sempty\squoted\sfield\sin\sCSV\simport\sin\sthe\sshell. -D 2013-08-28T11:57:52.201 +C Fix\san\soff-by-one\serror\sthat\scauses\sa\squoted\sempty\sstring\sat\sthe\send\sof\na\sCRNL-terminated\sline\sof\sCSV\sinput\sto\sbe\smisread\sby\sthe\sshell. +D 2013-08-28T13:33:40.242 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -217,7 +217,7 @@ F src/random.c 0b2dbc37fdfbfa6bd455b091dfcef5bdb32dba68 F src/resolve.c 9d53899cc6e1f4ec0b4632d07e97d57827bf63b9 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0 F src/select.c 8b148eb851f384412aea57091659d14b369918ca -F src/shell.c 00a23311466829d9b77f0be4f7cedee9328279db +F src/shell.c f726500a48dbc64d42e82b439ca7690a1e1a46cf F src/sqlite.h.in bd1451ba1ab681022a53bccc3c39580ba094a3ff F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc @@ -784,7 +784,7 @@ F test/shell1.test 928547277d385038c696428e9d791cbbad098974 F test/shell2.test 037d6ad16e873354195d30bb2dc4b5321788154a F test/shell3.test 9196c42772d575685e722c92b4b39053c6ebba59 F test/shell4.test aa4eef8118b412d1a01477a53426ece169ea86a9 -F test/shell5.test 174a2a3df73ae5caf2cf74d5232fd869a841ecec +F test/shell5.test bfa21ecc173adcbc15db2c075baa468778f67f88 F test/shortread1.test bb591ef20f0fd9ed26d0d12e80eee6d7ac8897a3 F test/shrink.test 8c70f62b6e8eb4d54533de6d65bd06b1b9a17868 F test/sidedelete.test f0ad71abe6233e3b153100f3b8d679b19a488329 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b5ccf6e995ac2fe010f49d46d40b487ddbd28e5a -R df6774582c305a98ada1f84d02d89329 +P 3df56e849164811117f9ac8ee2aee1eee188bb96 +R eb7817bc730740034eb3fac54ffa268e U drh -Z e58789328f00ee55080b8aec43c720b4 +Z 72d0cf2bcd39280bc82cec788bca9ff5 diff --git a/manifest.uuid b/manifest.uuid index bfb4c5f384..89c381c25a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -3df56e849164811117f9ac8ee2aee1eee188bb96 \ No newline at end of file +b5617e4fdadc4cded93c985008e90982dd48bc3b \ No newline at end of file diff --git a/src/shell.c b/src/shell.c index f8ee08a803..455fa5ca22 100644 --- a/src/shell.c +++ b/src/shell.c @@ -1717,7 +1717,7 @@ static char *csv_read_one_field(CSVReader *p){ } if( (c==cSep && pc==cQuote) || (c=='\n' && pc==cQuote) - || (c=='\n' && pc=='\r' && p->n>2 && p->z[p->n-2]==cQuote) + || (c=='\n' && pc=='\r' && p->n>=2 && p->z[p->n-2]==cQuote) || (c==EOF && pc==cQuote) ){ do{ p->n--; }while( p->z[p->n]!=cQuote ); diff --git a/test/shell5.test b/test/shell5.test index d881b96d14..bee4563372 100644 --- a/test/shell5.test +++ b/test/shell5.test @@ -249,7 +249,7 @@ do_test shell5-1.9 { puts $out {2,"x",22} puts $out {3,"""",33} puts $out {4,"hello",44} - puts $out {5,55,""} + puts $out "5,55,\"\"\r" puts $out {6,66,"x"} puts $out {7,77,""""} puts $out {8,88,"hello"} From 38e587779d53733244bc4df81e03afd3da3b7b32 Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Aug 2013 13:46:09 +0000 Subject: [PATCH 59/67] Increase the version number to 3.8.1 due to the addition of STAT4 support. FossilOrigin-Name: 41c089e2a21f52b3d6d850eccba60be17010c9fe --- VERSION | 2 +- configure | 18 +++++++++--------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/VERSION b/VERSION index 19811903a7..f280719674 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.8.0 +3.8.1 diff --git a/configure b/configure index a7281f089c..046cbcd5ec 100755 --- a/configure +++ b/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.62 for sqlite 3.8.0. +# Generated by GNU Autoconf 2.62 for sqlite 3.8.1. # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, # 2002, 2003, 2004, 2005, 2006, 2007, 2008 Free Software Foundation, Inc. @@ -743,8 +743,8 @@ SHELL=${CONFIG_SHELL-/bin/sh} # Identity of this package. PACKAGE_NAME='sqlite' PACKAGE_TARNAME='sqlite' -PACKAGE_VERSION='3.8.0' -PACKAGE_STRING='sqlite 3.8.0' +PACKAGE_VERSION='3.8.1' +PACKAGE_STRING='sqlite 3.8.1' PACKAGE_BUGREPORT='' # Factoring default headers for most tests. @@ -1484,7 +1484,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures sqlite 3.8.0 to adapt to many kinds of systems. +\`configure' configures sqlite 3.8.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1549,7 +1549,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of sqlite 3.8.0:";; + short | recursive ) echo "Configuration of sqlite 3.8.1:";; esac cat <<\_ACEOF @@ -1665,7 +1665,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -sqlite configure 3.8.0 +sqlite configure 3.8.1 generated by GNU Autoconf 2.62 Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, @@ -1679,7 +1679,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by sqlite $as_me 3.8.0, which was +It was created by sqlite $as_me 3.8.1, which was generated by GNU Autoconf 2.62. Invocation command line was $ $0 $@ @@ -14032,7 +14032,7 @@ exec 6>&1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by sqlite $as_me 3.8.0, which was +This file was extended by sqlite $as_me 3.8.1, which was generated by GNU Autoconf 2.62. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -14085,7 +14085,7 @@ Report bugs to ." _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_version="\\ -sqlite config.status 3.8.0 +sqlite config.status 3.8.1 configured by $0, generated by GNU Autoconf 2.62, with options \\"`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`\\" diff --git a/manifest b/manifest index 8a6b4333e9..a1f9c24ac8 100644 --- a/manifest +++ b/manifest @@ -1,12 +1,12 @@ -C Fix\san\soff-by-one\serror\sthat\scauses\sa\squoted\sempty\sstring\sat\sthe\send\sof\na\sCRNL-terminated\sline\sof\sCSV\sinput\sto\sbe\smisread\sby\sthe\sshell. -D 2013-08-28T13:33:40.242 +C Increase\sthe\sversion\snumber\sto\s3.8.1\sdue\sto\sthe\saddition\sof\sSTAT4\ssupport. +D 2013-08-28T13:46:09.756 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 F Makefile.msc e9f41f89111627baaabd95cab4988b8d1c3e47c9 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 -F VERSION f135b651727f978b7191bd6fa12c7fc1e13e13ac +F VERSION a8d1f6839521130dc73c5408cdd24bcfd791df34 F aclocal.m4 a5c22d164aff7ed549d53a90fa56d56955281f50 F addopcodes.awk 17dc593f791f874d2c23a0f9360850ded0286531 F art/sqlite370.eps aa97a671332b432a54e1d74ff5e8775be34200c2 @@ -38,7 +38,7 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 27e9279a219b652bd1c53439353a5e5be70214b2 x +F configure 911d9281dfc2c862d0af9089686ec57ea9414c7b x F configure.ac 81c43d151d0b0e406be056394cc9ff4cb3fd0444 F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 3df56e849164811117f9ac8ee2aee1eee188bb96 -R eb7817bc730740034eb3fac54ffa268e +P b5617e4fdadc4cded93c985008e90982dd48bc3b +R 58baec9db318ef05dcefcc1c1d1fdd0e U drh -Z 72d0cf2bcd39280bc82cec788bca9ff5 +Z 467137fcf3621d347c2f68aa3941d468 diff --git a/manifest.uuid b/manifest.uuid index 89c381c25a..7443da8734 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -b5617e4fdadc4cded93c985008e90982dd48bc3b \ No newline at end of file +41c089e2a21f52b3d6d850eccba60be17010c9fe \ No newline at end of file From d41d39f24d1db854e1249abf9ac6e3a2fbb6575a Mon Sep 17 00:00:00 2001 From: drh Date: Wed, 28 Aug 2013 16:27:01 +0000 Subject: [PATCH 60/67] In the query optimizer, when converting BETWEEN and LIKE/GLOB expressions into simpler forms for processing, be sure to transfer the LEFT JOIN markings. Fix for ticket [bc878246eafe0f52c]. FossilOrigin-Name: caab361ebee5f5c3fdafd9b1abe3d1ab7c5b4db9 --- manifest | 14 +++++++------- manifest.uuid | 2 +- src/where.c | 9 +++++++-- test/where9.test | 24 ++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/manifest b/manifest index a1f9c24ac8..8c58e2adb7 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Increase\sthe\sversion\snumber\sto\s3.8.1\sdue\sto\sthe\saddition\sof\sSTAT4\ssupport. -D 2013-08-28T13:46:09.756 +C In\sthe\squery\soptimizer,\swhen\sconverting\sBETWEEN\sand\sLIKE/GLOB\sexpressions\ninto\ssimpler\sforms\sfor\sprocessing,\sbe\ssure\sto\stransfer\sthe\sLEFT\sJOIN\smarkings.\nFix\sfor\sticket\s[bc878246eafe0f52c]. +D 2013-08-28T16:27:01.576 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vtab.c 165ce0e797c2cd23badb104c9f2ae9042d6d942c F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c 289bfa239207945e27d4689a22e838aea4afcc49 +F src/where.c 6a82cd9edf3f53336489062531bff4c9f08d6d15 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1049,7 +1049,7 @@ F test/where6.test 5da5a98cec820d488e82708301b96cb8c18a258b F test/where7.test 5a4b0abc207d71da4deecd734ad8579e8dd40aa8 F test/where8.test 6f95896633cf2d307b5263145b942b7d33e837c6 F test/where8m.test da346596e19d54f0aba35ebade032a7c47d79739 -F test/where9.test 74245dea86592a744b758dff2e7daf0a07bade7d +F test/where9.test 4f3eab951353a3ae164befc521c777dfa903e46c F test/whereA.test 4d253178d135ec46d1671e440cd8f2b916aa6e6b F test/whereB.test 0def95db3bdec220a731c7e4bec5930327c1d8c5 F test/whereC.test d6f4ecd4fa2d9429681a5b22a25d2bda8e86ab8a @@ -1107,7 +1107,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P b5617e4fdadc4cded93c985008e90982dd48bc3b -R 58baec9db318ef05dcefcc1c1d1fdd0e +P 41c089e2a21f52b3d6d850eccba60be17010c9fe +R fd635dde5a0061c3bcc42dceb0b7a7e4 U drh -Z 467137fcf3621d347c2f68aa3941d468 +Z f445406f63378451eee3ece49de14aa7 diff --git a/manifest.uuid b/manifest.uuid index 7443da8734..5370783404 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -41c089e2a21f52b3d6d850eccba60be17010c9fe \ No newline at end of file +caab361ebee5f5c3fdafd9b1abe3d1ab7c5b4db9 \ No newline at end of file diff --git a/src/where.c b/src/where.c index dc944612a3..38cf714386 100644 --- a/src/where.c +++ b/src/where.c @@ -1209,8 +1209,10 @@ static int isMatchOfColumn( ** a join, then transfer the appropriate markings over to derived. */ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ - pDerived->flags |= pBase->flags & EP_FromJoin; - pDerived->iRightJoinTable = pBase->iRightJoinTable; + if( pDerived ){ + pDerived->flags |= pBase->flags & EP_FromJoin; + pDerived->iRightJoinTable = pBase->iRightJoinTable; + } } #if !defined(SQLITE_OMIT_OR_OPTIMIZATION) && !defined(SQLITE_OMIT_SUBQUERY) @@ -1667,6 +1669,7 @@ static void exprAnalyze( pNewExpr = sqlite3PExpr(pParse, ops[i], sqlite3ExprDup(db, pExpr->pLeft, 0), sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0); + transferJoinMarkings(pNewExpr, pExpr); idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); @@ -1734,6 +1737,7 @@ static void exprAnalyze( pNewExpr1 = sqlite3PExpr(pParse, TK_GE, sqlite3ExprAddCollateToken(pParse,pNewExpr1,&sCollSeqName), pStr1, 0); + transferJoinMarkings(pNewExpr1, pExpr); idxNew1 = whereClauseInsert(pWC, pNewExpr1, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew1==0 ); exprAnalyze(pSrc, pWC, idxNew1); @@ -1741,6 +1745,7 @@ static void exprAnalyze( pNewExpr2 = sqlite3PExpr(pParse, TK_LT, sqlite3ExprAddCollateToken(pParse,pNewExpr2,&sCollSeqName), pStr2, 0); + transferJoinMarkings(pNewExpr2, pExpr); idxNew2 = whereClauseInsert(pWC, pNewExpr2, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew2==0 ); exprAnalyze(pSrc, pWC, idxNew2); diff --git a/test/where9.test b/test/where9.test index f3f67dbdce..1c32ff82f7 100644 --- a/test/where9.test +++ b/test/where9.test @@ -956,4 +956,28 @@ do_test where9-9.1 { } } {1 2 3 4 8 9} +# Fix for ticket [bc878246eafe0f52c519e29049b2fe4a99491b27] +# Incorrect result when OR is used in a join to the right of a LEFT JOIN +# +do_test where9-10.1 { + db eval { + CREATE TABLE t101 (id INTEGER PRIMARY KEY); + INSERT INTO t101 VALUES (1); + SELECT * FROM t101 AS t0 + LEFT JOIN t101 AS t1 ON t1.id BETWEEN 10 AND 20 + JOIN t101 AS t2 ON (t2.id = t0.id OR (t2.id<>555 AND t2.id=t1.id)); + } +} {1 {} 1} +do_test where9-10.2 { + db eval { + CREATE TABLE t102 (id TEXT UNIQUE NOT NULL); + INSERT INTO t102 VALUES ('1'); + SELECT * FROM t102 AS t0 + LEFT JOIN t102 AS t1 ON t1.id GLOB 'abc%' + JOIN t102 AS t2 ON (t2.id = t0.id OR (t2.id<>555 AND t2.id=t1.id)); + } +} {1 {} 1} + + + finish_test From 3259fe7910756b49592f07d9dc6de3d588e975e6 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 28 Aug 2013 17:59:38 +0000 Subject: [PATCH 61/67] Fix a couple comments. FossilOrigin-Name: 945cb89ecb92a6c2075b3e58ce1ba3d419b1b6ca --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/test1.c | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manifest b/manifest index 4529c41c94..f7f5eccc92 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C In\sthe\swin32longpath\stest,\smove\sthe\sjournal\smode\schange\sdown\swhere\sit\swas\sintended\sto\sbe. -D 2013-08-28T07:42:43.637 +C Fix\sa\scouple\scomments. +D 2013-08-28T17:59:38.115 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -226,7 +226,7 @@ F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e F src/tclsqlite.c 659dad8ef30b54831306a244b43e37af4725a444 -F src/test1.c 031d00dbf1b48378e6d47af1a0f728c767320b96 +F src/test1.c 26226cfd2b6dc3f77d2eb27f07ffcf236b4e728b F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c F src/test4.c 9b32d22f5f150abe23c1830e2057c4037c45b3df @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 37e85e444cde18f061049437980b965d4485f43c -R 56dde498dcecb04c21e3755845c4691c +P 5cead293bbbcddd7d0976db41133e40704efcc99 +R 27548a8da20b2d6d66d852b1f7e7eb3a U mistachkin -Z 6c6b47fa5ba3b59f97ecaf4251b21c76 +Z 711ba7d958e91e7fbe6d2b1f344832a5 diff --git a/manifest.uuid b/manifest.uuid index 102f430b4e..cd702a8c68 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -5cead293bbbcddd7d0976db41133e40704efcc99 \ No newline at end of file +945cb89ecb92a6c2075b3e58ce1ba3d419b1b6ca \ No newline at end of file diff --git a/src/test1.c b/src/test1.c index 6727d48899..b99efa7106 100644 --- a/src/test1.c +++ b/src/test1.c @@ -5939,7 +5939,7 @@ static int win32_file_lock( ** exists_win32_path PATH ** ** Returns non-zero if the specified path exists, whose fully qualified name -** may exceed 248 characters if it is prefixed with "\\?\". +** may exceed 260 characters if it is prefixed with "\\?\". */ static int win32_exists_path( void *clientData, @@ -6005,7 +6005,7 @@ static int win32_find_file( /* ** delete_win32_file FILENAME ** -** Deletes the specified file, whose fully qualified name may exceed 248 +** Deletes the specified file, whose fully qualified name may exceed 260 ** characters if it is prefixed with "\\?\". */ static int win32_delete_file( From fe27b08934a06ffd87044d6342ba9603125ecc68 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Wed, 28 Aug 2013 18:06:26 +0000 Subject: [PATCH 62/67] Update list of supported Tcl shells to include 8.6. Remove stray OS2 references. FossilOrigin-Name: ecd52ccf650ce2a4ab93c7c482289c1bc4768eeb --- configure | 18 ++++-------------- configure.ac | 18 ++++-------------- manifest | 14 +++++++------- manifest.uuid | 2 +- 4 files changed, 16 insertions(+), 36 deletions(-) diff --git a/configure b/configure index a7281f089c..ca2188ee9d 100755 --- a/configure +++ b/configure @@ -874,7 +874,6 @@ TEMP_STORE BUILD_EXEEXT SQLITE_OS_UNIX SQLITE_OS_WIN -SQLITE_OS_OS2 TARGET_EXEEXT TCL_VERSION TCL_BIN_DIR @@ -12257,7 +12256,7 @@ USE_AMALGAMATION=1 # if not, then we fall back to plain tclsh. # TODO: try other versions before falling back? # -for ac_prog in tclsh8.5 tclsh +for ac_prog in tclsh8.6 tclsh8.5 tclsh do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 @@ -12713,21 +12712,12 @@ else TARGET_EXEEXT=$config_TARGET_EXEEXT fi if test "$TARGET_EXEEXT" = ".exe"; then - if test $OS2_SHELL ; then - SQLITE_OS_UNIX=0 - SQLITE_OS_WIN=0 - SQLITE_OS_OS2=1 - CFLAGS="$CFLAGS -DSQLITE_OS_OS2=1" - else - SQLITE_OS_UNIX=0 - SQLITE_OS_WIN=1 - SQLITE_OS_OS2=0 - CFLAGS="$CFLAGS -DSQLITE_OS_WIN=1" - fi + SQLITE_OS_UNIX=0 + SQLITE_OS_WIN=1 + CFLAGS="$CFLAGS -DSQLITE_OS_WIN=1" else SQLITE_OS_UNIX=1 SQLITE_OS_WIN=0 - SQLITE_OS_OS2=0 CFLAGS="$CFLAGS -DSQLITE_OS_UNIX=1" fi diff --git a/configure.ac b/configure.ac index 4a7043ede1..2e70f2c235 100644 --- a/configure.ac +++ b/configure.ac @@ -139,7 +139,7 @@ USE_AMALGAMATION=1 # if not, then we fall back to plain tclsh. # TODO: try other versions before falling back? # -AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.5 tclsh], none) +AC_CHECK_PROGS(TCLSH_CMD, [tclsh8.6 tclsh8.5 tclsh], none) if test "$TCLSH_CMD" = "none"; then # If we can't find a local tclsh, then building the amalgamation will fail. # We act as though --disable-amalgamation has been used. @@ -340,28 +340,18 @@ else TARGET_EXEEXT=$config_TARGET_EXEEXT fi if test "$TARGET_EXEEXT" = ".exe"; then - if test $OS2_SHELL ; then - SQLITE_OS_UNIX=0 - SQLITE_OS_WIN=0 - SQLITE_OS_OS2=1 - CFLAGS="$CFLAGS -DSQLITE_OS_OS2=1" - else - SQLITE_OS_UNIX=0 - SQLITE_OS_WIN=1 - SQLITE_OS_OS2=0 - CFLAGS="$CFLAGS -DSQLITE_OS_WIN=1" - fi + SQLITE_OS_UNIX=0 + SQLITE_OS_WIN=1 + CFLAGS="$CFLAGS -DSQLITE_OS_WIN=1" else SQLITE_OS_UNIX=1 SQLITE_OS_WIN=0 - SQLITE_OS_OS2=0 CFLAGS="$CFLAGS -DSQLITE_OS_UNIX=1" fi AC_SUBST(BUILD_EXEEXT) AC_SUBST(SQLITE_OS_UNIX) AC_SUBST(SQLITE_OS_WIN) -AC_SUBST(SQLITE_OS_OS2) AC_SUBST(TARGET_EXEEXT) ########## diff --git a/manifest b/manifest index f7f5eccc92..74f2145549 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Fix\sa\scouple\scomments. -D 2013-08-28T17:59:38.115 +C Update\slist\sof\ssupported\sTcl\sshells\sto\sinclude\s8.6.\s\sRemove\sstray\sOS2\sreferences. +D 2013-08-28T18:06:26.411 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -38,8 +38,8 @@ F autoconf/tea/win/rules.vc c511f222b80064096b705dbeb97060ee1d6b6d63 F config.guess 226d9a188c6196f3033ffc651cbc9dcee1a42977 F config.h.in 0921066a13130082764ab4ab6456f7b5bebe56de F config.sub 9ebe4c3b3dab6431ece34f16828b594fb420da55 -F configure 27e9279a219b652bd1c53439353a5e5be70214b2 x -F configure.ac 81c43d151d0b0e406be056394cc9ff4cb3fd0444 +F configure af1a47c2c0d019462888cc6e55b387c2ca9b14f8 x +F configure.ac 4cf9f60785143fa141b10962ccc885d973792e9a F contrib/sqlitecon.tcl 210a913ad63f9f991070821e599d600bd913e0ad F doc/lemon.html 334dbf6621b8fb8790297ec1abf3cfa4621709d1 F doc/pager-invariants.txt 27fed9a70ddad2088750c4a2b493b63853da2710 @@ -1106,7 +1106,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 5cead293bbbcddd7d0976db41133e40704efcc99 -R 27548a8da20b2d6d66d852b1f7e7eb3a +P 945cb89ecb92a6c2075b3e58ce1ba3d419b1b6ca +R 8c92bd5c8e47ded17fee33a36ad92fa0 U mistachkin -Z 711ba7d958e91e7fbe6d2b1f344832a5 +Z e27f33536507066f2096aabdddbccd7c diff --git a/manifest.uuid b/manifest.uuid index cd702a8c68..fae0a2700a 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -945cb89ecb92a6c2075b3e58ce1ba3d419b1b6ca \ No newline at end of file +ecd52ccf650ce2a4ab93c7c482289c1bc4768eeb \ No newline at end of file From 1d2b70e3d21e5b6afedb0911b27f9a1db6266ebd Mon Sep 17 00:00:00 2001 From: dan Date: Wed, 28 Aug 2013 19:01:07 +0000 Subject: [PATCH 63/67] Re-enable test file wild001.test. FossilOrigin-Name: 4f182ddc36944fa54f1a34c1f0527db0ebb39c96 --- manifest | 14 +++++++------- manifest.uuid | 2 +- test/wild001.test | 4 ---- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/manifest b/manifest index 511e1108fe..04d21005ef 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Add\sthe\swin32-longpath\sVFS\sallowing\swindows\sfilenames\sup\sto\s32K\scharacters\sin\nlength.\s\sRemove\sunused\scode\swhen\s-DSQLITE_MAX_MMAP_SIZE=0.\s\sFix\ssome\ncompiler\swarnings\son\swindows. -D 2013-08-28T18:18:12.076 +C Re-enable\stest\sfile\swild001.test. +D 2013-08-28T19:01:07.361 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -1057,7 +1057,7 @@ F test/whereD.test 6c2feb79ef1f68381b07f39017fe5f9b96da8d62 F test/whereE.test b3a055eef928c992b0a33198a7b8dc10eea5ad2f F test/whereF.test 136a7301512d72a08a272806c8767066311b7bc1 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 -F test/wild001.test 384db4b30fbe82eaaefe8921f8a702c09e26db27 +F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 F test/win32longpath.test f888106783fc26515f393c8848c94cd6166addbb F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 @@ -1108,7 +1108,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P caab361ebee5f5c3fdafd9b1abe3d1ab7c5b4db9 ecd52ccf650ce2a4ab93c7c482289c1bc4768eeb -R 11a68a0ec35d591bf742b2cdf1cfbf81 -U drh -Z 9c29f776d777867b35099fa354ac7430 +P 12d0a8859de0a9d823997cfeccc77bd572cb6d13 +R fdbadc0d00e9a87112ecfe73cb5842df +U dan +Z 0f1dd9ce1608f2cfacffe15f693d1885 diff --git a/manifest.uuid b/manifest.uuid index 5220527afb..759b43a610 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -12d0a8859de0a9d823997cfeccc77bd572cb6d13 \ No newline at end of file +4f182ddc36944fa54f1a34c1f0527db0ebb39c96 \ No newline at end of file diff --git a/test/wild001.test b/test/wild001.test index e7cfdaae50..7fe1404294 100644 --- a/test/wild001.test +++ b/test/wild001.test @@ -42,10 +42,6 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl -# TODO: Reenable this test. - finish_test - return - ifcapable !stat3 { finish_test return From 8bcd3fa75a9893c9a8c9a19734b60f360a4bf67f Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 29 Aug 2013 01:03:38 +0000 Subject: [PATCH 64/67] Enable finer control of optimizations when compiling with the MSVC makefile. Also, several modularity enhancements to the MSVC makefile. FossilOrigin-Name: 6c709338bc77fbed24a2597eabd88dd8c29b38d7 --- Makefile.msc | 41 ++++++++++++++++++++++++++++++++++------- manifest | 14 +++++++------- manifest.uuid | 2 +- 3 files changed, 42 insertions(+), 15 deletions(-) diff --git a/Makefile.msc b/Makefile.msc index 23c5b17f08..6e32eb5124 100644 --- a/Makefile.msc +++ b/Makefile.msc @@ -94,6 +94,14 @@ WIN32HEAP = 0 DEBUG = 0 !ENDIF +# Enable use of available compiler optimizations? Normally, this should be +# non-zero. Setting this to zero, thus disabling all compiler optimizations, +# can be useful for testing. +# +!IFNDEF OPTIMIZATIONS +OPTIMIZATIONS = 2 +!ENDIF + # Check for the predefined command macro CC. This should point to the compiler # binary for the target platform. If it is not defined, simply define it to # the legacy default value 'cl.exe'. @@ -343,11 +351,15 @@ TCLSH_CMD = tclsh85 # Compiler options needed for programs that use the readline() library. # +!IFNDEF READLINE_FLAGS READLINE_FLAGS = -DHAVE_READLINE=0 +!ENDIF # The library that programs using readline() must link against. # +!IFNDEF LIBREADLINE LIBREADLINE = +!ENDIF # Should the database engine be compiled threadsafe # @@ -397,17 +409,30 @@ RCC = $(RCC) $(OPT_FEATURE_FLAGS) TCC = $(TCC) $(OPTS) RCC = $(RCC) $(OPTS) -# If symbols are enabled, enable PDBs. -# If debugging is enabled, disable all optimizations and enable PDBs. +# If compiling for debugging, add some defines. !IF $(DEBUG)>0 -TCC = $(TCC) -Od -D_DEBUG -BCC = $(BCC) -Od -D_DEBUG +TCC = $(TCC) -D_DEBUG +BCC = $(BCC) -D_DEBUG RCC = $(RCC) -D_DEBUG -!ELSE -TCC = $(TCC) -O2 -BCC = $(BCC) -O2 !ENDIF +# If optimizations are enabled or disabled (either implicitly or +# explicitly), add the necessary flags. +!IF $(DEBUG)>0 || $(OPTIMIZATIONS)==0 +TCC = $(TCC) -Od +BCC = $(BCC) -Od +!ELSEIF $(OPTIMIZATIONS)>=3 +TCC = $(TCC) -Ox +BCC = $(BCC) -Ox +!ELSEIF $(OPTIMIZATIONS)==2 +TCC = $(TCC) -O2 +BCC = $(BCC) -O2 +!ELSEIF $(OPTIMIZATIONS)==1 +TCC = $(TCC) -O1 +BCC = $(BCC) -O1 +!ENDIF + +# If symbols are enabled (or compiling for debugging), enable PDBs. !IF $(DEBUG)>0 || $(SYMBOLS)!=0 TCC = $(TCC) -Zi BCC = $(BCC) -Zi @@ -465,7 +490,9 @@ LTLIBS = $(LTLIBS) $(LIBICU) !ENDIF # nawk compatible awk. +!IFNDEF NAWK NAWK = gawk.exe +!ENDIF # You should not have to change anything below this line ############################################################################### diff --git a/manifest b/manifest index 04d21005ef..ed0912abbb 100644 --- a/manifest +++ b/manifest @@ -1,9 +1,9 @@ -C Re-enable\stest\sfile\swild001.test. -D 2013-08-28T19:01:07.361 +C Enable\sfiner\scontrol\sof\soptimizations\swhen\scompiling\swith\sthe\sMSVC\smakefile.\s\sAlso,\sseveral\smodularity\senhancements\sto\sthe\sMSVC\smakefile. +D 2013-08-29T01:03:38.501 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 -F Makefile.msc e9f41f89111627baaabd95cab4988b8d1c3e47c9 +F Makefile.msc a97163524522cd829cb91bcf900d07608e025502 F Makefile.vxworks db21ed42a01d5740e656b16f92cb5d8d5e5dd315 F README cd04a36fbc7ea56932a4052d7d0b7f09f27c33d6 F VERSION a8d1f6839521130dc73c5408cdd24bcfd791df34 @@ -1108,7 +1108,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 12d0a8859de0a9d823997cfeccc77bd572cb6d13 -R fdbadc0d00e9a87112ecfe73cb5842df -U dan -Z 0f1dd9ce1608f2cfacffe15f693d1885 +P 4f182ddc36944fa54f1a34c1f0527db0ebb39c96 +R d44711613e434eea3b5489989b8c56f0 +U mistachkin +Z 6c7751f0648dc6c9f1495284f8906a9f diff --git a/manifest.uuid b/manifest.uuid index 759b43a610..7f638abd90 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -4f182ddc36944fa54f1a34c1f0527db0ebb39c96 \ No newline at end of file +6c709338bc77fbed24a2597eabd88dd8c29b38d7 \ No newline at end of file From 1d406e0dfde1e383aa38d9702fa244919f91b239 Mon Sep 17 00:00:00 2001 From: mistachkin Date: Thu, 29 Aug 2013 01:09:14 +0000 Subject: [PATCH 65/67] Small enhancements to unit testing infrastructure. FossilOrigin-Name: 9229aeb361f9805894321327d05aba855b8799f3 --- manifest | 14 ++-- manifest.uuid | 2 +- test/tester.tcl | 138 ++++++++++++++++++++++++++++++---------- test/win32longpath.test | 58 ----------------- 4 files changed, 112 insertions(+), 100 deletions(-) diff --git a/manifest b/manifest index ed0912abbb..b049cf4d48 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Enable\sfiner\scontrol\sof\soptimizations\swhen\scompiling\swith\sthe\sMSVC\smakefile.\s\sAlso,\sseveral\smodularity\senhancements\sto\sthe\sMSVC\smakefile. -D 2013-08-29T01:03:38.501 +C Small\senhancements\sto\sunit\stesting\sinfrastructure. +D 2013-08-29T01:09:14.083 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -817,7 +817,7 @@ F test/tclsqlite.test 37a61c2da7e3bfe3b8c1a2867199f6b860df5d43 F test/tempdb.test 19d0f66e2e3eeffd68661a11c83ba5e6ace9128c F test/temptable.test d2c9b87a54147161bcd1822e30c1d1cd891e5b30 F test/temptrigger.test 26670ed7a39cf2296a7f0a9e0a1d7bdb7abe936d -F test/tester.tcl 63b24679c75a952c51f924de2802b2b57cddd22d +F test/tester.tcl 5e97d1fe08f45fa3cc2320cee437e315c75ce995 F test/thread001.test 9f22fd3525a307ff42a326b6bc7b0465be1745a5 F test/thread002.test e630504f8a06c00bf8bbe68528774dd96aeb2e58 F test/thread003.test ee4c9efc3b86a6a2767516a37bd64251272560a7 @@ -1059,7 +1059,7 @@ F test/whereF.test 136a7301512d72a08a272806c8767066311b7bc1 F test/wherelimit.test 5e9fd41e79bb2b2d588ed999d641d9c965619b31 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361 -F test/win32longpath.test f888106783fc26515f393c8848c94cd6166addbb +F test/win32longpath.test e2aafc07e6990fe86c69be22a3d1a0e210cd329b F test/zeroblob.test caaecfb4f908f7bc086ed238668049f96774d688 F test/zerodamage.test 209d7ed441f44cc5299e4ebffbef06fd5aabfefd F tool/build-all-msvc.bat c55f64ca200308fb5fa5c1ee751ea95a13977b5a x @@ -1108,7 +1108,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 4f182ddc36944fa54f1a34c1f0527db0ebb39c96 -R d44711613e434eea3b5489989b8c56f0 +P 6c709338bc77fbed24a2597eabd88dd8c29b38d7 +R c45150e3b7004e4a0ed3f6fa3be34cd1 U mistachkin -Z 6c7751f0648dc6c9f1495284f8906a9f +Z 89e12299b0f96674ea40270259273327 diff --git a/manifest.uuid b/manifest.uuid index 7f638abd90..03763eea4b 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -6c709338bc77fbed24a2597eabd88dd8c29b38d7 \ No newline at end of file +9229aeb361f9805894321327d05aba855b8799f3 \ No newline at end of file diff --git a/test/tester.tcl b/test/tester.tcl index 32dca4cb78..e4b5edeb80 100644 --- a/test/tester.tcl +++ b/test/tester.tcl @@ -14,7 +14,7 @@ # $Id: tester.tcl,v 1.143 2009/04/09 01:23:49 drh Exp $ #------------------------------------------------------------------------- -# The commands provided by the code in this file to help with creating +# The commands provided by the code in this file to help with creating # test cases are as follows: # # Commands to manipulate the db and the file-system at a high level: @@ -42,6 +42,7 @@ # # Commands to execute/explain SQL statements: # +# memdbsql SQL # stepsql DB SQL # execsql2 SQL # explain_no_trace SQL @@ -80,7 +81,7 @@ # presql # -# Set the precision of FP arithmatic used by the interpreter. And +# Set the precision of FP arithmatic used by the interpreter. And # configure SQLite to take database file locks on the page that begins # 64KB into the database file instead of the one 1GB in. This means # the code that handles that special case can be tested without creating @@ -90,7 +91,7 @@ set tcl_precision 15 sqlite3_test_control_pending_byte 0x0010000 -# If the pager codec is available, create a wrapper for the [sqlite3] +# If the pager codec is available, create a wrapper for the [sqlite3] # command that appends "-key {xyzzy}" to the command line. i.e. this: # # sqlite3 db test.db @@ -122,7 +123,7 @@ if {[info command sqlite_orig]==""} { } set res } else { - # This command is not opening a new database connection. Pass the + # This command is not opening a new database connection. Pass the # arguments through to the C implementation as the are. # uplevel 1 sqlite_orig $args @@ -291,6 +292,66 @@ proc do_delete_file {force args} { } } +if {$::tcl_platform(platform) eq "windows"} { + proc do_remove_win32_dir {args} { + set nRetry [getFileRetries] ;# Maximum number of retries. + set nDelay [getFileRetryDelay] ;# Delay in ms before retrying. + + foreach dirName $args { + # On windows, sometimes even a [remove_win32_dir] can fail just after + # a directory is emptied. The cause is usually "tag-alongs" - programs + # like anti-virus software, automatic backup tools and various explorer + # extensions that keep a file open a little longer than we expect, + # causing the delete to fail. + # + # The solution is to wait a short amount of time before retrying the + # removal. + # + if {$nRetry > 0} { + for {set i 0} {$i < $nRetry} {incr i} { + set rc [catch { + remove_win32_dir $dirName + } msg] + if {$rc == 0} break + if {$nDelay > 0} { after $nDelay } + } + if {$rc} { error $msg } + } else { + remove_win32_dir $dirName + } + } + } + + proc do_delete_win32_file {args} { + set nRetry [getFileRetries] ;# Maximum number of retries. + set nDelay [getFileRetryDelay] ;# Delay in ms before retrying. + + foreach fileName $args { + # On windows, sometimes even a [delete_win32_file] can fail just after + # a file is closed. The cause is usually "tag-alongs" - programs like + # anti-virus software, automatic backup tools and various explorer + # extensions that keep a file open a little longer than we expect, + # causing the delete to fail. + # + # The solution is to wait a short amount of time before retrying the + # delete. + # + if {$nRetry > 0} { + for {set i 0} {$i < $nRetry} {incr i} { + set rc [catch { + delete_win32_file $fileName + } msg] + if {$rc == 0} break + if {$nDelay > 0} { after $nDelay } + } + if {$rc} { error $msg } + } else { + delete_win32_file $fileName + } + } + } +} + proc execpresql {handle args} { trace remove execution $handle enter [list execpresql $handle] if {[info exists ::G(perm:presql)]} { @@ -312,8 +373,8 @@ proc do_not_use_codec {} { # if {[info exists cmdlinearg]==0} { - # Parse any options specified in the $argv array. This script accepts the - # following options: + # Parse any options specified in the $argv array. This script accepts the + # following options: # # --pause # --soft-heap-limit=NN @@ -342,7 +403,7 @@ if {[info exists cmdlinearg]==0} { foreach a $argv { switch -regexp -- $a { {^-+pause$} { - # Wait for user input before continuing. This is to give the user an + # Wait for user input before continuing. This is to give the user an # opportunity to connect profiling tools to the process. puts -nonewline "Press RETURN to begin..." flush stdout @@ -405,8 +466,8 @@ if {[info exists cmdlinearg]==0} { # Install the malloc layer used to inject OOM errors. And the 'automatic' # extensions. This only needs to be done once for the process. # - sqlite3_shutdown - install_malloc_faultsim 1 + sqlite3_shutdown + install_malloc_faultsim 1 sqlite3_initialize autoinstall_test_functions @@ -516,7 +577,7 @@ proc incr_ntest {} { } -# Invoke the do_test procedure to run a single test +# Invoke the do_test procedure to run a single test # proc do_test {name cmd expected} { global argv cmdlinearg @@ -525,7 +586,7 @@ proc do_test {name cmd expected} { sqlite3_memdebug_settitle $name -# if {[llength $argv]==0} { +# if {[llength $argv]==0} { # set go 1 # } else { # set go 0 @@ -628,13 +689,13 @@ proc do_realnum_test {name cmd expected} { proc fix_testname {varname} { upvar $varname testname - if {[info exists ::testprefix] + if {[info exists ::testprefix] && [string is digit [string range $testname 0 0]] } { set testname "${::testprefix}-$testname" } } - + proc do_execsql_test {testname sql {result {}}} { fix_testname testname uplevel do_test [list $testname] [list "execsql {$sql}"] [list [list {*}$result]] @@ -720,7 +781,7 @@ proc delete_all_data {} { } } -# Run an SQL script. +# Run an SQL script. # Return the number of microseconds per statement. # proc speed_trial {name numstmt units sql} { @@ -984,6 +1045,15 @@ proc execsql2 {sql} { return $result } +# Use a temporary in-memory database to execute SQL statements +# +proc memdbsql {sql} { + sqlite3 memdb :memory: + set result [memdb eval $sql] + memdb close + return $result +} + # Use the non-callback API to execute multiple SQL statements # proc stepsql {dbptr sql} { @@ -1098,7 +1168,7 @@ proc crashsql {args} { set crashfile "" set dc "" set sql [lindex $args end] - + for {set ii 0} {$ii < [llength $args]-1} {incr ii 2} { set z [lindex $args $ii] set n [string length $z] @@ -1117,7 +1187,7 @@ proc crashsql {args} { error "Compulsory option -file missing" } - # $crashfile gets compared to the native filename in + # $crashfile gets compared to the native filename in # cfSync(), which can be different then what TCL uses by # default, so here we force it to the "nativename" format. set cfile [string map {\\ \\\\} [file nativename [file join [get_pwd] $crashfile]]] @@ -1152,7 +1222,7 @@ proc crashsql {args} { set r [catch { exec [info nameofexec] crash.tcl >@stdout } msg] - + # Windows/ActiveState TCL returns a slightly different # error message. We map that to the expected message # so that we don't have to change all of the test @@ -1162,7 +1232,7 @@ proc crashsql {args} { set msg "child process exited abnormally" } } - + lappend r $msg } @@ -1188,7 +1258,7 @@ proc run_ioerr_prep {} { # Usage: do_ioerr_test # # This proc is used to implement test cases that check that IO errors -# are correctly handled. The first argument, , is an integer +# are correctly handled. The first argument, , is an integer # used to name the tests executed by this proc. Options are as follows: # # -tclprep TCL script to run to prepare test. @@ -1217,7 +1287,7 @@ proc do_ioerr_test {testname args} { # TEMPORARY: For 3.5.9, disable testing of extended result codes. There are # a couple of obscure IO errors that do not return them. set ::ioerropts(-erc) 0 - + # Create a single TCL script from the TCL and SQL specified # as the body of the test. set ::ioerrorbody {} @@ -1241,7 +1311,7 @@ proc do_ioerr_test {testname args} { set ::TN $n incr ::ioerropts(-count) -1 if {$::ioerropts(-count)<0} break - + # Skip this IO error if it was specified with the "-exclude" option. if {[info exists ::ioerropts(-exclude)]} { if {[lsearch $::ioerropts(-exclude) $n]!=-1} continue @@ -1250,7 +1320,7 @@ proc do_ioerr_test {testname args} { restore_prng_state } - # Delete the files test.db and test2.db, then execute the TCL and + # Delete the files test.db and test2.db, then execute the TCL and # SQL (in that order) to prepare for the test case. do_test $testname.$n.1 { run_ioerr_prep @@ -1268,7 +1338,7 @@ proc do_ioerr_test {testname args} { }] $n # Execute the TCL script created for the body of this test. If - # at least N IO operations performed by SQLite as a result of + # at least N IO operations performed by SQLite as a result of # the script, the Nth will fail. do_test $testname.$n.3 { set ::sqlite_io_error_hit 0 @@ -1322,12 +1392,12 @@ proc do_ioerr_test {testname args} { set ::sqlite_io_error_hit 0 set ::sqlite_io_error_pending 0 - # Check that no page references were leaked. There should be - # a single reference if there is still an active transaction, + # Check that no page references were leaked. There should be + # a single reference if there is still an active transaction, # or zero otherwise. # # UPDATE: If the IO error occurs after a 'BEGIN' but before any - # locks are established on database files (i.e. if the error + # locks are established on database files (i.e. if the error # occurs while attempting to detect a hot-journal file), then # there may 0 page references and an active transaction according # to [sqlite3_get_autocommit]. @@ -1343,7 +1413,7 @@ proc do_ioerr_test {testname args} { } {1} } - # If there is an open database handle and no open transaction, + # If there is an open database handle and no open transaction, # and the pager is not running in exclusive-locking mode, # check that the pager is in "unlocked" state. Theoretically, # if a call to xUnlock() failed due to an IO error the underlying @@ -1447,7 +1517,7 @@ proc allcksum {{db db}} { } # Generate a checksum based on the contents of a single database with -# a database connection. The name of the database is $dbname. +# a database connection. The name of the database is $dbname. # Examples of $dbname are "temp" or "main". # proc dbcksum {db dbname} { @@ -1541,8 +1611,8 @@ proc drop_all_tables {{db db}} { #------------------------------------------------------------------------- # If a test script is executed with global variable $::G(perm:name) set to -# "wal", then the tests are run in WAL mode. Otherwise, they should be run -# in rollback mode. The following Tcl procs are used to make this less +# "wal", then the tests are run in WAL mode. Otherwise, they should be run +# in rollback mode. The following Tcl procs are used to make this less # intrusive: # # wal_set_journal_mode ?DB? @@ -1557,9 +1627,9 @@ proc drop_all_tables {{db db}} { # Otherwise (if not running a WAL permutation) this is a no-op. # # wal_is_wal_mode -# +# # Returns true if this test should be run in WAL mode. False otherwise. -# +# proc wal_is_wal_mode {} { expr {[permutation] eq "wal"} } @@ -1660,10 +1730,10 @@ proc slave_test_file {zFile} { } set ::sqlite_open_file_count 0 - # Test that the global "shared-cache" setting was not altered by + # Test that the global "shared-cache" setting was not altered by # the test script. # - ifcapable shared_cache { + ifcapable shared_cache { set res [expr {[sqlite3_enable_shared_cache] == $scs}] do_test ${tail}-sharedcachesetting [list set {} $res] 1 } diff --git a/test/win32longpath.test b/test/win32longpath.test index 45f8825858..0a6a8f98e6 100644 --- a/test/win32longpath.test +++ b/test/win32longpath.test @@ -19,64 +19,6 @@ set testdir [file dirname $argv0] source $testdir/tester.tcl set testprefix win32longpath -proc do_remove_win32_dir {args} { - set nRetry [getFileRetries] ;# Maximum number of retries. - set nDelay [getFileRetryDelay] ;# Delay in ms before retrying. - - foreach dirName $args { - # On windows, sometimes even a [remove_win32_dir] can fail just after - # a directory is emptied. The cause is usually "tag-alongs" - programs - # like anti-virus software, automatic backup tools and various explorer - # extensions that keep a file open a little longer than we expect, - # causing the delete to fail. - # - # The solution is to wait a short amount of time before retrying the - # removal. - # - if {$nRetry > 0} { - for {set i 0} {$i < $nRetry} {incr i} { - set rc [catch { - remove_win32_dir $dirName - } msg] - if {$rc == 0} break - if {$nDelay > 0} { after $nDelay } - } - if {$rc} { error $msg } - } else { - remove_win32_dir $dirName - } - } -} - -proc do_delete_win32_file {args} { - set nRetry [getFileRetries] ;# Maximum number of retries. - set nDelay [getFileRetryDelay] ;# Delay in ms before retrying. - - foreach fileName $args { - # On windows, sometimes even a [delete_win32_file] can fail just after - # a file is closed. The cause is usually "tag-alongs" - programs like - # anti-virus software, automatic backup tools and various explorer - # extensions that keep a file open a little longer than we expect, - # causing the delete to fail. - # - # The solution is to wait a short amount of time before retrying the - # delete. - # - if {$nRetry > 0} { - for {set i 0} {$i < $nRetry} {incr i} { - set rc [catch { - delete_win32_file $fileName - } msg] - if {$rc == 0} break - if {$nDelay > 0} { after $nDelay } - } - if {$rc} { error $msg } - } else { - delete_win32_file $fileName - } - } -} - db close set path [file nativename [get_pwd]] sqlite3 db [file join $path test.db] -vfs win32-longpath From cd40abb297737d50e32d9f60aa55515226f42cc2 Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 29 Aug 2013 10:46:05 +0000 Subject: [PATCH 66/67] Candidate fix for [9f2eb3abac]: Have the whereShortCut() planner ignore indexes with more than four columns. FossilOrigin-Name: c1152bdcbb5ac185f743015fff63769effa81b4b --- manifest | 15 ++++---- manifest.uuid | 2 +- src/where.c | 8 +++- test/tkt-9f2eb3abac.test | 79 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 test/tkt-9f2eb3abac.test diff --git a/manifest b/manifest index b049cf4d48..c896281d95 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Small\senhancements\sto\sunit\stesting\sinfrastructure. -D 2013-08-29T01:09:14.083 +C Candidate\sfix\sfor\s[9f2eb3abac]:\sHave\sthe\swhereShortCut()\splanner\signore\sindexes\swith\smore\sthan\sfour\scolumns. +D 2013-08-29T10:46:05.710 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vtab.c 165ce0e797c2cd23badb104c9f2ae9042d6d942c F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c e0b70ec707e208af37b436eaa829b038cc55e545 +F src/where.c dab4d4b8f415e876e5acf63e55f7023fef42316b F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -859,6 +859,7 @@ F test/tkt-868145d012.test a5f941107ece6a64410ca4755c6329b7eb57a356 F test/tkt-91e2e8ba6f.test 08c4f94ae07696b05c9b822da0b4e5337a2f54c5 F test/tkt-94c04eaadb.test fa9c71192f7e2ea2d51bf078bc34e8da6088bf71 F test/tkt-9d68c883.test 458f7d82a523d7644b54b497c986378a7d8c8b67 +F test/tkt-9f2eb3abac.test 85bc63e749f050e6a61c8f9207f1eee65c9d3395 F test/tkt-a7b7803e.test 159ef554234fa1f9fb318c751b284bd1cf858da4 F test/tkt-b1d3a2e531.test 610ef582413171b379652663111b1f996d9f8f78 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0 @@ -1108,7 +1109,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 6c709338bc77fbed24a2597eabd88dd8c29b38d7 -R c45150e3b7004e4a0ed3f6fa3be34cd1 -U mistachkin -Z 89e12299b0f96674ea40270259273327 +P 9229aeb361f9805894321327d05aba855b8799f3 +R cdb2a97ae7b02b85f8d964c7dea1a83c +U dan +Z 5d34b767de2885354c0587f1fafecb4c diff --git a/manifest.uuid b/manifest.uuid index 03763eea4b..1641c63a70 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -9229aeb361f9805894321327d05aba855b8799f3 \ No newline at end of file +c1152bdcbb5ac185f743015fff63769effa81b4b \ No newline at end of file diff --git a/src/where.c b/src/where.c index d9a795e4ea..a8584a275f 100644 --- a/src/where.c +++ b/src/where.c @@ -5536,11 +5536,15 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ pLoop->rRun = 33; /* 33==whereCost(10) */ }else{ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->onError==OE_None || pIdx->pPartIdxWhere!=0 ) continue; + assert( pLoop->aLTermSpace==pLoop->aLTerm ); + assert( ArraySize(pLoop->aLTermSpace)==4 ); + if( pIdx->onError==OE_None + || pIdx->pPartIdxWhere!=0 + || pIdx->nColumn>ArraySize(pLoop->aLTermSpace) + ) continue; for(j=0; jnColumn; j++){ pTerm = findTerm(pWC, iCur, pIdx->aiColumn[j], 0, WO_EQ, pIdx); if( pTerm==0 ) break; - whereLoopResize(pWInfo->pParse->db, pLoop, j); pLoop->aLTerm[j] = pTerm; } if( j!=pIdx->nColumn ) continue; diff --git a/test/tkt-9f2eb3abac.test b/test/tkt-9f2eb3abac.test new file mode 100644 index 0000000000..5b93733afe --- /dev/null +++ b/test/tkt-9f2eb3abac.test @@ -0,0 +1,79 @@ + +# 2013 August 29 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# This file implements regression tests for SQLite library. +# + +set testdir [file dirname $argv0] +source $testdir/tester.tcl +source $testdir/malloc_common.tcl +set ::testprefix tkt-9f2eb3abac + +do_execsql_test 1.1 { + CREATE TABLE t1(a,b,c,d,e, PRIMARY KEY(a,b,c,d,e)); + SELECT * FROM t1 WHERE a=? AND b=? AND c=? AND d=? AND e=?; +} {} + +do_execsql_test 1.2 { + CREATE TABLE "a" ( + "b" integer NOT NULL, + "c" integer NOT NULL, + PRIMARY KEY ("b", "c") + ); + + CREATE TABLE "d" ( + "e" integer NOT NULL, + "g" integer NOT NULL, + "f" integer NOT NULL, + "h" integer NOT NULL, + "i" character(10) NOT NULL, + "j" int, + PRIMARY KEY ("e", "g", "f", "h") + ); + + CREATE TABLE "d_to_a" ( + "f_e" integer NOT NULL, + "f_g" integer NOT NULL, + "f_f" integer NOT NULL, + "f_h" integer NOT NULL, + "t_b" integer NOT NULL, + "t_c" integer NOT NULL, + "r" character NOT NULL, + "s" integer, + PRIMARY KEY ("f_e", "f_g", "f_f", "f_h", "t_b", "t_c") + ); + + INSERT INTO d (g, e, h, f, j, i) VALUES ( 1, 1, 1, 1, 1, 1 ); + INSERT INTO a (b, c) VALUES ( 1, 1 ); + INSERT INTO d_to_a VALUES (1, 1, 1, 1, 1, 1, 1, 1); + + DELETE FROM d_to_a + WHERE f_g = 1 AND f_e = 1 AND f_h = 1 AND f_f = 1 AND t_b = 1 AND t_c = 1; + + SELECT * FROM d_to_a; +} {} + +faultsim_delete_and_reopen +do_execsql_test 2.0 { CREATE TABLE t1(a,b,c,d,e, PRIMARY KEY(a,b,c,d,e)) } +do_execsql_test 2.1 { CREATE TABLE t2(x) } +faultsim_save_and_close + +do_faultsim_test 3 -faults oom* -prep { + faultsim_restore_and_reopen + execsql { SELECT 1 FROM sqlite_master } +} -body { + execsql { SELECT * FROM t1,t2 WHERE a=? AND b=? AND c=? AND d=? AND e=? } +} -test { + faultsim_test_result {0 {}} +} + +finish_test + From 8bff07a55f559223526c93fcf013192577e777dc Mon Sep 17 00:00:00 2001 From: dan Date: Thu, 29 Aug 2013 14:56:14 +0000 Subject: [PATCH 67/67] Restore fix [f15591f802], which was accidentally clobbered by the stat4 merge. FossilOrigin-Name: d4b6ad3333cc3bad500c2ebf7a6ea552b6762b69 --- manifest | 12 ++++++------ manifest.uuid | 2 +- src/where.c | 9 +++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/manifest b/manifest index c896281d95..86dc44c7b9 100644 --- a/manifest +++ b/manifest @@ -1,5 +1,5 @@ -C Candidate\sfix\sfor\s[9f2eb3abac]:\sHave\sthe\swhereShortCut()\splanner\signore\sindexes\swith\smore\sthan\sfour\scolumns. -D 2013-08-29T10:46:05.710 +C Restore\sfix\s[f15591f802],\swhich\swas\saccidentally\sclobbered\sby\sthe\sstat4\smerge. +D 2013-08-29T14:56:14.988 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23 @@ -290,7 +290,7 @@ F src/vtab.c 165ce0e797c2cd23badb104c9f2ae9042d6d942c F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4 F src/walker.c 4fa43583d0a84b48f93b1e88f11adf2065be4e73 -F src/where.c dab4d4b8f415e876e5acf63e55f7023fef42316b +F src/where.c cad6497f79307c2c94aaf72e64866bb33cf99d59 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6 @@ -1109,7 +1109,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac -P 9229aeb361f9805894321327d05aba855b8799f3 -R cdb2a97ae7b02b85f8d964c7dea1a83c +P c1152bdcbb5ac185f743015fff63769effa81b4b +R ef3cf6caed80e05bf3e44c08e4ed05a0 U dan -Z 5d34b767de2885354c0587f1fafecb4c +Z a6b4620a23effa55e7cb7c1762a8f7ee diff --git a/manifest.uuid b/manifest.uuid index 1641c63a70..9346487f5c 100644 --- a/manifest.uuid +++ b/manifest.uuid @@ -1 +1 @@ -c1152bdcbb5ac185f743015fff63769effa81b4b \ No newline at end of file +d4b6ad3333cc3bad500c2ebf7a6ea552b6762b69 \ No newline at end of file diff --git a/src/where.c b/src/where.c index a8584a275f..9e3a72e2d8 100644 --- a/src/where.c +++ b/src/where.c @@ -4309,11 +4309,12 @@ static int whereLoopAddBtreeIndex( int nIn = 0; #ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int nRecValid = pBuilder->nRecValid; - assert( pNew->nOut==saved_nOut ); - if( (pTerm->wtFlags & TERM_VNULL)!=0 && pSrc->pTab->aCol[iCol].notNull ){ - continue; /* skip IS NOT NULL constraints on a NOT NULL column */ - } #endif + if( (pTerm->eOperator==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) + && (iCol<0 || pSrc->pTab->aCol[iCol].notNull) + ){ + continue; /* ignore IS [NOT] NULL constraints on NOT NULL columns */ + } if( pTerm->prereqRight & pNew->maskSelf ) continue; assert( pNew->nOut==saved_nOut );