Merge the stat3-enhancement branch with trunk, but keep the resulting merge

in a separate branch for now.

FossilOrigin-Name: 63f2c7859fa6e5d0e2cdd218ff52a3ec2d44c61d

manifest

@@ -1,5 +1,5 @@
-C Update\scompiler\serror\smessage\sregarding\sthe\schoice\sof\smemory\sallocator\sdefines.
+C Merge\sthe\sstat3-enhancement\sbranch\swith\strunk,\sbut\skeep\sthe\sresulting\smerge\nin\sa\sseparate\sbranch\sfor\snow.
-D 2011-08-26T11:25:02.455
+D 2011-08-26T13:16:33.563
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 8c930e7b493d59099ea1304bd0f2aed152eb3315
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -118,7 +118,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
 F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc
 F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad
 F src/alter.c ac80a0f31189f8b4a524ebf661e47e84536ee7f5
-F src/analyze.c a425d62e8fa9ebcb4359ab84ff0c62c6563d2e2a
+F src/analyze.c 3fbffcfbc606d73fa996ded1f874eddffbb06d09
 F src/attach.c 12c6957996908edc31c96d7c68d4942c2474405f
 F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34
 F src/backup.c 28a4fe55327ff708bfaf9d4326d02686f7a553c3
@@ -127,10 +127,10 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
 F src/btree.c ed13fdefdbe671d5777773dcfb3a162ddb4623ae
 F src/btree.h 9ddf04226eac592d4cc3709c5a8b33b2351ff5f7
 F src/btreeInt.h 67978c014fa4f7cc874032dd3aacadd8db656bc3
-F src/build.c 2d5de52df616a3bf5a659cbca85211c46e2ba9bd
+F src/build.c ea854c9d5e86098c721574e7c0fda3552227f54f
 F src/callback.c 0425c6320730e6d3981acfb9202c1bed9016ad1a
 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
-F src/ctime.c caf51429be3e0d4114056a8273b0fff812ff8ae9
+F src/ctime.c 3e961190276d1a48c5224e9558253f8e1f31bf50
 F src/date.c a3c6842bad7ae632281811de112a8ba63ff08ab3
 F src/delete.c ff68e5ef23aee08c0ff528f699a19397ed8bbed8
 F src/expr.c 4bbdfaf66bc614be9254ce0c26a17429067a3e07
@@ -181,9 +181,9 @@ F src/resolve.c 36368f44569208fa074e61f4dd0b6c4fb60ca2b4
 F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697
 F src/select.c d219c4b68d603cc734b6f9b1e2780fee12a1fa0d
 F src/shell.c bbe7818ff5bc8614105ceb81ad67b8bdc0b671dd
-F src/sqlite.h.in 0a6c9c23337fd1352c5c75a613ff9533aa7d91cb
+F src/sqlite.h.in 3f531daa04457e83bc13765c98c06c7d71c27fa5
 F src/sqlite3ext.h 1a1a4f784aa9c3b00edd287940197de52487cd93
-F src/sqliteInt.h 86a4fdb3ba9ab31d98b266797606f30fefe5b8a9
+F src/sqliteInt.h 6dedc0587dff7dede458000557ac8fc5fbedf5aa
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c 7ac64842c86cec2fc1a1d0e5c16d3beb8ad332bf
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -201,7 +201,7 @@ F src/test_async.c 0612a752896fad42d55c3999a5122af10dcf22ad
 F src/test_autoext.c 30e7bd98ab6d70a62bb9ba572e4c7df347fe645e
 F src/test_backup.c c129c91127e9b46e335715ae2e75756e25ba27de
 F src/test_btree.c 47cd771250f09cdc6e12dda5bc71bc0b3abc96e2
-F src/test_config.c e342660556d64365aacea2b23cbb5e6654d7278f
+F src/test_config.c afc0bbf8fe048d79ae03469542e826c76a2175e5
 F src/test_demovfs.c 20a4975127993f4959890016ae9ce5535a880094
 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc
 F src/test_func.c cbdec5cededa0761daedde5baf06004a9bf416b5
@@ -235,39 +235,40 @@ F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/tokenize.c c819d9f72168a035d545a5bdafe9b085b20df705
 F src/trigger.c 1cfb80e2290ef66ea89cb4e821caae65a02c0d56
 F src/update.c 74a6cfb34e9732c1e2a86278b229913b4b51eeec
-F src/utf.c c53eb7404b3eb5c1cbb5655c6a7a0e0ce6bd50f0
+F src/utf.c 890c67dcfcc7a74623c95baac7535aadfe265e84
 F src/util.c 06302ffd2b80408d4f6c7af71f7090e0cf8d8ff7
 F src/vacuum.c 05513dca036a1e7848fe18d5ed1265ac0b32365e
 F src/vdbe.c 4a7191c0f8e918b74e8c84cbdd77746d6b7e3bcf
 F src/vdbe.h 2bf6ec77d8b9980fc19da6e0b0a36d0dbf884ce4
 F src/vdbeInt.h f9250326f264ca5f100acc19e9c07096bb889096
 F src/vdbeapi.c 11dc47987abacb76ad016dcf5abc0dc422482a98
-F src/vdbeaux.c 11b0df8822ecf61e543562247207df75e2ebb617
+F src/vdbeaux.c 1bd0fbeadede0c07d78e25008db61bc90ac3e412
 F src/vdbeblob.c f024f0bf420f36b070143c32b15cc7287341ffd3
-F src/vdbemem.c 0498796b6ffbe45e32960d6a1f5adfb6e419883b
+F src/vdbemem.c 74410d1639869b309d6fe1e8cbc02a557157a7c2
 F src/vdbesort.c 8a61a6d731cbe612217edf9eece6197f37c9489e
 F src/vdbetrace.c 5d0dc3d5fd54878cc8d6d28eb41deb8d5885b114
 F src/vtab.c 901791a47318c0562cd0c676a2c6ff1bc530e582
 F src/wal.c 3154756177d6219e233d84291d5b05f4e06ff5e9
 F src/wal.h 66b40bd91bc29a5be1c88ddd1f5ade8f3f48728a
 F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
-F src/where.c 7d09f4c1512affb60cc1190a4b33d121d4ce039a
+F src/where.c 3d9a78a422726c1b3a57188c3099e537cb18e18a
 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87
 F test/all.test 52fc8dee494092031a556911d404ca30a749a30b
-F test/alter.test 5314fc01ef51ab8af0b8890725b710ed48d4806b
+F test/alter.test 54912d932309df2e4f62aeb47169c2ff740e53ed
 F test/alter2.test 7ea05c7d92ac99349a802ef7ada17294dd647060
 F test/alter3.test 49c9d9fba2b8fcdce2dedeca97bbf1f369cc548d
 F test/alter4.test b2debc14d8cbe4c1d12ccd6a41eef88a8c1f15d5
 F test/altermalloc.test e81ac9657ed25c6c5bb09bebfa5a047cd8e4acfc
-F test/analyze.test 68b43c1f9cd6ffc3bbb30d27a23712b38c413eca
+F test/analyze.test f8ab7d15858b4093b06caf5e57e2a5ff7104bdae
 F test/analyze2.test 8f2b1534d43f5547ce9a6b736c021d4192c75be3
-F test/analyze3.test d61f55d8b472fc6e713160b1e577f7a68e63f38b
+F test/analyze3.test 7bcadc47589fd730f9a12ffc9b30a520d7f6931b
 F test/analyze4.test 757b37875cf9bb528d46f74497bc789c88365045
-F test/analyze5.test 1de8d66b11aae5a1453aa042d62e834a476bac9c
+F test/analyze5.test 713354664c5ff1853ab2cbcb740f0cf5cb7c802e
-F test/analyze6.test c125622a813325bba1b4999040ddc213773c2290
+F test/analyze6.test bd3625806a5ee6f7bef72d06295bd319f0290af2
-F test/analyze7.test 5508e7828164ea0b518ed219bed7320a481863d4
+F test/analyze7.test d3587aa5af75c9048d031b94fceca2534fa75d1d
+F test/analyze8.test 4ca170de2ba30ccb1af2c0406803db72262f9691
 F test/async.test 1d0e056ba1bb9729283a0f22718d3a25e82c277b
 F test/async2.test c0a9bd20816d7d6a2ceca7b8c03d3d69c28ffb8b
 F test/async3.test d73a062002376d7edc1fe3edff493edbec1fc2f7
@@ -278,7 +279,7 @@ F test/attach2.test e54436ed956d3d88bdee61221da59bf3935a0966
 F test/attach3.test d89ccfe4fe6e2b5e368d480fcdfe4b496c54cf4e
 F test/attach4.test 31f9eb0ca7bdbc393cc4657b877903a226a83d4b
 F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0
-F test/auth.test b047105c32da7db70b842fd24056723125ecc2ff
+F test/auth.test ac996c81ad910148606f5c7e3b3f85d47c29960f
 F test/auth2.test 270baddc8b9c273682760cffba6739d907bd2882
 F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5
 F test/autoinc.test bd30d372d00045252f6c2e41b5f41455e1975acf
@@ -360,7 +361,7 @@ F test/createtab.test b5de160630b209c4b8925bdcbbaf48cc90b67fe8
 F test/cse.test 277350a26264495e86b1785f34d2d0c8600e021c
 F test/ctime.test 7f0bd5084d9dd7da9ad46901810896edd2ebb463
 F test/date.test a18a2ce81add84b17b06559e82ad7bb91bc6ddff
-F test/dbstatus.test a719af0f226bd280748a4bb9054c0a5a9fc1b16c
+F test/dbstatus.test 9eb484ba837c6f3f9bbcaecc29e6060a8c3ba6d2
 F test/default.test 6faf23ccb300114924353007795aa9a8ec0aa9dc
 F test/delete.test a065b05d2ebf60fd16639c579a4adfb7c381c701
 F test/delete2.test 3a03f2cca1f9a67ec469915cb8babd6485db43fa
@@ -685,6 +686,7 @@ F test/speed4p.explain 6b5f104ebeb34a038b2f714150f51d01143e59aa
 F test/speed4p.test 0e51908951677de5a969b723e03a27a1c45db38b
 F test/sqllimits1.test b1aae27cc98eceb845e7f7adf918561256e31298
 F test/stat.test 0997f6a57a35866b14111ed361ed8851ce7978ae
+F test/stat3.test 986d735f70ef62a1daf98e8762f35fa3b062c5c3
 F test/stmt.test 25d64e3dbf9a3ce89558667d7f39d966fe2a71b9
 F test/subquery.test b524f57c9574b2c0347045b4510ef795d4686796
 F test/subselect.test d24fd8757daf97dafd2e889c73ea4c4272dcf4e4
@@ -737,7 +739,7 @@ F test/tkt-b1d3a2e531.test 610ef582413171b379652663111b1f996d9f8f78
 F test/tkt-b351d95f9.test d14a503c414c5c58fdde3e80f9a3cfef986498c0
 F test/tkt-b72787b1.test e6b62b2b2785c04d0d698d6a603507e384165049
 F test/tkt-bd484a090c.test 60460bf946f79a79712b71f202eda501ca99b898
-F test/tkt-cbd054fa6b.test f14f97ea43662e6f70c9e63287081e8be5d9d589
+F test/tkt-cbd054fa6b.test bd9fb546f63bc0c79d1776978d059fa51c5b1c63
 F test/tkt-d11f09d36e.test fb44f7961aa6d4b632fb7b9768239832210b5fc7
 F test/tkt-d635236375.test 9d37e988b47d87505bc9445be0ca447002df5d09
 F test/tkt-d82e3f3721.test 731359dfdcdb36fea0559cd33fec39dd0ceae8e6
@@ -854,7 +856,7 @@ F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84
 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a
 F test/unique.test 083c7fff74695bcc27a71d75699deba3595bc9c2
 F test/unixexcl.test 9d80a54d86d2261f660758928959368ffc36151e
-F test/unordered.test e81169ce2a8f31b2c6b66af691887e1376ab3ced
+F test/unordered.test f53095cee37851bf30130fa1bf299a8845e837bb
 F test/update.test 8bc86fd7ef1a00014f76dc6a6a7c974df4aef172
 F test/uri.test 53de9a2549cbda9c343223236918ef502f6a9051
 F test/utf16align.test 54cd35a27c005a9b6e7815d887718780b6a462ae
@@ -960,8 +962,11 @@ F tool/split-sqlite3c.tcl d9be87f1c340285a3e081eb19b4a247981ed290c
 F tool/symbols.sh caaf6ccc7300fd43353318b44524853e222557d5
 F tool/tostr.awk 11760e1b94a5d3dcd42378f3cc18544c06cfa576
 F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-F tool/warnings.sh b7fdb2cc525f5ef4fa43c80e771636dd3690f9d2
+F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
-P 5238a74987507f27af4080e2842f53e049b5b111
+P 1dada5158215d1816edb69ff2610f9d2259ce19d eaf447ea87b0ff29ae06283204f522fcd005b284
-R 68d004e4a18ddd9b15416c3d56993539
+R eae2c984d609130c06d9d7e2a5d61977
-U mistachkin
+T *branch * stat3-trunk
-Z cf5bdd8db49b4512fd908f85b844bfec
+T *sym-stat3-trunk *
+T -sym-trunk *
+U drh
+Z cd6356200476d143c6be69191b0e280a

manifest.uuid

@@ -1 +1 @@
-1dada5158215d1816edb69ff2610f9d2259ce19d
+63f2c7859fa6e5d0e2cdd218ff52a3ec2d44c61d

src/analyze.c

@@ -10,6 +10,96 @@
 **
 *************************************************************************
 ** This file contains code associated with the ANALYZE command.
+**
+** The ANALYZE command gather statistics about the content of tables
+** and indices.  These statistics are made available to the query planner
+** to help it make better decisions about how to perform queries.
+**
+** The following system tables are or have been supported:
+**
+**    CREATE TABLE sqlite_stat1(tbl, idx, stat);
+**    CREATE TABLE sqlite_stat2(tbl, idx, sampleno, sample);
+**    CREATE TABLE sqlite_stat3(tbl, idx, nEq, nLt, nDLt, sample);
+**
+** Additional tables might be added in future releases of SQLite.
+** The sqlite_stat2 table is not created or used unless the SQLite version
+** is between 3.6.18 and 3.7.7, inclusive, and unless SQLite is compiled
+** with SQLITE_ENABLE_STAT2.  The sqlite_stat2 table is deprecated.
+** The sqlite_stat2 table is superceded by sqlite_stat3, which is only
+** created and used by SQLite versions after 2011-08-09 with
+** SQLITE_ENABLE_STAT3 defined.  The fucntionality of sqlite_stat3
+** is a superset of sqlite_stat2.  
+**
+** Format of sqlite_stat1:
+**
+** There is normally one row per index, with the index identified by the
+** name in the idx column.  The tbl column is the name of the table to
+** which the index belongs.  In each such row, the stat column will be
+** a string consisting of a list of integers.  The first integer in this
+** list is the number of rows in the index and in the table.  The second
+** integer is the average number of rows in the index that have the same
+** value in the first column of the index.  The third integer is the average
+** number of rows in the index that have the same value for the first two
+** columns.  The N-th integer (for N>1) is the average number of rows in 
+** the index which have the same value for the first N-1 columns.  For
+** a K-column index, there will be K+1 integers in the stat column.  If
+** the index is unique, then the last integer will be 1.
+**
+** The list of integers in the stat column can optionally be followed
+** by the keyword "unordered".  The "unordered" keyword, if it is present,
+** must be separated from the last integer by a single space.  If the
+** "unordered" keyword is present, then the query planner assumes that
+** the index is unordered and will not use the index for a range query.
+** 
+** If the sqlite_stat1.idx column is NULL, then the sqlite_stat1.stat
+** column contains a single integer which is the (estimated) number of
+** rows in the table identified by sqlite_stat1.tbl.
+**
+** Format of sqlite_stat2:
+**
+** The sqlite_stat2 is only created and is only used if SQLite is compiled
+** with SQLITE_ENABLE_STAT2 and if the SQLite version number is between
+** 3.6.18 and 3.7.7.  The "stat2" table contains additional information
+** about the distribution of keys within an index.  The index is identified by
+** the "idx" column and the "tbl" column is the name of the table to which
+** the index belongs.  There are usually 10 rows in the sqlite_stat2
+** table for each index.
+**
+** The sqlite_stat2 entries for an index that have sampleno between 0 and 9
+** inclusive are samples of the left-most key value in the index taken at
+** evenly spaced points along the index.  Let the number of samples be S
+** (10 in the standard build) and let C be the number of rows in the index.
+** Then the sampled rows are given by:
+**
+**     rownumber = (i*C*2 + C)/(S*2)
+**
+** For i between 0 and S-1.  Conceptually, the index space is divided into
+** S uniform buckets and the samples are the middle row from each bucket.
+**
+** The format for sqlite_stat2 is recorded here for legacy reference.  This
+** version of SQLite does not support sqlite_stat2.  It neither reads nor
+** writes the sqlite_stat2 table.  This version of SQLite only supports
+** sqlite_stat3.
+**
+** Format for sqlite_stat3:
+**
+** The sqlite_stat3 is an enhancement to sqlite_stat2.  A new name is
+** used to avoid compatibility problems.  
+**
+** The format of the sqlite_stat3 table is similar to the format for
+** the sqlite_stat2 table, with the following changes:  (1)
+** The sampleno column is removed.  (2) Every sample has nEq, nLt, and nDLt
+** columns which hold the approximate number of rows in the table that
+** exactly match the sample, the approximate number of rows with values
+** less than the sample, and the approximate number of distinct key values
+** less than the sample, respectively.  (3) The number of samples can vary 
+** from one table to the next; the sample count does not have to be 
+** exactly 10 as it is with sqlite_stat2.
+**
+** The ANALYZE command will typically generate sqlite_stat3 tables
+** that contain between 10 and 40 samples which are distributed across
+** the key space, though not uniformly, and which include samples with
+** largest possible nEq values.
 */
 #ifndef SQLITE_OMIT_ANALYZE
 #include "sqliteInt.h"
@@ -42,8 +132,14 @@ static void openStatTable(
     const char *zCols;
   } aTable[] = {
     { "sqlite_stat1", "tbl,idx,stat" },
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
-    { "sqlite_stat2", "tbl,idx,sampleno,sample" },
+    { "sqlite_stat3", "tbl,idx,neq,nlt,ndlt,sample" },
+#endif
+  };
+  static const char *azToDrop[] = { 
+    "sqlite_stat2",
+#ifndef SQLITE_ENABLE_STAT3
+    "sqlite_stat3",
 #endif
   };
 
@@ -59,6 +155,17 @@ static void openStatTable(
   assert( sqlite3VdbeDb(v)==db );
   pDb = &db->aDb[iDb];
 
+  /* Drop all statistics tables that this version of SQLite does not
+  ** understand.
+  */
+  for(i=0; i<ArraySize(azToDrop); i++){
+    Table *pTab = sqlite3FindTable(db, azToDrop[i], pDb->zName);
+    if( pTab ) sqlite3CodeDropTable(pParse, pTab, iDb, 0);
+  }
+
+  /* Create new statistic tables if they do not exist, or clear them
+  ** if they do already exist.
+  */
   for(i=0; i<ArraySize(aTable); i++){
     const char *zTab = aTable[i].zName;
     Table *pStat;
@@ -89,7 +196,7 @@ static void openStatTable(
     }
   }
 
-  /* Open the sqlite_stat[12] tables for writing. */
+  /* Open the sqlite_stat[13] tables for writing. */
   for(i=0; i<ArraySize(aTable); i++){
     sqlite3VdbeAddOp3(v, OP_OpenWrite, iStatCur+i, aRoot[i], iDb);
     sqlite3VdbeChangeP4(v, -1, (char *)3, P4_INT32);
@@ -97,6 +204,227 @@ static void openStatTable(
   }
 }
 
+/*
+** Recommended number of samples for sqlite_stat3
+*/
+#ifndef SQLITE_STAT3_SAMPLES
+# define SQLITE_STAT3_SAMPLES 24
+#endif
+
+/*
+** Three SQL functions - stat3_init(), stat3_push(), and stat3_pop() -
+** share an instance of the following structure to hold their state
+** information.
+*/
+typedef struct Stat3Accum Stat3Accum;
+struct Stat3Accum {
+  tRowcnt nRow;             /* Number of rows in the entire table */
+  tRowcnt nPSample;         /* How often to do a periodic sample */
+  int iMin;                 /* Index of entry with minimum nEq and hash */
+  int mxSample;             /* Maximum number of samples to accumulate */
+  int nSample;              /* Current number of samples */
+  u32 iPrn;                 /* Pseudo-random number used for sampling */
+  struct Stat3Sample {
+    i64 iRowid;                /* Rowid in main table of the key */
+    tRowcnt nEq;               /* sqlite_stat3.nEq */
+    tRowcnt nLt;               /* sqlite_stat3.nLt */
+    tRowcnt nDLt;              /* sqlite_stat3.nDLt */
+    u8 isPSample;              /* True if a periodic sample */
+    u32 iHash;                 /* Tiebreaker hash */
+  } *a;                     /* An array of samples */
+};
+
+#ifdef SQLITE_ENABLE_STAT3
+/*
+** Implementation of the stat3_init(C,S) SQL function.  The two parameters
+** are the number of rows in the table or index (C) and the number of samples
+** to accumulate (S).
+**
+** This routine allocates the Stat3Accum object.
+**
+** The return value is the Stat3Accum object (P).
+*/
+static void stat3Init(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  Stat3Accum *p;
+  tRowcnt nRow;
+  int mxSample;
+  int n;
+
+  UNUSED_PARAMETER(argc);
+  nRow = (tRowcnt)sqlite3_value_int64(argv[0]);
+  mxSample = sqlite3_value_int(argv[1]);
+  n = sizeof(*p) + sizeof(p->a[0])*mxSample;
+  p = sqlite3_malloc( n );
+  if( p==0 ){
+    sqlite3_result_error_nomem(context);
+    return;
+  }
+  memset(p, 0, n);
+  p->a = (struct Stat3Sample*)&p[1];
+  p->nRow = nRow;
+  p->mxSample = mxSample;
+  p->nPSample = p->nRow/(mxSample/3+1) + 1;
+  sqlite3_randomness(sizeof(p->iPrn), &p->iPrn);
+  sqlite3_result_blob(context, p, sizeof(p), sqlite3_free);
+}
+static const FuncDef stat3InitFuncdef = {
+  2,                /* nArg */
+  SQLITE_UTF8,      /* iPrefEnc */
+  0,                /* flags */
+  0,                /* pUserData */
+  0,                /* pNext */
+  stat3Init,        /* xFunc */
+  0,                /* xStep */
+  0,                /* xFinalize */
+  "stat3_init",     /* zName */
+  0,                /* pHash */
+  0                 /* pDestructor */
+};
+
+
+/*
+** Implementation of the stat3_push(nEq,nLt,nDLt,rowid,P) SQL function.  The
+** arguments describe a single key instance.  This routine makes the 
+** decision about whether or not to retain this key for the sqlite_stat3
+** table.
+**
+** The return value is NULL.
+*/
+static void stat3Push(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  Stat3Accum *p = (Stat3Accum*)sqlite3_value_blob(argv[4]);
+  tRowcnt nEq = sqlite3_value_int64(argv[0]);
+  tRowcnt nLt = sqlite3_value_int64(argv[1]);
+  tRowcnt nDLt = sqlite3_value_int64(argv[2]);
+  i64 rowid = sqlite3_value_int64(argv[3]);
+  u8 isPSample = 0;
+  u8 doInsert = 0;
+  int iMin = p->iMin;
+  struct Stat3Sample *pSample;
+  int i;
+  u32 h;
+
+  UNUSED_PARAMETER(context);
+  UNUSED_PARAMETER(argc);
+  if( nEq==0 ) return;
+  h = p->iPrn = p->iPrn*1103515245 + 12345;
+  if( (nLt/p->nPSample)!=((nEq+nLt)/p->nPSample) ){
+    doInsert = isPSample = 1;
+  }else if( p->nSample<p->mxSample ){
+    doInsert = 1;
+  }else{
+    if( nEq>p->a[iMin].nEq || (nEq==p->a[iMin].nEq && h>p->a[iMin].iHash) ){
+      doInsert = 1;
+    }
+  }
+  if( !doInsert ) return;
+  if( p->nSample==p->mxSample ){
+    if( iMin<p->nSample ){
+      memcpy(&p->a[iMin], &p->a[iMin+1], sizeof(p->a[0])*(p->nSample-iMin));
+    }
+    pSample = &p->a[p->nSample-1];
+  }else{
+    pSample = &p->a[p->nSample++];
+  }
+  pSample->iRowid = rowid;
+  pSample->nEq = nEq;
+  pSample->nLt = nLt;
+  pSample->nDLt = nDLt;
+  pSample->iHash = h;
+  pSample->isPSample = isPSample;
+
+  /* Find the new minimum */
+  if( p->nSample==p->mxSample ){
+    pSample = p->a;
+    i = 0;
+    while( pSample->isPSample ){
+      i++;
+      pSample++;
+      assert( i<p->nSample );
+    }
+    nEq = pSample->nEq;
+    h = pSample->iHash;
+    iMin = i;
+    for(i++, pSample++; i<p->nSample; i++, pSample++){
+      if( pSample->isPSample ) continue;
+      if( pSample->nEq<nEq
+       || (pSample->nEq==nEq && pSample->iHash<h)
+      ){
+        iMin = i;
+        nEq = pSample->nEq;
+        h = pSample->iHash;
+      }
+    }
+    p->iMin = iMin;
+  }
+}
+static const FuncDef stat3PushFuncdef = {
+  5,                /* nArg */
+  SQLITE_UTF8,      /* iPrefEnc */
+  0,                /* flags */
+  0,                /* pUserData */
+  0,                /* pNext */
+  stat3Push,        /* xFunc */
+  0,                /* xStep */
+  0,                /* xFinalize */
+  "stat3_push",     /* zName */
+  0,                /* pHash */
+  0                 /* pDestructor */
+};
+
+/*
+** Implementation of the stat3_get(P,N,...) SQL function.  This routine is
+** used to query the results.  Content is returned for the Nth sqlite_stat3
+** row where N is between 0 and S-1 and S is the number of samples.  The
+** value returned depends on the number of arguments.
+**
+**   argc==2    result:  rowid
+**   argc==3    result:  nEq
+**   argc==4    result:  nLt
+**   argc==5    result:  nDLt
+*/
+static void stat3Get(
+  sqlite3_context *context,
+  int argc,
+  sqlite3_value **argv
+){
+  int n = sqlite3_value_int(argv[1]);
+  Stat3Accum *p = (Stat3Accum*)sqlite3_value_blob(argv[0]);
+
+  assert( p!=0 );
+  if( p->nSample<=n ) return;
+  switch( argc ){
+    case 2: sqlite3_result_int64(context, p->a[n].iRowid); break;
+    case 3: sqlite3_result_int64(context, p->a[n].nEq);    break;
+    case 4: sqlite3_result_int64(context, p->a[n].nLt);    break;
+    case 5: sqlite3_result_int64(context, p->a[n].nDLt);   break;
+  }
+}
+static const FuncDef stat3GetFuncdef = {
+  -1,               /* nArg */
+  SQLITE_UTF8,      /* iPrefEnc */
+  0,                /* flags */
+  0,                /* pUserData */
+  0,                /* pNext */
+  stat3Get,         /* xFunc */
+  0,                /* xStep */
+  0,                /* xFinalize */
+  "stat3_get",     /* zName */
+  0,                /* pHash */
+  0                 /* pDestructor */
+};
+#endif /* SQLITE_ENABLE_STAT3 */
+
+
+
+
 /*
 ** Generate code to do an analysis of all indices associated with
 ** a single table.
@@ -119,20 +447,27 @@ static void analyzeOneTable(
   int iDb;                     /* Index of database containing pTab */
   int regTabname = iMem++;     /* Register containing table name */
   int regIdxname = iMem++;     /* Register containing index name */
-  int regSampleno = iMem++;    /* Register containing next sample number */
+  int regStat1 = iMem++;       /* The stat column of sqlite_stat1 */
-  int regCol = iMem++;         /* Content of a column analyzed table */
+#ifdef SQLITE_ENABLE_STAT3
+  int regNumEq = regStat1;     /* Number of instances.  Same as regStat1 */
+  int regNumLt = iMem++;       /* Number of keys less than regSample */
+  int regNumDLt = iMem++;      /* Number of distinct keys less than regSample */
+  int regSample = iMem++;      /* The next sample value */
+  int regRowid = regSample;    /* Rowid of a sample */
+  int regAccum = iMem++;       /* Register to hold Stat3Accum object */
+  int regLoop = iMem++;        /* Loop counter */
+  int regCount = iMem++;       /* Number of rows in the table or index */
+  int regTemp1 = iMem++;       /* Intermediate register */
+  int regTemp2 = iMem++;       /* Intermediate register */
+  int once = 1;                /* One-time initialization */
+  int shortJump = 0;           /* Instruction address */
+  int iTabCur = pParse->nTab++; /* Table cursor */
+#endif
+  int regCol = iMem++;         /* Content of a column in analyzed table */
   int regRec = iMem++;         /* Register holding completed record */
   int regTemp = iMem++;        /* Temporary use register */
-  int regRowid = iMem++;       /* Rowid for the inserted record */
+  int regNewRowid = iMem++;    /* Rowid for the inserted record */
 
-#ifdef SQLITE_ENABLE_STAT2
-  int addr = 0;                /* Instruction address */
-  int regTemp2 = iMem++;       /* Temporary use register */
-  int regSamplerecno = iMem++; /* Index of next sample to record */
-  int regRecno = iMem++;       /* Current sample index */
-  int regLast = iMem++;        /* Index of last sample to record */
-  int regFirst = iMem++;       /* Index of first sample to record */
-#endif
 
   v = sqlite3GetVdbe(pParse);
   if( v==0 || NEVER(pTab==0) ){
@@ -165,9 +500,14 @@ static void analyzeOneTable(
   for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
     int nCol;
     KeyInfo *pKey;
+    int addrIfNot = 0;           /* address of OP_IfNot */
+    int *aChngAddr;              /* Array of jump instruction addresses */
 
     if( pOnlyIdx && pOnlyIdx!=pIdx ) continue;
+    VdbeNoopComment((v, "Begin analysis of %s", pIdx->zName));
     nCol = pIdx->nColumn;
+    aChngAddr = sqlite3DbMallocRaw(db, sizeof(int)*nCol);
+    if( aChngAddr==0 ) continue;
     pKey = sqlite3IndexKeyinfo(pParse, pIdx);
     if( iMem+1+(nCol*2)>pParse->nMem ){
       pParse->nMem = iMem+1+(nCol*2);
@@ -182,31 +522,20 @@ static void analyzeOneTable(
     /* Populate the register containing the index name. */
     sqlite3VdbeAddOp4(v, OP_String8, 0, regIdxname, 0, pIdx->zName, 0);
 
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
-
+    if( once ){
-    /* If this iteration of the loop is generating code to analyze the
+      once = 0;
-    ** first index in the pTab->pIndex list, then register regLast has
+      sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead);
-    ** not been populated. In this case populate it now.  */
-    if( pTab->pIndex==pIdx ){
-      sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES, regSamplerecno);
-      sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES*2-1, regTemp);
-      sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES*2, regTemp2);
-
-      sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regLast);
-      sqlite3VdbeAddOp2(v, OP_Null, 0, regFirst);
-      addr = sqlite3VdbeAddOp3(v, OP_Lt, regSamplerecno, 0, regLast);
-      sqlite3VdbeAddOp3(v, OP_Divide, regTemp2, regLast, regFirst);
-      sqlite3VdbeAddOp3(v, OP_Multiply, regLast, regTemp, regLast);
-      sqlite3VdbeAddOp2(v, OP_AddImm, regLast, SQLITE_INDEX_SAMPLES*2-2);
-      sqlite3VdbeAddOp3(v, OP_Divide,  regTemp2, regLast, regLast);
-      sqlite3VdbeJumpHere(v, addr);
     }
-
+    sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regCount);
-    /* Zero the regSampleno and regRecno registers. */
+    sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_STAT3_SAMPLES, regTemp1);
-    sqlite3VdbeAddOp2(v, OP_Integer, 0, regSampleno);
+    sqlite3VdbeAddOp2(v, OP_Integer, 0, regNumEq);
-    sqlite3VdbeAddOp2(v, OP_Integer, 0, regRecno);
+    sqlite3VdbeAddOp2(v, OP_Integer, 0, regNumLt);
-    sqlite3VdbeAddOp2(v, OP_Copy, regFirst, regSamplerecno);
+    sqlite3VdbeAddOp2(v, OP_Integer, -1, regNumDLt);
-#endif
+    sqlite3VdbeAddOp4(v, OP_Function, 1, regCount, regAccum,
+                      (char*)&stat3InitFuncdef, P4_FUNCDEF);
+    sqlite3VdbeChangeP5(v, 2);
+#endif /* SQLITE_ENABLE_STAT3 */
 
     /* The block of memory cells initialized here is used as follows.
     **
@@ -236,75 +565,83 @@ static void analyzeOneTable(
     endOfLoop = sqlite3VdbeMakeLabel(v);
     sqlite3VdbeAddOp2(v, OP_Rewind, iIdxCur, endOfLoop);
     topOfLoop = sqlite3VdbeCurrentAddr(v);
-    sqlite3VdbeAddOp2(v, OP_AddImm, iMem, 1);
+    sqlite3VdbeAddOp2(v, OP_AddImm, iMem, 1);  /* Increment row counter */
 
     for(i=0; i<nCol; i++){
       CollSeq *pColl;
       sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regCol);
       if( i==0 ){
-#ifdef SQLITE_ENABLE_STAT2
-        /* Check if the record that cursor iIdxCur points to contains a
-        ** value that should be stored in the sqlite_stat2 table. If so,
-        ** store it.  */
-        int ne = sqlite3VdbeAddOp3(v, OP_Ne, regRecno, 0, regSamplerecno);
-        assert( regTabname+1==regIdxname 
-             && regTabname+2==regSampleno
-             && regTabname+3==regCol
-        );
-        sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL);
-        sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 4, regRec, "aaab", 0);
-        sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regRowid);
-        sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regRec, regRowid);
-
-        /* Calculate new values for regSamplerecno and regSampleno.
-        **
-        **   sampleno = sampleno + 1
-        **   samplerecno = samplerecno+(remaining records)/(remaining samples)
-        */
-        sqlite3VdbeAddOp2(v, OP_AddImm, regSampleno, 1);
-        sqlite3VdbeAddOp3(v, OP_Subtract, regRecno, regLast, regTemp);
-        sqlite3VdbeAddOp2(v, OP_AddImm, regTemp, -1);
-        sqlite3VdbeAddOp2(v, OP_Integer, SQLITE_INDEX_SAMPLES, regTemp2);
-        sqlite3VdbeAddOp3(v, OP_Subtract, regSampleno, regTemp2, regTemp2);
-        sqlite3VdbeAddOp3(v, OP_Divide, regTemp2, regTemp, regTemp);
-        sqlite3VdbeAddOp3(v, OP_Add, regSamplerecno, regTemp, regSamplerecno);
-
-        sqlite3VdbeJumpHere(v, ne);
-        sqlite3VdbeAddOp2(v, OP_AddImm, regRecno, 1);
-#endif
-
         /* Always record the very first row */
-        sqlite3VdbeAddOp1(v, OP_IfNot, iMem+1);
+        addrIfNot = sqlite3VdbeAddOp1(v, OP_IfNot, iMem+1);
       }
       assert( pIdx->azColl!=0 );
       assert( pIdx->azColl[i]!=0 );
       pColl = sqlite3LocateCollSeq(pParse, pIdx->azColl[i]);
-      sqlite3VdbeAddOp4(v, OP_Ne, regCol, 0, iMem+nCol+i+1,
+      aChngAddr[i] = sqlite3VdbeAddOp4(v, OP_Ne, regCol, 0, iMem+nCol+i+1,
                                       (char*)pColl, P4_COLLSEQ);
       sqlite3VdbeChangeP5(v, SQLITE_NULLEQ);
+      VdbeComment((v, "jump if column %d changed", i));
+#ifdef SQLITE_ENABLE_STAT3
+      if( i==0 ){
+        sqlite3VdbeAddOp2(v, OP_AddImm, regNumEq, 1);
+        VdbeComment((v, "incr repeat count"));
       }
-    if( db->mallocFailed ){
+#endif
-      /* If a malloc failure has occurred, then the result of the expression 
-      ** passed as the second argument to the call to sqlite3VdbeJumpHere() 
-      ** below may be negative. Which causes an assert() to fail (or an
-      ** out-of-bounds write if SQLITE_DEBUG is not defined).  */
-      return;
     }
     sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfLoop);
     for(i=0; i<nCol; i++){
-      int addr2 = sqlite3VdbeCurrentAddr(v) - (nCol*2);
+      sqlite3VdbeJumpHere(v, aChngAddr[i]);  /* Set jump dest for the OP_Ne */
       if( i==0 ){
-        sqlite3VdbeJumpHere(v, addr2-1);  /* Set jump dest for the OP_IfNot */
+        sqlite3VdbeJumpHere(v, addrIfNot);   /* Jump dest for OP_IfNot */
+#ifdef SQLITE_ENABLE_STAT3
+        sqlite3VdbeAddOp4(v, OP_Function, 1, regNumEq, regTemp2,
+                          (char*)&stat3PushFuncdef, P4_FUNCDEF);
+        sqlite3VdbeChangeP5(v, 5);
+        sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, pIdx->nColumn, regRowid);
+        sqlite3VdbeAddOp3(v, OP_Add, regNumEq, regNumLt, regNumLt);
+        sqlite3VdbeAddOp2(v, OP_AddImm, regNumDLt, 1);
+        sqlite3VdbeAddOp2(v, OP_Integer, 1, regNumEq);
+#endif        
       }
-      sqlite3VdbeJumpHere(v, addr2);      /* Set jump dest for the OP_Ne */
       sqlite3VdbeAddOp2(v, OP_AddImm, iMem+i+1, 1);
       sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, iMem+nCol+i+1);
     }
+    sqlite3DbFree(db, aChngAddr);
 
-    /* End of the analysis loop. */
+    /* Always jump here after updating the iMem+1...iMem+1+nCol counters */
     sqlite3VdbeResolveLabel(v, endOfLoop);
+
     sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, topOfLoop);
     sqlite3VdbeAddOp1(v, OP_Close, iIdxCur);
+#ifdef SQLITE_ENABLE_STAT3
+    sqlite3VdbeAddOp4(v, OP_Function, 1, regNumEq, regTemp2,
+                      (char*)&stat3PushFuncdef, P4_FUNCDEF);
+    sqlite3VdbeChangeP5(v, 5);
+    sqlite3VdbeAddOp2(v, OP_Integer, -1, regLoop);
+    shortJump = 
+    sqlite3VdbeAddOp2(v, OP_AddImm, regLoop, 1);
+    sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regTemp1,
+                      (char*)&stat3GetFuncdef, P4_FUNCDEF);
+    sqlite3VdbeChangeP5(v, 2);
+    sqlite3VdbeAddOp1(v, OP_IsNull, regTemp1);
+    sqlite3VdbeAddOp3(v, OP_NotExists, iTabCur, shortJump, regTemp1);
+    sqlite3VdbeAddOp3(v, OP_Column, iTabCur, pIdx->aiColumn[0], regSample);
+    sqlite3ColumnDefault(v, pTab, pIdx->aiColumn[0], regSample);
+    sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regNumEq,
+                      (char*)&stat3GetFuncdef, P4_FUNCDEF);
+    sqlite3VdbeChangeP5(v, 3);
+    sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regNumLt,
+                      (char*)&stat3GetFuncdef, P4_FUNCDEF);
+    sqlite3VdbeChangeP5(v, 4);
+    sqlite3VdbeAddOp4(v, OP_Function, 1, regAccum, regNumDLt,
+                      (char*)&stat3GetFuncdef, P4_FUNCDEF);
+    sqlite3VdbeChangeP5(v, 5);
+    sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 6, regRec, "bbbbbb", 0);
+    sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regNewRowid);
+    sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regRec, regNewRowid);
+    sqlite3VdbeAddOp2(v, OP_Goto, 0, shortJump);
+    sqlite3VdbeJumpHere(v, shortJump+2);
+#endif        
 
     /* Store the results in sqlite_stat1.
     **
@@ -324,22 +661,22 @@ static void analyzeOneTable(
     ** If K>0 then it is always the case the D>0 so division by zero
     ** is never possible.
     */
-    sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regSampleno);
+    sqlite3VdbeAddOp2(v, OP_SCopy, iMem, regStat1);
     if( jZeroRows<0 ){
       jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, iMem);
     }
     for(i=0; i<nCol; i++){
       sqlite3VdbeAddOp4(v, OP_String8, 0, regTemp, 0, " ", 0);
-      sqlite3VdbeAddOp3(v, OP_Concat, regTemp, regSampleno, regSampleno);
+      sqlite3VdbeAddOp3(v, OP_Concat, regTemp, regStat1, regStat1);
       sqlite3VdbeAddOp3(v, OP_Add, iMem, iMem+i+1, regTemp);
       sqlite3VdbeAddOp2(v, OP_AddImm, regTemp, -1);
       sqlite3VdbeAddOp3(v, OP_Divide, iMem+i+1, regTemp, regTemp);
       sqlite3VdbeAddOp1(v, OP_ToInt, regTemp);
-      sqlite3VdbeAddOp3(v, OP_Concat, regTemp, regSampleno, regSampleno);
+      sqlite3VdbeAddOp3(v, OP_Concat, regTemp, regStat1, regStat1);
     }
     sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0);
-    sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid);
+    sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid);
-    sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid);
+    sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regNewRowid);
     sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
   }
 
@@ -349,22 +686,23 @@ static void analyzeOneTable(
   if( pTab->pIndex==0 ){
     sqlite3VdbeAddOp3(v, OP_OpenRead, iIdxCur, pTab->tnum, iDb);
     VdbeComment((v, "%s", pTab->zName));
-    sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regSampleno);
+    sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat1);
     sqlite3VdbeAddOp1(v, OP_Close, iIdxCur);
-    jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regSampleno);
+    jZeroRows = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1);
   }else{
     sqlite3VdbeJumpHere(v, jZeroRows);
     jZeroRows = sqlite3VdbeAddOp0(v, OP_Goto);
   }
   sqlite3VdbeAddOp2(v, OP_Null, 0, regIdxname);
   sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regRec, "aaa", 0);
-  sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid);
+  sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid);
-  sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid);
+  sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regNewRowid);
   sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
   if( pParse->nMem<regRec ) pParse->nMem = regRec;
   sqlite3VdbeJumpHere(v, jZeroRows);
 }
 
+
 /*
 ** Generate code that will cause the most recent index analysis to
 ** be loaded into internal hash tables where is can be used.
@@ -388,7 +726,7 @@ static void analyzeDatabase(Parse *pParse, int iDb){
 
   sqlite3BeginWriteOperation(pParse, 0, iDb);
   iStatCur = pParse->nTab;
-  pParse->nTab += 2;
+  pParse->nTab += 3;
   openStatTable(pParse, iDb, iStatCur, 0, 0);
   iMem = pParse->nMem+1;
   assert( sqlite3SchemaMutexHeld(db, iDb, 0) );
@@ -413,7 +751,7 @@ static void analyzeTable(Parse *pParse, Table *pTab, Index *pOnlyIdx){
   iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
   sqlite3BeginWriteOperation(pParse, 0, iDb);
   iStatCur = pParse->nTab;
-  pParse->nTab += 2;
+  pParse->nTab += 3;
   if( pOnlyIdx ){
     openStatTable(pParse, iDb, iStatCur, pOnlyIdx->zName, "idx");
   }else{
@@ -518,7 +856,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){
   Index *pIndex;
   Table *pTable;
   int i, c, n;
-  unsigned int v;
+  tRowcnt v;
   const char *z;
 
   assert( argc==3 );
@@ -561,10 +899,10 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){
 ** and its contents.
 */
 void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
   if( pIdx->aSample ){
     int j;
-    for(j=0; j<SQLITE_INDEX_SAMPLES; j++){
+    for(j=0; j<pIdx->nSample; j++){
       IndexSample *p = &pIdx->aSample[j];
       if( p->eType==SQLITE_TEXT || p->eType==SQLITE_BLOB ){
         sqlite3DbFree(db, p->u.z);
@@ -572,25 +910,159 @@ void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){
     }
     sqlite3DbFree(db, pIdx->aSample);
   }
+  if( db && db->pnBytesFreed==0 ){
+    pIdx->nSample = 0;
+    pIdx->aSample = 0;
+  }
 #else
   UNUSED_PARAMETER(db);
   UNUSED_PARAMETER(pIdx);
 #endif
 }
 
+#ifdef SQLITE_ENABLE_STAT3
 /*
-** Load the content of the sqlite_stat1 and sqlite_stat2 tables. The
+** Load content from the sqlite_stat3 table into the Index.aSample[]
+** arrays of all indices.
+*/
+static int loadStat3(sqlite3 *db, const char *zDb){
+  int rc;                       /* Result codes from subroutines */
+  sqlite3_stmt *pStmt = 0;      /* An SQL statement being run */
+  char *zSql;                   /* Text of the SQL statement */
+  Index *pPrevIdx = 0;          /* Previous index in the loop */
+  int idx = 0;                  /* slot in pIdx->aSample[] for next sample */
+  int eType;                    /* Datatype of a sample */
+  IndexSample *pSample;         /* A slot in pIdx->aSample[] */
+
+  if( !sqlite3FindTable(db, "sqlite_stat3", zDb) ){
+    return SQLITE_OK;
+  }
+
+  zSql = sqlite3MPrintf(db, 
+      "SELECT idx,count(*) FROM %Q.sqlite_stat3"
+      " GROUP BY idx", zDb);
+  if( !zSql ){
+    return SQLITE_NOMEM;
+  }
+  rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
+  sqlite3DbFree(db, zSql);
+  if( rc ) return rc;
+
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    char *zIndex;   /* Index name */
+    Index *pIdx;    /* Pointer to the index object */
+    int nSample;    /* Number of samples */
+
+    zIndex = (char *)sqlite3_column_text(pStmt, 0);
+    if( zIndex==0 ) continue;
+    nSample = sqlite3_column_int(pStmt, 1);
+    if( nSample>255 ) continue;
+    pIdx = sqlite3FindIndex(db, zIndex, zDb);
+    if( pIdx==0 ) continue;
+    assert( pIdx->nSample==0 );
+    pIdx->nSample = (u8)nSample;
+    pIdx->aSample = sqlite3MallocZero( nSample*sizeof(IndexSample) );
+    pIdx->avgEq = pIdx->aiRowEst[1];
+    if( pIdx->aSample==0 ){
+      db->mallocFailed = 1;
+      sqlite3_finalize(pStmt);
+      return SQLITE_NOMEM;
+    }
+  }
+  rc = sqlite3_finalize(pStmt);
+  if( rc ) return rc;
+
+  zSql = sqlite3MPrintf(db, 
+      "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat3", zDb);
+  if( !zSql ){
+    return SQLITE_NOMEM;
+  }
+  rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
+  sqlite3DbFree(db, zSql);
+  if( rc ) return rc;
+
+  while( sqlite3_step(pStmt)==SQLITE_ROW ){
+    char *zIndex;   /* Index name */
+    Index *pIdx;    /* Pointer to the index object */
+    int i;          /* Loop counter */
+    tRowcnt sumEq;  /* Sum of the nEq values */
+
+    zIndex = (char *)sqlite3_column_text(pStmt, 0);
+    if( zIndex==0 ) continue;
+    pIdx = sqlite3FindIndex(db, zIndex, zDb);
+    if( pIdx==0 ) continue;
+    if( pIdx==pPrevIdx ){
+      idx++;
+    }else{
+      pPrevIdx = pIdx;
+      idx = 0;
+    }
+    assert( idx<pIdx->nSample );
+    pSample = &pIdx->aSample[idx];
+    pSample->nEq = (tRowcnt)sqlite3_column_int64(pStmt, 1);
+    pSample->nLt = (tRowcnt)sqlite3_column_int64(pStmt, 2);
+    pSample->nDLt = (tRowcnt)sqlite3_column_int64(pStmt, 3);
+    if( idx==pIdx->nSample-1 ){
+      if( pSample->nDLt>0 ){
+        for(i=0, sumEq=0; i<=idx-1; i++) sumEq += pIdx->aSample[i].nEq;
+        pIdx->avgEq = (pSample->nLt - sumEq)/pSample->nDLt;
+      }
+      if( pIdx->avgEq<=0 ) pIdx->avgEq = 1;
+    }
+    eType = sqlite3_column_type(pStmt, 4);
+    pSample->eType = (u8)eType;
+    switch( eType ){
+      case SQLITE_INTEGER: {
+        pSample->u.i = sqlite3_column_int64(pStmt, 4);
+        break;
+      }
+      case SQLITE_FLOAT: {
+        pSample->u.r = sqlite3_column_double(pStmt, 4);
+        break;
+      }
+      case SQLITE_NULL: {
+        break;
+      }
+      default: assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); {
+        const char *z = (const char *)(
+              (eType==SQLITE_BLOB) ?
+              sqlite3_column_blob(pStmt, 4):
+              sqlite3_column_text(pStmt, 4)
+           );
+        int n = z ? sqlite3_column_bytes(pStmt, 4) : 0;
+        if( n>0xffff ) n = 0xffff;
+        pSample->nByte = (u16)n;
+        if( n < 1){
+          pSample->u.z = 0;
+        }else{
+          pSample->u.z = sqlite3Malloc(n);
+          if( pSample->u.z==0 ){
+            db->mallocFailed = 1;
+            sqlite3_finalize(pStmt);
+            return SQLITE_NOMEM;
+          }
+          memcpy(pSample->u.z, z, n);
+        }
+      }
+    }
+  }
+  return sqlite3_finalize(pStmt);
+}
+#endif /* SQLITE_ENABLE_STAT3 */
+
+/*
+** Load the content of the sqlite_stat1 and sqlite_stat3 tables. The
 ** contents of sqlite_stat1 are used to populate the Index.aiRowEst[]
-** arrays. The contents of sqlite_stat2 are used to populate the
+** arrays. The contents of sqlite_stat3 are used to populate the
 ** Index.aSample[] arrays.
 **
 ** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR
-** is returned. In this case, even if SQLITE_ENABLE_STAT2 was defined 
+** is returned. In this case, even if SQLITE_ENABLE_STAT3 was defined 
-** during compilation and the sqlite_stat2 table is present, no data is 
+** during compilation and the sqlite_stat3 table is present, no data is 
 ** read from it.
 **
-** If SQLITE_ENABLE_STAT2 was defined during compilation and the 
+** If SQLITE_ENABLE_STAT3 was defined during compilation and the 
-** sqlite_stat2 table is not present in the database, SQLITE_ERROR is
+** sqlite_stat3 table is not present in the database, SQLITE_ERROR is
 ** returned. However, in this case, data is read from the sqlite_stat1
 ** table (if it is present) before returning.
 **
@@ -612,8 +1084,10 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
   for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){
     Index *pIdx = sqliteHashData(i);
     sqlite3DefaultRowEst(pIdx);
+#ifdef SQLITE_ENABLE_STAT3
     sqlite3DeleteIndexSamples(db, pIdx);
     pIdx->aSample = 0;
+#endif
   }
 
   /* Check to make sure the sqlite_stat1 table exists */
@@ -625,7 +1099,7 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
 
   /* Load new statistics out of the sqlite_stat1 table */
   zSql = sqlite3MPrintf(db, 
-      "SELECT tbl, idx, stat FROM %Q.sqlite_stat1", sInfo.zDatabase);
+      "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase);
   if( zSql==0 ){
     rc = SQLITE_NOMEM;
   }else{
@@ -634,78 +1108,10 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
   }
 
 
-  /* Load the statistics from the sqlite_stat2 table. */
+  /* Load the statistics from the sqlite_stat3 table. */
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
-  if( rc==SQLITE_OK && !sqlite3FindTable(db, "sqlite_stat2", sInfo.zDatabase) ){
-    rc = SQLITE_ERROR;
-  }
   if( rc==SQLITE_OK ){
-    sqlite3_stmt *pStmt = 0;
+    rc = loadStat3(db, sInfo.zDatabase);
-
-    zSql = sqlite3MPrintf(db, 
-        "SELECT idx,sampleno,sample FROM %Q.sqlite_stat2", sInfo.zDatabase);
-    if( !zSql ){
-      rc = SQLITE_NOMEM;
-    }else{
-      rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
-      sqlite3DbFree(db, zSql);
-    }
-
-    if( rc==SQLITE_OK ){
-      while( sqlite3_step(pStmt)==SQLITE_ROW ){
-        char *zIndex;   /* Index name */
-        Index *pIdx;    /* Pointer to the index object */
-
-        zIndex = (char *)sqlite3_column_text(pStmt, 0);
-        pIdx = zIndex ? sqlite3FindIndex(db, zIndex, sInfo.zDatabase) : 0;
-        if( pIdx ){
-          int iSample = sqlite3_column_int(pStmt, 1);
-          if( iSample<SQLITE_INDEX_SAMPLES && iSample>=0 ){
-            int eType = sqlite3_column_type(pStmt, 2);
-
-            if( pIdx->aSample==0 ){
-              static const int sz = sizeof(IndexSample)*SQLITE_INDEX_SAMPLES;
-              pIdx->aSample = (IndexSample *)sqlite3DbMallocRaw(0, sz);
-              if( pIdx->aSample==0 ){
-                db->mallocFailed = 1;
-                break;
-              }
-	      memset(pIdx->aSample, 0, sz);
-            }
-
-            assert( pIdx->aSample );
-            {
-              IndexSample *pSample = &pIdx->aSample[iSample];
-              pSample->eType = (u8)eType;
-              if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
-                pSample->u.r = sqlite3_column_double(pStmt, 2);
-              }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){
-                const char *z = (const char *)(
-                    (eType==SQLITE_BLOB) ?
-                    sqlite3_column_blob(pStmt, 2):
-                    sqlite3_column_text(pStmt, 2)
-                );
-                int n = sqlite3_column_bytes(pStmt, 2);
-                if( n>24 ){
-                  n = 24;
-                }
-                pSample->nByte = (u8)n;
-                if( n < 1){
-                  pSample->u.z = 0;
-                }else{
-                  pSample->u.z = sqlite3DbStrNDup(0, z, n);
-                  if( pSample->u.z==0 ){
-                    db->mallocFailed = 1;
-                    break;
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-      rc = sqlite3_finalize(pStmt);
-    }
   }
 #endif
 

src/build.c

@@ -1990,7 +1990,11 @@ static void sqlite3ClearStatTables(
   const char *zType,     /* "idx" or "tbl" */
   const char *zName      /* Name of index or table */
 ){
-  static const char *azStatTab[] = { "sqlite_stat1", "sqlite_stat2" };
+  static const char *azStatTab[] = { 
+    "sqlite_stat1",
+    "sqlite_stat2",
+    "sqlite_stat3",
+  };
   int i;
   const char *zDbName = pParse->db->aDb[iDb].zName;
   for(i=0; i<ArraySize(azStatTab); i++){
@@ -2003,6 +2007,76 @@ static void sqlite3ClearStatTables(
   }
 }
 
+/*
+** Generate code to drop a table.
+*/
+void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, int isView){
+  Vdbe *v;
+  sqlite3 *db = pParse->db;
+  Trigger *pTrigger;
+  Db *pDb = &db->aDb[iDb];
+
+  v = sqlite3GetVdbe(pParse);
+  assert( v!=0 );
+  sqlite3BeginWriteOperation(pParse, 1, iDb);
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+  if( IsVirtual(pTab) ){
+    sqlite3VdbeAddOp0(v, OP_VBegin);
+  }
+#endif
+
+  /* Drop all triggers associated with the table being dropped. Code
+  ** is generated to remove entries from sqlite_master and/or
+  ** sqlite_temp_master if required.
+  */
+  pTrigger = sqlite3TriggerList(pParse, pTab);
+  while( pTrigger ){
+    assert( pTrigger->pSchema==pTab->pSchema || 
+        pTrigger->pSchema==db->aDb[1].pSchema );
+    sqlite3DropTriggerPtr(pParse, pTrigger);
+    pTrigger = pTrigger->pNext;
+  }
+
+#ifndef SQLITE_OMIT_AUTOINCREMENT
+  /* Remove any entries of the sqlite_sequence table associated with
+  ** the table being dropped. This is done before the table is dropped
+  ** at the btree level, in case the sqlite_sequence table needs to
+  ** move as a result of the drop (can happen in auto-vacuum mode).
+  */
+  if( pTab->tabFlags & TF_Autoincrement ){
+    sqlite3NestedParse(pParse,
+      "DELETE FROM %Q.sqlite_sequence WHERE name=%Q",
+      pDb->zName, pTab->zName
+    );
+  }
+#endif
+
+  /* Drop all SQLITE_MASTER table and index entries that refer to the
+  ** table. The program name loops through the master table and deletes
+  ** every row that refers to a table of the same name as the one being
+  ** dropped. Triggers are handled seperately because a trigger can be
+  ** created in the temp database that refers to a table in another
+  ** database.
+  */
+  sqlite3NestedParse(pParse, 
+      "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'",
+      pDb->zName, SCHEMA_TABLE(iDb), pTab->zName);
+  if( !isView && !IsVirtual(pTab) ){
+    destroyTable(pParse, pTab);
+  }
+
+  /* Remove the table entry from SQLite's internal schema and modify
+  ** the schema cookie.
+  */
+  if( IsVirtual(pTab) ){
+    sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0);
+  }
+  sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0);
+  sqlite3ChangeCookie(pParse, iDb);
+  sqliteViewResetAll(db, iDb);
+}
+
 /*
 ** This routine is called to do the work of a DROP TABLE statement.
 ** pName is the name of the table to be dropped.
@@ -2071,7 +2145,7 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){
     }
   }
 #endif
-  if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){
+  if( !pParse->nested && sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){
     sqlite3ErrorMsg(pParse, "table %s may not be dropped", pTab->zName);
     goto exit_drop_table;
   }
@@ -2095,69 +2169,12 @@ void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, int noErr){
   */
   v = sqlite3GetVdbe(pParse);
   if( v ){
-    Trigger *pTrigger;
-    Db *pDb = &db->aDb[iDb];
     sqlite3BeginWriteOperation(pParse, 1, iDb);
-
-#ifndef SQLITE_OMIT_VIRTUALTABLE
-    if( IsVirtual(pTab) ){
-      sqlite3VdbeAddOp0(v, OP_VBegin);
-    }
-#endif
-    sqlite3FkDropTable(pParse, pName, pTab);
-
-    /* Drop all triggers associated with the table being dropped. Code
-    ** is generated to remove entries from sqlite_master and/or
-    ** sqlite_temp_master if required.
-    */
-    pTrigger = sqlite3TriggerList(pParse, pTab);
-    while( pTrigger ){
-      assert( pTrigger->pSchema==pTab->pSchema || 
-          pTrigger->pSchema==db->aDb[1].pSchema );
-      sqlite3DropTriggerPtr(pParse, pTrigger);
-      pTrigger = pTrigger->pNext;
-    }
-
-#ifndef SQLITE_OMIT_AUTOINCREMENT
-    /* Remove any entries of the sqlite_sequence table associated with
-    ** the table being dropped. This is done before the table is dropped
-    ** at the btree level, in case the sqlite_sequence table needs to
-    ** move as a result of the drop (can happen in auto-vacuum mode).
-    */
-    if( pTab->tabFlags & TF_Autoincrement ){
-      sqlite3NestedParse(pParse,
-        "DELETE FROM %s.sqlite_sequence WHERE name=%Q",
-        pDb->zName, pTab->zName
-      );
-    }
-#endif
-
-    /* Drop all SQLITE_MASTER table and index entries that refer to the
-    ** table. The program name loops through the master table and deletes
-    ** every row that refers to a table of the same name as the one being
-    ** dropped. Triggers are handled seperately because a trigger can be
-    ** created in the temp database that refers to a table in another
-    ** database.
-    */
-    sqlite3NestedParse(pParse, 
-        "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'",
-        pDb->zName, SCHEMA_TABLE(iDb), pTab->zName);
     sqlite3ClearStatTables(pParse, iDb, "tbl", pTab->zName);
-    if( !isView && !IsVirtual(pTab) ){
+    sqlite3FkDropTable(pParse, pName, pTab);
-      destroyTable(pParse, pTab);
+    sqlite3CodeDropTable(pParse, pTab, iDb, isView);
   }
 
-    /* Remove the table entry from SQLite's internal schema and modify
-    ** the schema cookie.
-    */
-    if( IsVirtual(pTab) ){
-      sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0);
-    }
-    sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0);
-    sqlite3ChangeCookie(pParse, iDb);
-  }
-  sqliteViewResetAll(db, iDb);
-
 exit_drop_table:
   sqlite3SrcListDelete(db, pName);
 }
@@ -2634,8 +2651,8 @@ Index *sqlite3CreateIndex(
   nCol = pList->nExpr;
   pIndex = sqlite3DbMallocZero(db, 
       sizeof(Index) +              /* Index structure  */
+      sizeof(tRowcnt)*(nCol+1) +   /* Index.aiRowEst   */
       sizeof(int)*nCol +           /* Index.aiColumn   */
-      sizeof(int)*(nCol+1) +       /* Index.aiRowEst   */
       sizeof(char *)*nCol +        /* Index.azColl     */
       sizeof(u8)*nCol +            /* Index.aSortOrder */
       nName + 1 +                  /* Index.zName      */
@@ -2644,10 +2661,10 @@ Index *sqlite3CreateIndex(
   if( db->mallocFailed ){
     goto exit_create_index;
   }
-  pIndex->azColl = (char**)(&pIndex[1]);
+  pIndex->aiRowEst = (tRowcnt*)(&pIndex[1]);
+  pIndex->azColl = (char**)(&pIndex->aiRowEst[nCol+1]);
   pIndex->aiColumn = (int *)(&pIndex->azColl[nCol]);
-  pIndex->aiRowEst = (unsigned *)(&pIndex->aiColumn[nCol]);
+  pIndex->aSortOrder = (u8 *)(&pIndex->aiColumn[nCol]);
-  pIndex->aSortOrder = (u8 *)(&pIndex->aiRowEst[nCol+1]);
   pIndex->zName = (char *)(&pIndex->aSortOrder[nCol]);
   zExtra = (char *)(&pIndex->zName[nName+1]);
   memcpy(pIndex->zName, zName, nName+1);
@@ -2924,9 +2941,9 @@ exit_create_index:
 ** are based on typical values found in actual indices.
 */
 void sqlite3DefaultRowEst(Index *pIdx){
-  unsigned *a = pIdx->aiRowEst;
+  tRowcnt *a = pIdx->aiRowEst;
   int i;
-  unsigned n;
+  tRowcnt n;
   assert( a!=0 );
   a[0] = pIdx->pTable->nRowEst;
   if( a[0]<10 ) a[0] = 10;

src/ctime.c

@@ -117,6 +117,9 @@ static const char * const azCompileOpt[] = {
 #ifdef SQLITE_ENABLE_STAT2
   "ENABLE_STAT2",
 #endif
+#ifdef SQLITE_ENABLE_STAT3
+  "ENABLE_STAT3",
+#endif
 #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY
   "ENABLE_UNLOCK_NOTIFY",
 #endif

src/sqlite.h.in

@@ -2845,7 +2845,7 @@ int sqlite3_limit(sqlite3*, int id, int newVal);
 ** ^The specific value of WHERE-clause [parameter] might influence the 
 ** choice of query plan if the parameter is the left-hand side of a [LIKE]
 ** or [GLOB] operator or if the parameter is compared to an indexed column
-** and the [SQLITE_ENABLE_STAT2] compile-time option is enabled.
+** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled.
 ** the 
 ** </li>
 ** </ol>

src/sqliteInt.h

@@ -459,6 +459,18 @@ typedef INT8_TYPE i8;              /* 1-byte signed integer */
 */
 #define SQLITE_MAX_U32  ((((u64)1)<<32)-1)
 
+/*
+** The datatype used to store estimates of the number of rows in a
+** table or index.  This is an unsigned integer type.  For 99.9% of
+** the world, a 32-bit integer is sufficient.  But a 64-bit integer
+** can be used at compile-time if desired.
+*/
+#ifdef SQLITE_64BIT_STATS
+ typedef u64 tRowcnt;    /* 64-bit only if requested at compile-time */
+#else
+ typedef u32 tRowcnt;    /* 32-bit is the default */
+#endif
+
 /*
 ** Macros to determine whether the machine is big or little endian,
 ** evaluated at runtime.
@@ -1292,7 +1304,7 @@ struct Table {
   Column *aCol;        /* Information about each column */
   Index *pIndex;       /* List of SQL indexes on this table. */
   int tnum;            /* Root BTree node for this table (see note above) */
-  unsigned nRowEst;    /* Estimated rows in table - from sqlite_stat1 table */
+  tRowcnt nRowEst;     /* Estimated rows in table - from sqlite_stat1 table */
   Select *pSelect;     /* NULL for tables.  Points to definition if a view. */
   u16 nRef;            /* Number of pointers to this Table */
   u8 tabFlags;         /* Mask of TF_* values */
@@ -1491,18 +1503,22 @@ struct Index {
   char *zName;     /* Name of this index */
   int nColumn;     /* Number of columns in the table used by this index */
   int *aiColumn;   /* Which columns are used by this index.  1st is 0 */
-  unsigned *aiRowEst; /* Result of ANALYZE: Est. rows selected by each column */
+  tRowcnt *aiRowEst; /* Result of ANALYZE: Est. rows selected by each column */
   Table *pTable;   /* The SQL table being indexed */
   int tnum;        /* Page containing root of this index in database file */
   u8 onError;      /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */
   u8 autoIndex;    /* True if is automatically created (ex: by UNIQUE) */
   u8 bUnordered;   /* Use this index for == or IN queries only */
+  u8 nSample;      /* Number of elements in aSample[] */
   char *zColAff;   /* String defining the affinity of each column */
   Index *pNext;    /* The next index associated with the same table */
   Schema *pSchema; /* Schema containing this index */
   u8 *aSortOrder;  /* Array of size Index.nColumn. True==DESC, False==ASC */
   char **azColl;   /* Array of collation sequence names for index */
-  IndexSample *aSample;    /* Array of SQLITE_INDEX_SAMPLES samples */
+#ifdef SQLITE_ENABLE_STAT3
+  tRowcnt avgEq;           /* Average nEq value for key values not in aSample */
+  IndexSample *aSample;    /* Samples of the left-most key */
+#endif
 };
 
 /*
@@ -1512,10 +1528,14 @@ struct Index {
 struct IndexSample {
   union {
     char *z;        /* Value if eType is SQLITE_TEXT or SQLITE_BLOB */
-    double r;       /* Value if eType is SQLITE_FLOAT or SQLITE_INTEGER */
+    double r;       /* Value if eType is SQLITE_FLOAT */
+    i64 i;          /* Value if eType is SQLITE_INTEGER */
   } u;
   u8 eType;         /* SQLITE_NULL, SQLITE_INTEGER ... etc. */
-  u8 nByte;         /* Size in byte of text or blob. */
+  u16 nByte;        /* Size in byte of text or blob. */
+  tRowcnt nEq;      /* Est. number of rows where the key equals this sample */
+  tRowcnt nLt;      /* Est. number of rows where key is less than this sample */
+  tRowcnt nDLt;     /* Est. number of distinct keys less than this sample */
 };
 
 /*
@@ -2721,6 +2741,7 @@ void sqlite3CreateView(Parse*,Token*,Token*,Token*,Select*,int,int);
 #endif
 
 void sqlite3DropTable(Parse*, SrcList*, int, int);
+void sqlite3CodeDropTable(Parse*, Table*, int, int);
 void sqlite3DeleteTable(sqlite3*, Table*);
 #ifndef SQLITE_OMIT_AUTOINCREMENT
   void sqlite3AutoincrementBegin(Parse *pParse);
@@ -2977,7 +2998,7 @@ void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8,
 void sqlite3ValueFree(sqlite3_value*);
 sqlite3_value *sqlite3ValueNew(sqlite3 *);
 char *sqlite3Utf16to8(sqlite3 *, const void*, int, u8);
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
 char *sqlite3Utf8to16(sqlite3 *, u8, char *, int, int *);
 #endif
 int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value **);

src/test_config.c

@@ -424,6 +424,12 @@ Tcl_SetVar2(interp, "sqlite_options", "long_double",
   Tcl_SetVar2(interp, "sqlite_options", "stat2", "0", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_ENABLE_STAT3
+  Tcl_SetVar2(interp, "sqlite_options", "stat3", "1", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "stat3", "0", TCL_GLOBAL_ONLY);
+#endif
+
 #if !defined(SQLITE_ENABLE_LOCKING_STYLE)
 #  if defined(__APPLE__)
 #    define SQLITE_ENABLE_LOCKING_STYLE 1

src/utf.c

@@ -464,7 +464,7 @@ char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte, u8 enc){
 ** If a malloc failure occurs, NULL is returned and the db.mallocFailed
 ** flag set.
 */
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
 char *sqlite3Utf8to16(sqlite3 *db, u8 enc, char *z, int n, int *pnOut){
   Mem m;
   memset(&m, 0, sizeof(m));

src/vdbeaux.c

@@ -569,8 +569,8 @@ void sqlite3VdbeChangeP5(Vdbe *p, u8 val){
 ** the address of the next instruction to be coded.
 */
 void sqlite3VdbeJumpHere(Vdbe *p, int addr){
-  assert( addr>=0 );
+  assert( addr>=0 || p->db->mallocFailed );
-  sqlite3VdbeChangeP2(p, addr, p->nOp);
+  if( addr>=0 ) sqlite3VdbeChangeP2(p, addr, p->nOp);
 }
 
 

src/vdbemem.c

@@ -1032,11 +1032,11 @@ int sqlite3ValueFromExpr(
   }
   op = pExpr->op;
 
-  /* op can only be TK_REGISTER if we have compiled with SQLITE_ENABLE_STAT2.
+  /* op can only be TK_REGISTER if we have compiled with SQLITE_ENABLE_STAT3.
   ** The ifdef here is to enable us to achieve 100% branch test coverage even
-  ** when SQLITE_ENABLE_STAT2 is omitted.
+  ** when SQLITE_ENABLE_STAT3 is omitted.
   */
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
   if( op==TK_REGISTER ) op = pExpr->op2;
 #else
   if( NEVER(op==TK_REGISTER) ) op = pExpr->op2;

src/where.c

@@ -118,7 +118,7 @@ struct WhereTerm {
 #define TERM_ORINFO     0x10   /* Need to free the WhereTerm.u.pOrInfo object */
 #define TERM_ANDINFO    0x20   /* Need to free the WhereTerm.u.pAndInfo obj */
 #define TERM_OR_OK      0x40   /* Used during OR-clause processing */
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
 #  define TERM_VNULL    0x80   /* Manufactured x>NULL or x<=NULL term */
 #else
 #  define TERM_VNULL    0x00   /* Disabled if not using stat2 */
@@ -1332,7 +1332,7 @@ static void exprAnalyze(
   }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
   /* When sqlite_stat2 histogram data is available an operator of the
   ** form "x IS NOT NULL" can sometimes be evaluated more efficiently
   ** as "x>NULL" if x is not an INTEGER PRIMARY KEY.  So construct a
@@ -1371,7 +1371,7 @@ static void exprAnalyze(
       pNewTerm->prereqAll = pTerm->prereqAll;
     }
   }
-#endif /* SQLITE_ENABLE_STAT2 */
+#endif /* SQLITE_ENABLE_STAT */
 
   /* Prevent ON clause terms of a LEFT JOIN from being used to drive
   ** an index for tables to the left of the join.
@@ -2420,67 +2420,85 @@ static void bestVirtualIndex(
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
+#ifdef SQLITE_ENABLE_STAT3
 /*
-** Argument pIdx is a pointer to an index structure that has an array of
+** Estimate the location of a particular key among all keys in an
-** SQLITE_INDEX_SAMPLES evenly spaced samples of the first indexed column
+** index.  Store the results in aStat as follows:
-** stored in Index.aSample. These samples divide the domain of values stored
-** the index into (SQLITE_INDEX_SAMPLES+1) regions.
-** Region 0 contains all values less than the first sample value. Region
-** 1 contains values between the first and second samples.  Region 2 contains
-** values between samples 2 and 3.  And so on.  Region SQLITE_INDEX_SAMPLES
-** contains values larger than the last sample.
 **
-** If the index contains many duplicates of a single value, then it is
+**    aStat[0]      Est. number of rows less than pVal
-** possible that two or more adjacent samples can hold the same value.
+**    aStat[1]      Est. number of rows equal to pVal
-** When that is the case, the smallest possible region code is returned
-** when roundUp is false and the largest possible region code is returned
-** when roundUp is true.
 **
-** If successful, this function determines which of the regions value 
+** Return SQLITE_OK on success.
-** pVal lies in, sets *piRegion to the region index (a value between 0
-** and SQLITE_INDEX_SAMPLES+1, inclusive) and returns SQLITE_OK.
-** Or, if an OOM occurs while converting text values between encodings,
-** SQLITE_NOMEM is returned and *piRegion is undefined.
 */
-#ifdef SQLITE_ENABLE_STAT2
+static int whereKeyStats(
-static int whereRangeRegion(
   Parse *pParse,              /* Database connection */
   Index *pIdx,                /* Index to consider domain of */
   sqlite3_value *pVal,        /* Value to consider */
-  int roundUp,                /* Return largest valid region if true */
+  int roundUp,                /* Round up if true.  Round down if false */
-  int *piRegion               /* OUT: Region of domain in which value lies */
+  tRowcnt *aStat              /* OUT: stats written here */
 ){
-  assert( roundUp==0 || roundUp==1 );
+  tRowcnt n;
-  if( ALWAYS(pVal) ){
+  IndexSample *aSample;
-    IndexSample *aSample = pIdx->aSample;
+  int i, eType;
-    int i = 0;
+  int isEq = 0;
-    int eType = sqlite3_value_type(pVal);
+  i64 v;
+  double r, rS;
 
-    if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
+  assert( roundUp==0 || roundUp==1 );
-      double r = sqlite3_value_double(pVal);
+  if( pVal==0 ) return SQLITE_ERROR;
-      for(i=0; i<SQLITE_INDEX_SAMPLES; i++){
+  n = pIdx->aiRowEst[0];
+  aSample = pIdx->aSample;
+  i = 0;
+  eType = sqlite3_value_type(pVal);
+
+  if( eType==SQLITE_INTEGER ){
+    v = sqlite3_value_int64(pVal);
+    r = (i64)v;
+    for(i=0; i<pIdx->nSample; i++){
       if( aSample[i].eType==SQLITE_NULL ) continue;
       if( aSample[i].eType>=SQLITE_TEXT ) break;
-        if( roundUp ){
+      if( aSample[i].eType==SQLITE_INTEGER ){
-          if( aSample[i].u.r>r ) break;
+        if( aSample[i].u.i>=v ){
+          isEq = aSample[i].u.i==v;
+          break;
+        }
       }else{
-          if( aSample[i].u.r>=r ) break;
+        assert( aSample[i].eType==SQLITE_FLOAT );
+        if( aSample[i].u.r>=r ){
+          isEq = aSample[i].u.r==r;
+          break;
+        }
+      }
+    }
+  }else if( eType==SQLITE_FLOAT ){
+    r = sqlite3_value_double(pVal);
+    for(i=0; i<pIdx->nSample; i++){
+      if( aSample[i].eType==SQLITE_NULL ) continue;
+      if( aSample[i].eType>=SQLITE_TEXT ) break;
+      if( aSample[i].eType==SQLITE_FLOAT ){
+        rS = aSample[i].u.r;
+      }else{
+        rS = aSample[i].u.i;
+      }
+      if( rS>=r ){
+        isEq = rS==r;
+        break;
       }
     }
   }else if( eType==SQLITE_NULL ){
     i = 0;
-      if( roundUp ){
+    if( pIdx->nSample>=1 && aSample[0].eType==SQLITE_NULL ) isEq = 1;
-        while( i<SQLITE_INDEX_SAMPLES && aSample[i].eType==SQLITE_NULL ) i++;
-      }
   }else{
+    assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB );
+    for(i=0; i<pIdx->nSample; i++){
+      if( aSample[i].eType==SQLITE_TEXT || aSample[i].eType==SQLITE_BLOB ){
+        break;
+      }
+    }
+    if( i<pIdx->nSample ){      
       sqlite3 *db = pParse->db;
       CollSeq *pColl;
       const u8 *z;
-      int n;
-
-      /* pVal comes from sqlite3ValueFromExpr() so the type cannot be NULL */
-      assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB );
-
       if( eType==SQLITE_BLOB ){
         z = (const u8 *)sqlite3_value_blob(pVal);
         pColl = db->pDfltColl;
@@ -2500,11 +2518,11 @@ static int whereRangeRegion(
       }
       n = sqlite3ValueBytes(pVal, pColl->enc);
   
-      for(i=0; i<SQLITE_INDEX_SAMPLES; i++){
+      for(; i<pIdx->nSample; i++){
         int c;
         int eSampletype = aSample[i].eType;
-        if( eSampletype==SQLITE_NULL || eSampletype<eType ) continue;
+        if( eSampletype<eType ) continue;
-        if( (eSampletype!=eType) ) break;
+        if( eSampletype!=eType ) break;
 #ifndef SQLITE_OMIT_UTF16
         if( pColl->enc!=SQLITE_UTF8 ){
           int nSample;
@@ -2522,16 +2540,48 @@ static int whereRangeRegion(
         {
           c = pColl->xCmp(pColl->pUser, aSample[i].nByte, aSample[i].u.z, n, z);
         }
-        if( c-roundUp>=0 ) break;
+        if( c>=0 ){
+          if( c==0 ) isEq = 1;
+          break;
+        }
+      }
     }
   }
 
-    assert( i>=0 && i<=SQLITE_INDEX_SAMPLES );
+  /* At this point, aSample[i] is the first sample that is greater than
-    *piRegion = i;
+  ** or equal to pVal.  Or if i==pIdx->nSample, then all samples are less
+  ** than pVal.  If aSample[i]==pVal, then isEq==1.
+  */
+  if( isEq ){
+    assert( i<pIdx->nSample );
+    aStat[0] = aSample[i].nLt;
+    aStat[1] = aSample[i].nEq;
+  }else{
+    tRowcnt iLower, iUpper, iGap;
+    if( i==0 ){
+      iLower = 0;
+      iUpper = aSample[0].nLt;
+    }else{
+      iUpper = i>=pIdx->nSample ? n : aSample[i].nLt;
+      iLower = aSample[i-1].nEq + aSample[i-1].nLt;
+    }
+    aStat[1] = pIdx->avgEq;
+    if( iLower>=iUpper ){
+      iGap = 0;
+    }else{
+      iGap = iUpper - iLower;
+      if( iGap>=aStat[1]/2 ) iGap -= aStat[1]/2;
+    }
+    if( roundUp ){
+      iGap = (iGap*2)/3;
+    }else{
+      iGap = iGap/3;
+    }
+    aStat[0] = iLower + iGap;
   }
   return SQLITE_OK;
 }
-#endif   /* #ifdef SQLITE_ENABLE_STAT2 */
+#endif /* SQLITE_ENABLE_STAT3 */
 
 /*
 ** If expression pExpr represents a literal value, set *pp to point to
@@ -2549,7 +2599,7 @@ static int whereRangeRegion(
 **
 ** If an error occurs, return an error code. Otherwise, SQLITE_OK.
 */
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
 static int valueFromExpr(
   Parse *pParse, 
   Expr *pExpr, 
@@ -2597,17 +2647,15 @@ static int valueFromExpr(
 **
 ** then nEq should be passed 0.
 **
-** The returned value is an integer between 1 and 100, inclusive. A return
+** The returned value is an integer divisor to reduce the estimated
-** value of 1 indicates that the proposed range scan is expected to visit
+** search space.  A return value of 1 means that range constraints are
-** approximately 1/100th (1%) of the rows selected by the nEq equality
+** no help at all.  A return value of 2 means range constraints are
-** constraints (if any). A return value of 100 indicates that it is expected
+** expected to reduce the search space by half.  And so forth...
-** that the range scan will visit every row (100%) selected by the equality
-** constraints.
 **
-** In the absence of sqlite_stat2 ANALYZE data, each range inequality
+** In the absence of sqlite_stat3 ANALYZE data, each range inequality
-** reduces the search space by 3/4ths.  Hence a single constraint (x>?)
+** reduces the search space by a factor of 4.  Hence a single constraint (x>?)
-** results in a return of 25 and a range constraint (x>? AND x<?) results
+** results in a return of 4 and a range constraint (x>? AND x<?) results
-** in a return of 6.
+** in a return of 16.
 */
 static int whereRangeScanEst(
   Parse *pParse,       /* Parsing & code generating context */
@@ -2615,84 +2663,72 @@ static int whereRangeScanEst(
   int nEq,             /* index into p->aCol[] of the range-compared column */
   WhereTerm *pLower,   /* Lower bound on the range. ex: "x>123" Might be NULL */
   WhereTerm *pUpper,   /* Upper bound on the range. ex: "x<455" Might be NULL */
-  int *piEst           /* OUT: Return value */
+  double *pRangeDiv   /* OUT: Reduce search space by this divisor */
 ){
   int rc = SQLITE_OK;
 
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
 
-  if( nEq==0 && p->aSample ){
+  if( nEq==0 && p->nSample ){
-    sqlite3_value *pLowerVal = 0;
+    sqlite3_value *pRangeVal;
-    sqlite3_value *pUpperVal = 0;
+    tRowcnt iLower = 0;
-    int iEst;
+    tRowcnt iUpper = p->aiRowEst[0];
-    int iLower = 0;
+    tRowcnt a[2];
-    int iUpper = SQLITE_INDEX_SAMPLES;
-    int roundUpUpper = 0;
-    int roundUpLower = 0;
     u8 aff = p->pTable->aCol[p->aiColumn[0]].affinity;
 
     if( pLower ){
       Expr *pExpr = pLower->pExpr->pRight;
-      rc = valueFromExpr(pParse, pExpr, aff, &pLowerVal);
+      rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal);
       assert( pLower->eOperator==WO_GT || pLower->eOperator==WO_GE );
-      roundUpLower = (pLower->eOperator==WO_GT) ?1:0;
+      if( rc==SQLITE_OK
+       && whereKeyStats(pParse, p, pRangeVal, 0, a)==SQLITE_OK
+      ){
+        iLower = a[0];
+        if( pLower->eOperator==WO_GT ) iLower += a[1];
+      }
+      sqlite3ValueFree(pRangeVal);
     }
     if( rc==SQLITE_OK && pUpper ){
       Expr *pExpr = pUpper->pExpr->pRight;
-      rc = valueFromExpr(pParse, pExpr, aff, &pUpperVal);
+      rc = valueFromExpr(pParse, pExpr, aff, &pRangeVal);
       assert( pUpper->eOperator==WO_LT || pUpper->eOperator==WO_LE );
-      roundUpUpper = (pUpper->eOperator==WO_LE) ?1:0;
+      if( rc==SQLITE_OK
+       && whereKeyStats(pParse, p, pRangeVal, 1, a)==SQLITE_OK
+      ){
+        iUpper = a[0];
+        if( pUpper->eOperator==WO_LE ) iUpper += a[1];
+      }
+      sqlite3ValueFree(pRangeVal);
     }
-
-    if( rc!=SQLITE_OK || (pLowerVal==0 && pUpperVal==0) ){
-      sqlite3ValueFree(pLowerVal);
-      sqlite3ValueFree(pUpperVal);
-      goto range_est_fallback;
-    }else if( pLowerVal==0 ){
-      rc = whereRangeRegion(pParse, p, pUpperVal, roundUpUpper, &iUpper);
-      if( pLower ) iLower = iUpper/2;
-    }else if( pUpperVal==0 ){
-      rc = whereRangeRegion(pParse, p, pLowerVal, roundUpLower, &iLower);
-      if( pUpper ) iUpper = (iLower + SQLITE_INDEX_SAMPLES + 1)/2;
-    }else{
-      rc = whereRangeRegion(pParse, p, pUpperVal, roundUpUpper, &iUpper);
     if( rc==SQLITE_OK ){
-        rc = whereRangeRegion(pParse, p, pLowerVal, roundUpLower, &iLower);
+      if( iUpper<=iLower ){
-      }
+        *pRangeDiv = (double)p->aiRowEst[0];
-    }
-    WHERETRACE(("range scan regions: %d..%d\n", iLower, iUpper));
-
-    iEst = iUpper - iLower;
-    testcase( iEst==SQLITE_INDEX_SAMPLES );
-    assert( iEst<=SQLITE_INDEX_SAMPLES );
-    if( iEst<1 ){
-      *piEst = 50/SQLITE_INDEX_SAMPLES;
       }else{
-      *piEst = (iEst*100)/SQLITE_INDEX_SAMPLES;
+        *pRangeDiv = (double)p->aiRowEst[0]/(double)(iUpper - iLower);
+      }
+      WHERETRACE(("range scan regions: %u..%u  div=%g\n",
+                  (u32)iLower, (u32)iUpper, *pRangeDiv));
+      return SQLITE_OK;
     }
-    sqlite3ValueFree(pLowerVal);
-    sqlite3ValueFree(pUpperVal);
-    return rc;
   }
-range_est_fallback:
 #else
   UNUSED_PARAMETER(pParse);
   UNUSED_PARAMETER(p);
   UNUSED_PARAMETER(nEq);
 #endif
   assert( pLower || pUpper );
-  *piEst = 100;
+  *pRangeDiv = (double)1;
-  if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ) *piEst /= 4;
+  if( pLower && (pLower->wtFlags & TERM_VNULL)==0 ) *pRangeDiv *= (double)4;
-  if( pUpper ) *piEst /= 4;
+  if( pUpper ) *pRangeDiv *= (double)4;
   return rc;
 }
 
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
 /*
 ** Estimate the number of rows that will be returned based on
 ** an equality constraint x=VALUE and where that VALUE occurs in
 ** the histogram data.  This only works when x is the left-most
-** column of an index and sqlite_stat2 histogram data is available
+** column of an index and sqlite_stat3 histogram data is available
 ** for that index.  When pExpr==NULL that means the constraint is
 ** "x IS NULL" instead of "x=VALUE".
 **
@@ -2712,10 +2748,9 @@ static int whereEqualScanEst(
   double *pnRow        /* Write the revised row estimate here */
 ){
   sqlite3_value *pRhs = 0;  /* VALUE on right-hand side of pTerm */
-  int iLower, iUpper;       /* Range of histogram regions containing pRhs */
   u8 aff;                   /* Column affinity */
   int rc;                   /* Subfunction return code */
-  double nRowEst;           /* New estimate of the number of rows */
+  tRowcnt a[2];             /* Statistics */
 
   assert( p->aSample!=0 );
   aff = p->pTable->aCol[p->aiColumn[0]].affinity;
@@ -2726,26 +2761,18 @@ static int whereEqualScanEst(
     pRhs = sqlite3ValueNew(pParse->db);
   }
   if( pRhs==0 ) return SQLITE_NOTFOUND;
-  rc = whereRangeRegion(pParse, p, pRhs, 0, &iLower);
+  rc = whereKeyStats(pParse, p, pRhs, 0, a);
-  if( rc ) goto whereEqualScanEst_cancel;
+  if( rc==SQLITE_OK ){
-  rc = whereRangeRegion(pParse, p, pRhs, 1, &iUpper);
+    WHERETRACE(("equality scan regions: %d\n", (int)a[1]));
-  if( rc ) goto whereEqualScanEst_cancel;
+    *pnRow = a[1];
-  WHERETRACE(("equality scan regions: %d..%d\n", iLower, iUpper));
-  if( iLower>=iUpper ){
-    nRowEst = p->aiRowEst[0]/(SQLITE_INDEX_SAMPLES*2);
-    if( nRowEst<*pnRow ) *pnRow = nRowEst;
-  }else{
-    nRowEst = (iUpper-iLower)*p->aiRowEst[0]/SQLITE_INDEX_SAMPLES;
-    *pnRow = nRowEst;
   }
-
 whereEqualScanEst_cancel:
   sqlite3ValueFree(pRhs);
   return rc;
 }
-#endif /* defined(SQLITE_ENABLE_STAT2) */
+#endif /* defined(SQLITE_ENABLE_STAT3) */
 
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
 /*
 ** Estimate the number of rows that will be returned based on
 ** an IN constraint where the right-hand side of the IN operator
@@ -2768,60 +2795,25 @@ static int whereInScanEst(
   ExprList *pList,     /* The value list on the RHS of "x IN (v1,v2,v3,...)" */
   double *pnRow        /* Write the revised row estimate here */
 ){
-  sqlite3_value *pVal = 0;  /* One value from list */
-  int iLower, iUpper;       /* Range of histogram regions containing pRhs */
-  u8 aff;                   /* Column affinity */
   int rc = SQLITE_OK;         /* Subfunction return code */
-  double nRowEst;           /* New estimate of the number of rows */
+  double nEst;                /* Number of rows for a single term */
-  int nSpan = 0;            /* Number of histogram regions spanned */
+  double nRowEst = (double)0; /* New estimate of the number of rows */
-  int nSingle = 0;          /* Histogram regions hit by a single value */
-  int nNotFound = 0;        /* Count of values that are not constants */
   int i;                      /* Loop counter */
-  u8 aSpan[SQLITE_INDEX_SAMPLES+1];    /* Histogram regions that are spanned */
-  u8 aSingle[SQLITE_INDEX_SAMPLES+1];  /* Histogram regions hit once */
 
   assert( p->aSample!=0 );
-  aff = p->pTable->aCol[p->aiColumn[0]].affinity;
+  for(i=0; rc==SQLITE_OK && i<pList->nExpr; i++){
-  memset(aSpan, 0, sizeof(aSpan));
+    nEst = p->aiRowEst[0];
-  memset(aSingle, 0, sizeof(aSingle));
+    rc = whereEqualScanEst(pParse, p, pList->a[i].pExpr, &nEst);
-  for(i=0; i<pList->nExpr; i++){
+    nRowEst += nEst;
-    sqlite3ValueFree(pVal);
-    rc = valueFromExpr(pParse, pList->a[i].pExpr, aff, &pVal);
-    if( rc ) break;
-    if( pVal==0 || sqlite3_value_type(pVal)==SQLITE_NULL ){
-      nNotFound++;
-      continue;
-    }
-    rc = whereRangeRegion(pParse, p, pVal, 0, &iLower);
-    if( rc ) break;
-    rc = whereRangeRegion(pParse, p, pVal, 1, &iUpper);
-    if( rc ) break;
-    if( iLower>=iUpper ){
-      aSingle[iLower] = 1;
-    }else{
-      assert( iLower>=0 && iUpper<=SQLITE_INDEX_SAMPLES );
-      while( iLower<iUpper ) aSpan[iLower++] = 1;
-    }
   }
   if( rc==SQLITE_OK ){
-    for(i=nSpan=0; i<=SQLITE_INDEX_SAMPLES; i++){
-      if( aSpan[i] ){
-        nSpan++;
-      }else if( aSingle[i] ){
-        nSingle++;
-      }
-    }
-    nRowEst = (nSpan*2+nSingle)*p->aiRowEst[0]/(2*SQLITE_INDEX_SAMPLES)
-               + nNotFound*p->aiRowEst[1];
     if( nRowEst > p->aiRowEst[0] ) nRowEst = p->aiRowEst[0];
     *pnRow = nRowEst;
-    WHERETRACE(("IN row estimate: nSpan=%d, nSingle=%d, nNotFound=%d, est=%g\n",
+    WHERETRACE(("IN row estimate: est=%g\n", nRowEst));
-                 nSpan, nSingle, nNotFound, nRowEst));
   }
-  sqlite3ValueFree(pVal);
   return rc;
 }
-#endif /* defined(SQLITE_ENABLE_STAT2) */
+#endif /* defined(SQLITE_ENABLE_STAT3) */
 
 
 /*
@@ -2868,7 +2860,7 @@ static void bestBtreeIndex(
   int eqTermMask;             /* Current mask of valid equality operators */
   int idxEqTermMask;          /* Index mask of valid equality operators */
   Index sPk;                  /* A fake index object for the primary key */
-  unsigned int aiRowEstPk[2]; /* The aiRowEst[] value for the sPk index */
+  tRowcnt aiRowEstPk[2];      /* The aiRowEst[] value for the sPk index */
   int aiColumnPk = -1;        /* The aColumn[] value for the sPk index */
   int wsFlagMask;             /* Allowed flags in pCost->plan.wsFlag */
 
@@ -2923,7 +2915,7 @@ static void bestBtreeIndex(
   /* Loop over all indices looking for the best one to use
   */
   for(; pProbe; pIdx=pProbe=pProbe->pNext){
-    const unsigned int * const aiRowEst = pProbe->aiRowEst;
+    const tRowcnt * const aiRowEst = pProbe->aiRowEst;
     double cost;                /* Cost of using pProbe */
     double nRow;                /* Estimated number of rows in result set */
     double log10N;              /* base-10 logarithm of nRow (inexact) */
@@ -2966,14 +2958,12 @@ static void bestBtreeIndex(
     **    IN operator must be a SELECT, not a value list, for this variable
     **    to be true.
     **
-    **  estBound:
+    **  rangeDiv:
-    **    An estimate on the amount of the table that must be searched.  A
+    **    An estimate of a divisor by which to reduce the search space due
-    **    value of 100 means the entire table is searched.  Range constraints
+    **    to inequality constraints.  In the absence of sqlite_stat3 ANALYZE
-    **    might reduce this to a value less than 100 to indicate that only
+    **    data, a single inequality reduces the search space to 1/4rd its
-    **    a fraction of the table needs searching.  In the absence of
+    **    original size (rangeDiv==4).  Two inequalities reduce the search
-    **    sqlite_stat2 ANALYZE data, a single inequality reduces the search
+    **    space to 1/16th of its original size (rangeDiv==16).
-    **    space to 1/4rd its original size.  So an x>? constraint reduces
-    **    estBound to 25.  Two constraints (x>? AND x<?) reduce estBound to 6.
     **
     **  bSort:   
     **    Boolean. True if there is an ORDER BY clause that will require an 
@@ -2998,13 +2988,13 @@ static void bestBtreeIndex(
     int nEq;                      /* Number of == or IN terms matching index */
     int bInEst = 0;               /* True if "x IN (SELECT...)" seen */
     int nInMul = 1;               /* Number of distinct equalities to lookup */
-    int estBound = 100;           /* Estimated reduction in search space */
+    double rangeDiv = (double)1;  /* Estimated reduction in search space */
     int nBound = 0;               /* Number of range constraints seen */
     int bSort = !!pOrderBy;       /* True if external sort required */
     int bDist = !!pDistinct;      /* True if index cannot help with DISTINCT */
     int bLookup = 0;              /* True if not a covering index */
     WhereTerm *pTerm;             /* A single term of the WHERE clause */
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
     WhereTerm *pFirstTerm = 0;    /* First term matching the index */
 #endif
 
@@ -3028,19 +3018,19 @@ static void bestBtreeIndex(
       }else if( pTerm->eOperator & WO_ISNULL ){
         wsFlags |= WHERE_COLUMN_NULL;
       }
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
       if( nEq==0 && pProbe->aSample ) pFirstTerm = pTerm;
 #endif
       used |= pTerm->prereqRight;
     }
 
-    /* Determine the value of estBound. */
+    /* Determine the value of rangeDiv */
     if( nEq<pProbe->nColumn && pProbe->bUnordered==0 ){
       int j = pProbe->aiColumn[nEq];
       if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){
         WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pIdx);
         WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pIdx);
-        whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &estBound);
+        whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &rangeDiv);
         if( pTop ){
           nBound = 1;
           wsFlags |= WHERE_TOP_LIMIT;
@@ -3112,7 +3102,7 @@ static void bestBtreeIndex(
       nInMul = (int)(nRow / aiRowEst[nEq]);
     }
 
-#ifdef SQLITE_ENABLE_STAT2
+#ifdef SQLITE_ENABLE_STAT3
     /* If the constraint is of the form x=VALUE or x IN (E1,E2,...)
     ** and we do not think that values of x are unique and if histogram
     ** data is available for column x, then it might be possible
@@ -3128,12 +3118,12 @@ static void bestBtreeIndex(
         whereInScanEst(pParse, pProbe, pFirstTerm->pExpr->x.pList, &nRow);
       }
     }
-#endif /* SQLITE_ENABLE_STAT2 */
+#endif /* SQLITE_ENABLE_STAT3 */
 
     /* Adjust the number of output rows and downward to reflect rows
     ** that are excluded by range constraints.
     */
-    nRow = (nRow * (double)estBound) / (double)100;
+    nRow = nRow/rangeDiv;
     if( nRow<1 ) nRow = 1;
 
     /* Experiments run on real SQLite databases show that the time needed
@@ -3262,10 +3252,10 @@ static void bestBtreeIndex(
 
 
     WHERETRACE((
-      "%s(%s): nEq=%d nInMul=%d estBound=%d bSort=%d bLookup=%d wsFlags=0x%x\n"
+      "%s(%s): nEq=%d nInMul=%d rangeDiv=%d bSort=%d bLookup=%d wsFlags=0x%x\n"
       "         notReady=0x%llx log10N=%.1f nRow=%.1f cost=%.1f used=0x%llx\n",
       pSrc->pTab->zName, (pIdx ? pIdx->zName : "ipk"), 
-      nEq, nInMul, estBound, bSort, bLookup, wsFlags,
+      nEq, nInMul, (int)rangeDiv, bSort, bLookup, wsFlags,
       notReady, log10N, nRow, cost, used
     ));
 

test/alter.test

@@ -847,6 +847,7 @@ set system_table_list {1 sqlite_master}
 catchsql ANALYZE
 ifcapable analyze { lappend system_table_list 2 sqlite_stat1 }
 ifcapable stat2   { lappend system_table_list 3 sqlite_stat2 }
+ifcapable stat3   { lappend system_table_list 4 sqlite_stat3 }
 
 foreach {tn tbl} $system_table_list {
   do_test alter-15.$tn.1 {

test/analyze.test

@@ -288,7 +288,7 @@ do_test analyze-4.3 {
 } {}
 
 # Verify that DROP TABLE and DROP INDEX remove entries from the 
-# sqlite_stat1 and sqlite_stat2 tables.
+# sqlite_stat1 and sqlite_stat3 tables.
 #
 do_test analyze-5.0 {
   execsql {
@@ -306,11 +306,11 @@ do_test analyze-5.0 {
     SELECT DISTINCT tbl FROM sqlite_stat1 ORDER BY 1;
   }
 } {t3i1 t3i2 t3i3 t4i1 t4i2 t3 t4}
-ifcapable stat2 {
+ifcapable stat3 {
   do_test analyze-5.1 {
     execsql {
-      SELECT DISTINCT idx FROM sqlite_stat2 ORDER BY 1;
+      SELECT DISTINCT idx FROM sqlite_stat3 ORDER BY 1;
-      SELECT DISTINCT tbl FROM sqlite_stat2 ORDER BY 1;
+      SELECT DISTINCT tbl FROM sqlite_stat3 ORDER BY 1;
     }
   } {t3i1 t3i2 t3i3 t4i1 t4i2 t3 t4}
 }
@@ -321,11 +321,11 @@ do_test analyze-5.2 {
     SELECT DISTINCT tbl FROM sqlite_stat1 ORDER BY 1;
   }
 } {t3i1 t3i3 t4i1 t4i2 t3 t4}
-ifcapable stat2 {
+ifcapable stat3 {
   do_test analyze-5.3 {
     execsql {
-      SELECT DISTINCT idx FROM sqlite_stat2 ORDER BY 1;
+      SELECT DISTINCT idx FROM sqlite_stat3 ORDER BY 1;
-      SELECT DISTINCT tbl FROM sqlite_stat2 ORDER BY 1;
+      SELECT DISTINCT tbl FROM sqlite_stat3 ORDER BY 1;
     }
   } {t3i1 t3i3 t4i1 t4i2 t3 t4}
 }
@@ -336,11 +336,11 @@ do_test analyze-5.4 {
     SELECT DISTINCT tbl FROM sqlite_stat1 ORDER BY 1;
   }
 } {t4i1 t4i2 t4}
-ifcapable stat2 {
+ifcapable stat3 {
   do_test analyze-5.5 {
     execsql {
-      SELECT DISTINCT idx FROM sqlite_stat2 ORDER BY 1;
+      SELECT DISTINCT idx FROM sqlite_stat3 ORDER BY 1;
-      SELECT DISTINCT tbl FROM sqlite_stat2 ORDER BY 1;
+      SELECT DISTINCT tbl FROM sqlite_stat3 ORDER BY 1;
     }
   } {t4i1 t4i2 t4}
 }

test/analyze3.test

@@ -17,7 +17,7 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
-ifcapable !stat2 {
+ifcapable !stat3 {
   finish_test
   return
 }
@@ -97,10 +97,10 @@ do_test analyze3-1.1.1 {
 
 do_eqp_test analyze3-1.1.2 {
   SELECT sum(y) FROM t1 WHERE x>200 AND x<300
-} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x<?) (~100 rows)}}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x<?) (~179 rows)}}
 do_eqp_test analyze3-1.1.3 {
   SELECT sum(y) FROM t1 WHERE x>0 AND x<1100 
-} {0 0 0 {SCAN TABLE t1 (~111 rows)}}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (x>? AND x<?) (~959 rows)}}
 
 do_test analyze3-1.1.4 {
   sf_execsql { SELECT sum(y) FROM t1 WHERE x>200 AND x<300 }
@@ -117,17 +117,17 @@ do_test analyze3-1.1.6 {
 } {199 0 14850}
 do_test analyze3-1.1.7 {
   sf_execsql { SELECT sum(y) FROM t1 WHERE x>0 AND x<1100 }
-} {999 999 499500}
+} {2000 0 499500}
 do_test analyze3-1.1.8 {
   set l [string range "0" 0 end]
   set u [string range "1100" 0 end]
   sf_execsql { SELECT sum(y) FROM t1 WHERE x>$l AND x<$u }
-} {999 999 499500}
+} {2000 0 499500}
 do_test analyze3-1.1.9 {
   set l [expr int(0)]
   set u [expr int(1100)]
   sf_execsql { SELECT sum(y) FROM t1 WHERE x>$l AND x<$u }
-} {999 999 499500}
+} {2000 0 499500}
 
 
 # The following tests are similar to the block above. The difference is
@@ -146,10 +146,10 @@ do_test analyze3-1.2.1 {
 } {}
 do_eqp_test analyze3-1.2.2 {
   SELECT sum(y) FROM t2 WHERE x>1 AND x<2
-} {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x<?) (~200 rows)}}
+} {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x<?) (~196 rows)}}
 do_eqp_test analyze3-1.2.3 {
   SELECT sum(y) FROM t2 WHERE x>0 AND x<99
-} {0 0 0 {SCAN TABLE t2 (~111 rows)}}
+} {0 0 0 {SEARCH TABLE t2 USING INDEX i2 (x>? AND x<?) (~982 rows)}}
 do_test analyze3-1.2.4 {
   sf_execsql { SELECT sum(y) FROM t2 WHERE x>12 AND x<20 }
 } {161 0 4760}
@@ -165,17 +165,17 @@ do_test analyze3-1.2.6 {
 } {161 0 integer integer 4760}
 do_test analyze3-1.2.7 {
   sf_execsql { SELECT sum(y) FROM t2 WHERE x>0 AND x<99 }
-} {999 999 490555}
+} {1981 0 490555}
 do_test analyze3-1.2.8 {
   set l [string range "0" 0 end]
   set u [string range "99" 0 end]
   sf_execsql {SELECT typeof($l), typeof($u), sum(y) FROM t2 WHERE x>$l AND x<$u}
-} {999 999 text text 490555}
+} {1981 0 text text 490555}
 do_test analyze3-1.2.9 {
   set l [expr int(0)]
   set u [expr int(99)]
   sf_execsql {SELECT typeof($l), typeof($u), sum(y) FROM t2 WHERE x>$l AND x<$u}
-} {999 999 integer integer 490555}
+} {1981 0 integer integer 490555}
 
 # Same tests a third time. This time, column x has INTEGER affinity and
 # is not the leftmost column of the table. This triggered a bug causing
@@ -193,10 +193,10 @@ do_test analyze3-1.3.1 {
 } {}
 do_eqp_test analyze3-1.3.2 {
   SELECT sum(y) FROM t3 WHERE x>200 AND x<300
-} {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x<?) (~100 rows)}}
+} {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x<?) (~156 rows)}}
 do_eqp_test analyze3-1.3.3 {
   SELECT sum(y) FROM t3 WHERE x>0 AND x<1100
-} {0 0 0 {SCAN TABLE t3 (~111 rows)}}
+} {0 0 0 {SEARCH TABLE t3 USING INDEX i3 (x>? AND x<?) (~989 rows)}}
 
 do_test analyze3-1.3.4 {
   sf_execsql { SELECT sum(y) FROM t3 WHERE x>200 AND x<300 }
@@ -213,17 +213,17 @@ do_test analyze3-1.3.6 {
 } {199 0 14850}
 do_test analyze3-1.3.7 {
   sf_execsql { SELECT sum(y) FROM t3 WHERE x>0 AND x<1100 }
-} {999 999 499500}
+} {2000 0 499500}
 do_test analyze3-1.3.8 {
   set l [string range "0" 0 end]
   set u [string range "1100" 0 end]
   sf_execsql { SELECT sum(y) FROM t3 WHERE x>$l AND x<$u }
-} {999 999 499500}
+} {2000 0 499500}
 do_test analyze3-1.3.9 {
   set l [expr int(0)]
   set u [expr int(1100)]
   sf_execsql { SELECT sum(y) FROM t3 WHERE x>$l AND x<$u }
-} {999 999 499500}
+} {2000 0 499500}
 
 #-------------------------------------------------------------------------
 # Test that the values of bound SQL variables may be used for the LIKE
@@ -248,7 +248,7 @@ do_test analyze3-2.1 {
 } {}
 do_eqp_test analyze3-2.2 {
   SELECT count(a) FROM t1 WHERE b LIKE 'a%'
-} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (b>? AND b<?) (~30000 rows)}}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX i1 (b>? AND b<?) (~31250 rows)}}
 do_eqp_test analyze3-2.3 {
   SELECT count(a) FROM t1 WHERE b LIKE '%a'
 } {0 0 0 {SCAN TABLE t1 (~500000 rows)}}

test/analyze5.test

@@ -10,14 +10,14 @@
 #***********************************************************************
 #
 # This file implements tests for SQLite library.  The focus of the tests
-# in this file is the use of the sqlite_stat2 histogram data on tables
+# in this file is the use of the sqlite_stat3 histogram data on tables
 # with many repeated values and only a few distinct values.
 #
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
-ifcapable !stat2 {
+ifcapable !stat3 {
   finish_test
   return
 }
@@ -55,114 +55,102 @@ do_test analyze5-1.0 {
     CREATE INDEX t1y ON t1(y);  -- integers 0 and very few 1s
     CREATE INDEX t1z ON t1(z);  -- integers 0, 1, 2, and 3
     ANALYZE;
-    SELECT sample FROM sqlite_stat2 WHERE idx='t1u' ORDER BY sampleno;
+    SELECT sample FROM sqlite_stat3 WHERE idx='t1u' ORDER BY nlt;
   }
-} {alpha alpha alpha alpha bravo bravo bravo charlie charlie delta}
+} {alpha bravo charlie delta}
-do_test analyze5-1.1 {
-  string tolower \
-   [db eval {SELECT sample from sqlite_stat2 WHERE idx='t1v' ORDER BY sampleno}]
-} {alpha alpha alpha alpha bravo bravo bravo charlie charlie delta}
-do_test analyze5-1.2 {
-  db eval {SELECT sample from sqlite_stat2 WHERE idx='t1w' ORDER BY sampleno}
-} {{} 0 0 0 0 1 1 1 2 2}
-do_test analyze5-1.3 {
-  db eval {SELECT sample from sqlite_stat2 WHERE idx='t1x' ORDER BY sampleno}
-} {{} {} {} {} 1 1 1 2 2 3}
-do_test analyze5-1.4 {
-  db eval {SELECT sample from sqlite_stat2 WHERE idx='t1y' ORDER BY sampleno}
-} {0 0 0 0 0 0 0 0 0 0}
-do_test analyze5-1.5 {
-  db eval {SELECT sample from sqlite_stat2 WHERE idx='t1z' ORDER BY sampleno}
-} {0 0 0 0 1 1 1 2 2 3}
-do_test analyze5-1.6 {
-  db eval {SELECT sample from sqlite_stat2 WHERE idx='t1t' ORDER BY sampleno}
-} {0.5 0.5 0.5 0.5 1.5 1.5 1.5 2.5 2.5 3.5}
 
+do_test analyze5-1.1 {
+  db eval {SELECT DISTINCT lower(sample) FROM sqlite_stat3 WHERE idx='t1v'
+             ORDER BY 1}
+} {alpha bravo charlie delta}
+do_test analyze5-1.2 {
+  db eval {SELECT idx, count(*) FROM sqlite_stat3 GROUP BY 1 ORDER BY 1}
+} {t1t 4 t1u 4 t1v 4 t1w 4 t1x 4 t1y 2 t1z 4}
 
 # Verify that range queries generate the correct row count estimates
 #
 foreach {testid where index rows} {
     1  {z>=0 AND z<=0}       t1z  400
     2  {z>=1 AND z<=1}       t1z  300
-    3  {z>=2 AND z<=2}       t1z  200
+    3  {z>=2 AND z<=2}       t1z  175
-    4  {z>=3 AND z<=3}       t1z  100
+    4  {z>=3 AND z<=3}       t1z  125
-    5  {z>=4 AND z<=4}       t1z   50
+    5  {z>=4 AND z<=4}       t1z    1
-    6  {z>=-1 AND z<=-1}     t1z   50
+    6  {z>=-1 AND z<=-1}     t1z    1
-    7  {z>1 AND z<3}         t1z  200
+    7  {z>1 AND z<3}         t1z  175
     8  {z>0 AND z<100}       t1z  600
     9  {z>=1 AND z<100}      t1z  600
    10  {z>1 AND z<100}       t1z  300
    11  {z>=2 AND z<100}      t1z  300
-   12  {z>2 AND z<100}       t1z  100
+   12  {z>2 AND z<100}       t1z  125
-   13  {z>=3 AND z<100}      t1z  100
+   13  {z>=3 AND z<100}      t1z  125
-   14  {z>3 AND z<100}       t1z   50
+   14  {z>3 AND z<100}       t1z    1
-   15  {z>=4 AND z<100}      t1z   50
+   15  {z>=4 AND z<100}      t1z    1
-   16  {z>=-100 AND z<=-1}   t1z   50
+   16  {z>=-100 AND z<=-1}   t1z    1
    17  {z>=-100 AND z<=0}    t1z  400
-   18  {z>=-100 AND z<0}     t1z   50
+   18  {z>=-100 AND z<0}     t1z    1
    19  {z>=-100 AND z<=1}    t1z  700
    20  {z>=-100 AND z<2}     t1z  700
-   21  {z>=-100 AND z<=2}    t1z  900
+   21  {z>=-100 AND z<=2}    t1z  875
-   22  {z>=-100 AND z<3}     t1z  900
+   22  {z>=-100 AND z<3}     t1z  875
   
    31  {z>=0.0 AND z<=0.0}   t1z  400
    32  {z>=1.0 AND z<=1.0}   t1z  300
-   33  {z>=2.0 AND z<=2.0}   t1z  200
+   33  {z>=2.0 AND z<=2.0}   t1z  175
-   34  {z>=3.0 AND z<=3.0}   t1z  100
+   34  {z>=3.0 AND z<=3.0}   t1z  125
-   35  {z>=4.0 AND z<=4.0}   t1z   50
+   35  {z>=4.0 AND z<=4.0}   t1z    1
-   36  {z>=-1.0 AND z<=-1.0} t1z   50
+   36  {z>=-1.0 AND z<=-1.0} t1z    1
-   37  {z>1.5 AND z<3.0}     t1z  200
+   37  {z>1.5 AND z<3.0}     t1z  174
-   38  {z>0.5 AND z<100}     t1z  600
+   38  {z>0.5 AND z<100}     t1z  599
    39  {z>=1.0 AND z<100}    t1z  600
-   40  {z>1.5 AND z<100}     t1z  300
+   40  {z>1.5 AND z<100}     t1z  299
    41  {z>=2.0 AND z<100}    t1z  300
-   42  {z>2.1 AND z<100}     t1z  100
+   42  {z>2.1 AND z<100}     t1z  124
-   43  {z>=3.0 AND z<100}    t1z  100
+   43  {z>=3.0 AND z<100}    t1z  125
-   44  {z>3.2 AND z<100}     t1z   50
+   44  {z>3.2 AND z<100}     t1z    1
-   45  {z>=4.0 AND z<100}    t1z   50
+   45  {z>=4.0 AND z<100}    t1z    1
-   46  {z>=-100 AND z<=-1.0} t1z   50
+   46  {z>=-100 AND z<=-1.0} t1z    1
    47  {z>=-100 AND z<=0.0}  t1z  400
-   48  {z>=-100 AND z<0.0}   t1z   50
+   48  {z>=-100 AND z<0.0}   t1z    1
    49  {z>=-100 AND z<=1.0}  t1z  700
    50  {z>=-100 AND z<2.0}   t1z  700
-   51  {z>=-100 AND z<=2.0}  t1z  900
+   51  {z>=-100 AND z<=2.0}  t1z  875
-   52  {z>=-100 AND z<3.0}   t1z  900
+   52  {z>=-100 AND z<3.0}   t1z  875
   
-  101  {z=-1}                t1z   50
+  101  {z=-1}                t1z    1
   102  {z=0}                 t1z  400
   103  {z=1}                 t1z  300
-  104  {z=2}                 t1z  200
+  104  {z=2}                 t1z  175
-  105  {z=3}                 t1z  100
+  105  {z=3}                 t1z  125
-  106  {z=4}                 t1z   50
+  106  {z=4}                 t1z    1
-  107  {z=-10.0}             t1z   50
+  107  {z=-10.0}             t1z    1
   108  {z=0.0}               t1z  400
   109  {z=1.0}               t1z  300
-  110  {z=2.0}               t1z  200
+  110  {z=2.0}               t1z  175
-  111  {z=3.0}               t1z  100
+  111  {z=3.0}               t1z  125
-  112  {z=4.0}               t1z   50
+  112  {z=4.0}               t1z    1
-  113  {z=1.5}               t1z   50
+  113  {z=1.5}               t1z    1
-  114  {z=2.5}               t1z   50
+  114  {z=2.5}               t1z    1
   
-  201  {z IN (-1)}           t1z   50
+  201  {z IN (-1)}           t1z    1
   202  {z IN (0)}            t1z  400
   203  {z IN (1)}            t1z  300
-  204  {z IN (2)}            t1z  200
+  204  {z IN (2)}            t1z  175
-  205  {z IN (3)}            t1z  100
+  205  {z IN (3)}            t1z  125
-  206  {z IN (4)}            t1z   50
+  206  {z IN (4)}            t1z    1
-  207  {z IN (0.5)}          t1z   50
+  207  {z IN (0.5)}          t1z    1
   208  {z IN (0,1)}          t1z  700
-  209  {z IN (0,1,2)}        t1z  900
+  209  {z IN (0,1,2)}        t1z  875
   210  {z IN (0,1,2,3)}      {}   100
   211  {z IN (0,1,2,3,4,5)}  {}   100
-  212  {z IN (1,2)}          t1z  500
+  212  {z IN (1,2)}          t1z  475
   213  {z IN (2,3)}          t1z  300
   214  {z=3 OR z=2}          t1z  300
-  215  {z IN (-1,3)}         t1z  150
+  215  {z IN (-1,3)}         t1z  126
-  216  {z=-1 OR z=3}         t1z  150
+  216  {z=-1 OR z=3}         t1z  126
 
-  300  {y=0}                 {}   100
+  300  {y=0}                 t1y  974
-  301  {y=1}                 t1y   50
+  301  {y=1}                 t1y   26
-  302  {y=0.1}               t1y   50
+  302  {y=0.1}               t1y    1
 
   400  {x IS NULL}           t1x  400
 
@@ -204,16 +192,17 @@ db eval {
 # Verify that range queries generate the correct row count estimates
 #
 foreach {testid where index rows} {
-  500  {x IS NULL AND u='charlie'}         t1u  20
+  500  {x IS NULL AND u='charlie'}         t1u  17
-  501  {x=1 AND u='charlie'}               t1x   5
+  501  {x=1 AND u='charlie'}               t1x   1
-  502  {x IS NULL}                          {} 100
+  502  {x IS NULL}                         t1x 995
-  503  {x=1}                               t1x  50
+  503  {x=1}                               t1x   1
-  504  {x IS NOT NULL}                     t1x  25
+  504  {x IS NOT NULL}                     t1x   2
   505  {+x IS NOT NULL}                     {} 500
   506  {upper(x) IS NOT NULL}               {} 500
 
 } {
   # Verify that the expected index is used with the expected row count
+if {$testid==50299} {breakpoint; set sqlite_where_trace 1}
   do_test analyze5-1.${testid}a {
     set x [lindex [eqp "SELECT * FROM t1 WHERE $where"] 3]
     set idx {}
@@ -221,6 +210,7 @@ foreach {testid where index rows} {
     regexp {~([0-9]+) rows} $x all nrow
     list $idx $nrow
   } [list $index $rows]
+if {$testid==50299} exit
 
   # Verify that the same result is achieved regardless of whether or not
   # the index is used

test/analyze6.test

@@ -17,7 +17,7 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
-ifcapable !stat2 {
+ifcapable !stat3 {
   finish_test
   return
 }

test/analyze7.test

@@ -82,14 +82,14 @@ do_test analyze7-3.1 {
 do_test analyze7-3.2.1 {
   execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=?;}
 } {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=?) (~86 rows)}}
-ifcapable stat2 {
+ifcapable stat3 {
-  # If ENABLE_STAT2 is defined, SQLite comes up with a different estimated
+  # If ENABLE_STAT3 is defined, SQLite comes up with a different estimated
   # row count for (c=2) than it does for (c=?).
   do_test analyze7-3.2.2 {
     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=2;}
-  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=?) (~51 rows)}}
+  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=?) (~57 rows)}}
 } else {
-  # If ENABLE_STAT2 is not defined, the expected row count for (c=2) is the
+  # If ENABLE_STAT3 is not defined, the expected row count for (c=2) is the
   # same as that for (c=?).
   do_test analyze7-3.2.3 {
     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=2;}
@@ -98,12 +98,14 @@ ifcapable stat2 {
 do_test analyze7-3.3 {
   execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=123 AND b=123}
 } {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?) (~1 rows)}}
-do_test analyze7-3.4 {
+ifcapable {!stat3} {
+  do_test analyze7-3.4 {
     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=123 AND b=123}
-} {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?) (~2 rows)}}
+  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?) (~2 rows)}}
-do_test analyze7-3.5 {
+  do_test analyze7-3.5 {
     execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE a=123 AND c=123}
-} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?) (~1 rows)}}
+  } {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?) (~1 rows)}}
+}
 do_test analyze7-3.6 {
   execsql {EXPLAIN QUERY PLAN SELECT * FROM t1 WHERE c=123 AND d=123 AND b=123}
 } {0 0 0 {SEARCH TABLE t1 USING INDEX t1cd (c=? AND d=?) (~1 rows)}}

test/analyze8.test

@@ -0,0 +1,103 @@
+# 2011 August 13
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file implements tests for SQLite library.  The focus of the tests
+# in this file is testing the capabilities of sqlite_stat3.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+ifcapable !stat3 {
+  finish_test
+  return
+}
+
+set testprefix analyze8
+
+proc eqp {sql {db db}} {
+  uplevel execsql [list "EXPLAIN QUERY PLAN $sql"] $db
+}
+
+# Scenario:
+#
+#    Two indices.  One has mostly singleton entries, but for a few
+#    values there are hundreds of entries.  The other has 10-20
+#    entries per value.
+#
+# Verify that the query planner chooses the first index for the singleton
+# entries and the second index for the others.
+#
+do_test 1.0 {
+  db eval {
+    CREATE TABLE t1(a,b,c,d);
+    CREATE INDEX t1a ON t1(a);
+    CREATE INDEX t1b ON t1(b);
+    CREATE INDEX t1c ON t1(c);
+  }
+  for {set i 0} {$i<1000} {incr i} {
+    if {$i%2==0} {set a $i} {set a [expr {($i%8)*100}]}
+    set b [expr {$i/10}]
+    set c [expr {$i/8}]
+    set c [expr {$c*$c*$c}]
+    db eval {INSERT INTO t1 VALUES($a,$b,$c,$i)}
+  }
+  db eval {ANALYZE}
+} {}
+
+# The a==100 comparison is expensive because there are many rows
+# with a==100.  And so for those cases, choose the t1b index.
+#
+# Buf ro a==99 and a==101, there are far fewer rows so choose
+# the t1a index.
+#
+do_test 1.1 {
+  eqp {SELECT * FROM t1 WHERE a=100 AND b=55}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?) (~2 rows)}}
+do_test 1.2 {
+  eqp {SELECT * FROM t1 WHERE a=99 AND b=55}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?) (~1 rows)}}
+do_test 1.3 {
+  eqp {SELECT * FROM t1 WHERE a=101 AND b=55}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?) (~1 rows)}}
+do_test 1.4 {
+  eqp {SELECT * FROM t1 WHERE a=100 AND b=56}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b=?) (~2 rows)}}
+do_test 1.5 {
+  eqp {SELECT * FROM t1 WHERE a=99 AND b=56}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?) (~1 rows)}}
+do_test 1.6 {
+  eqp {SELECT * FROM t1 WHERE a=101 AND b=56}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?) (~1 rows)}}
+do_test 2.1 {
+  eqp {SELECT * FROM t1 WHERE a=100 AND b BETWEEN 50 AND 54}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?) (~2 rows)}}
+
+# There are many more values of c between 0 and 100000 than there are
+# between 800000 and 900000.  So t1c is more selective for the latter
+# range.
+#
+do_test 3.1 {
+  eqp {SELECT * FROM t1 WHERE b BETWEEN 50 AND 54 AND c BETWEEN 0 AND 100000}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX t1b (b>? AND b<?) (~6 rows)}}
+do_test 3.2 {
+  eqp {SELECT * FROM t1
+       WHERE b BETWEEN 50 AND 54 AND c BETWEEN 800000 AND 900000}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX t1c (c>? AND c<?) (~4 rows)}}
+do_test 3.3 {
+  eqp {SELECT * FROM t1 WHERE a=100 AND c BETWEEN 0 AND 100000}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX t1a (a=?) (~63 rows)}}
+do_test 3.4 {
+  eqp {SELECT * FROM t1
+       WHERE a=100 AND c BETWEEN 800000 AND 900000}
+} {0 0 0 {SEARCH TABLE t1 USING INDEX t1c (c>? AND c<?) (~2 rows)}}
+
+finish_test

test/auth.test

@@ -2323,9 +2323,13 @@ ifcapable compound&&subquery {
   }
   ifcapable stat2 {
     set stat2 "sqlite_stat2 "
+  } else {
+    ifcapable stat3 {
+      set stat2 "sqlite_stat3 "
     } else {
       set stat2 ""
     }
+  }
   do_test auth-5.2 {
     execsql {
       SELECT name FROM (

test/dbstatus.test

@@ -56,6 +56,12 @@ proc lookaside {db} {
   }
 }
 
+ifcapable stat3 {
+  set STAT3 1
+} else {
+  set STAT3 0
+}
+
 #---------------------------------------------------------------------------
 # Run the dbstatus-2 and dbstatus-3 tests with several of different
 # lookaside buffer sizes.
@@ -118,7 +124,7 @@ foreach ::lookaside_buffer_size {0 64 120} {
       CREATE TABLE t2(c, d);
       CREATE VIEW v1 AS SELECT * FROM t1 UNION SELECT * FROM t2;
     }
-    6 {
+    6y {
       CREATE TABLE t1(a, b);
       CREATE INDEX i1 ON t1(a);
       CREATE INDEX i2 ON t1(a,b);
@@ -198,7 +204,11 @@ foreach ::lookaside_buffer_size {0 64 120} {
     # much greater than just that reported by DBSTATUS_SCHEMA_USED in this
     # case.
     #
-    if {[string match *x $tn] || $AUTOVACUUM} {
+    # Some of the memory used for sqlite_stat3 is unaccounted for by
+    # dbstatus.
+    #
+    if {[string match *x $tn] || $AUTOVACUUM
+         || ([string match *y $tn] && $STAT3)} {
       do_test dbstatus-2.$tn.ax { expr {($nSchema1-$nSchema2)<=$nFree} } 1
     } else {
       do_test dbstatus-2.$tn.a { expr {$nSchema1-$nSchema2} } $nFree

test/stat3.test

@@ -0,0 +1,56 @@
+# 2011 August 08
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# This file implements regression tests for SQLite library. This file 
+# implements tests for the extra functionality provided by the ANALYZE 
+# command when the library is compiled with SQLITE_ENABLE_STAT3 defined.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+
+set testprefix stat3
+
+
+# Verify that if not compiled with SQLITE_ENABLE_STAT2 that the ANALYZE
+# command will delete the sqlite_stat2 table.  Likewise, if not compiled
+# with SQLITE_ENABLE_STAT3, the sqlite_stat3 table is deleted.
+#
+do_test 1.1 {
+  db eval {
+    PRAGMA writable_schema=ON;
+    CREATE TABLE sqlite_stat2(tbl,idx,sampleno,sample);
+    CREATE TABLE sqlite_stat3(tbl,idx,neq,nlt,ndlt,sample);
+    SELECT name FROM sqlite_master ORDER BY 1;
+  }
+} {sqlite_stat2 sqlite_stat3}
+do_test 1.2 {
+  db close
+  sqlite3 db test.db
+  db eval {SELECT name FROM sqlite_master ORDER BY 1}
+} {sqlite_stat2 sqlite_stat3}
+
+ifcapable {stat3} {
+  do_test 1.3 {
+    db eval {ANALYZE; SELECT name FROM sqlite_master ORDER BY 1}
+  } {sqlite_stat1 sqlite_stat3}
+} else {
+  do_test 1.4 {
+    db eval {ANALYZE; SELECT name FROM sqlite_master ORDER BY 1}
+  } {sqlite_stat1}
+  finish_test
+  return
+}
+
+
+
+
+finish_test

test/tkt-cbd054fa6b.test

@@ -16,7 +16,7 @@
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
 
-ifcapable !stat2 {
+ifcapable !stat3 {
   finish_test
   return
 }
@@ -46,7 +46,7 @@ do_test tkt-cbd05-1.2 {
 do_test tkt-cbd05-1.3 {
   execsql { 
     SELECT tbl,idx,group_concat(sample,' ') 
-    FROM sqlite_stat2 
+    FROM sqlite_stat3 
     WHERE idx = 't1_x' 
     GROUP BY tbl,idx
   }
@@ -78,7 +78,7 @@ do_test tkt-cbd05-2.2 {
 do_test tkt-cbd05-2.3 {
   execsql { 
     SELECT tbl,idx,group_concat(sample,' ') 
-    FROM sqlite_stat2 
+    FROM sqlite_stat3 
     WHERE idx = 't1_x' 
     GROUP BY tbl,idx
   }

test/unordered.test

@@ -31,11 +31,13 @@ do_execsql_test 1.0 {
 } {}
 
 foreach idxmode {ordered unordered} {
+  catchsql { DELETE FROM sqlite_stat2 }
+  catchsql { DELETE FROM sqlite_stat3 }
   if {$idxmode == "unordered"} {
     execsql { UPDATE sqlite_stat1 SET stat = stat || ' unordered' }
+  }
   db close
   sqlite3 db test.db
-  }
   foreach {tn sql r(ordered) r(unordered)} {
     1   "SELECT * FROM t1 ORDER BY a"
         {0 0 0 {SCAN TABLE t1 USING INDEX i1 (~128 rows)}}

tool/warnings.sh

@@ -9,9 +9,9 @@ echo '********** No optimizations.  Includes FTS4 and RTREE *********'
 gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \
       -ansi -DHAVE_STDINT_H -DSQLITE_ENABLE_FTS4 -DSQLITE_ENABLE_RTREE \
       sqlite3.c
-echo '********** No optimizations. ENABLE_STAT2. THREADSAFE=0 *******'
+echo '********** No optimizations. ENABLE_STAT3. THREADSAFE=0 *******'
 gcc -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \
-      -ansi -DSQLITE_ENABLE_STAT2 -DSQLITE_THREADSAFE=0 \
+      -ansi -DSQLITE_ENABLE_STAT3 -DSQLITE_THREADSAFE=0 \
       sqlite3.c
 echo '********** Optimized -O3.  Includes FTS4 and RTREE ************'
 gcc -O3 -c -Wshadow -Wall -Wextra -pedantic-errors -Wno-long-long -std=c89 \