First version of sqlite_stat2 (schema forces exactly 10 samples).

FossilOrigin-Name: dd96bda2a85c1d94fb4a0bf5f27e2977f7f7e42e

manifest

@@ -1,8 +1,5 @@
------BEGIN PGP SIGNED MESSAGE-----
+C First\sversion\sof\ssqlite_stat2\s(schema\sforces\sexactly\s10\ssamples).
-Hash: SHA1
+D 2009-08-17T17:06:59
-
-C Update\sthe\samalgamation\sbuilder\sso\sthat\sit\savoids\sputting\sredundant\nSQLITE_API\smacros\son\sdeclarations.
-D 2009-08-14T18:18:04
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in 0f7761c5d1c62ae7a841e3393ffaff1fa0f5c00a
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -103,7 +100,7 @@ F sqlite.pc.in 42b7bf0d02e08b9e77734a47798d1a55a9e0716b
 F sqlite3.1 6be1ad09113570e1fc8dcaff84c9b0b337db5ffc
 F sqlite3.pc.in ae6f59a76e862f5c561eb32a380228a02afc3cad
 F src/alter.c 8b42cace4f8e312de596807ba2685179da64fec4
-F src/analyze.c e239496cfb5394ac8867f1c112905ddab8d01cd9
+F src/analyze.c c3f1ea347d5a2c90aec66a510e4c7b29d79fbca2
 F src/attach.c 13995348fc5a26cdd136a50806faf292aabc173f
 F src/auth.c 802a9439dfa0b8c208b10055cba400e82ef18025
 F src/backup.c 6f1c2d9862c8a3feb7739dfcca02c1f5352e37f3
@@ -112,12 +109,12 @@ F src/btmutex.c 0f43a75bb5b8147b386e8e1c3e71ba734e3863b7
 F src/btree.c 49212ddaee8d7d12b4f1e17b9de62f7ea91ca59d
 F src/btree.h 577448a890c2ab9b21e6ab74f073526184bceebe
 F src/btreeInt.h 1c86297e69380f6577e7ae67452597dd8d5c2705
-F src/build.c a15de7c5d020a778b641fca0b2510126843f4b30
+F src/build.c 09389ab5d5a5997dbefa8f15fc4755f6c3243f4c
 F src/callback.c cb68b21b0d4ae7d11ae0e487933bce3323784dcf
 F src/complete.c 5ad5c6cd4548211867c204c41a126d73a9fbcea0
 F src/date.c ab5f7137656652a48434d64f96bdcdc823bb23b3
 F src/delete.c dcf07632d8ca3d4086df8b65ea907a47278e6382
-F src/expr.c d069ba1e060f296ea4f18fb85198fafefd00b22f
+F src/expr.c ea04de0bf495eb899ba0c8c7af6561afb1dfac2d
 F src/fault.c dc88c821842157460750d2d61a8a8b4197d047ff
 F src/func.c e536218d193b8d326aab91120bc4c6f28aa2b606
 F src/global.c 448419c44ce0701104c2121b0e06919b44514c0c
@@ -166,12 +163,12 @@ F src/select.c 67b0778c9585905c8aa75aaa469e76ef3c1d315a
 F src/shell.c db2643650b9268df89a4bedca3f1c6d9e786f1bb
 F src/sqlite.h.in a6850e9034df1336e8139c4d6964d7d2f0f52337
 F src/sqlite3ext.h 1db7d63ab5de4b3e6b83dd03d1a4e64fef6d2a17
-F src/sqliteInt.h 6a90791138ba3447572d184d0798c24f3cbbec98
+F src/sqliteInt.h 21722d546c8a93bf079564d49e628d5e66d3244a
 F src/sqliteLimit.h ffe93f5a0c4e7bd13e70cd7bf84cfb5c3465f45d
 F src/status.c 237b193efae0cf6ac3f0817a208de6c6c6ef6d76
 F src/table.c cc86ad3d6ad54df7c63a3e807b5783c90411a08d
 F src/tclsqlite.c e18e5013dc6bca9f25e6022fbe17ba3ccb821f95
-F src/test1.c 0e882812c94cf35fce30fc25fbf952a33a86d70b
+F src/test1.c eacb3456a9419191f42a0f601e12ca8a424a6de1
 F src/test2.c 0de743ec8890ca4f09e0bce5d6d5a681f5957fec
 F src/test3.c 2445c2beb5e7a0c91fd8136dc1339ec369a24898
 F src/test4.c f79ab52d27ff49b784b631a42e2ccd52cfd5c84c
@@ -204,19 +201,19 @@ F src/test_wsd.c 3ae5101de6cbfda2720152ab659ea84079719241
 F src/tokenize.c af8a56e6a50c5042fc305bfa796275e9bf26ff2b
 F src/trigger.c 9bc5278d509d81ff0f9b52f0ce7239563d188e32
 F src/update.c 4da327f706c0d0dfedf4d92154b1b5688bdea0ac
-F src/utf.c 9541d28f40441812c0b40f00334372a0542c00ff
+F src/utf.c 9b022ac1c1f306733d57daa0df0b8beb7c17e95e
 F src/util.c c2416f60ae704a8c4990e4909aa810f90cbffa07
 F src/vacuum.c 3fe0eebea6d2311c1c2ab2962887d11f7a4dcfb0
-F src/vdbe.c 0ce57f8211899b59d1d6f1642f79e75fc212d6d0
+F src/vdbe.c f2c07c6440826f69fc6d39083ac7fe5ba98fe3be
 F src/vdbe.h 457b6c70f02885cec1f5225b5e6441d067b55d3f
 F src/vdbeInt.h 831c254a6eef237ef4664c8381a0137586567007
 F src/vdbeapi.c 0ab8ada7260b32031ca97f338caecf0812460624
 F src/vdbeaux.c 4956536a636468fd07284028c39aab65ea99777e
 F src/vdbeblob.c a3f3e0e877fc64ea50165eec2855f5ada4477611
-F src/vdbemem.c ff40efaa2772e7aa66cf7501bf4142fd1a44bf51
+F src/vdbemem.c afd6ce02945e659f65642f290a37ccf4a88c4dcb
 F src/vtab.c aedd76e8670d5a5379f93804398d3ba960125547
 F src/walker.c 1edca756275f158b80f20eb6f104c8d3fcc96a04
-F src/where.c 7573120c1f2fe6d4c246f138f1e30fbcda3db241
+F src/where.c 33a3aa8bef9594002300b4bc9aa2a7b37c71345c
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/alias.test 4529fbc152f190268a15f9384a5651bbbabc9d87
 F test/all.test 14165b3e32715b700b5f0cbf8f6e3833dda0be45
@@ -235,7 +232,7 @@ F test/attach.test 1d1be27b9e4c654f9bb14d011a4a87753c0b197a
 F test/attach2.test a295d2d7061adcee5884ef4a93c7c96a82765437
 F test/attach3.test 7b92dc8e40c1ebca9732ca6f2d3fefbd46f196df
 F test/attachmalloc.test cf8cf17d183de357b1147a9baacbdfc85b940b61
-F test/auth.test b2813abf4ae55f179fbd6db486ed8a6599de0b73
+F test/auth.test 393be593c72bc452cd2fe6e026682752aa258559
 F test/auth2.test ee3ba272e2b975e913afc9b041ee75706e190005
 F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5
 F test/autoinc.test 71bc5183c93ed5e2b8b3a71c218d777b55e4fffc
@@ -746,14 +743,7 @@ F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
 F tool/vdbe-compress.tcl 672f81d693a03f80f5ae60bfefacd8a349e76746
-P 9cbe3654055a78c09ea1ecd5dc599bcd888b57e3
+P 0d5b058717858c9cda8ca120a3d814453a94a0e6
-R 9142cd5088584d6322dd160e9df9a386
+R 942eb5447bdff248ec9f0cbf9f54bb64
-U drh
+U dan
-Z 2a7604fcf5cc328cdfa6d300890f778c
+Z d16a7c84025a55a7e2426c9a05cd868d
------BEGIN PGP SIGNATURE-----
-Version: GnuPG v1.4.6 (GNU/Linux)
-
-iD8DBQFKhapgoxKgR168RlERAkH2AJoC/gxQtUL74VspfOFK2+qp6iCVFQCfRUDA
-GwXZvHleDts/BZKq+4hsve0=
-=Wsj5
------END PGP SIGNATURE-----

manifest.uuid

@@ -1 +1 @@
-0d5b058717858c9cda8ca120a3d814453a94a0e6
+dd96bda2a85c1d94fb4a0bf5f27e2977f7f7e42e

src/analyze.c

@@ -17,8 +17,9 @@
 #include "sqliteInt.h"
 
 /*
-** This routine generates code that opens the sqlite_stat1 table on cursor
+** This routine generates code that opens the sqlite_stat1 table for
-** iStatCur.
+** writing with cursor iStatCur. The sqlite_stat2 table is opened
+** for writing using cursor (iStatCur+1).
 **
 ** If the sqlite_stat1 tables does not previously exist, it is created.
 ** If it does previously exist, all entires associated with table zWhere
@@ -30,53 +31,55 @@ static void openStatTable(
   int iStatCur,           /* Open the sqlite_stat1 table on this cursor */
   const char *zWhere      /* Delete entries associated with this table */
 ){
+  const char *aName[] = { "sqlite_stat1", "sqlite_stat2" };
+  const char *aCols[] = { "tbl,idx,stat", "tbl,idx," SQLITE_INDEX_SAMPLE_COLS };
+  int aRoot[] = {0, 0};
+  int aCreateTbl[] = {0, 0};
+
+  int i;
   sqlite3 *db = pParse->db;
   Db *pDb;
-  int iRootPage;
-  u8 createStat1 = 0;
-  Table *pStat;
   Vdbe *v = sqlite3GetVdbe(pParse);
-
   if( v==0 ) return;
   assert( sqlite3BtreeHoldsAllMutexes(db) );
   assert( sqlite3VdbeDb(v)==db );
   pDb = &db->aDb[iDb];
-  if( (pStat = sqlite3FindTable(db, "sqlite_stat1", pDb->zName))==0 ){
+
-    /* The sqlite_stat1 tables does not exist.  Create it.  
+  for(i=0; i<ArraySize(aName); i++){
-    ** Note that a side-effect of the CREATE TABLE statement is to leave
+    Table *pStat;
-    ** the rootpage of the new table in register pParse->regRoot.  This is
+    if( (pStat = sqlite3FindTable(db, aName[i], pDb->zName))==0 ){
-    ** important because the OpenWrite opcode below will be needing it. */
+      /* The sqlite_stat[12] table does not exist. Create it. Note that a 
+      ** side-effect of the CREATE TABLE statement is to leave the rootpage 
+      ** of the new table in register pParse->regRoot. This is important 
+      ** because the OpenWrite opcode below will be needing it. */
       sqlite3NestedParse(pParse,
-      "CREATE TABLE %Q.sqlite_stat1(tbl,idx,stat)",
+          "CREATE TABLE %Q.%s(%s)", pDb->zName, aName[i], aCols[i]
-      pDb->zName
       );
-    iRootPage = pParse->regRoot;
+      aRoot[i] = pParse->regRoot;
-    createStat1 = 1;  /* Cause rootpage to be taken from top of stack */
+      aCreateTbl[i] = 1;
-  }else if( zWhere ){
-    /* The sqlite_stat1 table exists.  Delete all entries associated with
-    ** the table zWhere. */
-    sqlite3NestedParse(pParse,
-       "DELETE FROM %Q.sqlite_stat1 WHERE tbl=%Q",
-       pDb->zName, zWhere
-    );
-    iRootPage = pStat->tnum;
     }else{
-    /* The sqlite_stat1 table already exists.  Delete all rows. */
+      /* The table already exists. If zWhere is not NULL, delete all entries 
-    iRootPage = pStat->tnum;
+      ** associated with the table zWhere. If zWhere is NULL, delete the
-    sqlite3VdbeAddOp2(v, OP_Clear, pStat->tnum, iDb);
+      ** entire contents of the table. */
+      aRoot[i] = pStat->tnum;
+      sqlite3TableLock(pParse, iDb, aRoot[i], 1, aName[i]);
+      if( zWhere ){
+        sqlite3NestedParse(pParse,
+           "DELETE FROM %Q.%s WHERE tbl=%Q", pDb->zName, aName[i], zWhere
+        );
+      }else{
+        /* The sqlite_stat[12] table already exists.  Delete all rows. */
+        sqlite3VdbeAddOp2(v, OP_Clear, aRoot[i], iDb);
+      }
+    }
   }
 
-  /* Open the sqlite_stat1 table for writing. Unless it was created
+  /* Open the sqlite_stat[12] tables for writing. */
-  ** by this vdbe program, lock it for writing at the shared-cache level. 
+  for(i=0; i<ArraySize(aName); i++){
-  ** If this vdbe did create the sqlite_stat1 table, then it must have 
+    sqlite3VdbeAddOp3(v, OP_OpenWrite, iStatCur+i, aRoot[i], iDb);
-  ** already obtained a schema-lock, making the write-lock redundant.
-  */
-  if( !createStat1 ){
-    sqlite3TableLock(pParse, iDb, iRootPage, 1, "sqlite_stat1");
-  }
-  sqlite3VdbeAddOp3(v, OP_OpenWrite, iStatCur, iRootPage, iDb);
     sqlite3VdbeChangeP4(v, -1, (char *)3, P4_INT32);
-  sqlite3VdbeChangeP5(v, createStat1);
+    sqlite3VdbeChangeP5(v, aCreateTbl[i]);
+  }
 }
 
 /*
@@ -117,6 +120,7 @@ static void analyzeOneTable(
   /* Establish a read-lock on the table at the shared-cache level. */
   sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
 
+  iMem += 3;
   iIdxCur = pParse->nTab++;
   for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){
     KeyInfo *pKey = sqlite3IndexKeyinfo(pParse, pIdx);
@@ -126,6 +130,7 @@ static void analyzeOneTable(
     int regCol;       /* Content of a column from the table being analyzed */
     int regRowid;     /* Rowid for the inserted record */
     int regF2;
+    int regStat2;
 
     /* Open a cursor to the index to be analyzed
     */
@@ -134,22 +139,35 @@ static void analyzeOneTable(
     sqlite3VdbeAddOp4(v, OP_OpenRead, iIdxCur, pIdx->tnum, iDb,
         (char *)pKey, P4_KEYINFO_HANDOFF);
     VdbeComment((v, "%s", pIdx->zName));
-    regFields = iMem+nCol*2;
+    regStat2 = iMem+nCol*2+1;
+    regFields = regStat2+2+SQLITE_INDEX_SAMPLES;
     regTemp = regRowid = regCol = regFields+3;
     regRec = regCol+1;
     if( regRec>pParse->nMem ){
       pParse->nMem = regRec;
     }
 
-    /* Memory cells are used as follows:
+    /* Fill in the register with the total number of rows. */
+    if( pTab->pIndex==pIdx ){
+      sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, iMem-3);
+    }
+    sqlite3VdbeAddOp2(v, OP_Integer, 0, iMem-2);
+    sqlite3VdbeAddOp2(v, OP_Integer, 1, iMem-1);
+
+    /* Memory cells are used as follows. All memory cell addresses are
+    ** offset by iMem. That is, cell 0 below is actually cell iMem, cell
+    ** 1 is cell 1+iMem, etc.
     **
-    **    mem[iMem]:             The total number of rows in the table.
+    **    0:               The total number of rows in the table.
-    **    mem[iMem+1]:           Number of distinct values in column 1
+    **
-    **    ...
+    **    1..nCol:         Number of distinct entries in index considering the
-    **    mem[iMem+nCol]:        Number of distinct values in column N
+    **                     left-most N columns, where N is the same as the 
-    **    mem[iMem+nCol+1]       Last observed value of column 1
+    **                     memory cell number.
-    **    ...
+    **
-    **    mem[iMem+nCol+nCol]:   Last observed value of column N
+    **    nCol+1..2*nCol:  Previous value of indexed columns, from left to
+    **                     right.
+    **
+    **    2*nCol+1..2*nCol+10: 10 evenly spaced samples.
     **
     ** Cells iMem through iMem+nCol are initialized to 0.  The others
     ** are initialized to NULL.
@@ -161,29 +179,35 @@ static void analyzeOneTable(
       sqlite3VdbeAddOp2(v, OP_Null, 0, iMem+nCol+i+1);
     }
 
-    /* Do the analysis.
+    /* Start the analysis loop. This loop runs through all the entries inof
-    */
+    ** the index b-tree.  */
     endOfLoop = sqlite3VdbeMakeLabel(v);
     sqlite3VdbeAddOp2(v, OP_Rewind, iIdxCur, endOfLoop);
     topOfLoop = sqlite3VdbeCurrentAddr(v);
     sqlite3VdbeAddOp2(v, OP_AddImm, iMem, 1);
+
     for(i=0; i<nCol; i++){
       sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regCol);
+      if( i==0 ){
+        sqlite3VdbeAddOp3(v, OP_Sample, iMem-3, regCol, regStat2+2);
+      }
       sqlite3VdbeAddOp3(v, OP_Ne, regCol, 0, iMem+nCol+i+1);
       /**** TODO:  add collating sequence *****/
       sqlite3VdbeChangeP5(v, SQLITE_JUMPIFNULL);
     }
     sqlite3VdbeAddOp2(v, OP_Goto, 0, endOfLoop);
     for(i=0; i<nCol; i++){
-      sqlite3VdbeJumpHere(v, topOfLoop + 2*(i + 1));
+      sqlite3VdbeJumpHere(v, topOfLoop + 1 + 2*(i + 1));
       sqlite3VdbeAddOp2(v, OP_AddImm, iMem+i+1, 1);
       sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, iMem+nCol+i+1);
     }
+
+    /* End of the analysis loop. */
     sqlite3VdbeResolveLabel(v, endOfLoop);
     sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, topOfLoop);
     sqlite3VdbeAddOp1(v, OP_Close, iIdxCur);
 
-    /* Store the results.  
+    /* Store the results in sqlite_stat1.
     **
     ** The result is a single row of the sqlite_stat1 table.  The first
     ** two columns are the names of the table and index.  The third column
@@ -219,6 +243,16 @@ static void analyzeOneTable(
     sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regRowid);
     sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regRec, regRowid);
     sqlite3VdbeChangeP5(v, OPFLAG_APPEND);
+
+    /* Store the results in sqlite_stat2. */
+    sqlite3VdbeAddOp4(v, OP_String8, 0, regStat2, 0, pTab->zName, 0);
+    sqlite3VdbeAddOp4(v, OP_String8, 0, regStat2+1, 0, pIdx->zName, 0);
+    sqlite3VdbeAddOp4(v, OP_MakeRecord, regStat2, SQLITE_INDEX_SAMPLES+2,
+	regRec, "aabbbbbbbbbb", 0
+    );
+    sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur+1, regRowid);
+    sqlite3VdbeAddOp3(v, OP_Insert, iStatCur+1, regRec, regRowid);
+
     sqlite3VdbeJumpHere(v, addr);
   }
 }
@@ -245,7 +279,8 @@ static void analyzeDatabase(Parse *pParse, int iDb){
   int iMem;
 
   sqlite3BeginWriteOperation(pParse, 0, iDb);
-  iStatCur = pParse->nTab++;
+  iStatCur = pParse->nTab;
+  pParse->nTab += 2;
   openStatTable(pParse, iDb, iStatCur, 0);
   iMem = pParse->nMem+1;
   for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){
@@ -267,7 +302,8 @@ static void analyzeTable(Parse *pParse, Table *pTab){
   assert( sqlite3BtreeHoldsAllMutexes(pParse->db) );
   iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema);
   sqlite3BeginWriteOperation(pParse, 0, iDb);
-  iStatCur = pParse->nTab++;
+  iStatCur = pParse->nTab;
+  pParse->nTab += 2;
   openStatTable(pParse, iDb, iStatCur, pTab->zName);
   analyzeOneTable(pParse, pTab, iStatCur, pParse->nMem+1);
   loadAnalysis(pParse, iDb);
@@ -387,7 +423,8 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){
 }
 
 /*
-** Load the content of the sqlite_stat1 table into the index hash tables.
+** Load the content of the sqlite_stat1 and sqlite_stat2 tables into the 
+** index hash tables.
 */
 int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
   analysisInfo sInfo;
@@ -412,7 +449,6 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
      return SQLITE_ERROR;
   }
 
-
   /* Load new statistics out of the sqlite_stat1 table */
   zSql = sqlite3MPrintf(db, "SELECT idx, stat FROM %Q.sqlite_stat1",
                         sInfo.zDatabase);
@@ -423,8 +459,87 @@ int sqlite3AnalysisLoad(sqlite3 *db, int iDb){
     rc = sqlite3_exec(db, zSql, analysisLoader, &sInfo, 0);
     (void)sqlite3SafetyOn(db);
     sqlite3DbFree(db, zSql);
-    if( rc==SQLITE_NOMEM ) db->mallocFailed = 1;
   }
+
+  /* Load the statistics from the sqlite_stat2 table */
+  if( rc==SQLITE_OK ){
+    zSql = sqlite3MPrintf(db, 
+	"SELECT idx," SQLITE_INDEX_SAMPLE_COLS " FROM %Q.sqlite_stat2",
+        sInfo.zDatabase
+    );
+    if( zSql ){
+      sqlite3_stmt *pStmt = 0;
+      (void)sqlite3SafetyOff(db);
+      rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
+      if( rc==SQLITE_OK ){
+	while( SQLITE_ROW==sqlite3_step(pStmt) ){
+	  char *zIndex = (char *)sqlite3_column_text(pStmt, 0);
+	  Index *pIdx;
+          pIdx = sqlite3FindIndex(db, zIndex, sInfo.zDatabase);
+	  if( pIdx ){
+	    char *pSpace;
+	    IndexSample *pSample;
+	    int iCol;
+	    int nAlloc = SQLITE_INDEX_SAMPLES * sizeof(IndexSample);
+	    for(iCol=1; iCol<=SQLITE_INDEX_SAMPLES; iCol++){
+	      int eType = sqlite3_column_type(pStmt, iCol);
+	      if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){
+	        nAlloc += sqlite3_column_bytes(pStmt, iCol);
+	      }
+	    }
+	    pSample = sqlite3DbMallocRaw(db, nAlloc);
+	    if( !pSample ){
+	      rc = SQLITE_NOMEM;
+	      break;
+	    }
+	    sqlite3DbFree(db, pIdx->aSample);
+	    pIdx->aSample = pSample;
+	    pSpace = (char *)&pSample[SQLITE_INDEX_SAMPLES];
+	    for(iCol=1; iCol<=SQLITE_INDEX_SAMPLES; iCol++){
+	      int eType = sqlite3_column_type(pStmt, iCol);
+	      pSample[iCol-1].eType = eType;
+	      switch( eType ){
+                case SQLITE_BLOB:
+                case SQLITE_TEXT: {
+                  const char *z = (const char *)(
+		      (eType==SQLITE_BLOB) ?
+                      sqlite3_column_blob(pStmt, iCol):
+                      sqlite3_column_text(pStmt, iCol)
+		  );
+                  int n = sqlite3_column_bytes(pStmt, iCol);
+		  if( n>24 ){
+		    n = 24;
+		  }
+		  pSample[iCol-1].nByte = n;
+		  pSample[iCol-1].u.z = pSpace;
+		  memcpy(pSpace, z, n);
+		  pSpace += n;
+		  break;
+                }
+                case SQLITE_INTEGER:
+                case SQLITE_FLOAT:
+		  pSample[iCol-1].u.r = sqlite3_column_double(pStmt, iCol);
+		  break;
+                case SQLITE_NULL:
+		  break;
+	      }
+	    }
+	  }
+	}
+	if( rc==SQLITE_NOMEM ){
+	  sqlite3_finalize(pStmt);
+	}else{
+	  rc = sqlite3_finalize(pStmt);
+	}
+      }
+      (void)sqlite3SafetyOn(db);
+      sqlite3DbFree(db, zSql);
+    }else{
+      rc = SQLITE_NOMEM;
+    }
+  }
+
+  if( rc==SQLITE_NOMEM ) db->mallocFailed = 1;
   return rc;
 }
 

src/build.c

@@ -343,6 +343,7 @@ Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const char *zDb){
 static void freeIndex(Index *p){
   sqlite3 *db = p->pTable->dbMem;
   /* testcase( db==0 ); */
+  sqlite3DbFree(db, p->aSample);
   sqlite3DbFree(db, p->zColAff);
   sqlite3DbFree(db, p);
 }

src/expr.c

@@ -2835,6 +2835,7 @@ static int evalConstExpr(Walker *pWalker, Expr *pExpr){
     int r2;
     r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1);
     if( NEVER(r1!=r2) ) sqlite3ReleaseTempReg(pParse, r1);
+    pExpr->iColumn = pExpr->op;
     pExpr->op = TK_REGISTER;
     pExpr->iTable = r2;
     return WRC_Prune;

src/sqliteInt.h

@@ -77,6 +77,9 @@
 #include <inttypes.h>
 #endif
 
+#define SQLITE_INDEX_SAMPLES 10
+#define SQLITE_INDEX_SAMPLE_COLS "s1,s2,s3,s4,s5,s6,s7,s8,s9,s10"
+
 /*
 ** This macro is used to "hide" some ugliness in casting an int
 ** value to a ptr value under the MSVC 64-bit compiler.   Casting
@@ -595,6 +598,7 @@ typedef struct FuncDef FuncDef;
 typedef struct FuncDefHash FuncDefHash;
 typedef struct IdList IdList;
 typedef struct Index Index;
+typedef struct IndexSample IndexSample;
 typedef struct KeyClass KeyClass;
 typedef struct KeyInfo KeyInfo;
 typedef struct Lookaside Lookaside;
@@ -1410,6 +1414,20 @@ struct Index {
   Schema *pSchema; /* Schema containing this index */
   u8 *aSortOrder;  /* Array of size Index.nColumn. True==DESC, False==ASC */
   char **azColl;   /* Array of collation sequence names for index */
+  IndexSample *aSample;    /* Array of SQLITE_INDEX_SAMPLES samples */
+};
+
+/*
+** Each sample stored in the sqlite_stat2 table is represented in memory 
+** using a structure of this type.
+*/
+struct IndexSample {
+  union {
+    char *z;        /* Value if eType is SQLITE_TEXT or SQLITE_BLOB */
+    double r;       /* Value if eType is SQLITE_FLOAT or SQLITE_INTEGER */
+  } u;
+  u8 eType;         /* SQLITE_NULL, SQLITE_INTEGER ... etc. */
+  u8 nByte;         /* Size in byte of text or blob. */
 };
 
 /*
@@ -2781,6 +2799,7 @@ void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8,
 void sqlite3ValueFree(sqlite3_value*);
 sqlite3_value *sqlite3ValueNew(sqlite3 *);
 char *sqlite3Utf16to8(sqlite3 *, const void*, int);
+char *sqlite3Utf8to16(sqlite3 *, int, char *, int, int *);
 int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value **);
 void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8);
 #ifndef SQLITE_AMALGAMATION

src/test1.c

@@ -2309,7 +2309,9 @@ static int test_collate_func(
       assert(0);
   }
 
+  sqlite3BeginBenignMalloc();
   pVal = sqlite3ValueNew(0);
+  if( pVal ){
     sqlite3ValueSetStr(pVal, nA, zA, encin, SQLITE_STATIC);
     n = sqlite3_value_bytes(pVal);
     Tcl_ListObjAppendElement(i,pX,
@@ -2319,6 +2321,8 @@ static int test_collate_func(
     Tcl_ListObjAppendElement(i,pX,
         Tcl_NewStringObj((char*)sqlite3_value_text(pVal),n));
     sqlite3ValueFree(pVal);
+  }
+  sqlite3EndBenignMalloc();
 
   Tcl_EvalObjEx(i, pX, 0);
   Tcl_DecrRefCount(pX);

src/utf.c

@@ -454,6 +454,30 @@ char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte){
   return (m.flags & MEM_Dyn)!=0 ? m.z : sqlite3DbStrDup(db, m.z);
 }
 
+/*
+** Convert a UTF-8 string to the UTF-16 encoding specified by parameter
+** enc. A pointer to the new string is returned, and the value of *pnOut
+** is set to the length of the returned string in bytes. The call should
+** arrange to call sqlite3DbFree() on the returned pointer when it is
+** no longer required.
+** 
+** If a malloc failure occurs, NULL is returned and the db.mallocFailed
+** flag set.
+*/
+char *sqlite3Utf8to16(sqlite3 *db, int enc, char *z, int n, int *pnOut){
+  Mem m;
+  memset(&m, 0, sizeof(m));
+  m.db = db;
+  sqlite3VdbeMemSetStr(&m, z, n, SQLITE_UTF8, SQLITE_STATIC);
+  if( sqlite3VdbeMemTranslate(&m, enc) ){
+    assert( db->mallocFailed );
+    return 0;
+  }
+  assert( m.z==m.zMalloc );
+  *pnOut = m.n;
+  return m.z;
+}
+
 /*
 ** pZ is a UTF-16 encoded unicode string at least nChar characters long.
 ** Return the number of bytes in the first nChar unicode characters

src/vdbe.c

@@ -4974,6 +4974,49 @@ case OP_Expire: {
   break;
 }
 
+
+/* Opcode: Sample P1 P2 P3 * *
+**
+** Register P1 contains the total number of rows in the index being 
+** analyzed. Register P1+1 contains an integer between 0 and 9, the
+** index of the next sample required. Register P1+2 contains an index
+** between 1 and *P1, the number of the next sample required. Register
+** P1+3 contains the current row index.
+**
+** If the integer in register P1+3 is the same as the integer in register
+** P1+1, then the following takes place:
+**
+**   (a) the contents of register P1+1 is incremented.
+**
+**   (b) the contents of the register identified by parameter P2 is 
+**       copied to register number (P3 + X), where X is the newly
+**       incremented value of register P1+1.
+**
+**   (c) register P1+2 is set to the index of the next sample required.
+*/
+case OP_Sample: {
+  int p1 = pOp->p1;
+  i64 iReq = p->aMem[p1+2].u.i;
+  i64 iRow = p->aMem[p1+3].u.i;
+
+  while( iReq==iRow ){
+    i64 nRow = p->aMem[p1].u.i;
+    int iSample = ++p->aMem[p1+1].u.i;
+    Mem *pReg = &p->aMem[pOp->p3 + iSample - 1];
+
+    assert( pReg<&p->aMem[p->nMem] );
+    sqlite3VdbeMemShallowCopy(pReg, &p->aMem[pOp->p2], MEM_Ephem);
+    Deephemeralize(pReg);
+    if( iSample==SQLITE_INDEX_SAMPLES ){
+      iReq = 0;
+    }else{
+      iReq = iRow + (nRow-iRow)/(SQLITE_INDEX_SAMPLES - iSample);
+      p->aMem[p1+2].u.i = iReq;
+    }
+  }
+  break;
+}
+
 #ifndef SQLITE_OMIT_SHARED_CACHE
 /* Opcode: TableLock P1 P2 P3 P4 *
 **

src/vdbemem.c

@@ -999,6 +999,9 @@ int sqlite3ValueFromExpr(
     return SQLITE_OK;
   }
   op = pExpr->op;
+  if( op==TK_REGISTER ){
+    op = pExpr->iColumn;
+  }
 
   if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){
     pVal = sqlite3ValueNew(db);

src/where.c

@@ -1890,6 +1890,177 @@ static void bestVirtualIndex(
 }
 #endif /* SQLITE_OMIT_VIRTUALTABLE */
 
+/*
+** Argument pIdx is a pointer to an index structure that has an array of
+** SQLITE_INDEX_SAMPLES evenly spaced samples of the first indexed column
+** stored in Index.aSample. The domain of values stored in said column
+** may be thought of as divided into (SQLITE_INDEX_SAMPLES+1) regions.
+** Region 0 contains all values smaller than the first sample value. Region
+** 1 contains values larger than or equal to the value of the first sample,
+** but smaller than the value of the second. And so on.
+**
+** If successful, this function determines which of the regions value 
+** pVal lies in, sets *piRegion to the region index and returns SQLITE_OK.
+** Or, if an OOM occurs while converting text values between encodings,
+** SQLITE_NOMEM is returned.
+*/
+static int whereRangeRegion(
+  Parse *pParse,              /* Database connection */
+  Index *pIdx,                /* Index to consider domain of */
+  sqlite3_value *pVal,        /* Value to consider */
+  int *piRegion               /* OUT: Region of domain in which value lies */
+){
+  if( pVal ){
+    IndexSample *aSample = pIdx->aSample;
+    int i = 0;
+    int eType = sqlite3_value_type(pVal);
+
+    if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
+      double r = sqlite3_value_double(pVal);
+      for(i=0; i<SQLITE_INDEX_SAMPLES; i++){
+        if( aSample[i].eType==SQLITE_NULL ) continue;
+        if( aSample[i].eType>=SQLITE_TEXT || aSample[i].u.r>r ) break;
+      }
+    }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){
+      sqlite3 *db = pParse->db;
+      CollSeq *pColl;
+      const u8 *z;
+      int n;
+      if( eType==SQLITE_BLOB ){
+        z = (const u8 *)sqlite3_value_blob(pVal);
+        pColl = db->pDfltColl;
+	assert( pColl->enc==SQLITE_UTF8 );
+      }else{
+	pColl = sqlite3FindCollSeq(db, SQLITE_UTF8, *pIdx->azColl, 0);
+	if( sqlite3CheckCollSeq(pParse, pColl) ){
+	  return SQLITE_ERROR;
+	}
+        z = (const u8 *)sqlite3ValueText(pVal, pColl->enc);
+	if( !z ){
+	  return SQLITE_NOMEM;
+	}
+        assert( z && pColl && pColl->xCmp );
+      }
+      n = sqlite3ValueBytes(pVal, pColl->enc);
+
+      for(i=0; i<SQLITE_INDEX_SAMPLES; i++){
+	int r;
+        int eSampletype = aSample[i].eType;
+        if( eSampletype==SQLITE_NULL || eSampletype<eType ) continue;
+        if( (eSampletype!=eType) ) break;
+        if( pColl->enc==SQLITE_UTF8 ){
+	  r = pColl->xCmp(pColl->pUser, aSample[i].nByte, aSample[i].u.z, n, z);
+        }else{
+	  int nSample;
+	  char *zSample = sqlite3Utf8to16(
+              db, pColl->enc, aSample[i].u.z, aSample[i].nByte, &nSample
+          );
+	  if( !zSample ){
+	    assert( db->mallocFailed );
+	    return SQLITE_NOMEM;
+	  }
+	  r = pColl->xCmp(pColl->pUser, nSample, zSample, n, z);
+	  sqlite3DbFree(db, zSample);
+        }
+	if( r>0 ) break;
+      }
+    }
+
+    *piRegion = i;
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This function is used to estimate the number of rows that will be visited
+** by scanning an index for a range of values. The range may have an upper
+** bound, a lower bound, or both. The WHERE clause terms that set the upper
+** and lower bounds are represented by pLower and pUpper respectively. For
+** example, assuming that index p is on t1(a):
+**
+**   ... FROM t1 WHERE a > ? AND a < ? ...
+**                    |_____|   |_____|
+**                       |         |
+**                     pLower    pUpper
+**
+** If the upper or lower bound is not present, then NULL should be passed in
+** place of a WhereTerm.
+**
+** The nEq parameter is passed the index of the index column subject to the
+** range constraint. Or, equivalently, the number of equality constraints
+** optimized by the proposed index scan. For example, assuming index p is
+** on t1(a, b), and the SQL query is:
+**
+**   ... FROM t1 WHERE a = ? AND b > ? AND b < ? ...
+**
+** then nEq should be passed the value 1 (as the range restricted column,
+** b, is the second left-most column of the index). Or, if the query is:
+**
+**   ... FROM t1 WHERE a > ? AND a < ? ...
+**
+** then nEq should be passed 0.
+**
+** The returned value is an integer between 1 and 9, inclusive. A return
+** value of 1 indicates that the proposed range scan is expected to visit
+** approximately 1/9 (11%) of the rows selected by the nEq equality constraints
+** (if any). A return value of 9 indicates that it is expected that the
+** range scan will visit 9/9 (100%) of the rows selected by the equality
+** constraints.
+*/
+static int whereRangeScanEst(
+  Parse *pParse,
+  Index *p, 
+  int nEq, 
+  WhereTerm *pLower, 
+  WhereTerm *pUpper,
+  int *piEst                      /* OUT: Return value */
+){
+  sqlite3 *db = pParse->db;
+  sqlite3_value *pLowerVal = 0;
+  sqlite3_value *pUpperVal = 0;
+  int rc = SQLITE_OK;
+
+  if( nEq==0 && p->aSample ){
+    int iEst;
+    int iUpper = SQLITE_INDEX_SAMPLES;
+    int iLower = 0;
+    u8 aff = p->pTable->aCol[0].affinity;
+    if( pLower ){
+      Expr *pExpr = pLower->pExpr->pRight;
+      rc = sqlite3ValueFromExpr(db, pExpr, SQLITE_UTF8, aff, &pLowerVal);
+      if( !pLowerVal ) goto fallback;
+    }
+    if( pUpper ){
+      Expr *pExpr = pUpper->pExpr->pRight;
+      rc = sqlite3ValueFromExpr(db, pExpr, SQLITE_UTF8, aff, &pUpperVal);
+      if( !pUpperVal ){
+        sqlite3ValueFree(pLowerVal);
+        goto fallback;
+      }
+    }
+
+    rc = whereRangeRegion(pParse, p, pUpperVal, &iUpper);
+    if( rc==SQLITE_OK ){
+      rc = whereRangeRegion(pParse, p, pLowerVal, &iLower);
+    }
+
+    iEst = iUpper - iLower;
+    if( iEst>=SQLITE_INDEX_SAMPLES ) iEst = SQLITE_INDEX_SAMPLES-1;
+    else if( iEst<1 ) iEst = 1;
+
+    sqlite3ValueFree(pLowerVal);
+    sqlite3ValueFree(pUpperVal);
+    *piEst = iEst;
+    return rc;
+  }
+
+fallback:
+  assert( pLower || pUpper );
+  *piEst = (SQLITE_INDEX_SAMPLES-1) / ((pLower&&pUpper)?9:3);
+  return rc;
+}
+
+
 /*
 ** Find the query plan for accessing a particular table.  Write the
 ** best query plan and its cost into the WhereCost object supplied as the
@@ -2043,7 +2214,7 @@ static void bestBtreeIndex(
     int nEq;
     int bInEst = 0;
     int nInMul = 1;
-    int nBound = 1;
+    int nBound = 9;
     int bSort = 0;
     int bLookup = 0;
 
@@ -2075,14 +2246,13 @@ static void bestBtreeIndex(
       if( findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE|WO_GT|WO_GE, pIdx) ){
         WhereTerm *pTop = findTerm(pWC, iCur, j, notReady, WO_LT|WO_LE, pIdx);
         WhereTerm *pBtm = findTerm(pWC, iCur, j, notReady, WO_GT|WO_GE, pIdx);
+	whereRangeScanEst(pParse, pProbe, nEq, pBtm, pTop, &nBound);
         if( pTop ){
           wsFlags |= WHERE_TOP_LIMIT;
-          nBound *= 3;
           used |= pTop->prereqRight;
         }
         if( pBtm ){
           wsFlags |= WHERE_BTM_LIMIT;
-          nBound *= 3;
           used |= pBtm->prereqRight;
         }
         wsFlags |= (WHERE_COLUMN_RANGE|WHERE_ROWID_RANGE);
@@ -2152,8 +2322,8 @@ static void bestBtreeIndex(
       nInMul = nRow / aiRowEst[nEq];
     }
     cost = nRow + nInMul*estLog(aiRowEst[0]);
-    nRow /= nBound;
+    nRow = nRow * (double)nBound / 9.0;
-    cost /= nBound;
+    cost = cost * (double)nBound / 9.0;
     if( bSort ){
       cost += cost*estLog(cost);
     }

test/auth.test

@@ -2316,7 +2316,7 @@ ifcapable compound&&subquery {
       WHERE type='table'
       ORDER BY name
     }
-  } {sqlite_stat1 t1 t2 t3 t4}
+  } {sqlite_stat1 sqlite_stat2 t1 t2 t3 t4}
 }
 
 # Ticket #3944