Improved estimates of the relative speed of index scans based on declared

datatypes of columns in the table.  Add "r" column to PRAGMA index_info,
showing the estimated relative scan rate.

FossilOrigin-Name: 07462bb6059f023c22a6c84a4a02afbd84e69255

manifest

@@ -1,5 +1,5 @@
-C Progress\stoward\susing\sthe\siScanRatio\sinformation\son\sindices.\s\sMany\stests\nare\sstill\sfailing.
+C Improved\sestimates\sof\sthe\srelative\sspeed\sof\sindex\sscans\sbased\son\sdeclared\ndatatypes\sof\scolumns\sin\sthe\stable.\s\sAdd\s"r"\scolumn\sto\sPRAGMA\sindex_info,\nshowing\sthe\sestimated\srelative\sscan\srate.
-D 2013-10-04T02:36:19.375
+D 2013-10-04T15:30:21.142
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 5e41da95d92656a5004b03d3576e8b226858a28e
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -166,13 +166,13 @@ F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
 F src/btree.c 5ccbbaed7a32ba774306f610da4ab4f3e5348294
 F src/btree.h bfe0e8c5759b4ec77b0d18390064a6ef3cdffaaf
 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0
-F src/build.c 7926e17a5dc3f698dd8b15511a4902f48c73ac50
+F src/build.c 7a91b0db709404c8b606a6d8abcaf29b1bf75bbf
 F src/callback.c f99a8957ba2adf369645fac0db09ad8adcf1caa2
 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
 F src/ctime.c ea4b7f3623a0fcb1146e7f245d7410033e86859c
 F src/date.c 65196e95e69f36993659bd7781abe7c2f1994739
 F src/delete.c 45788c5e48734f2af4acd75a876466e5b9838e34
-F src/expr.c 1017f482217e093ecdaca91e8666681e83733252
+F src/expr.c e7338ccffdc391c53ba2d51c5eb6a2f5299e040e
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c be866cd8c4fa6cae98ba33109578fd1a3311ee5b
 F src/func.c 2c47b65e6e00e3e9374942f28254faf8adafe398
@@ -210,7 +210,7 @@ F src/parse.y a97566d6da75075589a7c716d1bda14b586cf8da
 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346
 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222
 F src/pcache1.c a467393909a4ed7ca9de066d85ba5c5b04a5be63
-F src/pragma.c 5a4d8d9c3c7d22ef920487a260ea3d4b59d3b3a6
+F src/pragma.c 3bcdc7a98ad9429218179d2d81fc4aa9e39d8592
 F src/prepare.c fa6988589f39af8504a61731614cd4f6ae71554f
 F src/printf.c da9119eb31a187a4b99f60aa4a225141c0ebb74b
 F src/random.c 0b2dbc37fdfbfa6bd455b091dfcef5bdb32dba68
@@ -221,7 +221,7 @@ F src/shell.c 5ee50ca3e35453bbd6ccdf1bdd0f6bbe9738e9fb
 F src/sqlite.h.in ec40aa958a270416fb04b4f72210357bf163d2c5
 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e
 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc
-F src/sqliteInt.h c32d524facfb39084cc1a830aef0c31aeb93c04e
+F src/sqliteInt.h da0e92df9550e57107aa43eab63b29d040fe9c80
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -722,7 +722,7 @@ F test/pcache.test b09104b03160aca0d968d99e8cd2c5b1921a993d
 F test/pcache2.test a83efe2dec0d392f814bfc998def1d1833942025
 F test/percentile.test b98fc868d71eb5619d42a1702e9ab91718cbed54
 F test/permutations.test 72f4f8881d388163ddbbeec0a6ed812e863ea2e6
-F test/pragma.test 5e7de6c32a5d764f09437d2025f07e4917b9e178
+F test/pragma.test de128f0253ee42a1edf6c80f21813babfb19bd3b
 F test/pragma2.test 224f0381f9411a78ae685cac24c13656a62021b7
 F test/printf.test ec9870c4dce8686a37818e0bf1aba6e6a1863552
 F test/progress.test a282973d1d17f08071bc58a77d6b80f2a81c354d
@@ -1119,7 +1119,7 @@ F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/wherecosttest.c f407dc4c79786982a475261866a161cd007947ae
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P d59d97b0a8d70dc31d45db61bbc11ebb5375a224
+P 6c352edbba85a15ca356b5e131f4b3b2723d1774
-R 753a9b9a56a7c0c63483772726286f21
+R bacabbd9616babddd027da683b20e83a
 U drh
-Z b513b854bdc830eeffd07ca885795f7d
+Z 469f9776554648d8f1435b44fd89545f

manifest.uuid

@@ -1 +1 @@
-6c352edbba85a15ca356b5e131f4b3b2723d1774
+07462bb6059f023c22a6c84a4a02afbd84e69255

src/build.c

@@ -1026,6 +1026,7 @@ void sqlite3AddColumn(Parse *pParse, Token *pName){
   ** be called next to set pCol->affinity correctly.
   */
   pCol->affinity = SQLITE_AFF_NONE;
+  pCol->szEst = 1;
   p->nCol++;
 }
 
@@ -1067,15 +1068,18 @@ void sqlite3AddNotNull(Parse *pParse, int onError){
 ** If none of the substrings in the above table are found,
 ** SQLITE_AFF_NUMERIC is returned.
 */
-char sqlite3AffinityType(const char *zIn){
+char sqlite3AffinityType(const char *zIn, u8 *pszEst){
   u32 h = 0;
   char aff = SQLITE_AFF_NUMERIC;
+  const char *zChar;
 
-  if( zIn ) while( zIn[0] ){
+  if( zIn==0 ) return aff;
+  while( zIn[0] ){
     h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff];
     zIn++;
     if( h==(('c'<<24)+('h'<<16)+('a'<<8)+'r') ){             /* CHAR */
       aff = SQLITE_AFF_TEXT;
+      zChar = zIn;
     }else if( h==(('c'<<24)+('l'<<16)+('o'<<8)+'b') ){       /* CLOB */
       aff = SQLITE_AFF_TEXT;
     }else if( h==(('t'<<24)+('e'<<16)+('x'<<8)+'t') ){       /* TEXT */
@@ -1099,7 +1103,25 @@ char sqlite3AffinityType(const char *zIn){
       break;
     }
   }
-
+  if( pszEst ){
+    if( aff>=SQLITE_AFF_NUMERIC ){
+      *pszEst = 1;
+    }else if( zChar ){
+      *pszEst = 1;
+      while( zChar[0] ){
+        int v;
+        if( sqlite3Isdigit(zChar[0]) && sqlite3GetInt32(zChar, &v) ){
+          v = v/4 + 1;
+          if( v>255 ) v = 255;
+          *pszEst = v;
+          break;
+        }
+        zChar++;
+      }
+    }else{
+      *pszEst = 3;
+    }
+  }
   return aff;
 }
 
@@ -1121,7 +1143,7 @@ void sqlite3AddColumnType(Parse *pParse, Token *pType){
   pCol = &p->aCol[p->nCol-1];
   assert( pCol->zType==0 );
   pCol->zType = sqlite3NameFromToken(pParse->db, pType);
-  pCol->affinity = sqlite3AffinityType(pCol->zType);
+  pCol->affinity = sqlite3AffinityType(pCol->zType, &pCol->szEst);
 }
 
 /*
@@ -1469,7 +1491,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){
     zType = azType[pCol->affinity - SQLITE_AFF_TEXT];
     len = sqlite3Strlen30(zType);
     assert( pCol->affinity==SQLITE_AFF_NONE 
-            || pCol->affinity==sqlite3AffinityType(zType) );
+            || pCol->affinity==sqlite3AffinityType(zType, 0) );
     memcpy(&zStmt[k], zType, len);
     k += len;
     assert( k<=n );
@@ -1478,6 +1500,39 @@ static char *createTableStmt(sqlite3 *db, Table *p){
   return zStmt;
 }
 
+/*
+** Estimate the total row width for a table.
+*/
+static unsigned estimatedTableWidth(const Table *pTab){
+  unsigned wTable = 0;
+  const Column *pTabCol;
+  int i;
+  for(i=pTab->nCol, pTabCol=pTab->aCol; i>0; i--, pTabCol++){
+    wTable += pTabCol->szEst;
+  }
+  if( pTab->iPKey<0 ) wTable++;
+  return wTable;
+}
+
+/*
+** Set the iScanRatio for an index based on estimates of the average
+** table row width and average index row width.  Estimates are derived
+** from the declared datatypes of the various columns.
+*/
+static void setIndexScanRatio(Index *pIdx, unsigned wTable){
+  unsigned wIndex = 1;
+  int i;
+  const Column *aCol = pIdx->pTable->aCol;
+  for(i=0; i<pIdx->nColumn; i++){
+    assert( pIdx->aiColumn[i]>=0 && pIdx->aiColumn[i]<pIdx->pTable->nCol );
+    wIndex += aCol[pIdx->aiColumn[i]].szEst;
+  }
+  assert( 100*wIndex/wTable <= 255 );
+  pIdx->iScanRatio = (u8)(128*wIndex/wTable);
+  /* printf("%s: wIndex=%d wTable=%d ratio=%d\n",
+  ** pIdx->zName, wIndex, wTable, (100*pIdx->iScanRatio)/128); */
+}
+
 /*
 ** This routine is called to report the final ")" that terminates
 ** a CREATE TABLE statement.
@@ -1504,9 +1559,11 @@ void sqlite3EndTable(
   Token *pEnd,            /* The final ')' token in the CREATE TABLE */
   Select *pSelect         /* Select from a "CREATE ... AS SELECT" */
 ){
-  Table *p;
+  Table *p;                 /* The new table */
-  sqlite3 *db = pParse->db;
+  sqlite3 *db = pParse->db; /* The database connection */
-  int iDb;
+  int iDb;                  /* Database in which the table lives */
+  Index *pIdx;              /* An implied index of the table */
+  unsigned wTable;          /* Estimated average width of a row in the table */
 
   if( (pEnd==0 && pSelect==0) || db->mallocFailed ){
     return;
@@ -1526,6 +1583,12 @@ void sqlite3EndTable(
   }
 #endif /* !defined(SQLITE_OMIT_CHECK) */
 
+  /* Compute the iScanRatio of implied indices */
+  wTable = estimatedTableWidth(p);
+  for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){
+    setIndexScanRatio(pIdx, wTable);
+  }
+
   /* If the db->init.busy is 1 it means we are reading the SQL off the
   ** "sqlite_master" or "sqlite_temp_master" table on the disk.
   ** So do not write to the disk again.  Extract the root page number
@@ -2442,15 +2505,6 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){
   sqlite3VdbeAddOp1(v, OP_Close, iSorter);
 }
 
-/*
-** Estimate the average width of a table column based on its affinity type.
-*/
-static unsigned estimatedColumnWidth(const Column *p){
-  if( p->affinity>=SQLITE_AFF_NUMERIC ) return 1;
-  if( p->affinity==SQLITE_AFF_TEXT ) return 3;
-  return 2;
-}
-
 /*
 ** Create a new index for an SQL table.  pName1.pName2 is the name of the index 
 ** and pTblList is the name of the table that is to be indexed.  Both will 
@@ -2493,8 +2547,6 @@ Index *sqlite3CreateIndex(
   int iDb;             /* Index of the database that is being written */
   Token *pName = 0;    /* Unqualified name of the index to create */
   struct ExprList_item *pListItem; /* For looping over pList */
-  unsigned wTable = 0;             /* Approximate "width" of the table */
-  unsigned wIndex = 0;             /* Approximate "width" of this index */
   const Column *pTabCol;           /* A column in the table */
   int nCol;                        /* Number of columns */
   int nExtra = 0;                  /* Space allocated for zExtra[] */
@@ -2738,7 +2790,6 @@ Index *sqlite3CreateIndex(
       goto exit_create_index;
     }
     pIndex->aiColumn[i] = j;
-    wIndex += estimatedColumnWidth(pTabCol);
     if( pListItem->pExpr ){
       int nColl;
       assert( pListItem->pExpr->op==TK_COLLATE );
@@ -2762,11 +2813,9 @@ Index *sqlite3CreateIndex(
     if( pTab->aCol[j].notNull==0 ) pIndex->uniqNotNull = 0;
   }
   sqlite3DefaultRowEst(pIndex);
-  for(j=pTab->nCol, pTabCol=pTab->aCol; j>0; j--, pTabCol++){
+  if( pParse->pNewTable==0 ){
-    wTable += estimatedColumnWidth(pTabCol);
+    setIndexScanRatio(pIndex, estimatedTableWidth(pTab));
   }
-  assert( 100*wIndex/wTable <= 255 );
-  pIndex->iScanRatio = (u8)(128*wIndex/wTable);
 
   if( pTab==pParse->pNewTable ){
     /* This routine has been called to create an automatic index as a

src/expr.c

@@ -41,7 +41,7 @@ char sqlite3ExprAffinity(Expr *pExpr){
 #ifndef SQLITE_OMIT_CAST
   if( op==TK_CAST ){
     assert( !ExprHasProperty(pExpr, EP_IntValue) );
-    return sqlite3AffinityType(pExpr->u.zToken);
+    return sqlite3AffinityType(pExpr->u.zToken, 0);
   }
 #endif
   if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER) 
@@ -2461,7 +2461,7 @@ int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target){
       int aff, to_op;
       inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target);
       assert( !ExprHasProperty(pExpr, EP_IntValue) );
-      aff = sqlite3AffinityType(pExpr->u.zToken);
+      aff = sqlite3AffinityType(pExpr->u.zToken, 0);
       to_op = aff - SQLITE_AFF_TEXT + OP_ToText;
       assert( to_op==OP_ToText    || aff!=SQLITE_AFF_TEXT    );
       assert( to_op==OP_ToBlob    || aff!=SQLITE_AFF_NONE    );
@@ -3128,7 +3128,7 @@ void sqlite3ExplainExpr(Vdbe *pOut, Expr *pExpr){
     case TK_CAST: {
       /* Expressions of the form:   CAST(pLeft AS token) */
       const char *zAff = "unk";
-      switch( sqlite3AffinityType(pExpr->u.zToken) ){
+      switch( sqlite3AffinityType(pExpr->u.zToken, 0) ){
         case SQLITE_AFF_TEXT:    zAff = "TEXT";     break;
         case SQLITE_AFF_NONE:    zAff = "NONE";     break;
         case SQLITE_AFF_NUMERIC: zAff = "NUMERIC";  break;

src/pragma.c

@@ -1455,17 +1455,19 @@ void sqlite3Pragma(
       pIdx = pTab->pIndex;
       if( pIdx ){
         int i = 0; 
-        sqlite3VdbeSetNumCols(v, 3);
+        sqlite3VdbeSetNumCols(v, 4);
-        pParse->nMem = 3;
+        pParse->nMem = 4;
         sqlite3CodeVerifySchema(pParse, iDb);
         sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "seq", SQLITE_STATIC);
         sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "name", SQLITE_STATIC);
         sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "unique", SQLITE_STATIC);
+        sqlite3VdbeSetColName(v, 3, COLNAME_NAME, "r", SQLITE_STATIC);
         while(pIdx){
           sqlite3VdbeAddOp2(v, OP_Integer, i, 1);
           sqlite3VdbeAddOp4(v, OP_String8, 0, 2, 0, pIdx->zName, 0);
           sqlite3VdbeAddOp2(v, OP_Integer, pIdx->onError!=OE_None, 3);
-          sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3);
+          sqlite3VdbeAddOp2(v, OP_Integer, pIdx->iScanRatio*100/128, 4);
+          sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4);
           ++i;
           pIdx = pIdx->pNext;
         }

src/sqliteInt.h

@@ -1197,7 +1197,8 @@ struct Column {
   char *zColl;     /* Collating sequence.  If NULL, use the default */
   u8 notNull;      /* An OE_ code for handling a NOT NULL constraint */
   char affinity;   /* One of the SQLITE_AFF_... values */
-  u16 colFlags;    /* Boolean properties.  See COLFLAG_ defines below */
+  u8 szEst;        /* Estimated size of this column.  INT==1 */
+  u8 colFlags;     /* Boolean properties.  See COLFLAG_ defines below */
 };
 
 /* Allowed values for Column.colFlags:
@@ -3082,7 +3083,7 @@ void sqlite3ColumnDefault(Vdbe *, Table *, int, int);
 void sqlite3AlterFinishAddColumn(Parse *, Token *);
 void sqlite3AlterBeginAddColumn(Parse *, SrcList *);
 CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*);
-char sqlite3AffinityType(const char*);
+char sqlite3AffinityType(const char*, u8*);
 void sqlite3Analyze(Parse*, Token*, Token*);
 int sqlite3InvokeBusyHandler(BusyHandler*);
 int sqlite3FindDb(sqlite3*, Token*);

test/pragma.test

@@ -574,7 +574,7 @@ ifcapable {foreignkey} {
     execsql {
       pragma index_list(t3);
     }
-  } {0 sqlite_autoindex_t3_1 1}
+  } {/0 sqlite_autoindex_t3_1 1 \d+/}
 }
 ifcapable {!foreignkey} {
   execsql {CREATE TABLE t3(a,b UNIQUE)}
@@ -647,7 +647,7 @@ do_test pragma-7.1.1 {
   execsql {
     pragma index_list(t3);
   }
-} {0 t3i1 0 1 sqlite_autoindex_t3_1 1}
+} {/0 t3i1 0 \d+ 1 sqlite_autoindex_t3_1 1 \d+/}
 do_test pragma-7.1.2 {
   execsql {
     pragma index_list(t3_bogus);
@@ -1661,7 +1661,7 @@ do_test 23.3 {
     CREATE INDEX i3 ON t1(d,b,c);
   }
   db2 eval {PRAGMA index_list(t1)}
-} {0 i3 0 1 i2 0 2 i1 0}
+} {/0 i3 0 \d+ 1 i2 0 \d+ 2 i1 0 \d+/}
 do_test 23.4 {
   db eval {
     ALTER TABLE t1 ADD COLUMN e;