Fix handling of covering indexes that use virtual columns.

FossilOrigin-Name: e0f7e321eca91c49102649f70728de69347cbd7b16599b854203c24cfdc348e6

manifest

@@ -1,5 +1,5 @@
-C Do\snot\sallow\sALTER\sTABLE\sADD\sCOLUMN\sfor\sa\sSTORED\scolumn.
+C Fix\shandling\sof\scovering\sindexes\sthat\suse\svirtual\scolumns.
-D 2019-10-24T01:04:10.370
+D 2019-10-24T20:29:25.035
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -469,7 +469,7 @@ F src/btmutex.c 8acc2f464ee76324bf13310df5692a262b801808984c1b79defb2503bbafadb6
 F src/btree.c a8a9c2ce62bdf54c8cf9795143d7cb10b7473a1230a0572f702d061ffcceefe5
 F src/btree.h f27a33c49280209a93385e218306c4ee5f46ba8d7649d2f81a7166b282232484
 F src/btreeInt.h 91806f01fd1145a9a86ba3042f25c38d8faf6002701bf5e780742cf88bcff437
-F src/build.c f7070af66656b75be79a79a8ac720f4f0f8b48ba56298f2d9bf0f982d07ed949
+F src/build.c 0c9704f95817aa585fdad2668c611280d0b62bc4c1c836cd1c797ba96879a7d6
 F src/callback.c 88615dfc0a82167b65b452b4b305dbf86be77200b3343c6ffc6d03e92a01d181
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/ctime.c 1b0724e66f95f33b160b1af85caaf9cceb325d22abf39bd24df4f54a73982251
@@ -530,7 +530,7 @@ F src/shell.c.in 3093bdf5eedd91da08f0268f1442aa510a60798c9441868149ddbecdf8bcaa7
 F src/sqlite.h.in 5725a6b20190a1e8d662077a1c1c8ea889ad7be90dd803f914c2de226f5fe6ab
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h cef696ce3293242c67b2339763608427bf72ee66f1f3a05389ac2a7b46001c31
-F src/sqliteInt.h f8609e5a04eec2e89f35520b31c0b506ef905a3485f09b4dc4e20583743cee31
+F src/sqliteInt.h 5b2d25ba23135ece06886d82f60d9a16869506592e5950f3c09257b3b5d28d5c
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -613,7 +613,7 @@ F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a
 F src/walker.c d5a94907dcac990e31976be9dc769d17f6a806782593d6aec9d760ee01ec22cd
 F src/where.c 6ff3ef076485dd71efbbabcc7ddc0473c04a9bdcb524128939757b002466f2d1
 F src/whereInt.h 4a296fd4fa79fdcbc2b5e8c1b898901617655811223e1082b899c23ecb092217
-F src/wherecode.c d96190c0b536339375846048ad3c41758b4bd6baaf8f8f350da5911d42bc4a61
+F src/wherecode.c c491ae0ce1de97bc4eea566350a2d1e7c2c5fcf97b9d4c89593c4f068e21a30d
 F src/whereexpr.c 0705f608f6dbbd4e95d440528d6c760b91b6f402ba4eb8b8d964c110e2010780
 F src/window.c 064f251451c8e2a1c76b6269229d911a651e119c6a5f522b6eaebf8dc8714041
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1847,7 +1847,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 30065716878d4058e75eb510b0b27b68e5193d04625eb173210de8061f20f499
+P 42fc08bc1528a34a603c2c085b515766a8d33ae7ea0350a52b0ca24b94ebcbc5
-R 7202e3e1ab0c99bc839229fb4e803cbe
+R 6a9e8fc570824aa923ac6838bd994212
 U drh
-Z 98506792ef3b33ac52584457b01d83d1
+Z b5f3ce7a354e5714077f87bfe3d7f12c

manifest.uuid

@@ -1 +1 @@
-42fc08bc1528a34a603c2c085b515766a8d33ae7ea0350a52b0ca24b94ebcbc5
+e0f7e321eca91c49102649f70728de69347cbd7b16599b854203c24cfdc348e6

src/build.c

@@ -1923,15 +1923,24 @@ static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){
 ** high-order bit of colNotIdxed is always 1.  All unindexed columns
 ** of the table have a 1.
 **
+** 2019-10-24:  For the purpose of this computation, virtual columns are
+** not considered to be covered by the index, even if they are in the
+** index, because we do not trust the logic in whereIndexExprTrans() to be
+** able to find all instances of a reference to the indexed table column
+** and convert them into references to the index.  Hence we always want
+** the actual table at hand in order to recompute the virtual column, if
+** necessary.
+**
 ** The colNotIdxed mask is AND-ed with the SrcList.a[].colUsed mask
 ** to determine if the index is covering index.
 */
 static void recomputeColumnsNotIndexed(Index *pIdx){
   Bitmask m = 0;
   int j;
+  Table *pTab = pIdx->pTable;
   for(j=pIdx->nColumn-1; j>=0; j--){
     int x = pIdx->aiColumn[j];
-    if( x>=0 ){
+    if( x>=0 && (pTab->aCol[x].colFlags & COLFLAG_VIRTUAL)==0 ){
       testcase( x==BMS-1 );
       testcase( x==BMS-2 );
       if( x<BMS-1 ) m |= MASKBIT(x);
@@ -3617,9 +3626,14 @@ void sqlite3CreateIndex(
       assert( j<=0x7fff );
       if( j<0 ){
         j = pTab->iPKey;
-      }else if( pTab->aCol[j].notNull==0 ){
+      }else{
+        if( pTab->aCol[j].notNull==0 ){
           pIndex->uniqNotNull = 0;
         }
+        if( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ){
+          pIndex->bHasVCol = 1;
+        }
+      }
       pIndex->aiColumn[i] = (i16)j;
     }
     zColl = 0;

src/sqliteInt.h

@@ -2295,6 +2295,7 @@ struct Index {
   unsigned hasStat1:1;     /* aiRowLogEst values come from sqlite_stat1 */
   unsigned bNoQuery:1;     /* Do not use this index to optimize queries */
   unsigned bAscKeyBug:1;   /* True if the bba7b69f9849b5bf bug applies */
+  unsigned bHasVCol:1;     /* Index references one or more VIRTUAL columns */
 #ifdef SQLITE_ENABLE_STAT4
   int nSample;             /* Number of elements in aSample[] */
   int nSampleCol;          /* Size of IndexSample.anEq[] and so on */

src/wherecode.c

@@ -1106,6 +1106,7 @@ typedef struct IdxExprTrans {
   int iTabCur;       /* The cursor of the corresponding table */
   int iIdxCur;       /* The cursor for the index */
   int iIdxCol;       /* The column for the index */
+  int iTabCol;       /* The column for the table */
 } IdxExprTrans;
 
 /* The walker node callback used to transform matching expressions into
@@ -1128,10 +1129,30 @@ static int whereIndexExprTransNode(Walker *p, Expr *pExpr){
   }
 }
 
+#ifndef SQLITE_OMIT_GENERATED_COLUMNS
+/* A walker node callback that translates a column reference to a table
+** into a corresponding column reference of an index.
+*/
+static int whereIndexExprTransColumn(Walker *p, Expr *pExpr){
+  if( pExpr->op==TK_COLUMN ){
+    IdxExprTrans *pX = p->u.pIdxTrans;
+    if( pExpr->iTable==pX->iTabCur && pExpr->iColumn==pX->iTabCol ){
+      pExpr->iTable = pX->iIdxCur;
+      pExpr->iColumn = pX->iIdxCol;
+    }
+  }
+  return WRC_Continue;
+}
+#endif /* SQLITE_OMIT_GENERATED_COLUMNS */
+
 /*
 ** For an indexes on expression X, locate every instance of expression X
 ** in pExpr and change that subexpression into a reference to the appropriate
 ** column of the index.
+**
+** 2019-10-24: Updated to also translate references to a VIRTUAL column in
+** the table into references to the corresponding (stored) column of the
+** index.
 */
 static void whereIndexExprTrans(
   Index *pIdx,      /* The Index */
@@ -1141,20 +1162,35 @@ static void whereIndexExprTrans(
 ){
   int iIdxCol;               /* Column number of the index */
   ExprList *aColExpr;        /* Expressions that are indexed */
+  Table *pTab;
   Walker w;
   IdxExprTrans x;
   aColExpr = pIdx->aColExpr;
-  if( aColExpr==0 ) return;  /* Not an index on expressions */
+  if( aColExpr==0 && !pIdx->bHasVCol ){
+    /* The index does not reference any expressions or virtual columns
+    ** so no translations are needed. */
+    return;
+  }
+  pTab = pIdx->pTable;
   memset(&w, 0, sizeof(w));
-  w.xExprCallback = whereIndexExprTransNode;
   w.u.pIdxTrans = &x;
   x.iTabCur = iTabCur;
   x.iIdxCur = iIdxCur;
-  for(iIdxCol=0; iIdxCol<aColExpr->nExpr; iIdxCol++){
+  for(iIdxCol=0; iIdxCol<pIdx->nColumn; iIdxCol++){
-    if( pIdx->aiColumn[iIdxCol]!=XN_EXPR ) continue;
+    i16 iRef = pIdx->aiColumn[iIdxCol];
+    if( iRef==XN_EXPR ){
       assert( aColExpr->a[iIdxCol].pExpr!=0 );
-    x.iIdxCol = iIdxCol;
       x.pIdxExpr = aColExpr->a[iIdxCol].pExpr;
+      w.xExprCallback = whereIndexExprTransNode;
+#ifndef SQLITE_OMIT_GENERATED_COLUMNS
+    }else if( iRef>=0 && (pTab->aCol[iRef].colFlags & COLFLAG_VIRTUAL)!=0 ){
+      x.iTabCol = iRef;
+      w.xExprCallback = whereIndexExprTransColumn;
+#endif /* SQLITE_OMIT_GENERATED_COLUMNS */
+    }else{
+      continue;
+    }
+    x.iIdxCol = iIdxCol;
     sqlite3WalkExpr(&w, pWInfo->pWhere);
     sqlite3WalkExprList(&w, pWInfo->pOrderBy);
     sqlite3WalkExprList(&w, pWInfo->pResultSet);
@@ -1835,7 +1871,9 @@ Bitmask sqlite3WhereCodeOneLoopStart(
 
     /* If pIdx is an index on one or more expressions, then look through
     ** all the expressions in pWInfo and try to transform matching expressions
-    ** into reference to index columns.
+    ** into reference to index columns.  Also attempt to translate references
+    ** to virtual columns in the table into references to (stored) columns
+    ** of the index.
     **
     ** Do not do this for the RHS of a LEFT JOIN. This is because the 
     ** expression may be evaluated after OP_NullRow has been executed on