Allow an index to be used for sorting even if prior terms of the index

are constrained by IN operators.

FossilOrigin-Name: 2cef8b68f0e1216cf68bb7dd45a5a9a330748070

manifest

@@ -1,5 +1,5 @@
-C Improve\smemory\sallocation\serror\shandling\son\sWinCE.
+C Allow\san\sindex\sto\sbe\sused\sfor\ssorting\seven\sif\sprior\sterms\sof\sthe\sindex\nare\sconstrained\sby\sIN\soperators.
-D 2013-02-12T22:20:01.870
+D 2013-02-13T01:00:35.115
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in a48faa9e7dd7d556d84f5456eabe5825dd8a6282
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -179,7 +179,7 @@ F src/shell.c 266791241d7add796ccce2317977ae6c3c67d77f
 F src/sqlite.h.in 6296506a8fba279d8fa31f4abf01ab0cc92738a6
 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0
 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477
-F src/sqliteInt.h 8e01aa31d5337ca0c0d0000745994f63762ec1bb
+F src/sqliteInt.h 758285f17a55bc37b6f042aa3a2464e74327f4a5
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c bedc37ec1a6bb9399944024d63f4c769971955a9
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -252,7 +252,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83
 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2
 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6
 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b
-F src/where.c 427c6ec07c2150e6d2bd28aefd98ab234cc374c3
+F src/where.c 43e05406f0e05960a62d4719ed77f551f8204d3f
 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test 45c0201e28045ad38a530b5a144b73cd4aa2cfd6
@@ -970,8 +970,8 @@ F test/walro.test a31deb621033442a76c3a61e44929250d06f81b1
 F test/walshared.test 6dda2293880c300baf5d791c307f653094585761
 F test/walslow.test e7be6d9888f83aa5d3d3c7c08aa9b5c28b93609a
 F test/walthread.test de8dbaf6d9e41481c460ba31ca61e163d7348f8e
-F test/where.test 9714e6f292d70c22e78e1cecb3d896457186b9b7
+F test/where.test 15ac8611c9439a2c5f4a6ac10cfe4c1ec9854c24
-F test/where2.test 43d4becaf5a5df854e6c21d624a1cb84c6904554
+F test/where2.test 399b3178289925a0aa976b3d60ef139740540ecd
 F test/where3.test 667e75642102c97a00bf9b23d3cb267db321d006
 F test/where4.test e9b9e2f2f98f00379e6031db6a6fca29bae782a2
 F test/where5.test fdf66f96d29a064b63eb543e28da4dfdccd81ad2
@@ -1034,7 +1034,7 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P b7222a2bd035e7a32dc9416b25a488d9d017aad1 09dfc0c915ec2f0c5f633a3485d47cad15eec4dc
+P cdbca259da80df901837034d2616da434cb82216 71b6c26053fdf2a5a84116e005bad1f2ca873a66
-R ce10bb0bb316ca17678000c2f3d6adb2
+R 77755448a8b45c569e7779e037e5b44c
 U drh
-Z 19468d73c330af48697ca035c027cc38
+Z c2bdfa94c74d9c022b301306a5110614

manifest.uuid

@@ -1 +1 @@
-cdbca259da80df901837034d2616da434cb82216
+2cef8b68f0e1216cf68bb7dd45a5a9a330748070

src/sqliteInt.h

@@ -1965,6 +1965,7 @@ struct WhereLevel {
       struct InLoop {
         int iCur;              /* The VDBE cursor used by this IN operator */
         int addrInTop;         /* Top of the IN loop */
+        u8 eEndLoopOp;         /* IN Loop terminator. OP_Next or OP_Prev */
       } *aInLoop;           /* Information about each nested IN operator */
     } in;                 /* Used when plan.wsFlags&WHERE_IN_ABLE */
     Index *pCovidx;       /* Possible covering index for WHERE_MULTI_OR */

src/where.c

@@ -140,7 +140,6 @@ struct WhereTerm {
 struct WhereClause {
   Parse *pParse;           /* The parser context */
   WhereMaskSet *pMaskSet;  /* Mapping of table cursor numbers to bitmasks */
-  Bitmask vmask;           /* Bitmask identifying virtual table cursors */
   WhereClause *pOuter;     /* Outer conjunction */
   u8 op;                   /* Split operator.  TK_AND or TK_OR */
   u16 wctrlFlags;          /* Might include WHERE_AND_ONLY */
@@ -317,7 +316,6 @@ static void whereClauseInit(
   pWC->nTerm = 0;
   pWC->nSlot = ArraySize(pWC->aStatic);
   pWC->a = pWC->aStatic;
-  pWC->vmask = 0;
   pWC->wctrlFlags = wctrlFlags;
 }
 
@@ -917,7 +915,7 @@ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){
 **
 ** CASE 1:
 **
-** If all subterms are of the form T.C=expr for some single column of C
+** If all subterms are of the form T.C=expr for some single column of C and
 ** a single table T (as shown in example B above) then create a new virtual
 ** term that is an equivalent IN expression.  In other words, if the term
 ** being analyzed is:
@@ -1005,7 +1003,7 @@ static void exprAnalyzeOrTerm(
   ** Compute the set of tables that might satisfy cases 1 or 2.
   */
   indexable = ~(Bitmask)0;
-  chngToIN = ~(pWC->vmask);
+  chngToIN = ~(Bitmask)0;
   for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0 && indexable; i--, pOrTerm++){
     if( (pOrTerm->eOperator & WO_SINGLE)==0 ){
       WhereAndInfo *pAndInfo;
@@ -2272,8 +2270,9 @@ static void bestVirtualIndex(WhereBestIdx *p){
   struct sqlite3_index_constraint *pIdxCons;
   struct sqlite3_index_constraint_usage *pUsage;
   WhereTerm *pTerm;
-  int i, j;
+  int i, j, k;
   int nOrderBy;
+  int sortOrder;                  /* Sort order for IN clauses */
   int bAllowIN;                   /* Allow IN optimizations */
   double rCost;
 
@@ -2372,18 +2371,27 @@ static void bestVirtualIndex(WhereBestIdx *p){
       return;
     }
   
+    sortOrder = SQLITE_SO_ASC;
     pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint;
     for(i=0; i<pIdxInfo->nConstraint; i++, pIdxCons++){
       if( pUsage[i].argvIndex>0 ){
         j = pIdxCons->iTermOffset;
         pTerm = &pWC->a[j];
         p->cost.used |= pTerm->prereqRight;
-        if( (pTerm->eOperator & WO_IN)!=0 && pUsage[i].omit==0 ){
+        if( (pTerm->eOperator & WO_IN)!=0 ){
-          /* Do not attempt to use an IN constraint if the virtual table
+          if( pUsage[i].omit==0 ){
-          ** says that the equivalent EQ constraint cannot be safely omitted.
+            /* Do not attempt to use an IN constraint if the virtual table
-          ** If we do attempt to use such a constraint, some rows might be
+            ** says that the equivalent EQ constraint cannot be safely omitted.
-          ** repeated in the output. */
+            ** If we do attempt to use such a constraint, some rows might be
-          break;
+            ** repeated in the output. */
+            break;
+          }
+          for(k=0; k<pIdxInfo->nOrderBy; k++){
+            if( pIdxInfo->aOrderBy[k].iColumn==pIdxCons->iColumn ){
+              sortOrder = pIdxInfo->aOrderBy[k].desc;
+              break;
+            }
+          }
         }
       }
     }
@@ -2413,7 +2421,8 @@ static void bestVirtualIndex(WhereBestIdx *p){
   }
   p->cost.plan.u.pVtabIdx = pIdxInfo;
   if( pIdxInfo->orderByConsumed ){
-    p->cost.plan.wsFlags |= WHERE_ORDERED;
+    assert( sortOrder==0 || sortOrder==1 );
+    p->cost.plan.wsFlags |= WHERE_ORDERED + sortOrder*WHERE_REVERSE;
     p->cost.plan.nOBSat = nOrderBy;
   }else{
     p->cost.plan.nOBSat = p->i ? p->aLevel[p->i-1].plan.nOBSat : 0;
@@ -3010,10 +3019,7 @@ static int isSortingIndex(
     if( pConstraint==0 ){
       isEq = 0;
     }else if( (pConstraint->eOperator & WO_IN)!=0 ){
-      /* Constraints of the form: "X IN ..." cannot be used with an ORDER BY
+      isEq = 0;
-      ** because we do not know in what order the values on the RHS of the IN
-      ** operator will occur. */
-      break;
     }else if( (pConstraint->eOperator & WO_ISNULL)!=0 ){
       uniqueNotNull = 0;
       isEq = 1;  /* "X IS NULL" means X has only a single value */
@@ -3317,8 +3323,8 @@ static void bestBtreeIndex(WhereBestIdx *p){
     ** indicate this to the caller.
     **
     ** Otherwise, if the search may find more than one row, test to see if
-    ** there is a range constraint on indexed column (pc.plan.nEq+1) that can be 
+    ** there is a range constraint on indexed column (pc.plan.nEq+1) that
-    ** optimized using the index. 
+    ** can be optimized using the index. 
     */
     if( pc.plan.nEq==pProbe->nColumn && pProbe->onError!=OE_None ){
       testcase( pc.plan.wsFlags & WHERE_COLUMN_IN );
@@ -3659,7 +3665,8 @@ static void bestIndex(WhereBestIdx *p){
     sqlite3_index_info *pIdxInfo = 0;
     p->ppIdxInfo = &pIdxInfo;
     bestVirtualIndex(p);
-    if( pIdxInfo->needToFreeIdxStr ){
+    assert( pIdxInfo!=0 || p->pParse->db->mallocFailed );
+    if( pIdxInfo && pIdxInfo->needToFreeIdxStr ){
       sqlite3_free(pIdxInfo->idxStr);
     }
     sqlite3DbFree(p->pParse->db, pIdxInfo);
@@ -3783,12 +3790,13 @@ static int codeEqualityTerm(
     int eType;
     int iTab;
     struct InLoop *pIn;
+    u8 bRev = (pLevel->plan.wsFlags & WHERE_REVERSE)!=0;
 
     assert( pX->op==TK_IN );
     iReg = iTarget;
     eType = sqlite3FindInIndex(pParse, pX, 0);
     iTab = pX->iTable;
-    sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0);
+    sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0);
     assert( pLevel->plan.wsFlags & WHERE_IN_ABLE );
     if( pLevel->u.in.nIn==0 ){
       pLevel->addrNxt = sqlite3VdbeMakeLabel(v);
@@ -3806,6 +3814,7 @@ static int codeEqualityTerm(
       }else{
         pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg);
       }
+      pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next;
       sqlite3VdbeAddOp1(v, OP_IsNull, iReg);
     }else{
       pLevel->u.in.nIn = 0;
@@ -5057,24 +5066,13 @@ WhereInfo *sqlite3WhereBegin(
   ** bitmask for all tables to the left of the join.  Knowing the bitmask
   ** for all tables to the left of a left join is important.  Ticket #3015.
   **
-  ** Configure the WhereClause.vmask variable so that bits that correspond
-  ** to virtual table cursors are set. This is used to selectively disable 
-  ** the OR-to-IN transformation in exprAnalyzeOrTerm(). It is not helpful 
-  ** with virtual tables.
-  **
   ** Note that bitmasks are created for all pTabList->nSrc tables in
   ** pTabList, not just the first nTabList tables.  nTabList is normally
   ** equal to pTabList->nSrc but might be shortened to 1 if the
   ** WHERE_ONETABLE_ONLY flag is set.
   */
-  assert( sWBI.pWC->vmask==0 && pMaskSet->n==0 );
   for(ii=0; ii<pTabList->nSrc; ii++){
     createMask(pMaskSet, pTabList->a[ii].iCursor);
-#ifndef SQLITE_OMIT_VIRTUALTABLE
-    if( ALWAYS(pTabList->a[ii].pTab) && IsVirtual(pTabList->a[ii].pTab) ){
-      sWBI.pWC->vmask |= ((Bitmask)1 << ii);
-    }
-#endif
   }
 #ifndef NDEBUG
   {
@@ -5558,7 +5556,7 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
       sqlite3VdbeResolveLabel(v, pLevel->addrNxt);
       for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){
         sqlite3VdbeJumpHere(v, pIn->addrInTop+1);
-        sqlite3VdbeAddOp2(v, OP_Next, pIn->iCur, pIn->addrInTop);
+        sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop);
         sqlite3VdbeJumpHere(v, pIn->addrInTop-1);
       }
       sqlite3DbFree(db, pLevel->u.in.aInLoop);

test/where.test

@@ -379,11 +379,26 @@ ifcapable subquery {
       SELECT * FROM t1 WHERE rowid+0 IN (1,2,3,1234) order by 1;
     }
   } {1 0 4 2 1 9 3 1 16 102}
-  do_test where-5.3 {
+  do_test where-5.3a {
     count {
       SELECT * FROM t1 WHERE w IN (-1,1,2,3) order by 1;
     }
-  } {1 0 4 2 1 9 3 1 16 14}
+  } {1 0 4 2 1 9 3 1 16 13}
+  do_test where-5.3b {
+    count {
+      SELECT * FROM t1 WHERE w IN (3,-1,1,2) order by 1;
+    }
+  } {1 0 4 2 1 9 3 1 16 13}
+  do_test where-5.3c {
+    count {
+      SELECT * FROM t1 WHERE w IN (3,2,-1,1,2) order by 1;
+    }
+  } {1 0 4 2 1 9 3 1 16 13}
+  do_test where-5.3d {
+    count {
+      SELECT * FROM t1 WHERE w IN (-1,1,2,3) order by 1 DESC;
+    }
+  } {3 1 16 2 1 9 1 0 4 12}
   do_test where-5.4 {
     count {
       SELECT * FROM t1 WHERE w+0 IN (-1,1,2,3) order by 1;
@@ -452,6 +467,30 @@ ifcapable subquery {
       SELECT * FROM t1 WHERE x IN (1,7) AND y IN (9,16) ORDER BY 1;
     }
   } {2 1 9 3 1 16 11}
+  do_test where-5.100 {
+    db eval {
+      SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969)
+       ORDER BY x, y
+    }
+  } {2 1 9 54 5 3025 62 5 3969}
+  do_test where-5.101 {
+    db eval {
+      SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969)
+       ORDER BY x DESC, y DESC
+    }
+  } {62 5 3969 54 5 3025 2 1 9}
+  do_test where-5.102 {
+    db eval {
+      SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969)
+       ORDER BY x DESC, y
+    }
+  } {54 5 3025 62 5 3969 2 1 9}
+  do_test where-5.103 {
+    db eval {
+      SELECT w, x, y FROM t1 WHERE x IN (1,5) AND y IN (9,8,3025,1000,3969)
+       ORDER BY x, y DESC
+    }
+  } {2 1 9 62 5 3969 54 5 3025}
 }
 
 # This procedure executes the SQL.  Then it checks to see if the OP_Sort
@@ -511,11 +550,16 @@ do_test where-6.7 {
   }
 } {1 100 4 2 99 9 3 98 16 nosort}
 ifcapable subquery {
-  do_test where-6.8 {
+  do_test where-6.8a {
     cksort {
       SELECT * FROM t3 WHERE a IN (3,5,7,1,9,4,2) ORDER BY a LIMIT 3
     }
-  } {1 100 4 2 99 9 3 98 16 sort}
+  } {1 100 4 2 99 9 3 98 16 nosort}
+  do_test where-6.8b {
+    cksort {
+      SELECT * FROM t3 WHERE a IN (3,5,7,1,9,4,2) ORDER BY a DESC LIMIT 3
+    }
+  } {9 92 100 7 94 64 5 96 36 nosort}
 }
 do_test where-6.9.1 {
   cksort {

test/where2.test

@@ -167,24 +167,54 @@ ifcapable subquery {
       }
     } {99 6 10000 10006 100 6 10201 10207 sort t1 i1zyx}
   }
-  do_test where2-4.6 {
+  do_test where2-4.6a {
     queryplan {
       SELECT * FROM t1
        WHERE x IN (1,2,3,4,5,6,7,8)
          AND y IN (10000,10001,10002,10003,10004,10005)
-       ORDER BY 2
+       ORDER BY x
+    }
+  } {99 6 10000 10006 nosort t1 i1xy}
+  do_test where2-4.6b {
+    queryplan {
+      SELECT * FROM t1
+       WHERE x IN (1,2,3,4,5,6,7,8)
+         AND y IN (10000,10001,10002,10003,10004,10005)
+       ORDER BY x DESC
+    }
+  } {99 6 10000 10006 nosort t1 i1xy}
+  do_test where2-4.6c {
+    queryplan {
+      SELECT * FROM t1
+       WHERE x IN (1,2,3,4,5,6,7,8)
+         AND y IN (10000,10001,10002,10003,10004,10005)
+       ORDER BY x, y
+    }
+  } {99 6 10000 10006 nosort t1 i1xy}
+  do_test where2-4.6d {
+    queryplan {
+      SELECT * FROM t1
+       WHERE x IN (1,2,3,4,5,6,7,8)
+         AND y IN (10000,10001,10002,10003,10004,10005)
+       ORDER BY x, y DESC
     }
   } {99 6 10000 10006 sort t1 i1xy}
 
   # Duplicate entires on the RHS of an IN operator do not cause duplicate
   # output rows.
   #
-  do_test where2-4.6 {
+  do_test where2-4.6x {
     queryplan {
       SELECT * FROM t1 WHERE z IN (10207,10006,10006,10207)
       ORDER BY w
     }
   } {99 6 10000 10006 100 6 10201 10207 sort t1 i1zyx}
+  do_test where2-4.6y {
+    queryplan {
+      SELECT * FROM t1 WHERE z IN (10207,10006,10006,10207)
+      ORDER BY w DESC
+    }
+  } {100 6 10201 10207 99 6 10000 10006 sort t1 i1zyx}
   ifcapable compound {
     do_test where2-4.7 {
       queryplan {
@@ -207,11 +237,16 @@ do_test where2-5.1 {
 } {99 6 10000 10006 nosort t1 i1w}
 
 ifcapable subquery {
-  do_test where2-5.2 {
+  do_test where2-5.2a {
     queryplan {
       SELECT * FROM t1 WHERE w IN (99) ORDER BY w
     }
-  } {99 6 10000 10006 sort t1 i1w}
+  } {99 6 10000 10006 nosort t1 i1w}
+  do_test where2-5.2b {
+    queryplan {
+      SELECT * FROM t1 WHERE w IN (99) ORDER BY w DESC
+    }
+  } {99 6 10000 10006 nosort t1 i1w}
 }
 
 # Verify that OR clauses get translated into IN operators.