In ORDER BY LIMIT queries, try to evaluate the ORDER BY terms first, and it

it becomes clear that the row will not come in under the LIMIT, then skip
evaluation of the other columns.

FossilOrigin-Name: c381f0ea57002a264fd958b28e4921cb9c9e73a10fb592f6bb64e6bc9bd16d39

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\sbranch\sthat\shas\sbecome\sunreachable\sdue\sto\srecent\senhancements.
+C In\sORDER\sBY\sLIMIT\squeries,\stry\sto\sevaluate\sthe\sORDER\sBY\sterms\sfirst,\sand\sit\nit\sbecomes\sclear\sthat\sthe\srow\swill\snot\scome\sin\sunder\sthe\sLIMIT,\sthen\sskip\nevaluation\sof\sthe\sother\scolumns.
-D 2018-05-03T22:52:56.038
+D 2018-05-03T23:20:06.057
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 5ce9343cba9c189046f1afe6d2bcc1f68079439febc05267b98aec6ecc752439
@@ -493,12 +493,12 @@ F src/printf.c d3b7844ddeb11fbbdd38dd84d09c9c1ac171d21fb038473c3aa97981201cc660
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 6415381a0e9d22c0e7cba33ca4a53f81474190862f5d4838190f5eb5b0b47bc9
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
-F src/select.c c937eb1fc32417fb0be022ab8372dd27ff8138242a8d377724996d817258bd8a
+F src/select.c bf9185f40cbb98a1342c0149f66dff3fbadf262081305519927523b4ec7b876e
 F src/shell.c.in 29309f2ab656c8817fbc3b7910b9af8464557b91cba75277a03669399c8e2730
 F src/sqlite.h.in 469aed42e75193b7f5d88d812befa961c04746869475f96fcf434c9f67079f66
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 83a3c4ce93d650bedfd1aa558cb85a516bd6d094445ee989740827d0d944368d
-F src/sqliteInt.h 7fbd839f07d873a877375138f2681453232e2e2de8f48be65888e295cd05ced3
+F src/sqliteInt.h 4b98a37f27033ca887f2027f9a60636625fb2a4a9ef1b96c17f0db8fd951891f
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -1727,7 +1727,8 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 260fc696538b195e8decabaab46771f664fb829b539efa86fb0b8170db01fa0a
+P 81ab5e0d106fb9f2de80d17c8167d8542c6721b8d7ec69c43e1571c2dcadeddd ce4ef46058f4aaea6623a41255a2e4b69bb24f16a287391df48f6bacdb4c4989
-R 9efc9c5b7f70976e1122ef5263d6cddb
+R df0ac85de3cdeeeca99ef241c3f826a6
+T +closed ce4ef46058f4aaea6623a41255a2e4b69bb24f16a287391df48f6bacdb4c4989
 U drh
-Z f48c0146f4c18019bae878d6a3510a09
+Z e934c97462a3a7876abf79aae9b0d67e

manifest.uuid

@@ -1 +1 @@
-81ab5e0d106fb9f2de80d17c8167d8542c6721b8d7ec69c43e1571c2dcadeddd
+c381f0ea57002a264fd958b28e4921cb9c9e73a10fb592f6bb64e6bc9bd16d39

src/select.c

@@ -78,6 +78,7 @@ struct SortCtx {
     int nKey;           /* Number of PK columns for table pTab (>=1) */
   } aDefer[4];
 #endif
+  struct RowLoadInfo *pDeferredRowLoad;  /* Deferred row loading info or NULL */
 };
 #define SORTFLAG_UseSorter  0x01   /* Use SorterOpen instead of OpenEphemeral */
 
@@ -536,6 +537,62 @@ static KeyInfo *keyInfoFromExprList(
   int nExtra           /* Add this many extra columns to the end */
 );
 
+/*
+** An instance of this object holds information (beyond pParse and pSelect)
+** needed to load the next result row that is to be added to the sorter.
+*/
+typedef struct RowLoadInfo RowLoadInfo;
+struct RowLoadInfo {
+  int regResult;               /* Store results in array of registers here */
+  u8 ecelFlags;                /* Flag argument to ExprCodeExprList() */
+#ifdef SQLITE_ENABLE_SORTER_REFERENCES
+  ExprList *pExtra;            /* Extra columns needed by sorter refs */
+  int regExtraResult;          /* Where to load the extra columns */
+#endif
+};
+
+/*
+** This routine does the work of loading query data into an array of
+** registers so that it can be added to the sorter.
+*/
+static void innerLoopLoadRow(
+  Parse *pParse,             /* Statement under construction */
+  Select *pSelect,           /* The query being coded */
+  RowLoadInfo *pInfo         /* Info needed to complete the row load */
+){
+  sqlite3ExprCodeExprList(pParse, pSelect->pEList, pInfo->regResult,
+                          0, pInfo->ecelFlags);
+#ifdef SQLITE_ENABLE_SORTER_REFERENCES
+  if( pInfo->pExtra ){
+    sqlite3ExprCodeExprList(pParse, pInfo->pExtra, pInfo->regExtraResult, 0, 0);
+    sqlite3ExprListDelete(pParse->db, pInfo->pExtra);
+  }
+#endif
+}
+
+/*
+** Code the OP_MakeRecord instruction that generates the entry to be
+** added into the sorter.
+**
+** Return the register in which the result is stored.
+*/
+static int makeSorterRecord(
+  Parse *pParse,
+  SortCtx *pSort,
+  Select *pSelect,
+  int regBase,
+  int nBase
+){
+  int nOBSat = pSort->nOBSat;
+  Vdbe *v = pParse->pVdbe;
+  int regOut = ++pParse->nMem;
+  if( pSort->pDeferredRowLoad ){
+    innerLoopLoadRow(pParse, pSelect, pSort->pDeferredRowLoad);
+  }
+  sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regOut);
+  return regOut;
+}
+
 /*
 ** Generate code that will push the record in registers regData
 ** through regData+nData-1 onto the sorter.
@@ -546,7 +603,7 @@ static void pushOntoSorter(
   Select *pSelect,       /* The whole SELECT statement */
   int regData,           /* First register holding data to be sorted */
   int regOrigData,       /* First register holding data before packing */
-  int nData,             /* Number of elements in the data array */
+  int nData,             /* Number of elements in the regData data array */
   int nPrefixReg         /* No. of reg prior to regData available for use */
 ){
   Vdbe *v = pParse->pVdbe;                         /* Stmt under construction */
@@ -554,16 +611,31 @@ static void pushOntoSorter(
   int nExpr = pSort->pOrderBy->nExpr;              /* No. of ORDER BY terms */
   int nBase = nExpr + bSeq + nData;                /* Fields in sorter record */
   int regBase;                                     /* Regs for sorter record */
-  int regRecord = ++pParse->nMem;                  /* Assembled sorter record */
+  int regRecord = 0;                               /* Assembled sorter record */
   int nOBSat = pSort->nOBSat;                      /* ORDER BY terms to skip */
   int op;                            /* Opcode to add sorter record to sorter */
   int iLimit;                        /* LIMIT counter */
+  int iSkip = 0;                     /* End of the sorter insert loop */
 
   assert( bSeq==0 || bSeq==1 );
+
+  /* Three cases:
+  **   (1) The data to be sorted has already been packed into a Record
+  **       by a prior OP_MakeRecord.  In this case nData==1 and regData
+  **       will be completely unrelated to regOrigData.
+  **   (2) All output columns are included in the sort record.  In that
+  **       case regData==regOrigData.
+  **   (3) Some output columns are omitted from the sort record due to
+  **       the SQLITE_ENABLE_SORTER_REFERENCE optimization, or due to the
+  **       SQLITE_ECEL_OMITREF optimization.  In that case, regOrigData==0
+  **       to prevent this routine from trying to copy values that might
+  **       not exist.
+  */
   assert( nData==1 || regData==regOrigData || regOrigData==0 );
+
   if( nPrefixReg ){
     assert( nPrefixReg==nExpr+bSeq );
-    regBase = regData - nExpr - bSeq;
+    regBase = regData - nPrefixReg;
   }else{
     regBase = pParse->nMem + 1;
     pParse->nMem += nBase;
@@ -587,7 +659,7 @@ static void pushOntoSorter(
     int nKey;         /* Number of sorting key columns, including OP_Sequence */
     KeyInfo *pKI;     /* Original KeyInfo on the sorter table */
 
-    sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat,regRecord);
+    regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase);
     regPrevKey = pParse->nMem+1;
     pParse->nMem += pSort->nOBSat;
     nKey = nExpr - pSort->nOBSat + bSeq;
@@ -638,17 +710,16 @@ static void pushOntoSorter(
     ** of the outer loop.
     */
     int iCsr = pSort->iECursor;
-    int iJmp = sqlite3VdbeCurrentAddr(v)+5+(nOBSat<=0)+pSort->bOrderedInnerLoop;
-    assert( pSort->bOrderedInnerLoop==0 || pSort->bOrderedInnerLoop==1 );
     sqlite3VdbeAddOp2(v, OP_IfNotZero, iLimit, sqlite3VdbeCurrentAddr(v)+4);
     VdbeCoverage(v);
     sqlite3VdbeAddOp2(v, OP_Last, iCsr, 0);
-    sqlite3VdbeAddOp4Int(v, OP_IdxLE, iCsr, iJmp, regBase+nOBSat, nExpr-nOBSat);
+    iSkip = sqlite3VdbeAddOp4Int(v, OP_IdxLE,
+                                 iCsr, 0, regBase+nOBSat, nExpr-nOBSat);
     VdbeCoverage(v);
     sqlite3VdbeAddOp1(v, OP_Delete, iCsr);
   }
-  if( nOBSat<=0 ){
+  if( regRecord==0 ){
-    sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat,regRecord);
+    regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase);
   }
   if( pSort->sortFlags & SORTFLAG_UseSorter ){
     op = OP_SorterInsert;
@@ -657,6 +728,11 @@ static void pushOntoSorter(
   }
   sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord,
                        regBase+nOBSat, nBase-nOBSat);
+  if( iSkip ){
+    assert( pSort->bOrderedInnerLoop==0 || pSort->bOrderedInnerLoop==1 );
+    sqlite3VdbeChangeP2(v, iSkip,
+         sqlite3VdbeCurrentAddr(v) + pSort->bOrderedInnerLoop);
+  }
 }
 
 /*
@@ -742,9 +818,6 @@ static void selectExprDefer(
       Table *pTab = pExpr->pTab;
       if( pExpr->op==TK_COLUMN && pTab && !IsVirtual(pTab)
        && (pTab->aCol[pExpr->iColumn].colFlags & COLFLAG_SORTERREF)
-#if 0
-          && pTab->pSchema && pTab->pSelect==0 && !IsVirtual(pTab)
-#endif
       ){
         int j;
         for(j=0; j<nDefer; j++){
@@ -811,6 +884,7 @@ static void selectInnerLoop(
   int iParm = pDest->iSDParm; /* First argument to disposal method */
   int nResultCol;             /* Number of result columns */
   int nPrefixReg = 0;         /* Number of extra registers before regResult */
+  RowLoadInfo sRowLoadInfo;   /* Info for deferred row loading */
 
   /* Usually, regResult is the first cell in an array of memory cells
   ** containing the current result row. In this case regOrig is set to the
@@ -863,7 +937,8 @@ static void selectInnerLoop(
     /* If the destination is an EXISTS(...) expression, the actual
     ** values returned by the SELECT are not required.
     */
-    u8 ecelFlags;
+    u8 ecelFlags;    /* "ecel" is an abbreviation of "ExprCodeExprList" */
+    ExprList *pEList;
     if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){
       ecelFlags = SQLITE_ECEL_DUP;
     }else{
@@ -877,6 +952,7 @@ static void selectInnerLoop(
       ** This allows the p->pEList field to be omitted from the sorted record,
       ** saving space and CPU cycles.  */
       ecelFlags |= (SQLITE_ECEL_OMITREF|SQLITE_ECEL_REF);
+
       for(i=pSort->nOBSat; i<pSort->pOrderBy->nExpr; i++){
         int j;
         if( (j = pSort->pOrderBy->a[i].u.x.iOrderByCol)>0 ){
@@ -897,20 +973,46 @@ static void selectInnerLoop(
         pParse->nMem += pExtra->nExpr;
       }
 #endif
-      regOrig = 0;
+
+      /* Adjust nResultCol to account for columns that are omitted
+      ** from the sorter by the optimizations in this branch */
+      pEList = p->pEList;
+      for(i=0; i<pEList->nExpr; i++){
+        if( pEList->a[i].u.x.iOrderByCol>0
+#ifdef SQLITE_ENABLE_SORTER_REFERENCES
+         || pEList->a[i].bSorterRef
+#endif
+        ){
+          nResultCol--;
+          regOrig = 0;
+        }
+      }
+
+      testcase( regOrig );
+      testcase( eDest==SRT_Set );
+      testcase( eDest==SRT_Mem );
+      testcase( eDest==SRT_Coroutine );
+      testcase( eDest==SRT_Output );
       assert( eDest==SRT_Set || eDest==SRT_Mem 
            || eDest==SRT_Coroutine || eDest==SRT_Output );
     }
-    nResultCol = sqlite3ExprCodeExprList(pParse,p->pEList,regResult,
+    sRowLoadInfo.regResult = regResult;
-                                         0,ecelFlags);
+    sRowLoadInfo.ecelFlags = ecelFlags;
 #ifdef SQLITE_ENABLE_SORTER_REFERENCES
-    if( pExtra ){
+    sRowLoadInfo.pExtra = pExtra;
-      nResultCol += sqlite3ExprCodeExprList(
+    sRowLoadInfo.regExtraResult = regResult + nResultCol;
-          pParse, pExtra, regResult + nResultCol, 0, 0
+    if( pExtra ) nResultCol += pExtra->nExpr;
-      );
-      sqlite3ExprListDelete(pParse->db, pExtra);
-    }
 #endif
+    if( p->iLimit
+     && (ecelFlags & SQLITE_ECEL_OMITREF)!=0 
+     && nPrefixReg>0
+    ){
+      assert( pSort!=0 );
+      assert( hasDistinct==0 );
+      pSort->pDeferredRowLoad = &sRowLoadInfo;
+    }else{
+      innerLoopLoadRow(pParse, p, &sRowLoadInfo);
+    }
   }
 
   /* If the DISTINCT keyword was present on the SELECT statement
@@ -1026,7 +1128,8 @@ static void selectInnerLoop(
       }
 #endif
       if( pSort ){
-        pushOntoSorter(pParse, pSort, p, r1+nPrefixReg,regResult,1,nPrefixReg);
+        assert( regResult==regOrig );
+        pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, regOrig, 1, nPrefixReg);
       }else{
         int r2 = sqlite3GetTempReg(pParse);
         sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2);

src/sqliteInt.h

@@ -2756,11 +2756,8 @@ struct Upsert {
 ** An instance of the following structure contains all information
 ** needed to generate code for a single SELECT statement.
 **
-** nLimit is set to -1 if there is no LIMIT clause.  nOffset is set to 0.
+** See the header comment on the computeLimitRegisters() routine for a
-** If there is a LIMIT clause, the parser sets nLimit to the value of the
+** detailed description of the meaning of the iLimit and iOffset fields.
-** limit and nOffset to the value of the offset (or 0 if there is not
-** offset).  But later on, nLimit and nOffset become the memory locations
-** in the VDBE that record the limit and offset counters.
 **
 ** addrOpenEphm[] entries contain the address of OP_OpenEphemeral opcodes.
 ** These addresses must be stored so that we can go back and fill in