Better optimize queries that use parameters in the LIMIT clause.

FossilOrigin-Name: b44f2676716be37d85eb6f207e187d997fc9d7493883bd7f1541cc7ae52475be

manifest

@@ -1,5 +1,5 @@
-C Fix\stypos\sin\scomments.\s\sProvided\s".wheretrace"\sdebugging\soutput\sfor\sthe\ninterstage\sheuristic\smodule.\s\sDo\somit\sautomatic\sindex\sloops\sin\sthe\ninterstage\sheuristic.
+C Better\soptimize\squeries\sthat\suse\sparameters\sin\sthe\sLIMIT\sclause.
-D 2024-04-02T18:48:25.553
+D 2024-06-06T16:06:21.901
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -470,7 +470,7 @@ F src/date.c ebe1dc7c8a347117bb02570f1a931c62dd78f4a2b1b516f4837d45b7d6426957
 F src/dbpage.c 135eb3b5e74f9ef74bde5cec2571192c90c86984fa534c88bf4a055076fa19b7
 F src/dbstat.c c12833de69cb655751487d2c5a59607e36be1c58ba1f4bd536609909ad47b319
 F src/delete.c fb363f793fc40df36843cc67fb692659164b6109690f4efdd254ab785f8e511d
-F src/expr.c 314b08e6502b9bc638fb8a0d686806e4eb4adebd2c728a28617780dcd1bc4c3b
+F src/expr.c 4bbaf2ca6d6531bc0f00777f9679ff0b94943707f683f81e226f5125a2b031e6
 F src/fault.c 460f3e55994363812d9d60844b2a6de88826e007
 F src/fkey.c e6408a869edeb222b049f6bb991a862a2c4a161da27794018897f196b1c2f584
 F src/func.c 2ccf4ae12430b1ae7096be5f0675887e1bd0732828af0ac0f7496339b7c6edee
@@ -516,14 +516,14 @@ F src/pragma.h a776bb9c915207e9d1117b5754743ddf1bf6a39cc092a4a44e74e6cb5fab1177
 F src/prepare.c 695a19948348b0202741a650af577c35ef2c8daad271bf818e3d0e4a5881ef3f
 F src/printf.c 67f79227273a9009d86a017619717c3f554f50b371294526da59faa6014ed2cd
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c 0965e1667d16634e24c3c268da4c7e10c6d902c95b40edbb10d4d6c41a80c679
+F src/resolve.c 5153c9fb3fa402cfdef7dc5a50779ce05f40632098b6bfb7d861bff4bfe15dee
 F src/rowset.c d977b011993aaea002cab3e0bb2ce50cf346000dff94e944d547b989f4b1fe93
-F src/select.c 7f5f44a6c6527d9dcf30ee16cc08420aa68e94040635b365ed7128e8ab2e2f1a
+F src/select.c cd4b49d6d8d8898bb26ed1d46537d67978b2e25a40baa928229b382344b62452
-F src/shell.c.in c1986496062f9dba4ed5b70db06b5e0f32e1954cdcfab0b30372c6c186796810
+F src/shell.c.in 7081fe81793df713d0c0fc5f82a908b84c7c30c43a8df67d6188bfb373332c91
 F src/sqlite.h.in 59f5e145b8d7a915ca29c6bf4a1f00e3112c1605c9ac5c627c45060110332ba2
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 9ecc93b8493bd20c0c07d52e2ac0ed8bab9b549c7f7955b59869597b650dd8b5
-F src/sqliteInt.h 47bb8365d8e78bfb527c151f968a1d09fa410347054dee54a4702019fa2ca2d7
+F src/sqliteInt.h 25a98035df1faf47a9945cfe0a8c0519dd255a1ef2c89d587b006daa727974ce
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -604,11 +604,11 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 9eccc7ebb532a7b0fd3cabc16cff576b9afa763472272db67d84fb8cec96f5c0
 F src/wal.h 606292549f5a7be50b6227bd685fa76e3a4affad71bb8ac5ce4cb5c79f6a176a
 F src/walker.c 7607f1a68130c028255d8d56094ea602fc402c79e1e35a46e6282849d90d5fe4
-F src/where.c 88042f93c5955c628ee77ff9b1ff675527c92ff5efbc7dbd63c01ae45c2e5069
+F src/where.c 7b276c3110231eebda4d537e32a560030762c2ce7dd44e8e89aa2898af04e263
 F src/whereInt.h 064a1508edcc9af400a3b79211ba55e553fab6451b797a31334c0d11cbb7debb
 F src/wherecode.c cf67460973119c7b2141ad67daf8368dfb4871f225e2489f95effaa139007bfd
-F src/whereexpr.c 9813b81c1bb5c9e90c093d3be7b842fec49072ef6b53c454756aa986735d1d36
+F src/whereexpr.c cb357cf1efd04ae2be4a0d4cf5e0b04c1d666c55f6af72574f593b4a0cf549f6
-F src/window.c d73354234d17bbc8916bf4cf65027854058cbb5d8497e083144522e90bb05c2f
+F src/window.c bb684e5e2824875dc1c79aa4df313cf7f80fc2a5906d3fbd663a0c93be20cd70
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/affinity2.test a6d901b436328bd67a79b41bb0ac2663918fe3bd
 F test/affinity3.test 6a101af2fc945ce2912f6fe54dd646018551710d
@@ -1679,7 +1679,8 @@ F test/whereN.test d205fe91148c4bfc50b668ce9f8ebcaafd4ad806447d5bfee8c482d4cd985
 F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864
 F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f6411db44e3
 F test/wherelimit.test 592081800806d297dd7449b1030c863d2883d6d42901837ccd2e5a9bd962edb0
-F test/wherelimit2.test 9bf0aa56cca40ea0e4c5e2915341355a2bbc0859ec4ce1589197fe2a9d94635f
+F test/wherelimit2.test a87c0456e9e9927449965313377d116af8acc8193cd5b9d40a913607babf964f
+F test/wherelimit3.test 23b24343ceedbe4e722c6055c63d1c65e604781caa4f622a3246619ddc7cb990
 F test/widetab1.test ea6e1d8ce3cf3fc9f0b6f5e4264a45d287944ee9becabbe8ad55ba80bb3cc225
 F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
 F test/win32heap.test 10fd891266bd00af68671e702317726375e5407561d859be1aa04696f2aeee74
@@ -1821,9 +1822,12 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 1f97086d628b6f52e6f9a87cd9d2c365f3c51f5b33c5c36a3f83889ec08765be
+P 357d9513d2bd13c4e9f227300d43a04d13ae1d34b0b5ec55e8b94eb3204d1921
-Q +186dcae19e249db36de15f295999cff25063b54ee3d5d481cd2ba99b6d13148e
+Q +e58cb304d1e0ec6e30260a165aaccc2cf096ce6c999efb06683c4ef600ee12ef
-R 72348ce53a15ea8deb2d0622361d010b
+R ef49b8a39e4003dd4f26e37870f517f3
+T *branch * branch-3.28-var-in-limit
+T *sym-branch-3.28-var-in-limit *
+T -sym-branch-3.28 *
 U drh
-Z 298c3dc9319a58678b6aa3cd002b5e96
+Z fe971a71ec90aa07a1a2ecd8a55c9fa5
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-357d9513d2bd13c4e9f227300d43a04d13ae1d34b0b5ec55e8b94eb3204d1921
+b44f2676716be37d85eb6f207e187d997fc9d7493883bd7f1541cc7ae52475be

src/expr.c

@@ -901,13 +901,13 @@ void sqlite3PExprAddSelect(Parse *pParse, Expr *pExpr, Select *pSelect){
 static int exprAlwaysTrue(Expr *p){
   int v = 0;
   if( ExprHasProperty(p, EP_FromJoin) ) return 0;
-  if( !sqlite3ExprIsInteger(p, &v) ) return 0;
+  if( !sqlite3ExprIsInteger(p, &v, 0) ) return 0;
   return v!=0;
 }
 static int exprAlwaysFalse(Expr *p){
   int v = 0;
   if( ExprHasProperty(p, EP_FromJoin) ) return 0;
-  if( !sqlite3ExprIsInteger(p, &v) ) return 0;
+  if( !sqlite3ExprIsInteger(p, &v, 0) ) return 0;
   return v==0;
 }
 
@@ -2085,8 +2085,12 @@ int sqlite3ExprContainsSubquery(Expr *p){
 ** to fit in a 32-bit integer, return 1 and put the value of the integer
 ** in *pValue.  If the expression is not an integer or if it is too big
 ** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged.
+**
+** If the pParse pointer is provided, then allow the expression p to be
+** a parameter (TK_VARIABLE) that is bound to an integer.
+** But if pParse is NULL, then p must be a pure integer literal.
 */
-int sqlite3ExprIsInteger(Expr *p, int *pValue){
+int sqlite3ExprIsInteger(const Expr *p, int *pValue, Parse *pParse){
   int rc = 0;
   if( p==0 ) return 0;  /* Can only happen following on OOM */
 
@@ -2101,18 +2105,38 @@ int sqlite3ExprIsInteger(Expr *p, int *pValue){
   }
   switch( p->op ){
     case TK_UPLUS: {
-      rc = sqlite3ExprIsInteger(p->pLeft, pValue);
+      rc = sqlite3ExprIsInteger(p->pLeft, pValue, 0);
       break;
     }
     case TK_UMINUS: {
       int v;
-      if( sqlite3ExprIsInteger(p->pLeft, &v) ){
+      if( sqlite3ExprIsInteger(p->pLeft, &v, 0) ){
         assert( v!=(-2147483647-1) );
         *pValue = -v;
         rc = 1;
       }
       break;
     }
+    case TK_VARIABLE: {
+      sqlite3_value *pVal;
+      if( pParse==0 ) break;
+      if( NEVER(pParse->pVdbe==0) ) break;
+      if( (pParse->db->flags & SQLITE_EnableQPSG)!=0 ) break;
+      sqlite3VdbeSetVarmask(pParse->pVdbe, p->iColumn);
+      pVal = sqlite3VdbeGetBoundValue(pParse->pReprepare, p->iColumn,
+                                      SQLITE_AFF_BLOB);
+      if( pVal ){
+        if( sqlite3_value_type(pVal)==SQLITE_INTEGER ){
+          sqlite3_int64 vv = sqlite3_value_int64(pVal);
+          if( vv == (vv & 0x7fffffff) ){ /* non-negative numbers only */
+            *pValue = (int)vv;
+            rc = 1;
+          }
+        }
+        sqlite3ValueFree(pVal);
+      }
+      break;
+    }
     default: break;
   }
   return rc;

src/resolve.c

@@ -1049,7 +1049,7 @@ static int resolveOrderByTermToExprList(
   int rc;            /* Return code from subprocedures */
   u8 savedSuppErr;   /* Saved value of db->suppressErr */
 
-  assert( sqlite3ExprIsInteger(pE, &i)==0 );
+  assert( sqlite3ExprIsInteger(pE, &i, 0)==0 );
   pEList = pSelect->pEList;
 
   /* Resolve all names in the ORDER BY term expression
@@ -1145,7 +1145,7 @@ static int resolveCompoundOrderBy(
       Expr *pE, *pDup;
       if( pItem->done ) continue;
       pE = sqlite3ExprSkipCollate(pItem->pExpr);
-      if( sqlite3ExprIsInteger(pE, &iCol) ){
+      if( sqlite3ExprIsInteger(pE, &iCol, 0) ){
         if( iCol<=0 || iCol>pEList->nExpr ){
           resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr);
           return 1;
@@ -1338,7 +1338,7 @@ static int resolveOrderGroupBy(
         continue;
       }
     }
-    if( sqlite3ExprIsInteger(pE2, &iCol) ){
+    if( sqlite3ExprIsInteger(pE2, &iCol, 0) ){
       /* The ORDER BY term is an integer constant.  Again, set the column
       ** number so that sqlite3ResolveOrderGroupBy() will convert the
       ** order-by term to a copy of the result-set expression */

src/select.c

@@ -2171,7 +2171,7 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){
     p->iLimit = iLimit = ++pParse->nMem;
     v = sqlite3GetVdbe(pParse);
     assert( v!=0 );
-    if( sqlite3ExprIsInteger(pLimit->pLeft, &n) ){
+    if( sqlite3ExprIsInteger(pLimit->pLeft, &n, pParse) ){
       sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit);
       VdbeComment((v, "LIMIT counter"));
       if( n==0 ){
@@ -2624,7 +2624,7 @@ static int multiSelect(
         p->pPrior = pPrior;
         p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow);
         if( pPrior->pLimit
-         && sqlite3ExprIsInteger(pPrior->pLimit->pLeft, &nLimit)
+         && sqlite3ExprIsInteger(pPrior->pLimit->pLeft, &nLimit, pParse)
          && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) 
         ){
           p->nSelectRow = sqlite3LogEst((u64)nLimit);

src/shell.c.in

@@ -3071,6 +3071,7 @@ static int shell_exec(
         zEQP = sqlite3_mprintf("EXPLAIN QUERY PLAN %s", zStmtSql);
         rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
         if( rc==SQLITE_OK ){
+          bind_prepared_stmt(pArg, pExplain);
           while( sqlite3_step(pExplain)==SQLITE_ROW ){
             const char *zEQPLine = (const char*)sqlite3_column_text(pExplain,3);
             int iEqpId = sqlite3_column_int(pExplain, 0);
@@ -3088,6 +3089,7 @@ static int shell_exec(
           rc = sqlite3_prepare_v2(db, zEQP, -1, &pExplain, 0);
           if( rc==SQLITE_OK ){
             pArg->cMode = MODE_Explain;
+            bind_prepared_stmt(pArg, pExplain);
             explain_data_prepare(pArg, pExplain);
             exec_prepared_stmt(pArg, pExplain);
             explain_data_delete(pArg);

src/sqliteInt.h

@@ -4093,7 +4093,7 @@ int sqlite3ExprIsTableConstant(Expr*,int);
 #ifdef SQLITE_ENABLE_CURSOR_HINTS
 int sqlite3ExprContainsSubquery(Expr*);
 #endif
-int sqlite3ExprIsInteger(Expr*, int*);
+int sqlite3ExprIsInteger(const Expr*, int*, Parse*);
 int sqlite3ExprCanBeNull(const Expr*);
 int sqlite3ExprNeedsNoAffinityChange(const Expr*, char);
 int sqlite3IsRowid(const char*);

src/where.c

@@ -2300,7 +2300,7 @@ static void whereLoopOutputAdjust(
         if( pTerm->eOperator&(WO_EQ|WO_IS) ){
           Expr *pRight = pTerm->pExpr->pRight;
           testcase( pTerm->pExpr->op==TK_IS );
-          if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){
+          if( sqlite3ExprIsInteger(pRight, &k, 0) && k>=(-1) && k<=1 ){
             k = 10;
           }else{
             k = 20;

src/whereexpr.c

@@ -1457,6 +1457,7 @@ void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){
   }
 }
 
+
 /*
 ** Initialize a preallocated WhereClause structure.
 */

src/window.c

@@ -875,6 +875,7 @@ static ExprList *exprListAppendList(
     int nInit = pList ? pList->nExpr : 0;
     for(i=0; i<pAppend->nExpr; i++){
       Expr *pDup = sqlite3ExprDup(pParse->db, pAppend->a[i].pExpr, 0);
+
       pList = sqlite3ExprListAppend(pParse, pList, pDup);
       if( pList ) pList->a[nInit+i].sortOrder = pAppend->a[i].sortOrder;
     }

test/wherelimit2.test

@@ -296,4 +296,35 @@ do_test 5.5 {
 } {ax a bx b cx c dx d ex a}
 
 
+# 2024-06-06
+# Use LIMIT if the argument is a variable.
+#
+reset_db
+do_execsql_test 8.0 {
+  CREATE TABLE t1(a INT, b INT);
+  WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<1000)
+    INSERT INTO t1 SELECT n, n FROM c;
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  ANALYZE;
+}
+do_eqp_test 8.1 {
+  SELECT * FROM t1 WHERE a>=100 AND b<300 ORDER BY b LIMIT 5;
+} {
+  QUERY PLAN
+  |--SEARCH t1 USING INDEX t1a (a>? AND a<?)
+  `--USE TEMP B-TREE FOR ORDER BY
+}
+do_eqp_test 8.2 {
+  SELECT * FROM t1 WHERE a>=100 AND b<300 ORDER BY b LIMIT -1;
+} {
+  QUERY PLAN
+  |--SEARCH t1 USING INDEX t1a (a>? AND a<?)
+  `--USE TEMP B-TREE FOR ORDER BY
+}
+
+
+
+
+
 finish_test

test/wherelimit3.test

@@ -0,0 +1,67 @@
+# 2024-06-06
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+# Test cases for query plans using LIMIT
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix wherelimit3
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a INT, b INT);
+  WITH RECURSIVE c(n) AS (VALUES(1) UNION ALL SELECT n+1 FROM c WHERE n<1000)
+    INSERT INTO t1 SELECT n, n FROM c;
+  CREATE INDEX t1a ON t1(a);
+  CREATE INDEX t1b ON t1(b);
+  ANALYZE;
+}
+
+do_eqp_test 1.1 {
+  SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT 5;
+} {
+  QUERY PLAN
+  |--SEARCH TABLE t1 USING INDEX t1a (a>? AND a<?)
+  `--USE TEMP B-TREE FOR ORDER BY
+}
+ifcapable stat4 {
+  do_eqp_test 1.2 {
+    SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT -1;
+  } {
+  QUERY PLAN
+  `--SCAN TABLE t1 USING INDEX t1b
+}
+}
+
+set N [expr 5]
+do_eqp_test 1.3 {
+  SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT $::N;
+} {
+  QUERY PLAN
+  |--SEARCH TABLE t1 USING INDEX t1a (a>? AND a<?)
+  `--USE TEMP B-TREE FOR ORDER BY
+}
+
+ifcapable stat4 {
+  set N [expr -1]
+  do_eqp_test 1.4 {
+    SELECT * FROM t1 WHERE a>=100 AND a<300 ORDER BY b LIMIT $::N;
+  } {
+  QUERY PLAN
+  `--SCAN TABLE t1 USING INDEX t1b
+}
+}
+
+
+
+
+
+finish_test