Merge in the exists-to-join optimization that has been modified

to relax the requirement of having an indexed join constraint.

FossilOrigin-Name: 1c1aef2b7feae29066d0330699ab634ef41f5b60cdcd479a60cb1a5409553138

manifest

@@ -1,5 +1,5 @@
-C Fix\sVDBE\scoverage
+C Merge\sin\sthe\sexists-to-join\soptimization\sthat\shas\sbeen\smodified\nto\srelax\sthe\srequirement\sof\shaving\san\sindexed\sjoin\sconstraint.
-D 2025-07-02T17:43:59.873
+D 2025-07-02T20:46:02.299
 F .fossil-settings/binary-glob 61195414528fb3ea9693577e1980230d78a1f8b0a54c78cf1b9b24d0a409ed6a x
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
@@ -729,7 +729,7 @@ F src/btmutex.c 30dada73a819a1ef5b7583786370dce1842e12e1ad941e4d05ac29695528daea
 F src/btree.c 96fcbe6db6af625e5a14c34d8f13688d1d22c5f924a436b12395aaf09ec65944
 F src/btree.h e823c46d87f63d904d735a24b76146d19f51f04445ea561f71cc3382fd1307f0
 F src/btreeInt.h 9c0f9ea5c9b5f4dcaea18111d43efe95f2ac276cd86d770dce10fd99ccc93886
-F src/build.c 67c1db4c5e89a8519fe9b6dafc287f6bc3627696b5b8536dc5e06db570d8c05f
+F src/build.c f0a3d935954b1b344b929f5092f94e535ac98cf72bb8bcd3e72fb3dfa982e169
 F src/callback.c acae8c8dddda41ee85cfdf19b926eefe830f371069f8aadca3aa39adf5b1c859
 F src/complete.c a3634ab1e687055cd002e11b8f43eb75c17da23e
 F src/date.c 9db4d604e699a73e10b8e85a44db074a1f04c0591a77e2abfd77703f50dce1e9
@@ -783,14 +783,14 @@ F src/pragma.c 30b535d0a66348df844ee36f890617b4cf45e9a22dcbc47ec3ca92909c50aaf1
 F src/prepare.c 1832be043fce7d489959aae6f994c452d023914714c4d5457beaed51c0f3d126
 F src/printf.c 71b6d3a0093bf23f473e25480ca0024e8962681506c75f4ffd3d343a3f0ab113
 F src/random.c 606b00941a1d7dd09c381d3279a058d771f406c5213c9932bbd93d5587be4b9c
-F src/resolve.c d40fe18d7c2fd0339f5846ffcf7d6809866e380acdf14c76fb2af87e9fe13f64
+F src/resolve.c 9583e6eb8066400d377db9b130ebf46d461b59fccf1d018a5010d55b9bea217c
 F src/rowset.c 8432130e6c344b3401a8874c3cb49fefe6873fec593294de077afea2dce5ec97
-F src/select.c fc2fe502971df1205a3231d3b3c8b0cc9ed4779cecbd060952c9558e22b6b02d
+F src/select.c 30d7e450179a123c2a4dd9000592da196f5a6766688773cb11b921294afa11bf
 F src/shell.c.in 4f14a1f5196b6006abc8e73cc8fd6c1a62cf940396f8ba909d6711f35f074bb6
 F src/sqlite.h.in 5c54f2461a1ea529bab8499148a2b238e2d4bb571d59e8ea5322d0c190abb693
 F src/sqlite3.rc 015537e6ac1eec6c7050e17b616c2ffe6f70fca241835a84a4f0d5937383c479
 F src/sqlite3ext.h 0bfd049bb2088cc44c2ad54f2079d1c6e43091a4e1ce8868779b75f6c1484f1e
-F src/sqliteInt.h 005542f8760edf9b62f014abccb876cf64533b64475a40a89402054d62535288
+F src/sqliteInt.h fcb2fd7eb47f6731ca8d4cbfbada95f24665bcc8a5ec44e2d24942220eb32453
 F src/sqliteLimit.h 6d817c28a8f19af95e6f4921933b7fbbca48a962bce0eb0ec81e8bb3ef38e68b
 F src/status.c 0e72e4f6be6ccfde2488eb63210297e75f569f3ce9920f6c3d77590ec6ce5ffd
 F src/table.c 0f141b58a16de7e2fbe81c308379e7279f4c6b50eb08efeec5892794a0ba30d1
@@ -867,9 +867,9 @@ F src/vxworks.h d2988f4e5a61a4dfe82c6524dd3d6e4f2ce3cdb9
 F src/wal.c 20be6f0a25a80b7897cf2a5369bfd37ef198e6f0b6cdef16d83eee856056b159
 F src/wal.h ba252daaa94f889f4b2c17c027e823d9be47ce39da1d3799886bbd51f0490452
 F src/walker.c d5006d6b005e4ea7302ad390957a8d41ed83faa177e412f89bc5600a7462a014
-F src/where.c b11e56a24d01ae9b293f702c9de6dd16ced6b886be0d7cccb8bdeb62c8d92362
+F src/where.c 458b2089adc9ad65b2585fdf705b716e74abef146f20472962368cd784898f65
 F src/whereInt.h 8d94cb116c9e06205c3d5ac87af065fc044f8cf08bfdccd94b6ea1c1308e65da
-F src/wherecode.c 504f3c1270c3ffd51ebcdf7a31de08aa51a63b33a2ccdf8f5736afe3dfa73d45
+F src/wherecode.c 2917e70e12f7b238285240e564329374f20e4270fe90c36e0d19b6770754a7c4
 F src/whereexpr.c 566ca4382e07a4ba1fd86c97ae0781cdf84004c7d9c59466bf5db75733548807
 F src/window.c d01227141f622f24fbe36ca105fbe6ef023f9fd98f1ccd65da95f88886565db5
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
@@ -1131,7 +1131,7 @@ F test/enc.test b5503a87b31cea8a5084c6e447383f9ca08933bd2f29d97b6b6201081b2343eb
 F test/enc2.test 872afe58db772e7dfa1ad8e0759f8cc820e9efc8172d460fae83023101c2e435
 F test/enc3.test 55ef64416d72975c66167310a51dc9fc544ba3ae4858b8d5ab22f4cb6500b087
 F test/enc4.test c8f1ce3618508fd0909945beb8b8831feef2c020
-F test/eqp.test 82f221e8cd588434d7f3bba9a0f4c78cbe7a541615a41632e12f50608bfb4a99
+F test/eqp.test 800fb69fae9086b76dc767931a9973355187d673f69cd2ccfd3c455528af7859
 F test/eqp2.test 6e8996148de88f0e7670491e92e712a2920a369b4406f21a27c3c9b6a46b68dd
 F test/errmsg.test eae9f091eb39ce7e20305de45d8e5d115b68fa856fba4ea6757b6ca3705ff7f9
 F test/errofst1.test 6da78363739ba8991f498396ab331b5d64e7ab5c4172c12b5884683ef523ac53
@@ -1140,6 +1140,9 @@ F test/exclusive.test 7ff63be7503990921838d5c9f77f6e33e68e48ed1a9d48cd28745bf650
 F test/exclusive2.test cd70b1d9c6fffd336f9795b711dcc5d9ceba133ad3f7001da3fda63615bdc91e
 F test/exec.test e949714dc127eaa5ecc7d723efec1ec27118fdd7
 F test/exists.test 79a75323c78f02bbe9c251ea502a092f9ef63dac
+F test/existsexpr.test d87e7ee394935f9b4a9a1a488f3faa55abd20dd85152efaf2034881a079c7ba1
+F test/existsexpr2.test dc23e76389eff3d29f6488ff733012a3560cd67ec8cfaecbecd52cced5d5af11
+F test/existsfault.test ff41c11f3052c1bbd4f8dd557802310026253d67d7c4e3a180c16d2f0862973e
 F test/expr.test 4ada8eb822c45ef27a36851a258004d43c1e95e7c82585a1217e732084e4482c
 F test/expr2.test c27327ae9c017a7ff6280123f67aff496f912da74d78c888926d68b46ec75fd8
 F test/exprfault.test da33606d799718e2f8e34efd0e5858884a1ad87f608774c552a7f5517cc27181
@@ -1387,7 +1390,7 @@ F test/json/json-generator.tcl dc0dd0f393800c98658fc4c47eaa6af29d4e17527380cd286
 F test/json/json-q1.txt 65f9d1cdcc4cffa9823fb73ed936aae5658700cd001fde448f68bfb91c807307
 F test/json/json-speed-check.sh 7d5898808ce7542762318306ae6075a30f5e7ee115c4a409f487e123afe91d88 x
 F test/json/jsonb-q1.txt 1e180fe6491efab307e318b22879e3a736ac9a96539bbde7911a13ee5b33abc7
-F test/json101.test 8237a484c256965eab1678fd950a32ac56325bb7d0dadbd095a46b0ddd95d62b
+F test/json101.test cf53254f0f0c1399a01b21fc58fee0e63a12a556be91b9ee9faccdb8b82c083c
 F test/json102.test 9b2e5ada10845ff84853b3feaae2ce51ce7145ae458f74c6a6cecc6ef6ee3ae1
 F test/json103.test 355746a6b66aa438f214b4fae454b13068fad2444b5f693e0d538ad1c059b264
 F test/json104.test 1b844a70cddcfa2e4cd81a5db0657b2e61e7f00868310f24f56a9ba0114348c1
@@ -1494,7 +1497,7 @@ F test/notify1.test 669b2b743618efdc18ca4b02f45423d5d2304abf
 F test/notify2.test 2ecabaa1305083856b7c39cf32816b612740c161
 F test/notify3.test 796c7b7157f55c93b4e672b724e9c923a6fc6aa72ac419379a623e2350472e22
 F test/notnull.test a37b663d5bb728d66fc182016613fb8e4a0a4bbf3d75b8876a7527f7d4ed3f18
-F test/notnull2.test 2ac7b4e04917148c7a1a9ed36df20150175ce942f07f5714375b29acbaca7106
+F test/notnull2.test 5b7dd6e82c409b2d011ad6acf19ae4bf0816a9c69ccf600b529d7405d7c49874
 F test/notnullfault.test fc4bb7845582a2b3db376001ef49118393b1b11abe0d24adb03db057ee2b73d5
 F test/null.test b7ff206a1c60fe01aa2abd33ef9ea83c93727d993ca8a613de86e925c9f2bc6f
 F test/nulls1.test 7a5e4346ee4285034100b4cd20e6784f16a9d6c927e44ecdf10034086bbee9c9
@@ -2208,8 +2211,8 @@ F tool/version-info.c 3b36468a90faf1bbd59c65fd0eb66522d9f941eedd364fabccd7227350
 F tool/warnings-clang.sh bbf6a1e685e534c92ec2bfba5b1745f34fb6f0bc2a362850723a9ee87c1b31a7
 F tool/warnings.sh 1ad0169b022b280bcaaf94a7fa231591be96b514230ab5c98fbf15cd7df842dd
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P eaad6ac707a5960d9518d4049b7b1759e7512727ce87be3c402408144bda0a97
+P ff593a16d61cc5c588d1737deb822abb90b1759475a4cabfcf608978b1191487 9084a4c8726a2c7ba1c381886e29c7b86121d531282be0d63d5988d84f6f448d
-R 19ae64c3af637e68252816fb2a35fc7d
+R f60eba1acfa64dc97c08e5ab9333e9b4
 U drh
-Z 04fda6f3c6a7b8123cd1480694a7568d
+Z 4c3d6c95c910f93bc2abb148fe548800
 # Remove this line to create a well-formed Fossil manifest.

manifest.uuid

@@ -1 +1 @@
-ff593a16d61cc5c588d1737deb822abb90b1759475a4cabfcf608978b1191487
+1c1aef2b7feae29066d0330699ab634ef41f5b60cdcd479a60cb1a5409553138

src/build.c

@@ -5138,14 +5138,17 @@ void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){
 ** are deleted by this function.
 */
 SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2){
-  assert( p1 && p1->nSrc==1 );
+  assert( p1 );
   if( p2 ){
-    SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, 1);
+    int nOld = p1->nSrc;
+    SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, nOld);
     if( pNew==0 ){
       sqlite3SrcListDelete(pParse->db, p2);
     }else{
       p1 = pNew;
-      memcpy(&p1->a[1], p2->a, p2->nSrc*sizeof(SrcItem));
+      memcpy(&p1->a[nOld], p2->a, p2->nSrc*sizeof(SrcItem));
+      assert( nOld==1 || (p2->nSrc==1 && (p2->a[0].fg.jointype&JT_LTORJ)==0) );
+      assert( p1->nSrc>=2 );
       sqlite3DbFree(pParse->db, p2);
       p1->a[0].fg.jointype |= (JT_LTORJ & p1->a[1].fg.jointype);
     }

src/resolve.c

@@ -1379,6 +1379,7 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
         if( nRef!=pNC->nRef ){
           ExprSetProperty(pExpr, EP_VarSelect);
           pExpr->x.pSelect->selFlags |= SF_Correlated;
+          if( pExpr->op==TK_EXISTS ) pParse->bHasExists = 1;
         }
         pNC->ncFlags |= NC_Subquery;
       }

src/select.c

@@ -384,7 +384,7 @@ static int tableAndColumnIndex(
   int iEnd,            /* Last member of pSrc->a[] to check */
   const char *zCol,    /* Name of the column we are looking for */
   int *piTab,          /* Write index of pSrc->a[] here */
-  int *piCol,          /* Write index of pSrc->a[*piTab].pTab->aCol[] here */
+  int *piCol,          /* Write index of pSrc->a[*piTab].pSTab->aCol[] here */
   int bIgnoreHidden    /* Ignore hidden columns */
 ){
   int i;               /* For looping over tables in pSrc */
@@ -4658,7 +4658,7 @@ static int flattenSubquery(
   ** complete, since there may still exist Expr.pTab entries that
   ** refer to the subquery even after flattening.  Ticket #3346.
   **
-  ** pSubitem->pTab is always non-NULL by test restrictions and tests above.
+  ** pSubitem->pSTab is always non-NULL by test restrictions and tests above.
   */
   if( ALWAYS(pSubitem->pSTab!=0) ){
     Table *pTabToDel = pSubitem->pSTab;
@@ -5771,7 +5771,7 @@ With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){
 ** CTE expression, through routine checks to see if the reference is
 ** a recursive reference to the CTE.
 **
-** If pFrom matches a CTE according to either of these two above, pFrom->pTab
+** If pFrom matches a CTE according to either of these two above, pFrom->pSTab
 ** and other fields are populated accordingly.
 **
 ** Return 0 if no match is found.
@@ -7398,6 +7398,74 @@ static int fromClauseTermCanBeCoroutine(
   return 1;
 }
 
+/*
+** Argument pWhere is the WHERE clause belonging to SELECT statement p. This
+** function attempts to transform expressions of the form:
+**
+**     EXISTS (SELECT ...)
+**
+** into joins. For example, given
+**
+**    CREATE TABLE sailors(sid INTEGER PRIMARY KEY, name TEXT);
+**    CREATE TABLE reserves(sid INT, day DATE, PRIMARY KEY(sid, day));
+**
+**    SELECT name FROM sailors AS S WHERE EXISTS (
+**      SELECT * FROM reserves AS R WHERE S.sid = R.sid AND R.day = '2022-10-25'
+**    );
+**
+** the SELECT statement may be transformed as follows:
+**
+**    SELECT name FROM sailors AS S, reserves AS R
+**      WHERE S.sid = R.sid AND R.day = '2022-10-25';
+*/
+static SQLITE_NOINLINE void existsToJoin(
+  Parse *pParse,
+  Select *p,
+  Expr *pWhere
+){
+  if( pWhere 
+   && !ExprHasProperty(pWhere, EP_OuterON|EP_InnerON) 
+   && p->pSrc->nSrc>0
+   && p->pSrc->nSrc<BMS
+   && pParse->db->mallocFailed==0 
+  ){
+    if( pWhere->op==TK_AND ){
+      Expr *pRight = pWhere->pRight;
+      existsToJoin(pParse, p, pWhere->pLeft);
+      existsToJoin(pParse, p, pRight);
+    }
+    else if( pWhere->op==TK_EXISTS ){
+      Select *pSub = pWhere->x.pSelect;
+      if( pSub->pSrc->nSrc==1 
+       && (pSub->selFlags & (SF_Aggregate|SF_Correlated))==SF_Correlated
+       && pSub->pWhere
+      ){
+        memset(pWhere, 0, sizeof(*pWhere));
+        pWhere->op = TK_INTEGER;
+        pWhere->u.iValue = 1;
+        ExprSetProperty(pWhere, EP_IntValue);
+
+        assert( p->pWhere!=0 );
+        pSub->pSrc->a[0].fg.fromExists = 1;
+        pSub->pSrc->a[0].fg.jointype |= JT_CROSS;
+        p->pSrc = sqlite3SrcListAppendList(pParse, p->pSrc, pSub->pSrc);
+        p->pWhere = sqlite3PExpr(pParse, TK_AND, p->pWhere, pSub->pWhere);
+
+        pSub->pWhere = 0;
+        pSub->pSrc = 0;
+        sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSub);
+#if TREETRACE_ENABLED
+        if( sqlite3TreeTrace & 0x100000 ){
+          TREETRACE(0x100000,pParse,p,
+                    ("After EXISTS-to-JOIN optimization:\n"));
+          sqlite3TreeViewSelect(0, p, 0);
+        }
+#endif
+      }
+    }
+  }
+}
+
 /*
 ** Generate byte-code for the SELECT statement given in the p argument. 
 **
@@ -7766,6 +7834,13 @@ int sqlite3Select(
   }
 #endif
 
+  /* If there may be an "EXISTS (SELECT ...)" in the WHERE clause, attempt
+  ** to change it into a join.  */
+  if( pParse->bHasExists && OptimizationEnabled(db,SQLITE_ExistsToJoin) ){
+    existsToJoin(pParse, p, p->pWhere);
+    pTabList = p->pSrc;
+  }
+
   /* Do the WHERE-clause constant propagation optimization if this is
   ** a join.  No need to spend time on this operation for non-join queries
   ** as the equivalent optimization will be handled by query planner in

src/sqliteInt.h

@@ -1154,6 +1154,7 @@ extern u32 sqlite3TreeTrace;
 **   0x00040000     SELECT tree dump after all code has been generated
 **   0x00080000     NOT NULL strength reduction
 **   0x00100000     Pointers are all shown as zero
+**   0x00200000     EXISTS-to-JOIN optimization
 */
 
 /*
@@ -1926,6 +1927,7 @@ struct sqlite3 {
 #define SQLITE_OnePass        0x08000000 /* Single-pass DELETE and UPDATE */
 #define SQLITE_OrderBySubq    0x10000000 /* ORDER BY in subquery helps outer */
 #define SQLITE_StarQuery      0x20000000 /* Heurists for star queries */
+#define SQLITE_ExistsToJoin   0x40000000 /* The EXISTS-to-JOIN optimization */
 #define SQLITE_AllOpts        0xffffffff /* All optimizations */
 
 /*
@@ -3370,6 +3372,7 @@ struct SrcItem {
     unsigned rowidUsed :1;     /* The ROWID of this table is referenced */
     unsigned fixedSchema :1;   /* Uses u4.pSchema, not u4.zDatabase */
     unsigned hadSchema :1;     /* Had u4.zDatabase before u4.pSchema */
+    unsigned fromExists :1;    /* Comes from WHERE EXISTS(...) */
   } fg;
   int iCursor;      /* The VDBE cursor number used to access this table */
   Bitmask colUsed;  /* Bit N set if column N used. Details above for N>62 */
@@ -3900,6 +3903,7 @@ struct Parse {
   u8 disableLookaside; /* Number of times lookaside has been disabled */
   u8 prepFlags;        /* SQLITE_PREPARE_* flags */
   u8 withinRJSubrtn;   /* Nesting level for RIGHT JOIN body subroutines */
+  u8 bHasExists;       /* Has a correlated "EXISTS (SELECT ....)" expression */
   u8 mSubrtnSig;       /* mini Bloom filter on available SubrtnSig.selId */
   u8 eTriggerOp;       /* TK_UPDATE, TK_INSERT or TK_DELETE */
   u8 bReturning;       /* Coding a RETURNING trigger */

src/where.c

@@ -3518,6 +3518,7 @@ static int whereLoopAddBtreeIndex(
    && pProbe->hasStat1!=0
    && OptimizationEnabled(db, SQLITE_SkipScan)
    && pProbe->aiRowLogEst[saved_nEq+1]>=42  /* TUNING: Minimum for skip-scan */
+   && pSrc->fg.fromExists==0
    && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK
   ){
     LogEst nIter;
@@ -7397,6 +7398,9 @@ void sqlite3WhereEnd(WhereInfo *pWInfo){
         sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2);
       }
 #endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */
+      if( pTabList->a[pLevel->iFrom].fg.fromExists ){
+        sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2);
+      }
       /* The common case: Advance to the next row */
       if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont);
       sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3);

src/wherecode.c

@@ -135,6 +135,7 @@ void sqlite3WhereAddExplainText(
 #if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_EXPLAIN)
     char *zMsg;                   /* Text to add to EQP output */
 #endif
+    const char *zFormat;
     StrAccum str;                 /* EQP output string */
     char zBuf[100];               /* Initial space for EQP output string */
 
@@ -149,7 +150,14 @@ void sqlite3WhereAddExplainText(
 
     sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH);
     str.printfFlags = SQLITE_PRINTF_INTERNAL;
-    sqlite3_str_appendf(&str, "%s %S", isSearch ? "SEARCH" : "SCAN", pItem);
+    if( pItem->fg.fromExists ){
+      zFormat = "SINGLETON %S";
+    }else if( isSearch ){
+      zFormat = "SEARCH %S";
+    }else{
+      zFormat = "SCAN %S";
+    }
+    sqlite3_str_appendf(&str, zFormat, pItem);
     if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){
       const char *zFmt = 0;
       Index *pIdx;

test/eqp.test

@@ -338,8 +338,7 @@ det 3.3.3 {
 } {
   QUERY PLAN
   |--SCAN t1
-  `--CORRELATED SCALAR SUBQUERY xxxxxx
+  `--SINGLETON t2
-     `--SCAN t2
 }
 
 #-------------------------------------------------------------------------

test/existsexpr.test

@@ -0,0 +1,426 @@
+# 2024 May 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set testprefix existsexpr
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE x1(a, b, PRIMARY KEY(a)) WITHOUT ROWID;
+  INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6);
+  CREATE INDEX x1b ON x1(b);
+
+  CREATE TABLE x2(x, y);
+  INSERT INTO x2 VALUES(1, 2), (3, 4), (5, 6);
+}
+
+do_execsql_test 1.1 {
+  SELECT 1 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=5)
+} {1}
+
+do_execsql_test 1.2 {
+  SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x)
+} {1 2 3 4 5 6}
+
+# With "a=x", the UNIQUE index means the EXIST can be transformed to a join.
+# So no "SUBQUERY". With "b=x", the index is not UNIQUE and so there is a
+# "SUBQUERY".
+do_execsql_test 1.3.1 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x)
+} {~/SUBQUERY/}
+do_execsql_test 1.3.2 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE b=x)
+} {~/SUBQUERY/}
+
+do_execsql_test 1.4.1 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM x2 WHERE x=1 AND EXISTS (SELECT 1 FROM x1 WHERE a=x)
+} {~/SUBQUERY/}
+do_execsql_test 1.4.2 {
+  EXPLAIN QUERY PLAN
+  SELECT * FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) AND y=2
+} {~/SUBQUERY/}
+
+do_execsql_test 1.5 {
+  SELECT count(*) FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x)
+} {3}
+
+#-------------------------------------------------------------------------
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a, b);
+  WITH s(i) AS (
+    SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<1000
+  ) INSERT INTO t1 SELECT i, i FROM s;
+
+  CREATE TABLE t2(c, d);
+  WITH s(i) AS (
+    SELECT 10 UNION ALL SELECT i+10 FROM s WHERE i<1000
+  ) INSERT INTO t2 SELECT i, i FROM s;
+}
+
+do_execsql_test 2.1 {
+  SELECT count(*) FROM t1;
+  SELECT count(*) FROM t2;
+} {1000 100}
+
+do_execsql_test 2.2 {
+  SELECT count(*) FROM t1, t2 WHERE a=c;
+} {100}
+
+do_execsql_test 2.3 {
+  SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a)
+} {100}
+do_eqp_test 2.4 {
+  SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a)
+} {SCAN t1}
+
+do_execsql_test 2.4.0 {
+  CREATE UNIQUE INDEX t2c ON t2(c);
+  CREATE UNIQUE INDEX t1a ON t1(a);
+}
+
+do_eqp_test 2.4.1 {
+  SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a);
+} {SCAN t1*SINGLETON t2}
+do_execsql_test 2.4.2 {
+  ANALYZE;
+}
+do_eqp_test 2.4.3 {
+  SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a);
+} {SCAN t1*SINGLETON t2}
+do_execsql_test 2.4.4 {
+  SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c=a);
+} {100}
+
+do_execsql_test 2.5.1 {
+  EXPLAIN QUERY PLAN
+  SELECT count(*) FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE t2.rowid=a);
+} {~/SUBQUERY/}
+
+#-------------------------------------------------------------------------
+proc do_subquery_test {tn bSub sql res} {
+  set r1(0) ~/SUBQUERY/
+  set r1(1) /SUBQUERY/
+  do_execsql_test $tn.1 "explain query plan $sql" $r1($bSub)
+  do_execsql_test $tn.2 $sql $res
+}
+
+do_execsql_test 3.0 {
+  CREATE TABLE y1(a, b, c);
+  CREATE TABLE y2(x, y, z);
+  CREATE UNIQUE INDEX y2zy ON y2(z, y);
+
+  INSERT INTO y1 VALUES(1, 1, 1);
+  INSERT INTO y1 VALUES(2, 2, 2);
+  INSERT INTO y1 VALUES(3, 3, 3);
+  INSERT INTO y1 VALUES(4, 4, 4);
+
+  INSERT INTO y2 VALUES(1, 1, 1);
+  INSERT INTO y2 VALUES(3, 3, 3);
+}
+
+do_subquery_test 3.1 0 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 WHERE z=a AND y=b AND x=z
+  )
+} {
+  1 1 1          3 3 3
+}    
+
+do_subquery_test 3.2 0 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 WHERE z=max(a,b) AND y=min(b,a) AND x=z
+  )
+} {
+  1 1 1          3 3 3
+}    
+
+do_subquery_test 3.3 0 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 WHERE z=max(a,b) AND y=min(b,a) AND c!=3
+  )
+} {
+  1 1 1
+}    
+
+do_subquery_test 3.4 0 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 WHERE z=max(a,b) AND b=3
+  )
+} {
+  3 3 3
+}    
+
+do_subquery_test 3.5 0 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 WHERE z=a-1 AND y=a-1
+  )
+} {
+  2 2 2
+  4 4 4
+}    
+
+do_subquery_test 3.6 0 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 WHERE z=a-1 AND y+1=a
+  )
+} {
+  2 2 2
+  4 4 4
+}    
+
+do_subquery_test 3.7 1 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT count(*) FROM y2 WHERE z=a-1 AND y=a-1
+  )
+} {
+  1 1 1
+  2 2 2
+  3 3 3
+  4 4 4
+}    
+
+do_subquery_test 3.8 1 {
+  SELECT * FROM y1 WHERE EXISTS ( SELECT a+1 FROM y2 )
+} {
+  1 1 1
+  2 2 2
+  3 3 3
+  4 4 4
+}    
+
+do_subquery_test 3.9 1 {
+  SELECT * FROM y1 WHERE EXISTS ( 
+      SELECT 1 FROM y2 one, y2 two WHERE one.z=a-1 AND one.y=a-1
+  )
+} {
+  2 2 2
+  4 4 4
+}    
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 4.0 {
+  CREATE TABLE tx1(a TEXT COLLATE nocase, b TEXT);
+  CREATE UNIQUE INDEX tx1ab ON tx1(a, b);
+
+  INSERT INTO tx1 VALUES('a', 'a');
+  INSERT INTO tx1 VALUES('B', 'b');
+  INSERT INTO tx1 VALUES('c', 'c');
+  INSERT INTO tx1 VALUES('D', 'd');
+  INSERT INTO tx1 VALUES('e', 'e');
+
+  CREATE TABLE tx2(x, y);
+  INSERT INTO tx2 VALUES('A', 'a');
+  INSERT INTO tx2 VALUES('b', 'b');
+  INSERT INTO tx2 VALUES('C', 'c');
+  INSERT INTO tx2 VALUES('D', 'd');
+}
+
+do_subquery_test 4.1 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE a=x AND b=y
+  )
+} {
+  A a
+  b b
+  C c
+  D d
+}
+
+do_subquery_test 4.1.1 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE (a COLLATE nocase)=x AND b=y
+  )
+} {
+  A a   b b   C c   D d
+}
+do_subquery_test 4.1.2 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE a=x AND (b COLLATE binary)=y
+  )
+} {
+  A a   b b   C c   D d
+}
+do_subquery_test 4.1.1 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE x=(a COLLATE nocase) AND b=y
+  )
+} {
+  A a   b b   C c   D d
+}
+do_subquery_test 4.1.2 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE a=x AND y=(b COLLATE binary)
+  )
+} {
+  A a   b b   C c   D d
+}
+
+do_subquery_test 4.2 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE a=x AND b=y COLLATE nocase
+  )
+} {
+  A a
+  b b
+  C c
+  D d
+}
+
+do_execsql_test 4.3 {
+  DROP INDEX tx1ab;
+  CREATE UNIQUE INDEX tx1ab ON tx1(a COLLATE binary, b);
+}
+
+do_subquery_test 4.4 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE a=x AND b=y
+  )
+} {
+  A a
+  b b
+  C c
+  D d
+}
+
+do_subquery_test 4.4 0 {
+  SELECT * FROM tx2 WHERE EXISTS (
+    SELECT 1 FROM tx1 WHERE a=x COLLATE binary AND b=y
+  )
+} {
+  D d
+}
+
+do_subquery_test 4.4 1 {
+  SELECT EXISTS ( SELECT x FROM tx1 ) FROM tx2
+} {
+  1 1 1 1
+}
+
+do_subquery_test 4.4 1 {
+  SELECT (SELECT EXISTS ( SELECT x FROM tx1 ) WHERE 1) FROM tx2
+} {
+  1 1 1 1
+}
+
+#-------------------------------------------------------------------------
+proc cols {s f} {
+  set lCols [list]
+  for {set i $s} {$i<=$f} {incr i} {
+    lappend lCols [format "c%02d" $i]
+  }
+  join $lCols ", "
+}
+proc vals {n val} {
+  set lVal [list]
+  for {set i 0} {$i<$n} {incr i} {
+    lappend lVal $val
+  }
+  join $lVal ", "
+}
+proc exprs {s f} {
+  set lExpr [list]
+  for {set i $s} {$i<=$f} {incr i} {
+    lappend lExpr [format "c%02d = o" $i]
+  }
+  join $lExpr " AND "
+}
+
+
+do_execsql_test 5.0 "
+  CREATE TABLE a1( [cols 0 99] );
+"
+do_execsql_test 5.1 "
+  -- 63 column index
+  CREATE UNIQUE INDEX a1idx1 ON a1( [cols 0 62] );
+"
+do_execsql_test 5.2 "
+  -- 64 column index
+  CREATE UNIQUE INDEX a1idx2 ON a1( [cols 10 73] );
+"
+do_execsql_test 5.2 "
+  -- 65 column index
+  CREATE UNIQUE INDEX a1idx3 ON a1( [cols 20 84] );
+"
+
+do_test 5.3 {
+  foreach v {1 2 3 4 5 6} {
+    execsql "INSERT INTO a1 VALUES( [vals 100 $v] )"
+  }
+} {}
+
+do_execsql_test 5.4 {
+  CREATE TABLE a2(o);
+  INSERT INTO a2 VALUES(2), (5); 
+}
+
+do_subquery_test 5.5 0 "
+  SELECT o FROM a2 WHERE EXISTS (
+    SELECT 1 FROM a1 WHERE [exprs 0 62]
+  )
+" {
+  2 5
+}
+
+do_subquery_test 5.6 0 "
+  SELECT o FROM a2 WHERE EXISTS (
+    SELECT 1 FROM a1 WHERE [exprs 10 73]
+  )
+" {
+  2 5
+}
+
+do_subquery_test 5.7 0 "
+  SELECT o FROM a2 WHERE EXISTS (
+    SELECT 1 FROM a1 WHERE [exprs 20 84]
+  )
+" {
+  2 5
+}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 6.0 {
+  CREATE TABLE t1(a, b UNIQUE, c UNIQUE);
+  CREATE TABLE t2(a INfEGER PRIMARY KEY, b);
+  CREATE UNIQUE INDEX t2b ON t2(b);
+}
+
+do_catchsql_test 6.1 {
+  SELECT a FROM t1 WHERE EXISTS (SELECT 1 FROM t2 WHERE c COLLATE f = a)
+} {1 {no such collation sequence: f}}
+
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 7.0 {
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(y UNIQUE);
+
+  INSERT INTO t1 VALUES(1), (2);
+  INSERT INTO t2 VALUES(1), (3);
+
+  SELECT * FROM t1 one LEFT JOIN t1 two ON (one.x=two.x AND EXISTS (
+    SELECT 1 FROM t2 WHERE y=one.x
+  ));
+} {
+  1 1
+  2 {}
+}
+
+
+
+finish_test

test/existsexpr2.test

@@ -0,0 +1,96 @@
+# 2024 June 14
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+set testprefix existsexpr2
+
+
+do_execsql_test 1.0 {
+  CREATE TABLE x1(a, b, PRIMARY KEY(a)) WITHOUT ROWID;
+  INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6);
+  CREATE INDEX x1b ON x1(b);
+
+  CREATE TABLE x2(x, y);
+  INSERT INTO x2 VALUES(1, 2), (3, 4), (5, 6);
+}
+
+do_execsql_test 1.1 {
+  SELECT * FROM x1 WHERE EXISTS (SELECT 1 FROM x2 WHERE a!=123)
+} {1 2   3 4   5 6}
+
+do_execsql_test 1.2 {
+  CREATE TABLE x3(u, v);
+  CREATE INDEX x3u ON x3(u);
+  INSERT INTO x3 VALUES
+    (1, 1), (1, 2), (1, 3),
+    (2, 1), (2, 2), (2, 3);
+}
+
+do_execsql_test 1.3 {
+  SELECT * FROM x1 WHERE EXISTS (
+    SELECT 1 FROM x3 WHERE u IN (1, 2, 3, 4) AND v=b
+  );
+} {
+  1 2
+}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+do_execsql_test 2.0 {
+  CREATE TABLE t1(a, b, c);
+  CREATE INDEX t1ab ON t1(a,b);
+
+  INSERT INTO t1 VALUES
+      ('abc', 1, 1),
+      ('abc', 2, 2),
+      ('abc', 2, 3),
+
+      ('def', 1, 1),
+      ('def', 2, 2),
+      ('def', 2, 3);
+
+  CREATE TABLE t2(x, y);
+  INSERT INTO t2 VALUES(1, 1), (2, 2), (3, 3);
+
+  ANALYZE;
+  DELETE FROM sqlite_stat1;
+  INSERT INTO sqlite_stat1 VALUES('t1','t1ab','10000 5000 2');
+  ANALYZE sqlite_master;
+}
+
+
+do_execsql_test 2.1 {
+  SELECT a,b,c FROM t1 WHERE b=2 ORDER BY a
+} {
+  abc 2 2
+  abc 2 3
+  def 2 2
+  def 2 3
+}
+
+do_execsql_test 2.2 {
+  SELECT x, y FROM t2 WHERE EXISTS (
+    SELECT 1 FROM t1 WHERE b=x
+  )
+} {
+  1 1
+  2 2
+}
+
+
+
+finish_test
+
+

test/existsfault.test

@@ -0,0 +1,49 @@
+# 2024 May 25
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/malloc_common.tcl
+set testprefix existsfault
+
+db close
+sqlite3_shutdown
+sqlite3_config_lookaside 0 0
+sqlite3_initialize
+autoinstall_test_functions
+sqlite3 db test.db
+
+do_execsql_test 1.0 {
+  CREATE TABLE x1(a, b);
+  INSERT INTO x1 VALUES(1, 2), (3, 4), (5, 6);
+  CREATE UNIQUE INDEX x1a ON x1(a);
+  CREATE INDEX x1b ON x1(b);
+
+  CREATE TABLE x2(x, y);
+  INSERT INTO x2 VALUES(1, 2), (3, 4), (5, 6);
+}
+
+do_faultsim_test 1 -faults oom* -prep {
+  sqlite3 db test.db
+  execsql { SELECT * FROM sqlite_schema }
+} -body {
+  execsql {
+    SELECT count(*) FROM x2 WHERE EXISTS (SELECT 1 FROM x1 WHERE a=x) AND y!=11
+  }
+} -test {
+  faultsim_test_result {0 3}
+}
+
+finish_test
+
+

test/json101.test

@@ -892,15 +892,15 @@ do_execsql_test json101-13.100 {
   INSERT INTO t2(id,json) VALUES(4,'{"value":4}');
   INSERT INTO t2(id,json) VALUES(5,'{"value":5}');
   INSERT INTO t2(id,json) VALUES(6,'{"value":6}');
-  SELECT * FROM t1 CROSS JOIN t2
+  SELECT *, 'NL' FROM t1 CROSS JOIN t2
    WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z
                  WHERE Z.value==t2.id);
-} {1 {{"items":[3,5]}} 3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}}}
+} {1 {{"items":[3,5]}} 3 {{"value":3}} NL 1 {{"items":[3,5]}} 5 {{"value":5}} NL}
 do_execsql_test json101-13.110 {
-  SELECT * FROM t2 CROSS JOIN t1
+  SELECT *, 'NL' FROM t2 CROSS JOIN t1
    WHERE EXISTS(SELECT 1 FROM json_each(t1.json,'$.items') AS Z
                  WHERE Z.value==t2.id);
-} {3 {{"value":3}} 1 {{"items":[3,5]}} 5 {{"value":5}} 1 {{"items":[3,5]}}}
+} {3 {{"value":3}} 1 {{"items":[3,5]}} NL 5 {{"value":5}} 1 {{"items":[3,5]}} NL}
 
 # 2018-05-16
 # Incorrect fullkey output from json_each()

test/notnull2.test

@@ -66,7 +66,7 @@ do_vmstep_test 1.5.2 {
   SELECT count(*) FROM t2 WHERE EXISTS(
     SELECT 1 FROM t1 WHERE t1.a=450 AND t2.c IS NULL
   )
-} +8000 {0}
+} 4000 {0}
 
 #-------------------------------------------------------------------------
 reset_db