Add an optimization that attempts to run a subquery as a coroutine rather

than manifesting it into a temporary table.

FossilOrigin-Name: 7af3acbbd49a2114fcdada9ec907c68ba9b053bd

manifest

@@ -1,5 +1,5 @@
-C Make\sthe\smultiplexor\sand\sthe\sIntArray\sextensions\scallable\sfrom\sC++.
+C Add\san\soptimization\sthat\sattempts\sto\srun\sa\ssubquery\sas\sa\scoroutine\srather\nthan\smanifesting\sit\sinto\sa\stemporary\stable.
-D 2012-10-29T14:27:26.624
+D 2012-10-30T00:29:07.224
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 82c41c0ed4cc94dd3cc7d498575b84c57c2c2384
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -129,8 +129,8 @@ F src/callback.c d7e46f40c3cf53c43550b7da7a1d0479910b62cc
 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
 F src/ctime.c 72a70dcfda75d3a1f81041ce4573e7afddcd8e4e
 F src/date.c 067a81c9942c497aafd2c260e13add8a7d0c7dd4
-F src/delete.c e35684ad93c741266b086610d2efd709b7946853
+F src/delete.c 9b8d308979114991e5dc7cee958316e07186941d
-F src/expr.c 57fb8e7a05d4147e40b9f4c439e37ed2abab9332
+F src/expr.c 152ba793a8747061c0c332857d0e4d4fd52d4397
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c c82a04e7a92bb728f9ab972b76590403283be2af
 F src/func.c cbb90dc84b22eea25caf39528d342279e61b8898
@@ -174,12 +174,12 @@ F src/printf.c 4a9f882f1c1787a8b494a2987765acf9d97ac21f
 F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50
 F src/resolve.c 7b986a715ac281643309c29257bb58cfae7aa810
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
-F src/select.c 9b759521f805e25e97baa2dd64037157fe365817
+F src/select.c d441b50c389dc0ffc50f0b6f469ff5117196dc86
 F src/shell.c 24cd0aa74aff73ea08594629faead564c4c2a286
 F src/sqlite.h.in c7be05ad191d2634292fcc77bdb2bcfa4526eb98
 F src/sqlite3.rc fea433eb0a59f4c9393c8e6d76a6e2596b1fe0c0
 F src/sqlite3ext.h 6904f4aadf976f95241311fbffb00823075d9477
-F src/sqliteInt.h b49674b1c92173e9796c94fd6f15c1679e3b10cc
+F src/sqliteInt.h c1927e2b0d596480445bcf3b7d679aae1069a8e0
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c 53463144deb5dfac0a66b3be4dd7844b8f8a4c98
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -236,9 +236,9 @@ F src/update.c 28d2d098b43a2c70dae399896ea8a02f622410ef
 F src/utf.c 8d819e2e5104a430fc2005f018db14347c95a38f
 F src/util.c 0af2e515dc0dabacec931bca39525f6c3f1c5455
 F src/vacuum.c 2727bdd08847fcb6b2d2da6d14f018910e8645d3
-F src/vdbe.c 31523df2b986fc6c959dd54ca640ba865884641b
+F src/vdbe.c f3310ec731dc43e58bbf871da0054a297ecada4b
 F src/vdbe.h b52887278cb173e66188da84dfab216bea61119d
-F src/vdbeInt.h 573a43ab5697b648a1e8f3dfc7d8667d5ca55729
+F src/vdbeInt.h 79abf9b31be406d35ca77d6999cb2d42aaf91e78
 F src/vdbeapi.c 4c2418161cf45392ba76a7ca92f9a5f06b96f89c
 F src/vdbeaux.c 8c8cfd30063e9b3664e2faa0e3785102bf131a10
 F src/vdbeblob.c 32f2a4899d67f69634ea4dd93e3f651936d732cb
@@ -249,7 +249,7 @@ F src/vtab.c b05e5f1f4902461ba9f5fc49bb7eb7c3a0741a83
 F src/wal.c f5c7b5027d0ed0e9bc9afeb4a3a8dfea762ec7d2
 F src/wal.h 29c197540b19044e6cd73487017e5e47a1d3dac6
 F src/walker.c 3d75ba73de15e0f8cd0737643badbeb0e002f07b
-F src/where.c bcd54574cffbfd17a1755d75c29759913f714ac4
+F src/where.c 73eb4c41f3e745bbaa7ecc49d3aa2688e4f298ba
 F test/8_3_names.test 631ea964a3edb091cf73c3b540f6bcfdb36ce823
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
 F test/aggnested.test 0be144b453e0622a085fae8665c32f5676708e00
@@ -611,8 +611,8 @@ F test/memdb.test 708a028d6d373e5b3842e4bdc8ba80998c9a4da6
 F test/memleak.test 10b9c6c57e19fc68c32941495e9ba1c50123f6e2
 F test/memsubsys1.test a8f9e37567453a5d1d9d37ec102d4d88ab6be33f
 F test/memsubsys2.test 3a1c1a9de48e5726faa85108b02459fae8cb9ee9
-F test/minmax.test 722d80816f7e096bf2c04f4111f1a6c1ba65453d
+F test/minmax.test c61518429e66e228efc79661fbd2dc3e4924ec44
-F test/minmax2.test 33504c01a03bd99226144e4b03f7631a274d66e0
+F test/minmax2.test b44bae787fc7b227597b01b0ca5575c7cb54d3bc
 F test/minmax3.test cc1e8b010136db0d01a6f2a29ba5a9f321034354
 F test/minmax4.test 536a3360470633a177e42fbc19660d146b51daef
 F test/misc1.test 889b40722442380a2f6575f30831b32b2372d70e
@@ -760,7 +760,7 @@ F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660
 F test/tkt-2a5629202f.test 1ab32e084e9fc3d36be6dee2617530846a0eb0b6
 F test/tkt-2d1a5c67d.test b028a811049eb472cb2d3a43fc8ce4f6894eebda
 F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28
-F test/tkt-31338dca7e.test 1f714c14b6682c5db715e0bda347926a3456f7a9
+F test/tkt-31338dca7e.test 6fb8807851964da0d24e942f2e19c7c705b9fb58
 F test/tkt-313723c356.test c47f8a9330523e6f35698bf4489bcb29609b53ac
 F test/tkt-385a5b56b9.test 7782a382912a51f09f1d1a1442bca1e75f9c549b
 F test/tkt-38cb5df375.test f3cc8671f1eb604d4ae9cf886ed4366bec656678
@@ -851,7 +851,7 @@ F test/tkt3461.test 228ea328a5a21e8663f80ee3d212a6ad92549a19
 F test/tkt3493.test 1686cbde85f8721fc1bdc0ee72f2ef2f63139218
 F test/tkt3508.test d75704db9501625c7f7deec119fcaf1696aefb7d
 F test/tkt3522.test 22ce2ebbcb04a6be56c0977d405c207967318fd6
-F test/tkt3527.test 9e8f28a706c772d5a7cd1020c946fab6c74e3ae0
+F test/tkt3527.test 1a6a48441b560bdc53aec581a868eb576234874d
 F test/tkt3541.test 5dc257bde9bc833ab9cc6844bf170b998dbb950a
 F test/tkt3554.test f599967f279077bace39220cbe76085c7b423725
 F test/tkt3581.test 1966b7193f1e3f14951cce8c66907ae69454e9a3
@@ -1021,7 +1021,10 @@ F tool/vdbe-compress.tcl f12c884766bd14277f4fcedcae07078011717381
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh fbc018d67fd7395f440c28f33ef0f94420226381
 F tool/win/sqlite.vsix 97894c2790eda7b5bce3cc79cb2a8ec2fde9b3ac
-P 99f27d642fbaba02264a470977cde38c3a526d13
+P cbd8fddf3b4cde840279502e1e9b151bb4bd42f0
-R 07bc3a42510a6d7b1ec8ddb3f50b6182
+R e804ca235c11687c1af8c5ce37532c75
+T *branch * subquery-as-coroutine
+T *sym-subquery-as-coroutine *
+T -sym-trunk *
 U drh
-Z 118483038e9bfcfd996fdf110e0050f4
+Z e25ab88b15575239ebd4117d30628530

manifest.uuid

@@ -1 +1 @@
-cbd8fddf3b4cde840279502e1e9b151bb4bd42f0
+7af3acbbd49a2114fcdada9ec907c68ba9b053bd

src/delete.c

@@ -112,6 +112,7 @@ void sqlite3MaterializeView(
       sqlite3SelectDelete(db, pDup);
     }
     pDup = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, 0, 0, 0);
+    if( pDup ) pDup->selFlags |= SF_Materialize;
   }
   sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur);
   sqlite3Select(pParse, pDup, &dest);

src/expr.c

@@ -939,6 +939,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){
     pNewItem->addrFillSub = pOldItem->addrFillSub;
     pNewItem->regReturn = pOldItem->regReturn;
     pNewItem->isCorrelated = pOldItem->isCorrelated;
+    pNewItem->viaCoroutine = pOldItem->viaCoroutine;
     pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex);
     pNewItem->notIndexed = pOldItem->notIndexed;
     pNewItem->pIndex = pOldItem->pIndex;

src/select.c

@@ -3909,8 +3909,17 @@ int sqlite3Select(
     int isAggSub;
 
     if( pSub==0 ) continue;
+
+    /* Sometimes the code for a subquery will be generated more than
+    ** once, if the subquery is part of the WHERE clause in a LEFT JOIN,
+    ** for example.  In that case, do not regenerate the code to manifest
+    ** a view or the co-routine to implement a view.  The first instance
+    ** is sufficient, though the subroutine to manifest the view does need
+    ** to be invoked again. */
     if( pItem->addrFillSub ){
-      sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub);
+      if( pItem->viaCoroutine==0 ){
+        sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub);
+      }
       continue;
     }
 
@@ -3931,6 +3940,33 @@ int sqlite3Select(
         p->selFlags |= SF_Aggregate;
       }
       i = -1;
+    }else if( pTabList->nSrc==1 && (p->selFlags & SF_Materialize)==0 ){
+      /* Implement a co-routine that will return a single row of the result
+      ** set on each invocation.
+      */
+      int addrTop;
+      int addrEof;
+      pItem->regReturn = ++pParse->nMem;
+      addrEof = ++pParse->nMem;
+      sqlite3VdbeAddOp0(v, OP_Goto);
+      addrTop = sqlite3VdbeAddOp1(v, OP_OpenPseudo, pItem->iCursor);
+      sqlite3VdbeChangeP5(v, 1);
+      VdbeComment((v, "coroutine for %s", pItem->pTab->zName));
+      pItem->addrFillSub = addrTop;
+      sqlite3VdbeAddOp2(v, OP_Integer, 0, addrEof);
+      sqlite3VdbeChangeP5(v, 1);
+      sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn);
+      explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId);
+      sqlite3Select(pParse, pSub, &dest);
+      pItem->pTab->nRowEst = (unsigned)pSub->nSelectRow;
+      pItem->viaCoroutine = 1;
+      sqlite3VdbeChangeP2(v, addrTop, dest.iSdst);
+      sqlite3VdbeChangeP3(v, addrTop, dest.nSdst);
+      sqlite3VdbeAddOp2(v, OP_Integer, 1, addrEof);
+      sqlite3VdbeAddOp1(v, OP_Yield, pItem->regReturn);
+      VdbeComment((v, "end %s", pItem->pTab->zName));
+      sqlite3VdbeJumpHere(v, addrTop-1);
+      sqlite3ClearTempRegCache(pParse);
     }else{
       /* Generate a subroutine that will fill an ephemeral table with
       ** the content of this subquery.  pItem->addrFillSub will point

src/sqliteInt.h

@@ -1878,8 +1878,9 @@ struct SrcList {
     int addrFillSub;  /* Address of subroutine to manifest a subquery */
     int regReturn;    /* Register holding return address of addrFillSub */
     u8 jointype;      /* Type of join between this able and the previous */
-    u8 notIndexed;    /* True if there is a NOT INDEXED clause */
+    unsigned notIndexed :1;    /* True if there is a NOT INDEXED clause */
-    u8 isCorrelated;  /* True if sub-query is correlated */
+    unsigned isCorrelated :1;  /* True if sub-query is correlated */
+    unsigned viaCoroutine :1;  /* Implemented as a co-routine */
 #ifndef SQLITE_OMIT_EXPLAIN
     u8 iSelectId;     /* If pSelect!=0, the id of the sub-select in EQP */
 #endif
@@ -2103,14 +2104,15 @@ struct Select {
 ** Allowed values for Select.selFlags.  The "SF" prefix stands for
 ** "Select Flag".
 */
-#define SF_Distinct        0x01  /* Output should be DISTINCT */
+#define SF_Distinct        0x0001  /* Output should be DISTINCT */
-#define SF_Resolved        0x02  /* Identifiers have been resolved */
+#define SF_Resolved        0x0002  /* Identifiers have been resolved */
-#define SF_Aggregate       0x04  /* Contains aggregate functions */
+#define SF_Aggregate       0x0004  /* Contains aggregate functions */
-#define SF_UsesEphemeral   0x08  /* Uses the OpenEphemeral opcode */
+#define SF_UsesEphemeral   0x0008  /* Uses the OpenEphemeral opcode */
-#define SF_Expanded        0x10  /* sqlite3SelectExpand() called on this */
+#define SF_Expanded        0x0010  /* sqlite3SelectExpand() called on this */
-#define SF_HasTypeInfo     0x20  /* FROM subqueries have Table metadata */
+#define SF_HasTypeInfo     0x0020  /* FROM subqueries have Table metadata */
-#define SF_UseSorter       0x40  /* Sort using a sorter */
+#define SF_UseSorter       0x0040  /* Sort using a sorter */
-#define SF_Values          0x80  /* Synthesized from VALUES clause */
+#define SF_Values          0x0080  /* Synthesized from VALUES clause */
+#define SF_Materialize     0x0100  /* Force materialization of views */
 
 
 /*

src/vdbe.c

@@ -2207,7 +2207,10 @@ case OP_Column: {
   ** the number of columns is stored in the VdbeCursor.nField element.
   */
   pC = p->apCsr[p1];
-  assert( pC!=0 );
+  if( pC==0 ){
+    sqlite3VdbeMemSetNull(pDest);
+    goto op_column_out;
+  }
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   assert( pC->pVtabCursor==0 );
 #endif
@@ -2237,6 +2240,11 @@ case OP_Column: {
     }
   }else if( ALWAYS(pC->pseudoTableReg>0) ){
     pReg = &aMem[pC->pseudoTableReg];
+    if( pC->multiPseudo ){
+      sqlite3VdbeMemShallowCopy(pDest, pReg+p2, MEM_Ephem);
+      Deephemeralize(pDest);
+      goto op_column_out;
+    }
     assert( pReg->flags & MEM_Blob );
     assert( memIsValid(pReg) );
     payloadSize = pReg->n;
@@ -3322,8 +3330,9 @@ case OP_SorterOpen: {
 **
 ** Open a new cursor that points to a fake table that contains a single
 ** row of data.  The content of that one row in the content of memory
-** register P2.  In other words, cursor P1 becomes an alias for the 
+** register P2 when P5==0.  In other words, cursor P1 becomes an alias for the 
-** MEM_Blob content contained in register P2.
+** MEM_Blob content contained in register P2.  When P5==1, then the
+** row is represented by P3 consecutive registers beginning with P2.
 **
 ** A pseudo-table created by this opcode is used to hold a single
 ** row output from the sorter so that the row can be decomposed into
@@ -3343,6 +3352,7 @@ case OP_OpenPseudo: {
   pCx->pseudoTableReg = pOp->p2;
   pCx->isTable = 1;
   pCx->isIndex = 0;
+  pCx->multiPseudo = pOp->p5;
   break;
 }
 
@@ -4307,7 +4317,7 @@ case OP_Rowid: {                 /* out2-prerelease */
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   pC = p->apCsr[pOp->p1];
   assert( pC!=0 );
-  assert( pC->pseudoTableReg==0 );
+  assert( pC->pseudoTableReg==0 || pC->nullRow );
   if( pC->nullRow ){
     pOut->flags = MEM_Null;
     break;

src/vdbeInt.h

@@ -63,6 +63,7 @@ struct VdbeCursor {
   Bool isIndex;         /* True if an index containing keys only - no data */
   Bool isOrdered;       /* True if the underlying table is BTREE_UNORDERED */
   Bool isSorter;        /* True if a new-style sorter */
+  Bool multiPseudo;     /* Multi-register pseudo-cursor */
   sqlite3_vtab_cursor *pVtabCursor;  /* The cursor for a virtual table */
   const sqlite3_module *pModule;     /* Module for cursor pVtabCursor */
   i64 seqCount;         /* Sequence counter */

src/where.c

@@ -2998,7 +2998,7 @@ static int isSortingIndex(
 ** SQLITE_BIG_DBL. If a plan is found that uses the named index, 
 ** then the cost is calculated in the usual way.
 **
-** If a NOT INDEXED clause (pSrc->notIndexed!=0) was attached to the table 
+** If a NOT INDEXED clause was attached to the table 
 ** in the SELECT statement, then no indexes are considered. However, the 
 ** selected plan may still take advantage of the built-in rowid primary key
 ** index.
@@ -4026,6 +4026,16 @@ static Bitmask codeOneLoopStart(
     VdbeComment((v, "init LEFT JOIN no-match flag"));
   }
 
+  /* Special case of a FROM clause subquery implemented as a co-routine */
+  if( pTabItem->viaCoroutine ){
+    int regYield = pTabItem->regReturn;
+    sqlite3VdbeAddOp2(v, OP_Integer, pTabItem->addrFillSub-1, regYield);
+    pLevel->p2 =  sqlite3VdbeAddOp1(v, OP_Yield, regYield);
+    VdbeComment((v, "next row of co-routine %s", pTabItem->pTab->zName));
+    sqlite3VdbeAddOp2(v, OP_If, regYield+1, addrBrk);
+    pLevel->op = OP_Goto;
+  }else
+
 #ifndef SQLITE_OMIT_VIRTUALTABLE
   if(  (pLevel->plan.wsFlags & WHERE_VIRTUALTABLE)!=0 ){
     /* Case 0:  The table is a virtual-table.  Use the VFilter and VNext

test/minmax.test

@@ -299,7 +299,7 @@ ifcapable {compound && subquery} {
         SELECT max(rowid) FROM t4 UNION SELECT max(rowid) FROM t5
       )
     }
-  } {1}
+  } {{}}
   do_test minmax-9.2 {
     execsql {
       SELECT max(rowid) FROM (

test/minmax2.test

@@ -289,7 +289,7 @@ ifcapable {compound && subquery} {
         SELECT max(rowid) FROM t4 UNION SELECT max(rowid) FROM t5
       )
     }
-  } {1}
+  } {{}}
   do_test minmax2-9.2 {
     execsql {
       SELECT max(rowid) FROM (

test/tkt-31338dca7e.test

@@ -91,7 +91,7 @@ do_test tkt-31338-3.1 {
     INSERT INTO t3 VALUES(4);
     CREATE TABLE t4(h);
     INSERT INTO t4 VALUES(5);
-    
+
     SELECT * FROM t3 LEFT JOIN t1 ON d=g LEFT JOIN t4 ON c=h
      WHERE (a=1 AND h=4)
          OR (b IN (

test/tkt3527.test

@@ -52,8 +52,8 @@ do_test tkt3527-1.1 {
     INSERT INTO Element VALUES(5,'Elem5');
     INSERT INTO ElemOr Values(3,4);
     INSERT INTO ElemOr Values(3,5);
-    INSERT INTO ElemAnd VALUES(1,3,1,1,1);
+    INSERT INTO ElemAnd VALUES(1,3,'a','b','c');
-    INSERT INTO ElemAnd VALUES(1,2,1,1,1);
+    INSERT INTO ElemAnd VALUES(1,2,'x','y','z');
     
     CREATE VIEW ElemView1 AS
     SELECT
@@ -112,12 +112,12 @@ do_test tkt3527-1.1 {
  
     SELECT * FROM ElemView1;
   }
-} {1 1 Elem1 2 1 1 1 0 0 1 1 Elem1 3 1 1 1 0 0 3 3 Elem3 4 {} {} {} 0 1 3 3 Elem3 5 {} {} {} 0 1}
+} {1 1 Elem1 2 x y z 0 0 1 1 Elem1 3 a b c 0 0 3 3 Elem3 4 {} {} {} 0 1 3 3 Elem3 5 {} {} {} 0 1}
    
 do_test tkt3527-1.2 {
   db eval {
     SELECT * FROM ElemView2;
   }
-} {1 1 Elem1 2 1 1 1 0 0 1 1 Elem1 3 1 1 1 0 0 1.3 3 Elem3 4 {} {} {} 1 1 1.3 3 Elem3 5 {} {} {} 1 1 3 3 Elem3 4 {} {} {} 0 1 3 3 Elem3 5 {} {} {} 0 1}
+} {1 1 Elem1 2 x y z 0 0 1 1 Elem1 3 a b c 0 0 1.3 3 Elem3 4 {} {} {} 1 1 1.3 3 Elem3 5 {} {} {} 1 1 3 3 Elem3 4 {} {} {} 0 1 3 3 Elem3 5 {} {} {} 0 1}
 
 finish_test