Allow sub-queries that use UNION ALL to be flattened, even if the parent query is a join. Still some problems on this branch.

FossilOrigin-Name: 00e4bf74d3dfb87666a2266905f7d1a2afc6eb088d22cfd4f38f048733d6b936

manifest

@@ -1,5 +1,5 @@
-C Remove\san\sunnecessary\sand\sincorrect\s#ifdef.\s\sFix\sharmless\scompiler\swarnings.
+C Allow\ssub-queries\sthat\suse\sUNION\sALL\sto\sbe\sflattened,\seven\sif\sthe\sparent\squery\sis\sa\sjoin.\sStill\ssome\sproblems\son\sthis\sbranch.
-D 2020-12-16T14:20:45.731
+D 2020-12-16T20:00:46.479
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F LICENSE.md df5091916dbb40e6e9686186587125e1b2ff51f022cc334e886c19a0e9982724
@@ -540,7 +540,7 @@ F src/printf.c 30e92b638fac71dcd85cdea1d12ecfae354c9adee2c71e8e1ae4727cde7c91ed
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
 F src/resolve.c 1948a92ca9eab776632816b97e57c61d933474a78aad4f4ef835c916a83dbb1c
 F src/rowset.c ba9515a922af32abe1f7d39406b9d35730ed65efab9443dc5702693b60854c92
-F src/select.c c9b68506e5d8cc8d0e4b307b97a9800b050ac37dada80ae9c66f680f8fac3e09
+F src/select.c 85f02c7d6bcc81c369da37e3c5decf45e326d8a4988b44f7111a06352d9c8830
 F src/shell.c.in e9f674ee4ec6c345679e8a5b16c869c6c59eb1540dd98ac69e4736ecddce0090
 F src/sqlite.h.in 5b7593bb0f3658e682a9fcd1cd8fcedf244ec45ca93d645055a53172f55eb783
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
@@ -1331,7 +1331,7 @@ F test/select8.test 8c8f5ae43894c891efc5755ed905467d1d67ad5d
 F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95
 F test/selectA.test 68de52409e45a3313d00b8461b48bef4fb729faf36ade9067a994eae55cc86f4
 F test/selectB.test 954e4e49cf1f896d61794e440669e03a27ceea25
-F test/selectC.test e25243f8ca503e06f252eb0218976d07cfeceac3
+F test/selectC.test fec14c9015ed4ec941508bbc144f30b42e40ac34a4bb33001450369865dd0b75
 F test/selectD.test fc20452847a01775710090383cfb4423275d2f745fed61f34fbf37573ac0d214
 F test/selectE.test a8730ca330fcf40ace158f134f4fe0eb00c7edbf
 F test/selectF.test 21c94e6438f76537b72532fa9fd4710cdd455fc3
@@ -1627,6 +1627,8 @@ F test/tt3_vacuum.c 1753f45917699c9c1f66b64c717a717c9379f776
 F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff
 F test/types2.test 1aeb81976841a91eef292723649b5c4fe3bc3cac
 F test/types3.test 99e009491a54f4dc02c06bdbc0c5eea56ae3e25a
+F test/unionall.test 52a4324f59c70df2525188298a0593d650d983ccbfa9441a8a411ac99e1d6644
+F test/unionallfault.test 8dcc3f680ace498d8d3110ddcfdaa6a3d8aa1843bc7c266b990f13815ee6d7fe
 F test/unionvtab.test e1704ab1b4c1bb3ffc9da4681f8e85a0b909fd80b937984fc94b27415ac8e5a4
 F test/unionvtabfault.test e8759f3d14fb938ce9657e2342db34aeac0fb9bc1692b0d1ebb0069630151d06
 F test/unique.test 93f8b2ef5ea51b9495f8d6493429b1fd0f465264
@@ -1741,7 +1743,7 @@ F test/whereH.test e4b07f7a3c2f5d31195cd33710054c78667573b2
 F test/whereI.test a2874062140ed4aba9ffae76e6190a3df6fc73d1373fdfa8fd632945082a5364
 F test/whereJ.test 88287550f6ee604422403b053455b1ad894eeaa5c35d348532dfa1439286cb9a
 F test/whereK.test f8e3cf26a8513ecc7f514f54df9f0572c046c42b
-F test/whereL.test e05cedc9389c6f09ad55bd5999a3fddccebec90672fb989433c145dcdaf26996
+F test/whereL.test 1afe47227f093dc0547236491fb37529b7be9724b8575925a321001b80e6a23a
 F test/wherefault.test 1374c3aa198388925246475f84ad4cd5f9528864
 F test/wherelfault.test 9012e4ef5259058b771606616bd007af5d154e64cc25fa9fd4170f6411db44e3
 F test/wherelimit.test 592081800806d297dd7449b1030c863d2883d6d42901837ccd2e5a9bd962edb0
@@ -1891,7 +1893,10 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 48301edc90fe5811df0394b106edce7726d0ea86ac562c9f4db511b812a76433
+P 31cd1bbfa5b06723288d99d1cb423f88353bdef770b82e9103f71a796d66f660
-R bc270d52af99cca590351162eea6886b
+R 1855283db3260c0b82311305192dc3a1
-U drh
+T *branch * union-all-flattener
-Z 53bb33e2ab1210ebb52c38429ca9bfe7
+T *sym-union-all-flattener *
+T -sym-trunk *
+U dan
+Z cdcd89c0fb8db082c205bd2de653ec39

manifest.uuid

@@ -1 +1 @@
-31cd1bbfa5b06723288d99d1cb423f88353bdef770b82e9103f71a796d66f660
+00e4bf74d3dfb87666a2266905f7d1a2afc6eb088d22cfd4f38f048733d6b936

src/select.c

@@ -3920,7 +3920,7 @@ static int flattenSubquery(
     if( pSub->pOrderBy ){
       return 0;  /* Restriction (20) */
     }
-    if( isAgg || (p->selFlags & SF_Distinct)!=0 || pSrc->nSrc!=1 ){
+    if( isAgg || (p->selFlags & SF_Distinct)!=0 || isLeftJoin>0 ){
       return 0; /* (17d1), (17d2), or (17d3) */
     }
     for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){
@@ -3947,15 +3947,10 @@ static int flattenSubquery(
         if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0;
       }
     }
-  }
 
-  /* Ex-restriction (23):
+    /* Restriction (23) */
-  ** The only way that the recursive part of a CTE can contain a compound
+    if( (p->selFlags & SF_Recursive) ) return 0;
-  ** subquery is for the subquery to be one term of a join.  But if the
+  }
-  ** subquery is a join, then the flattening has already been stopped by
-  ** restriction (17d3)
-  */
-  assert( (p->selFlags & SF_Recursive)==0 || pSub->pPrior==0 );
 
   /***** If we reach this point, flattening is permitted. *****/
   SELECTTRACE(1,pParse,p,("flatten %u.%p from term %d\n",
@@ -3967,6 +3962,17 @@ static int flattenSubquery(
   testcase( i==SQLITE_DENY );
   pParse->zAuthContext = zSavedAuthContext;
 
+  /* Delete the transient structures associated with thesubquery */
+  pSub1 = pSubitem->pSelect;
+  sqlite3DbFree(db, pSubitem->zDatabase);
+  sqlite3DbFree(db, pSubitem->zName);
+  sqlite3DbFree(db, pSubitem->zAlias);
+  pSubitem->zDatabase = 0;
+  pSubitem->zName = 0;
+  pSubitem->zAlias = 0;
+  pSubitem->pSelect = 0;
+  assert( pSubitem->pOn==0 );
+
   /* If the sub-query is a compound SELECT statement, then (by restrictions
   ** 17 and 18 above) it must be a UNION ALL and the parent query must 
   ** be of the form:
@@ -4005,15 +4011,16 @@ static int flattenSubquery(
     ExprList *pOrderBy = p->pOrderBy;
     Expr *pLimit = p->pLimit;
     Select *pPrior = p->pPrior;
+    Table *pItemTab = pSubitem->pTab;
+    pSubitem->pTab = 0;
     p->pOrderBy = 0;
-    p->pSrc = 0;
     p->pPrior = 0;
     p->pLimit = 0;
     pNew = sqlite3SelectDup(db, p, 0);
     p->pLimit = pLimit;
     p->pOrderBy = pOrderBy;
-    p->pSrc = pSrc;
     p->op = TK_ALL;
+    pSubitem->pTab = pItemTab;
     if( pNew==0 ){
       p->pPrior = pPrior;
     }else{
@@ -4024,25 +4031,13 @@ static int flattenSubquery(
       SELECTTRACE(2,pParse,p,("compound-subquery flattener"
                               " creates %u as peer\n",pNew->selId));
     }
-    if( db->mallocFailed ) return 1;
+    assert( pSubitem->pSelect==0 );
+    if( db->mallocFailed ){
+      pSubitem->pSelect = pSub1;
+      return 1;
+    }
   }
 
-  /* Begin flattening the iFrom-th entry of the FROM clause 
-  ** in the outer query.
-  */
-  pSub = pSub1 = pSubitem->pSelect;
-
-  /* Delete the transient table structure associated with the
-  ** subquery
-  */
-  sqlite3DbFree(db, pSubitem->zDatabase);
-  sqlite3DbFree(db, pSubitem->zName);
-  sqlite3DbFree(db, pSubitem->zAlias);
-  pSubitem->zDatabase = 0;
-  pSubitem->zName = 0;
-  pSubitem->zAlias = 0;
-  pSubitem->pSelect = 0;
-
   /* Defer deleting the Table object associated with the
   ** subquery until code generation is
   ** complete, since there may still exist Expr.pTab entries that
@@ -4075,6 +4070,7 @@ static int flattenSubquery(
   ** those references with expressions that resolve to the subquery FROM
   ** elements we are now copying in.
   */
+  pSub = pSub1;
   for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){
     int nSubSrc;
     u8 jointype = 0;
@@ -4083,16 +4079,10 @@ static int flattenSubquery(
     nSubSrc = pSubSrc->nSrc;  /* Number of terms in subquery FROM clause */
     pSrc = pParent->pSrc;     /* FROM clause of the outer query */
 
-    if( pSrc ){
+    if( pParent==p ){
-      assert( pParent==p );  /* First time through the loop */
+      jointype = pSubitem->fg.jointype;     /* First time through the loop */
-      jointype = pSubitem->fg.jointype;
-    }else{
-      assert( pParent!=p );  /* 2nd and subsequent times through the loop */
-      pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0);
-      if( pSrc==0 ) break;
-      pParent->pSrc = pSrc;
     }
-
+    
     /* The subquery uses a single slot of the FROM clause of the outer
     ** query.  If the subquery has more than one element in its FROM clause,
     ** then expand the outer query to make space for it to hold all elements

test/selectC.test

@@ -261,10 +261,10 @@ do_execsql_test 5.2 {
 }
 
 do_execsql_test 5.3 {
-  SELECT * FROM x1, (SELECT b FROM vvv UNION ALL SELECT c from x3);
+  SELECT * FROM x1, (SELECT b FROM vvv UNION ALL SELECT c from x3) ORDER BY 1,2;
 } {
-  a 21 a 22 a 23 a 24 a 25 a 302 a 303 a 301
+  a 21 a 22 a 23 a 24 a 25 a 301 a 302 a 303
-  b 21 b 22 b 23 b 24 b 25 b 302 b 303 b 301
+  b 21 b 22 b 23 b 24 b 25 b 301 b 302 b 303
 }
 
 finish_test

test/unionall.test

@@ -0,0 +1,58 @@
+# 2020-12-16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is flattening UNION ALL sub-queries.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix unionall
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1_a(a INTEGER PRIMARY KEY, b TEXT);
+  CREATE TABLE t1_b(c INTEGER PRIMARY KEY, d TEXT);
+  CREATE TABLE t1_c(e INTEGER PRIMARY KEY, f TEXT);
+
+  INSERT INTO t1_a VALUES(1, 'one'), (4, 'four');
+  INSERT INTO t1_b VALUES(2, 'two'), (5, 'five');
+  INSERT INTO t1_c VALUES(3, 'three'), (6, 'six');
+
+  CREATE VIEW t1 AS 
+    SELECT a, b FROM t1_a   UNION ALL
+    SELECT c, d FROM t1_b   UNION ALL
+    SELECT e, f FROM t1_c;
+
+  CREATE TABLE i1(x);
+  INSERT INTO i1 VALUES(2), (5), (6), (1);
+}
+
+do_execsql_test 1.1 {
+  SELECT a, b FROM (
+    SELECT a, b FROM t1_a   UNION ALL
+    SELECT c, d FROM t1_b   UNION ALL
+    SELECT e, f FROM t1_c
+  ) ORDER BY a
+} {
+  1 one 2 two 3 three 4 four 5 five 6 six
+}
+
+do_execsql_test 1.2 {
+  SELECT a, b FROM t1 ORDER BY a
+} {
+  1 one 2 two 3 three 4 four 5 five 6 six
+}
+
+do_execsql_test 1.3 {
+  SELECT a, b FROM i1, t1 WHERE a=x ORDER BY a
+} {1 one 2 two 5 five 6 six}
+
+
+finish_test

test/unionallfault.test

@@ -0,0 +1,36 @@
+# 2020-12-16
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix unionallfault
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x,y,z);
+  CREATE TABLE t3(x,y,z);
+}
+faultsim_save_and_close
+
+
+do_faultsim_test 1 -faults oom-t* -prep {
+  faultsim_restore_and_reopen
+} -body {
+  execsql {
+    SELECT * FROM (
+      SELECT x FROM t1 UNION ALL SELECT y FROM t1
+    )
+  }
+} -test {
+  faultsim_test_result {0 {}}
+}
+
+finish_test

test/whereL.test

@@ -26,14 +26,13 @@ do_eqp_test 110 {
   SELECT * FROM t1, v4 WHERE t1.a=?1 AND v4.a=t1.a;
 } {
   QUERY PLAN
-  |--MATERIALIZE xxxxxx
+  `--COMPOUND QUERY
-  |  `--COMPOUND QUERY
+     |--LEFT-MOST SUBQUERY
-  |     |--LEFT-MOST SUBQUERY
+     |  |--SEARCH TABLE t2 USING INDEX sqlite_autoindex_t2_1 (a=?)
-  |     |  `--SEARCH TABLE t2 USING INDEX sqlite_autoindex_t2_1 (a=?)
+     |  `--SEARCH TABLE t1 USING INDEX sqlite_autoindex_t1_1 (a=?)
-  |     `--UNION ALL
+     `--UNION ALL
-  |        `--SEARCH TABLE t3 USING INDEX sqlite_autoindex_t3_1 (a=?)
+        |--SEARCH TABLE t3 USING INDEX sqlite_autoindex_t3_1 (a=?)
-  |--SCAN SUBQUERY xxxxxx
+        `--SEARCH TABLE t1 USING INDEX sqlite_autoindex_t1_1 (a=?)
-  `--SEARCH TABLE t1 USING INDEX sqlite_autoindex_t1_1 (a=?)
 }
 
 # The scan of the t1 table goes first since that enables the ORDER BY