Cleanup in flattenSubquery. Add OOM tests for flattenSubquery. Fix issues

with OOM errors causes problems for flattenSubquery.  Ticket #3485. (CVS 5882)

FossilOrigin-Name: ea5f4baa041aed934600f0f96b84afb92a14bc47

manifest

@@ -1,5 +1,5 @@
-C Fixed\scrash\sin\sintegrity_check\swith\scorrupt\scontent\soffset\ssize\sin\spage\sheader.\s(CVS\s5881)
+C Cleanup\sin\sflattenSubquery.\s\sAdd\sOOM\stests\sfor\sflattenSubquery.\s\sFix\sissues\nwith\sOOM\serrors\scauses\sproblems\sfor\sflattenSubquery.\s\sTicket\s#3485.\s(CVS\s5882)
-D 2008-11-11T17:36:30
+D 2008-11-11T18:28:59
 F Makefile.arm-wince-mingw32ce-gcc fcd5e9cd67fe88836360bb4f9ef4cb7f8e2fb5a0
 F Makefile.in 48172b58e444a9725ec482e0c022a564749acab4
 F Makefile.linux-gcc d53183f4aa6a9192d249731c90dbdffbd2c68654
@@ -102,7 +102,7 @@ F src/btmutex.c 3a90096c3080b9057dc570b8e16e46511e1c788a
 F src/btree.c 7824bc89b38bafb8a87b7091e912ea789b60e428
 F src/btree.h 179c3ea813780df78a289a8f5130db18e6d4616e
 F src/btreeInt.h e38e9b2b285f40f5bc0a6664f630d4a141622f16
-F src/build.c 41464eb891eb3672b30a5188a352187cba038af9
+F src/build.c 98a6884d47c3cc12faeb2e9a926018d3a7382133
 F src/callback.c e970e5beddbdb23f89a6d05cb1a6419d9f755624
 F src/complete.c cb14e06dbe79dee031031f0d9e686ff306afe07c
 F src/date.c 6f4277fa56d8c1b8e70c0bde838c9e99609f5ec0
@@ -148,11 +148,11 @@ F src/prepare.c ae49b8298eca79acdbc964679962e089b943ec94
 F src/printf.c 785f87120589c1db672e37c6eb1087c456e6f84d
 F src/random.c a87afbd598aa877e23ac676ee92fd8ee5c786a51
 F src/resolve.c 266bb03d2b456fe68f5df2dd5687e7e88ff8088d
-F src/select.c 0d79c6c0b48b9d67a443853fd6add8c2967ba870
+F src/select.c 058d5c227953755cc393ad359f35b653b9337fee
 F src/shell.c 33ae5c8ce9e8b1d02e0cd510b3b9de18e3c414ba
 F src/sqlite.h.in b73e17f40b0dcfc4b17f5ef18ba90bca8d1e8bc5
 F src/sqlite3ext.h 1db7d63ab5de4b3e6b83dd03d1a4e64fef6d2a17
-F src/sqliteInt.h 1e112bd969e0b318c8ee13d7104d4ae0a30da822
+F src/sqliteInt.h f2b50cd40d847ccf2d9baf5a622e744a05f136e3
 F src/sqliteLimit.h f435e728c6b620ef7312814d660a81f9356eb5c8
 F src/status.c 237b193efae0cf6ac3f0817a208de6c6c6ef6d76
 F src/table.c 22744786199c9195720c15a7a42cb97b2e2728d8
@@ -428,7 +428,7 @@ F test/mallocF.test 2d5c590ebc2fc7f0dcebdf5aa8498b9aed69107e
 F test/mallocG.test 4584d0d8ddb8009f16ca0c8bab1fa37f6358efa2
 F test/mallocH.test 79b65aed612c9b3ed2dcdaa727c85895fd1bfbdb
 F test/mallocI.test 6e24fe6444bd2999ccc81f984977b44c0d6e5591
-F test/mallocJ.test 9ed2275a7857fe1d5f5e1debc68dfdc285fc262f
+F test/mallocJ.test 44dfbbaca731cb933818ad300b4566265d652609
 F test/malloc_common.tcl 984baeb6c6b185e798827d1187d426acc2bc4962
 F test/manydb.test 8de36b8d33aab5ef295b11d9e95310aeded31af8
 F test/memdb.test a67bda4ff90a38f2b19f6c7f95aa7289e051d893
@@ -654,7 +654,7 @@ F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
 F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
 F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
-P da869446c53ec6ed769bba01cdc2b6fd69a8b5c9
+P 0659a666ff0a9fc81ee4df3c35e535164c79e588
-R e4f0f76dc4c5672626cf25fdc1112cb0
+R 4feb3b063e6492d2c3095674c86d05b4
-U shane
+U drh
-Z 42e5ec31ef7e6a7867cc9c00711cc593
+Z ccf27dcefe10bea054f7ff323046db0d

manifest.uuid

@@ -1 +1 @@
-0659a666ff0a9fc81ee4df3c35e535164c79e588
+ea5f4baa041aed934600f0f96b84afb92a14bc47

src/build.c

@@ -22,7 +22,7 @@
 **     COMMIT
 **     ROLLBACK
 **
-** $Id: build.c,v 1.500 2008/11/03 20:55:07 drh Exp $
+** $Id: build.c,v 1.501 2008/11/11 18:28:59 drh Exp $
 */
 #include "sqliteInt.h"
 #include <ctype.h>
@@ -2991,11 +2991,81 @@ int sqlite3IdListIndex(IdList *pList, const char *zName){
   return -1;
 }
 
+/*
+** Expand the space allocated for the given SrcList object by
+** creating nExtra new slots beginning at iStart.  iStart is zero based.
+** New slots are zeroed.
+**
+** For example, suppose a SrcList initially contains two entries: A,B.
+** To append 3 new entries onto the end, do this:
+**
+**    sqlite3SrcListEnlarge(db, pSrclist, 3, 2);
+**
+** After the call above it would contain:  A, B, nil, nil, nil.
+** If the iStart argument had been 1 instead of 2, then the result
+** would have been:  A, nil, nil, nil, B.  To prepend the new slots,
+** the iStart value would be 0.  The result then would
+** be: nil, nil, nil, A, B.
+**
+** If a memory allocation fails the SrcList is unchanged.  The
+** db->mallocFailed flag will be set to true.
+*/
+SrcList *sqlite3SrcListEnlarge(
+  sqlite3 *db,       /* Database connection to notify of OOM errors */
+  SrcList *pSrc,     /* The SrcList to be enlarged */
+  int nExtra,        /* Number of new slots to add to pSrc->a[] */
+  int iStart         /* Index in pSrc->a[] of first new slot */
+){
+  int i;
+
+  /* Sanity checking on calling parameters */
+  assert( iStart>=0 );
+  assert( nExtra>=1 );
+  if( pSrc==0 || iStart>pSrc->nSrc ){
+    assert( db->mallocFailed );
+    return pSrc;
+  }
+
+  /* Allocate additional space if needed */
+  if( pSrc->nSrc+nExtra>pSrc->nAlloc ){
+    SrcList *pNew;
+    int nAlloc = pSrc->nSrc+nExtra;
+    pNew = sqlite3DbRealloc(db, pSrc,
+               sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) );
+    if( pNew==0 ){
+      assert( db->mallocFailed );
+      return pSrc;
+    }
+    pSrc = pNew;
+    pSrc->nAlloc = nAlloc;
+  }
+
+  /* Move existing slots that come after the newly inserted slots
+  ** out of the way */
+  for(i=pSrc->nSrc-1; i>=iStart; i--){
+    pSrc->a[i+nExtra] = pSrc->a[i];
+  }
+  pSrc->nSrc += nExtra;
+
+  /* Zero the newly allocated slots */
+  memset(&pSrc->a[iStart], 0, sizeof(pSrc->a[0])*nExtra);
+  for(i=iStart; i<iStart+nExtra; i++){
+    pSrc->a[i].iCursor = -1;
+  }
+
+  /* Return a pointer to the enlarged SrcList */
+  return pSrc;
+}
+
+
 /*
 ** Append a new table name to the given SrcList.  Create a new SrcList if
 ** need be.  A new entry is created in the SrcList even if pToken is NULL.
 **
-** A new SrcList is returned, or NULL if malloc() fails.
+** A SrcList is returned, or NULL if there is an OOM error.  The returned
+** SrcList might be the same as the SrcList that was input or it might be
+** a new one.  If an OOM error does occurs, then the prior value of pList
+** that is input to this routine is automatically freed.
 **
 ** If pDatabase is not null, it means that the table has an optional
 ** database name prefix.  Like this:  "database.table".  The pDatabase
@@ -3028,19 +3098,12 @@ SrcList *sqlite3SrcListAppend(
     if( pList==0 ) return 0;
     pList->nAlloc = 1;
   }
-  if( pList->nSrc>=pList->nAlloc ){
+  pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc);
-    SrcList *pNew;
+  if( db->mallocFailed ){
-    pList->nAlloc *= 2;
-    pNew = sqlite3DbRealloc(db, pList,
-               sizeof(*pList) + (pList->nAlloc-1)*sizeof(pList->a[0]) );
-    if( pNew==0 ){
     sqlite3SrcListDelete(db, pList);
     return 0;
   }
-    pList = pNew;
+  pItem = &pList->a[pList->nSrc-1];
-  }
-  pItem = &pList->a[pList->nSrc];
-  memset(pItem, 0, sizeof(pList->a[0]));
   if( pDatabase && pDatabase->z==0 ){
     pDatabase = 0;
   }
@@ -3051,8 +3114,6 @@ SrcList *sqlite3SrcListAppend(
   }
   pItem->zName = sqlite3NameFromToken(db, pTable);
   pItem->zDatabase = sqlite3NameFromToken(db, pDatabase);
-  pItem->iCursor = -1;
-  pList->nSrc++;
   return pList;
 }
 

src/select.c

@@ -12,7 +12,7 @@
 ** This file contains C code routines that are called by the parser
 ** to handle SELECT statements in SQLite.
 **
-** $Id: select.c,v 1.482 2008/10/31 10:53:23 danielk1977 Exp $
+** $Id: select.c,v 1.483 2008/11/11 18:28:59 drh Exp $
 */
 #include "sqliteInt.h"
 
@@ -2573,7 +2573,9 @@ static int flattenSubquery(
 
   /* Check to see if flattening is permitted.  Return 0 if not.
   */
+  assert( p!=0 );
   if( p==0 ) return 0;
+  assert( p->pPrior==0 );  /* Unable to flatten compound queries */
   pSrc = p->pSrc;
   assert( pSrc && iFrom>=0 && iFrom<pSrc->nSrc );
   pSubitem = &pSrc->a[iFrom];
@@ -2685,44 +2687,95 @@ static int flattenSubquery(
   **     SELECT <expr-list> FROM (<sub-query>) <where-clause> 
   **
   ** followed by any ORDER BY, LIMIT and/or OFFSET clauses. This block
-  ** creates N copies of the parent query without any ORDER BY, LIMIT or 
+  ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or 
   ** OFFSET clauses and joins them to the left-hand-side of the original
   ** using UNION ALL operators. In this case N is the number of simple
   ** select statements in the compound sub-query.
+  **
+  ** Example:
+  **
+  **     SELECT a+1 FROM (
+  **        SELECT x FROM tab
+  **        UNION ALL
+  **        SELECT y FROM tab
+  **        UNION ALL
+  **        SELECT abs(z*2) FROM tab2
+  **     ) WHERE a!=5 ORDER BY 1
+  **
+  ** Transformed into:
+  **
+  **     SELECT x+1 FROM tab WHERE x+1!=5
+  **     UNION ALL
+  **     SELECT y+1 FROM tab WHERE y+1!=5
+  **     UNION ALL
+  **     SELECT abs(z*2)+1 FROM tab2 WHERE abs(z*2)+1!=5
+  **     ORDER BY 1
+  **
+  ** We call this the "compound-subquery flattening".
   */
   for(pSub=pSub->pPrior; pSub; pSub=pSub->pPrior){
     Select *pNew;
     ExprList *pOrderBy = p->pOrderBy;
     Expr *pLimit = p->pLimit;
-    Expr *pOffset = p->pOffset;
     Select *pPrior = p->pPrior;
     p->pOrderBy = 0;
     p->pSrc = 0;
     p->pPrior = 0;
     p->pLimit = 0;
     pNew = sqlite3SelectDup(db, p);
-    pNew->pPrior = pPrior;
-    p->pPrior = pNew;
-    p->pOrderBy = pOrderBy;
-    p->op = TK_ALL;
-    p->pSrc = pSrc;
     p->pLimit = pLimit;
-    p->pOffset = pOffset;
+    p->pOrderBy = pOrderBy;
+    p->pSrc = pSrc;
+    p->op = TK_ALL;
     p->pRightmost = 0;
+    if( pNew==0 ){
+      pNew = pPrior;
+    }else{
+      pNew->pPrior = pPrior;
       pNew->pRightmost = 0;
     }
+    p->pPrior = pNew;
+    if( db->mallocFailed ) return 1;
+  }
 
   /* Begin flattening the iFrom-th entry of the FROM clause 
   ** in the outer query.
   */
   pSub = pSub1 = pSubitem->pSelect;
-  for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){
-    int nSubSrc = pSubSrc->nSrc;
-    int jointype = 0;
-    pSubSrc = pSub->pSrc;
-    pSrc = pParent->pSrc;
 
-    /* Move all of the FROM elements of the subquery into the
+  /* Delete the transient table structure associated with the
+  ** subquery
+  */
+  sqlite3DbFree(db, pSubitem->zDatabase);
+  sqlite3DbFree(db, pSubitem->zName);
+  sqlite3DbFree(db, pSubitem->zAlias);
+  pSubitem->zDatabase = 0;
+  pSubitem->zName = 0;
+  pSubitem->zAlias = 0;
+  pSubitem->pSelect = 0;
+
+  /* Defer deleting the Table object associated with the
+  ** subquery until code generation is
+  ** complete, since there may still exist Expr.pTab entries that
+  ** refer to the subquery even after flattening.  Ticket #3346.
+  */
+  if( pSubitem->pTab!=0 ){
+    Table *pTabToDel = pSubitem->pTab;
+    if( pTabToDel->nRef==1 ){
+      pTabToDel->pNextZombie = pParse->pZombieTab;
+      pParse->pZombieTab = pTabToDel;
+    }else{
+      pTabToDel->nRef--;
+    }
+    pSubitem->pTab = 0;
+  }
+
+  /* The following loop runs once for each term in a compound-subquery
+  ** flattening (as described above).  If we are doing a different kind
+  ** of flattening - a flattening other than a compound-subquery flattening -
+  ** then this loop only runs once.
+  **
+  ** This loop moves all of the FROM elements of the subquery into the
   ** the FROM clause of the outer query.  Before doing this, remember
   ** the cursor number for the original outer query FROM element in
   ** iParent.  The iParent cursor will never be used.  Subsequent code
@@ -2730,47 +2783,50 @@ static int flattenSubquery(
   ** those references with expressions that resolve to the subquery FROM
   ** elements we are now copying in.
   */
-    if( pSrc ){
+  for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){
-      Table *pTabToDel;
+    int nSubSrc;
-      pSubitem = &pSrc->a[iFrom];
+    int jointype = 0;
-      nSubSrc = pSubSrc->nSrc;
+    pSubSrc = pSub->pSrc;     /* FROM clause of subquery */
-      jointype = pSubitem->jointype;
+    nSubSrc = pSubSrc->nSrc;  /* Number of terms in subquery FROM clause */
-      sqlite3DbFree(db, pSubitem->zDatabase);
+    pSrc = pParent->pSrc;     /* FROM clause of the outer query */
-      sqlite3DbFree(db, pSubitem->zName);
-      sqlite3DbFree(db, pSubitem->zAlias);
-      pSubitem->zDatabase = 0;
-      pSubitem->zName = 0;
-      pSubitem->zAlias = 0;
 
-      /* If the FROM element is a subquery, defer deleting the Table
+    if( pSrc ){
-      ** object associated with that subquery until code generation is
+      assert( pParent==p );  /* First time through the loop */
-      ** complete, since there may still exist Expr.pTab entires that
+      jointype = pSubitem->jointype;
-      ** refer to the subquery even after flattening.  Ticket #3346.
-      */
-      if( (pTabToDel = pSubitem->pTab)!=0 ){
-        if( pTabToDel->nRef==1 ){
-          pTabToDel->pNextZombie = pParse->pZombieTab;
-          pParse->pZombieTab = pTabToDel;
     }else{
-          pTabToDel->nRef--;
+      assert( pParent!=p );  /* 2nd and subsequent times through the loop */
-        }
+      pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0);
-      }
-      pSubitem->pTab = 0;
-    }
-    if( nSubSrc!=1 || !pSrc ){
-      int extra = nSubSrc - 1;
-      for(i=(pSrc?1:0); i<nSubSrc; i++){
-        pSrc = sqlite3SrcListAppend(db, pSrc, 0, 0);
       if( pSrc==0 ){
-          pParent->pSrc = 0;
+        assert( db->mallocFailed );
-          return 1;
+        break;
       }
     }
-      pParent->pSrc = pSrc;
+
-      for(i=pSrc->nSrc-1; i-extra>=iFrom; i--){
+    /* The subquery uses a single slot of the FROM clause of the outer
-        pSrc->a[i] = pSrc->a[i-extra];
+    ** query.  If the subquery has more than one element in its FROM clause,
+    ** then expand the outer query to make space for it to hold all elements
+    ** of the subquery.
+    **
+    ** Example:
+    **
+    **    SELECT * FROM tabA, (SELECT * FROM sub1, sub2), tabB;
+    **
+    ** The outer query has 3 slots in its FROM clause.  One slot of the
+    ** outer query (the middle slot) is used by the subquery.  The next
+    ** block of code will expand the out query to 4 slots.  The middle
+    ** slot is expanded to two slots in order to make space for the
+    ** two elements in the FROM clause of the subquery.
+    */
+    if( nSubSrc>1 ){
+      pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1);
+      if( db->mallocFailed ){
+        break;
       }
     }
+
+    /* Transfer the FROM clause terms from the subquery into the
+    ** outer query.
+    */
     for(i=0; i<nSubSrc; i++){
       pSrc->a[i+iFrom] = pSubSrc->a[i];
       memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i]));

src/sqliteInt.h

@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.789 2008/11/10 18:05:36 shane Exp $
+** @(#) $Id: sqliteInt.h,v 1.790 2008/11/11 18:29:00 drh Exp $
 */
 #ifndef _SQLITEINT_H_
 #define _SQLITEINT_H_
@@ -2144,6 +2144,7 @@ void sqlite3Insert(Parse*, SrcList*, ExprList*, Select*, IdList*, int);
 void *sqlite3ArrayAllocate(sqlite3*,void*,int,int,int*,int*,int*);
 IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*);
 int sqlite3IdListIndex(IdList*,const char*);
+SrcList *sqlite3SrcListEnlarge(sqlite3*, SrcList*, int, int);
 SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*);
 SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*,
                                       Token*, Select*, Expr*, IdList*);

test/mallocJ.test

@@ -12,7 +12,7 @@
 # This test script checks malloc failures in LIMIT operations for 
 # UPDATE/DELETE statements.
 # 
-# $Id: mallocJ.test,v 1.4 2008/11/11 00:21:30 drh Exp $
+# $Id: mallocJ.test,v 1.5 2008/11/11 18:29:00 drh Exp $
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -39,7 +39,7 @@ ifcapable {update_delete_limit} {
 }
 
 # ticket #3467
-do_malloc_test mallocJ-2 -start 114 -sqlprep {
+do_malloc_test mallocJ-2 -sqlprep {
   CREATE TABLE t1(a,b);
   INSERT INTO t1 VALUES(1,2);
   PRAGMA vdbe_trace=ON;
@@ -55,4 +55,12 @@ do_malloc_test mallocJ-3 -sqlbody {
   EXPLAIN COMMIT
 }
 
+# ticket #3485
+do_malloc_test mallocJ-4 -sqlprep {
+  CREATE TABLE t1(a,b,c);
+  CREATE TABLE t2(x,y,z);
+} -sqlbody {
+  SELECT * FROM (SELECT a,b FROM t1 UNION ALL SELECT x, y FROM t2) ORDER BY 1
+}
+
 finish_test