Merge latest trunk changes with this branch.

FossilOrigin-Name: 0b63e8dcbaec5043e353734e684c2a46552a3409

ext/ota/sqlite3ota.c

@@ -2197,11 +2197,11 @@ sqlite3_int64 sqlite3ota_progress(sqlite3ota *pOta){
 ** Beginning of OTA VFS shim methods. The VFS shim modifies the behaviour
 ** of a standard VFS in the following ways:
 **
-**   1. Whenever the first page of an OTA target database file is read or 
+**   1. Whenever the first page of a main database file is read or 
 **      written, the value of the change-counter cookie is stored in
-**      sqlite3ota.iCookie. This ensures that, so long as a read transaction
+**      ota_file.iCookie. Similarly, the value of the "write-version"
-**      is held on the db file, the value of sqlite3ota.iCookie matches
+**      database header field is stored in ota_file.iWriteVer. This ensures
-**      that stored on disk.
+**      that the values are always trustworthy within an open transaction.
 **
 **   2. When the ota handle is in OTA_STAGE_OAL or OTA_STAGE_CKPT state, all
 **      EXCLUSIVE lock attempts on the target database fail. This prevents
@@ -2210,7 +2210,7 @@ sqlite3_int64 sqlite3ota_progress(sqlite3ota *pOta){
 **      checkpoint may be required to delete the *-wal file.
 **
 **   3. In OTA_STAGE_OAL, the *-shm file is stored in memory. All xShmLock()
-**      calls are noops.
+**      calls are noops. This is just an optimization.
 **
 **   4. In OTA_STAGE_OAL mode, when SQLite calls xAccess() to check if a
 **      *-wal file associated with the target database exists, the following
@@ -2243,6 +2243,7 @@ static int otaVfsClose(sqlite3_file *pFile){
   p->apShm = 0;
   sqlite3_free(p->zDel);
 
+  /* Close the underlying file handle */
   rc = p->pReal->pMethods->xClose(p->pReal);
   return rc;
 }

manifest

@@ -1,5 +1,5 @@
-C Ensure\sthat\san\serror\sis\sreported\sif\san\sattempt\sis\smade\sto\supdate\sa\swal\smode\sdatabase\svia\sota.
+C Merge\slatest\strunk\schanges\swith\sthis\sbranch.
-D 2015-02-11T16:25:27.816
+D 2015-02-11T16:54:48.127
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 6b9e7677829aa94b9f30949656e27312aefb9a46
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -135,7 +135,7 @@ F ext/ota/ota8.test cd70e63a0c29c45c0906692827deafa34638feda
 F ext/ota/ota9.test d3eee95dd836824d07a22e5efcdb7bf6e869358b
 F ext/ota/otaA.test 95566a8d193113867b960eadf85b310937f2fe03
 F ext/ota/otafault.test 508ba87c83d632670ac0f94371a465d4bb4d49dd
-F ext/ota/sqlite3ota.c 7400075206e6cb8cbb32fc7268cb2fcf6321ea66
+F ext/ota/sqlite3ota.c 0ef0f189344b169c9c166bd9fb85793ae2c5968b
 F ext/ota/sqlite3ota.h 1cc7201086fe65a36957740381485a24738c4077
 F ext/rtree/README 6315c0d73ebf0ec40dedb5aa0e942bc8b54e3761
 F ext/rtree/rtree.c 14e6239434d4e3f65d3e90320713f26aa24e167f
@@ -195,8 +195,8 @@ F src/callback.c 7b44ce59674338ad48b0e84e7b72f935ea4f68b0
 F src/complete.c 198a0066ba60ab06fc00fba1998d870a4d575463
 F src/ctime.c 98f89724adc891a1a4c655bee04e33e716e05887
 F src/date.c e4d50b3283696836ec1036b695ead9a19e37a5ac
-F src/delete.c bd1a91ddd247ce13004075251e0b7fe2bf9925ef
+F src/delete.c 37964e6c1d73ff49cbea9ff690c9605fb15f600e
-F src/expr.c abe930897ccafae3819fd2855cbc1b00c262fd12
+F src/expr.c 3ef111b88ae2941b84b6b6ea4be8d501ba1af0cb
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c e0444b61bed271a76840cbe6182df93a9baa3f12
 F src/func.c 6d3c4ebd72aa7923ce9b110a7dc15f9b8c548430
@@ -231,9 +231,9 @@ F src/os_setup.h c9d4553b5aaa6f73391448b265b89bed0b890faa
 F src/os_unix.c aefeaf915aaef9f81aa2645e0d5d06fa1bd83beb
 F src/os_win.c 8223e7db5b7c4a81d8b161098ac3959400434cdb
 F src/os_win.h eb7a47aa17b26b77eb97e4823f20a00b8bda12ca
-F src/pager.c 46bc7849b02c51e13f0165fa6d6faa452e91a957
+F src/pager.c 9d29fb3dfd99d16896d839a511b467784d72f4da
 F src/pager.h 20954a3fa1bbf05d39063d94e789ad9efd15e5d1
-F src/parse.y c5d0d964f9ac023e8154cad512e54b0b6058e086
+F src/parse.y 0f8e7d60f0ab3cb53d270adef69259ac307d83a8
 F src/pcache.c d210cf90d04365a74f85d21374dded65af67b0cb
 F src/pcache.h b44658c9c932d203510279439d891a2a83e12ba8
 F src/pcache1.c 1e77432b40b7d3288327d9cdf399dcdfd2b6d3bf
@@ -242,19 +242,19 @@ F src/pragma.h 09c89bca58e9a44de2116cc8272b8d454657129f
 F src/prepare.c 173a5a499138451b2561614ecb87d78f9f4644b9
 F src/printf.c 05edc41450d0eb2c05ef7db113bf32742ae65325
 F src/random.c ba2679f80ec82c4190062d756f22d0c358180696
-F src/resolve.c f6c46d3434439ab2084618d603e6d6dbeb0d6ada
+F src/resolve.c f4d79e31ffa5820c2e3d1740baa5e9b190425f2b
 F src/rowset.c eccf6af6d620aaa4579bd3b72c1b6395d9e9fa1e
-F src/select.c 1f2087523007c42900ffcbdeaef06a23ad9329fc
+F src/select.c e46cef4c224549b439384c88fc7f57ba064dad54
 F src/shell.c 82c25508dac802b32198af6f5256ca1597c6a1af
 F src/sqlite.h.in c49acd2daa6e54110ab0cc607eb73ff32720a269
 F src/sqlite3.rc 992c9f5fb8285ae285d6be28240a7e8d3a7f2bad
 F src/sqlite3ext.h 17d487c3c91b0b8c584a32fbeb393f6f795eea7d
-F src/sqliteInt.h 57f8f45028598cc2877fc08ac03b402242242c68
+F src/sqliteInt.h 57a405ae6d2ed10fff52de376d18f21e04d96609
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c 81712116e826b0089bb221b018929536b2b5406f
 F src/table.c e7a09215315a978057fb42c640f890160dbcc45e
 F src/tclsqlite.c b321464aba1fff1ed9317ebc82a1a94887f97af8
-F src/test1.c ce8ea168800d129acb2c0afdf2831ddf8667e082
+F src/test1.c 90fbedce75330d48d99eadb7d5f4223e86969585
 F src/test2.c 577961fe48961b2f2e5c8b56ee50c3f459d3359d
 F src/test3.c 64d2afdd68feac1bb5e2ffb8226c8c639f798622
 F src/test4.c d168f83cc78d02e8d35567bb5630e40dcd85ac1e
@@ -312,7 +312,7 @@ F src/vdbe.h 6fc69d9c5e146302c56e163cb4b31d1ee64a18c3
 F src/vdbeInt.h 9bb69ff2447c34b6ccc58b34ec35b615f86ead78
 F src/vdbeapi.c 4bc511a46b9839392ae0e90844a71dc96d9dbd71
 F src/vdbeaux.c 97911edb61074b871ec4aa2d6bb779071643dee5
-F src/vdbeblob.c 4af4bfb71f6df7778397b4a0ebc1879793276778
+F src/vdbeblob.c 4f2e8e075d238392df98c5e03a64342465b03f90
 F src/vdbemem.c 31d8eabb0cd78bfeab4e5124c7363c3e9e54db9f
 F src/vdbesort.c 6d64c5448b64851b99931ede980addc3af70d5e2
 F src/vdbetrace.c 7e4222955e07dd707a2f360c0eb73452be1cb010
@@ -417,7 +417,7 @@ F test/collate4.test f04d5168685f2eef637ecfa2d4ddf8ec0d600177
 F test/collate5.test 65d928034d30d2d263a80f6359f7549ee1598ec6
 F test/collate6.test 8be65a182abaac8011a622131486dafb8076e907
 F test/collate7.test 8ec29d98f3ee4ccebce6e16ce3863fb6b8c7b868
-F test/collate8.test df26649cfcbddf109c04122b340301616d3a88f6
+F test/collate8.test cd9b3d3f999b8520ffaa7cc1647061fc5bab1334
 F test/collate9.test 3adcc799229545940df2f25308dd1ad65869145a
 F test/collateA.test b8218ab90d1fa5c59dcf156efabb1b2599c580d6
 F test/colmeta.test 2c765ea61ee37bc43bbe6d6047f89004e6508eb1
@@ -652,7 +652,7 @@ F test/in3.test 3cbf58c87f4052cee3a58b37b6389777505aa0c0
 F test/in4.test d2b38cba404bc4320f4fe1b595b3d163f212c068
 F test/in5.test 1de657472fa9ac2924be25c2c959ac5ca1aae554
 F test/incrblob.test e81846d214f3637622620fbde7cd526781cfe328
-F test/incrblob2.test bf4d549aa4a466d7fbe3e3a3693d3861263d5600
+F test/incrblob2.test 0d8821730a84f90af78a9dd547fe7a2480a06240
 F test/incrblob3.test d8d036fde015d4a159cd3cbae9d29003b37227a4
 F test/incrblob4.test f26502a5697893e5acea268c910f16478c2f0fab
 F test/incrblob_err.test af1f12ba60d220c9752073ff2bda2ad59e88960d
@@ -849,7 +849,7 @@ F test/select2.test 352480e0e9c66eda9c3044e412abdf5be0215b56
 F test/select3.test 2ce595f8fb8e2ac10071d3b4e424cadd4634a054
 F test/select4.test 8c5a60d439e2df824aed56223566877a883c5c84
 F test/select5.test e758b8ef94f69b111df4cb819008856655dcd535
-F test/select6.test e76bd10a56988f15726c097a5d5a7966fe82d3b2
+F test/select6.test 39eac4a5c03650b2b473c532882273283ee8b7a0
 F test/select7.test 7fd2ef598cfabb6b9ff6ac13973b91d0527df49d
 F test/select8.test 391de11bdd52339c30580dabbbbe97e3e9a3c79d
 F test/select9.test aebc2bb0c3bc44606125033cbcaac2c8d1f33a95
@@ -1237,7 +1237,7 @@ F tool/showstat4.c 9515faa8ec176599d4a8288293ba8ec61f7b728a
 F tool/showwal.c 85cb36d4fe3e93e2fbd63e786e0d1ce42d0c4fad
 F tool/soak1.tcl 8d407956e1a45b485a8e072470a3e629a27037fe
 F tool/space_used.tcl f714c41a59e326b8b9042f415b628b561bafa06b
-F tool/spaceanal.tcl 8e50b217c56a6a086a1b47eac9d09c5cd65b996f
+F tool/spaceanal.tcl d5a09620c66a6c144576cb9d2bdfa9a6fbe362a5
 F tool/speedtest.tcl 06c76698485ccf597b9e7dbb1ac70706eb873355
 F tool/speedtest16.c c8a9c793df96db7e4933f0852abb7a03d48f2e81
 F tool/speedtest2.tcl ee2149167303ba8e95af97873c575c3e0fab58ff
@@ -1254,7 +1254,7 @@ F tool/vdbe_profile.tcl 67746953071a9f8f2f668b73fe899074e2c6d8c1
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh 0abfd78ceb09b7f7c27c688c8e3fe93268a13b32
 F tool/win/sqlite.vsix deb315d026cc8400325c5863eef847784a219a2f
-P 0f152416be792457c52417aeb531ac860d12a5bd
+P 6fc5d4d26a603b3906f02ceea0f507780d0c35eb 24e78b8d65734a6a8ae21a20542cd1839e756fb1
-R d9bfc5d0f25bbcabd7abf7a0a51ab0ed
+R fb939d4eb119b38004785cfc38a020ff
 U dan
-Z 85c9979bf4d433f3d16c5b8b0ec7a03f
+Z ef20780a933decd4d3a7bbad742f919b

manifest.uuid

@@ -1 +1 @@
-6fc5d4d26a603b3906f02ceea0f507780d0c35eb
+0b63e8dcbaec5043e353734e684c2a46552a3409

src/delete.c

@@ -189,7 +189,7 @@ Expr *sqlite3LimitWhere(
 
   pInClause->x.pSelect = pSelect;
   pInClause->flags |= EP_xIsSelect;
-  sqlite3ExprSetHeight(pParse, pInClause);
+  sqlite3ExprSetHeightAndFlags(pParse, pInClause);
   return pInClause;
 
   /* something went wrong. clean up anything allocated. */

src/expr.c

@@ -146,10 +146,25 @@ CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){
       break;
     }
     if( p->flags & EP_Collate ){
-      if( ALWAYS(p->pLeft) && (p->pLeft->flags & EP_Collate)!=0 ){
+      if( p->pLeft && (p->pLeft->flags & EP_Collate)!=0 ){
         p = p->pLeft;
       }else{
-        p = p->pRight;
+        Expr *pNext  = p->pRight;
+        /* The Expr.x union is never used at the same time as Expr.pRight */
+        assert( p->x.pList==0 || p->pRight==0 );
+        /* p->flags holds EP_Collate and p->pLeft->flags does not.  And
+        ** p->x.pSelect cannot.  So if p->x.pLeft exists, it must hold at
+        ** least one EP_Collate. Thus the following two ALWAYS. */
+        if( p->x.pList!=0 && ALWAYS(!ExprHasProperty(p, EP_xIsSelect)) ){
+          int i;
+          for(i=0; ALWAYS(i<p->x.pList->nExpr); i++){
+            if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){
+              pNext = p->x.pList->a[i].pExpr;
+              break;
+            }
+          }
+        }
+        p = pNext;
       }
     }else{
       break;
@@ -355,6 +370,9 @@ static void heightOfSelect(Select *p, int *pnHeight){
 ** Expr.pSelect member has a height of 1. Any other expression
 ** has a height equal to the maximum height of any other 
 ** referenced Expr plus one.
+**
+** Also propagate EP_Propagate flags up from Expr.x.pList to Expr.flags,
+** if appropriate.
 */
 static void exprSetHeight(Expr *p){
   int nHeight = 0;
@@ -362,8 +380,9 @@ static void exprSetHeight(Expr *p){
   heightOfExpr(p->pRight, &nHeight);
   if( ExprHasProperty(p, EP_xIsSelect) ){
     heightOfSelect(p->x.pSelect, &nHeight);
-  }else{
+  }else if( p->x.pList ){
     heightOfExprList(p->x.pList, &nHeight);
+    p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList);
   }
   p->nHeight = nHeight + 1;
 }
@@ -372,8 +391,11 @@ static void exprSetHeight(Expr *p){
 ** Set the Expr.nHeight variable using the exprSetHeight() function. If
 ** the height is greater than the maximum allowed expression depth,
 ** leave an error in pParse.
+**
+** Also propagate all EP_Propagate flags from the Expr.x.pList into
+** Expr.flags. 
 */
-void sqlite3ExprSetHeight(Parse *pParse, Expr *p){
+void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){
   exprSetHeight(p);
   sqlite3ExprCheckHeight(pParse, p->nHeight);
 }
@@ -387,8 +409,17 @@ int sqlite3SelectExprHeight(Select *p){
   heightOfSelect(p, &nHeight);
   return nHeight;
 }
-#else
+#else /* ABOVE:  Height enforcement enabled.  BELOW: Height enforcement off */
-  #define exprSetHeight(y)
+/*
+** Propagate all EP_Propagate flags from the Expr.x.pList into
+** Expr.flags. 
+*/
+void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){
+  if( p && p->x.pList && !ExprHasProperty(p, EP_xIsSelect) ){
+    p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList);
+  }
+}
+#define exprSetHeight(y)
 #endif /* SQLITE_MAX_EXPR_DEPTH>0 */
 
 /*
@@ -490,11 +521,11 @@ void sqlite3ExprAttachSubtrees(
   }else{
     if( pRight ){
       pRoot->pRight = pRight;
-      pRoot->flags |= EP_Collate & pRight->flags;
+      pRoot->flags |= EP_Propagate & pRight->flags;
     }
     if( pLeft ){
       pRoot->pLeft = pLeft;
-      pRoot->flags |= EP_Collate & pLeft->flags;
+      pRoot->flags |= EP_Propagate & pLeft->flags;
     }
     exprSetHeight(pRoot);
   }
@@ -594,7 +625,7 @@ Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){
   }
   pNew->x.pList = pList;
   assert( !ExprHasProperty(pNew, EP_xIsSelect) );
-  sqlite3ExprSetHeight(pParse, pNew);
+  sqlite3ExprSetHeightAndFlags(pParse, pNew);
   return pNew;
 }
 
@@ -1209,6 +1240,21 @@ void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){
   sqlite3DbFree(db, pList);
 }
 
+/*
+** Return the bitwise-OR of all Expr.flags fields in the given
+** ExprList.
+*/
+u32 sqlite3ExprListFlags(const ExprList *pList){
+  int i;
+  u32 m = 0;
+  if( pList ){
+    for(i=0; i<pList->nExpr; i++){
+       m |= pList->a[i].pExpr->flags;
+    }
+  }
+  return m;
+}
+
 /*
 ** These routines are Walker callbacks used to check expressions to
 ** see if they are "constant" for some definition of constant.  The
@@ -1249,7 +1295,7 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){
     ** and either pWalker->eCode==4 or 5 or the function has the
     ** SQLITE_FUNC_CONST flag. */
     case TK_FUNCTION:
-      if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_Constant) ){
+      if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_ConstFunc) ){
         return WRC_Continue;
       }else{
         pWalker->eCode = 0;

src/pager.c

@@ -615,18 +615,6 @@ struct PagerSavepoint {
 **   is set to zero in all other states. In PAGER_ERROR state, Pager.errCode 
 **   is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX 
 **   sub-codes.
-**
-** otaMode
-**   This variable is normally 0. It is set to 1 by the PagerSetOtaMode()
-**   function - as a result of a "PRAGMA pager_ota_mode=1" command. Once 
-**   the *-oal file has been opened and it has been determined that the 
-**   database file has not been modified since it was created, this variable 
-**   is set to 2.
-**
-**   It is also possible to use PagerSetOtaMode() to 2 if the database is
-**   already in WAL mode. In this case the only effect is to prevent the
-**   connection from checkpointing the db as part of an sqlite3PagerClose()
-**   call.
 */
 struct Pager {
   sqlite3_vfs *pVfs;          /* OS functions to use for IO */
@@ -838,8 +826,6 @@ static int pagerUseWal(Pager *pPager){
 # define pagerBeginReadTransaction(z) SQLITE_OK
 #endif
 
-static int pagerOpenWalInternal(Pager*, int*);
-
 #ifndef NDEBUG 
 /*
 ** Usage:
@@ -2039,7 +2025,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){
     if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK;
   }
 
-  if( !pPager->exclusiveMode
+  if( !pPager->exclusiveMode 
    && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0))
   ){
     rc2 = pagerUnlockDb(pPager, SHARED_LOCK);
@@ -7175,15 +7161,23 @@ static int pagerOpenWal(Pager *pPager){
   return rc;
 }
 
+
 /*
-** Open the WAL file if it is not open. If it is already open, set *pbOpen
+** The caller must be holding a SHARED lock on the database file to call
-** to 1 before returning. Return SQLITE_OK if successful, or an SQLite error
+** this function.
-** code otherwise.
 **
-** The difference between this function and sqlite3PagerOpenWal() is that
+** If the pager passed as the first argument is open on a real database
-** PagerOpenWal() does not open the WAL file if the pager is in OTA mode.
+** file (not a temp file or an in-memory database), and the WAL file
+** is not already open, make an attempt to open it now. If successful,
+** return SQLITE_OK. If an error occurs or the VFS used by the pager does 
+** not support the xShmXXX() methods, return an error code. *pbOpen is
+** not modified in either case.
+**
+** If the pager is open on a temp-file (or in-memory database), or if
+** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK
+** without doing anything.
 */
-static int pagerOpenWalInternal(
+int sqlite3PagerOpenWal(
   Pager *pPager,                  /* Pager object */
   int *pbOpen                     /* OUT: Set to true if call is a no-op */
 ){
@@ -7213,28 +7207,6 @@ static int pagerOpenWalInternal(
   return rc;
 }
 
-/*
-** The caller must be holding a SHARED lock on the database file to call
-** this function.
-**
-** If the pager passed as the first argument is open on a real database
-** file (not a temp file or an in-memory database), and the WAL file
-** is not already open, make an attempt to open it now. If successful,
-** return SQLITE_OK. If an error occurs or the VFS used by the pager does 
-** not support the xShmXXX() methods, return an error code. *pbOpen is
-** not modified in either case.
-**
-** If the pager is open on a temp-file (or in-memory database), or if
-** the WAL file is already open, set *pbOpen to 1 and return SQLITE_OK
-** without doing anything.
-*/
-int sqlite3PagerOpenWal(
-  Pager *pPager,                  /* Pager object */
-  int *pbOpen                     /* OUT: Set to true if call is a no-op */
-){
-  return pagerOpenWalInternal(pPager, pbOpen);
-}
-
 /*
 ** This function is called to close the connection to the log file prior
 ** to switching from WAL to rollback mode.

src/parse.y

@@ -1078,7 +1078,7 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
       A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0);
       if( A.pExpr ){
         A.pExpr->x.pList = Y;
-        sqlite3ExprSetHeight(pParse, A.pExpr);
+        sqlite3ExprSetHeightAndFlags(pParse, A.pExpr);
       }else{
         sqlite3ExprListDelete(pParse->db, Y);
       }
@@ -1091,8 +1091,8 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
     A.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0);
     if( A.pExpr ){
       A.pExpr->x.pSelect = X;
-      ExprSetProperty(A.pExpr, EP_xIsSelect);
+      ExprSetProperty(A.pExpr, EP_xIsSelect|EP_Subquery);
-      sqlite3ExprSetHeight(pParse, A.pExpr);
+      sqlite3ExprSetHeightAndFlags(pParse, A.pExpr);
     }else{
       sqlite3SelectDelete(pParse->db, X);
     }
@@ -1103,8 +1103,8 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
     A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0);
     if( A.pExpr ){
       A.pExpr->x.pSelect = Y;
-      ExprSetProperty(A.pExpr, EP_xIsSelect);
+      ExprSetProperty(A.pExpr, EP_xIsSelect|EP_Subquery);
-      sqlite3ExprSetHeight(pParse, A.pExpr);
+      sqlite3ExprSetHeightAndFlags(pParse, A.pExpr);
     }else{
       sqlite3SelectDelete(pParse->db, Y);
     }
@@ -1117,8 +1117,8 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
     A.pExpr = sqlite3PExpr(pParse, TK_IN, X.pExpr, 0, 0);
     if( A.pExpr ){
       A.pExpr->x.pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0);
-      ExprSetProperty(A.pExpr, EP_xIsSelect);
+      ExprSetProperty(A.pExpr, EP_xIsSelect|EP_Subquery);
-      sqlite3ExprSetHeight(pParse, A.pExpr);
+      sqlite3ExprSetHeightAndFlags(pParse, A.pExpr);
     }else{
       sqlite3SrcListDelete(pParse->db, pSrc);
     }
@@ -1130,8 +1130,8 @@ expr(A) ::= expr(W) between_op(N) expr(X) AND expr(Y). [BETWEEN] {
     Expr *p = A.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0);
     if( p ){
       p->x.pSelect = Y;
-      ExprSetProperty(p, EP_xIsSelect);
+      ExprSetProperty(p, EP_xIsSelect|EP_Subquery);
-      sqlite3ExprSetHeight(pParse, p);
+      sqlite3ExprSetHeightAndFlags(pParse, p);
     }else{
       sqlite3SelectDelete(pParse->db, Y);
     }
@@ -1145,7 +1145,7 @@ expr(A) ::= CASE(C) case_operand(X) case_exprlist(Y) case_else(Z) END(E). {
   A.pExpr = sqlite3PExpr(pParse, TK_CASE, X, 0, 0);
   if( A.pExpr ){
     A.pExpr->x.pList = Z ? sqlite3ExprListAppend(pParse,Y,Z) : Y;
-    sqlite3ExprSetHeight(pParse, A.pExpr);
+    sqlite3ExprSetHeightAndFlags(pParse, A.pExpr);
   }else{
     sqlite3ExprListDelete(pParse->db, Y);
     sqlite3ExprDelete(pParse->db, Z);

src/resolve.c

@@ -247,9 +247,10 @@ static int lookupName(
     testcase( pNC->ncFlags & NC_PartIdx );
     testcase( pNC->ncFlags & NC_IsCheck );
     if( (pNC->ncFlags & (NC_PartIdx|NC_IsCheck))!=0 ){
-      /* Silently ignore database qualifiers inside CHECK constraints and partial
+      /* Silently ignore database qualifiers inside CHECK constraints and
-      ** indices.  Do not raise errors because that might break legacy and
+      ** partial indices.  Do not raise errors because that might break
-      ** because it does not hurt anything to just ignore the database name. */
+      ** legacy and because it does not hurt anything to just ignore the
+      ** database name. */
       zDb = 0;
     }else{
       for(i=0; i<db->nDb; i++){
@@ -320,7 +321,8 @@ static int lookupName(
       if( pMatch ){
         pExpr->iTable = pMatch->iCursor;
         pExpr->pTab = pMatch->pTab;
-        assert( (pMatch->jointype & JT_RIGHT)==0 ); /* RIGHT JOIN not (yet) supported */
+        /* RIGHT JOIN not (yet) supported */
+        assert( (pMatch->jointype & JT_RIGHT)==0 );
         if( (pMatch->jointype & JT_LEFT)!=0 ){
           ExprSetProperty(pExpr, EP_CanBeNull);
         }
@@ -641,7 +643,8 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
       pExpr->affinity = SQLITE_AFF_INTEGER;
       break;
     }
-#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) */
+#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT)
+          && !defined(SQLITE_OMIT_SUBQUERY) */
 
     /* A lone identifier is the name of a column.
     */
@@ -706,19 +709,20 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
           if( n==2 ){
             pExpr->iTable = exprProbability(pList->a[1].pExpr);
             if( pExpr->iTable<0 ){
-              sqlite3ErrorMsg(pParse, "second argument to likelihood() must be a "
+              sqlite3ErrorMsg(pParse,
-                                      "constant between 0.0 and 1.0");
+                "second argument to likelihood() must be a "
+                "constant between 0.0 and 1.0");
               pNC->nErr++;
             }
           }else{
-            /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is equivalent to
+            /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is
-            ** likelihood(X, 0.0625).
+            ** equivalent to likelihood(X, 0.0625).
-            ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is short-hand for
+            ** EVIDENCE-OF: R-01283-11636 The unlikely(X) function is
-            ** likelihood(X,0.0625).
+            ** short-hand for likelihood(X,0.0625).
-            ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand for
+            ** EVIDENCE-OF: R-36850-34127 The likely(X) function is short-hand
-            ** likelihood(X,0.9375).
+            ** for likelihood(X,0.9375).
-            ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent to
+            ** EVIDENCE-OF: R-53436-40973 The likely(X) function is equivalent
-            ** likelihood(X,0.9375). */
+            ** to likelihood(X,0.9375). */
             /* TUNING: unlikely() probability is 0.0625.  likely() is 0.9375 */
             pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120;
           }             
@@ -735,7 +739,9 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
           return WRC_Prune;
         }
 #endif
-        if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ) ExprSetProperty(pExpr,EP_Constant);
+        if( pDef->funcFlags & SQLITE_FUNC_CONSTANT ){
+          ExprSetProperty(pExpr,EP_ConstFunc);
+        }
       }
       if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){
         sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId);
@@ -1046,7 +1052,8 @@ int sqlite3ResolveOrderGroupBy(
         resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr);
         return 1;
       }
-      resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr, zType,0);
+      resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr,
+                   zType,0);
     }
   }
   return 0;

src/select.c

@@ -3194,7 +3194,10 @@ static void substSelect(
 **
 **   (1)  The subquery and the outer query do not both use aggregates.
 **
-**   (2)  The subquery is not an aggregate or the outer query is not a join.
+**   (2)  The subquery is not an aggregate or (2a) the outer query is not a join
+**        and (2b) the outer query does not use subqueries other than the one
+**        FROM-clause subquery that is a candidate for flattening.  (2b is
+**        due to ticket [2f7170d73bf9abf80] from 2015-02-09.)
 **
 **   (3)  The subquery is not the right operand of a left outer join
 **        (Originally ticket #306.  Strengthened by ticket #3300)
@@ -3331,8 +3334,17 @@ static int flattenSubquery(
   iParent = pSubitem->iCursor;
   pSub = pSubitem->pSelect;
   assert( pSub!=0 );
-  if( isAgg && subqueryIsAgg ) return 0;                 /* Restriction (1)  */
+  if( subqueryIsAgg ){
-  if( subqueryIsAgg && pSrc->nSrc>1 ) return 0;          /* Restriction (2)  */
+    if( isAgg ) return 0;                                /* Restriction (1)   */
+    if( pSrc->nSrc>1 ) return 0;                         /* Restriction (2a)  */
+    if( (p->pWhere && ExprHasProperty(p->pWhere,EP_Subquery))
+     || (sqlite3ExprListFlags(p->pEList) & EP_Subquery)!=0
+     || (sqlite3ExprListFlags(p->pOrderBy) & EP_Subquery)!=0
+    ){
+      return 0;                                          /* Restriction (2b)  */
+    }
+  }
+    
   pSubSrc = pSub->pSrc;
   assert( pSubSrc );
   /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants,
@@ -4752,6 +4764,13 @@ int sqlite3Select(
   }
   isAgg = (p->selFlags & SF_Aggregate)!=0;
   assert( pEList!=0 );
+#if SELECTTRACE_ENABLED
+  if( sqlite3SelectTrace & 0x100 ){
+    SELECTTRACE(0x100,pParse,p, ("after name resolution:\n"));
+    sqlite3TreeViewSelect(0, p, 0);
+  }
+#endif
+
 
   /* Begin generating code.
   */
@@ -5497,9 +5516,9 @@ select_end:
 void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){
   int n = 0;
   pView = sqlite3TreeViewPush(pView, moreToFollow);
-  sqlite3TreeViewLine(pView, "SELECT%s%s",
+  sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p)",
     ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""),
-    ((p->selFlags & SF_Aggregate) ? " agg_flag" : "")
+    ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p
   );
   if( p->pSrc && p->pSrc->nSrc ) n++;
   if( p->pWhere ) n++;

src/sqliteInt.h

@@ -1207,6 +1207,7 @@ struct sqlite3 {
 #define SQLITE_QueryOnly      0x02000000  /* Disable database changes */
 #define SQLITE_VdbeEQP        0x04000000  /* Debug EXPLAIN QUERY PLAN */
 
+
 /*
 ** Bits of the sqlite3.dbOptFlags field that are used by the
 ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to
@@ -2043,8 +2044,14 @@ struct Expr {
 #define EP_MemToken  0x010000 /* Need to sqlite3DbFree() Expr.zToken */
 #define EP_NoReduce  0x020000 /* Cannot EXPRDUP_REDUCE this Expr */
 #define EP_Unlikely  0x040000 /* unlikely() or likelihood() function */
-#define EP_Constant  0x080000 /* Node is a constant */
+#define EP_ConstFunc 0x080000 /* Node is a SQLITE_FUNC_CONSTANT function */
 #define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */
+#define EP_Subquery  0x200000 /* Tree contains a TK_SELECT operator */
+
+/*
+** Combinations of two or more EP_* flags
+*/
+#define EP_Propagate (EP_Collate|EP_Subquery) /* Propagate these bits up tree */
 
 /*
 ** These macros can be used to test, set, or clear bits in the 
@@ -3152,6 +3159,7 @@ ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*);
 void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int);
 void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*);
 void sqlite3ExprListDelete(sqlite3*, ExprList*);
+u32 sqlite3ExprListFlags(const ExprList*);
 int sqlite3Init(sqlite3*, char**);
 int sqlite3InitCallback(void*, int, char**, char**);
 void sqlite3Pragma(Parse*,Token*,Token*,Token*,int);
@@ -3735,12 +3743,11 @@ void sqlite3MemJournalOpen(sqlite3_file *);
 int sqlite3MemJournalSize(void);
 int sqlite3IsMemJournal(sqlite3_file *);
 
+void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p);
 #if SQLITE_MAX_EXPR_DEPTH>0
-  void sqlite3ExprSetHeight(Parse *pParse, Expr *p);
   int sqlite3SelectExprHeight(Select *);
   int sqlite3ExprCheckHeight(Parse*, int);
 #else
-  #define sqlite3ExprSetHeight(x,y)
   #define sqlite3SelectExprHeight(x) 0
   #define sqlite3ExprCheckHeight(x,y)
 #endif
@@ -3818,7 +3825,6 @@ SQLITE_EXTERN void (*sqlite3IoTrace)(const char*,...);
 #define MEMTYPE_SCRATCH    0x04  /* Scratch allocations */
 #define MEMTYPE_PCACHE     0x08  /* Page cache allocations */
 
-
 /*
 ** Threading interface
 */

src/test1.c

@@ -6928,7 +6928,6 @@ int Sqlitetest1_Init(Tcl_Interp *interp){
 #endif
 
   };
-
   static int bitmask_size = sizeof(Bitmask)*8;
   int i;
   extern int sqlite3_sync_count, sqlite3_fullsync_count;

src/vdbeblob.c

@@ -154,12 +154,17 @@ int sqlite3_blob_open(
   Incrblob *pBlob = 0;
 
 #ifdef SQLITE_ENABLE_API_ARMOR
-  if( !sqlite3SafetyCheckOk(db) || ppBlob==0 || zTable==0 ){
+  if( ppBlob==0 ){
+    return SQLITE_MISUSE_BKPT;
+  }
+#endif
+  *ppBlob = 0;
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( !sqlite3SafetyCheckOk(db) || zTable==0 ){
     return SQLITE_MISUSE_BKPT;
   }
 #endif
   flags = !!flags;                /* flags = (flags ? 1 : 0); */
-  *ppBlob = 0;
 
   sqlite3_mutex_enter(db->mutex);
 
@@ -373,7 +378,7 @@ static int blobReadWrite(
   sqlite3_mutex_enter(db->mutex);
   v = (Vdbe*)p->pStmt;
 
-  if( n<0 || iOffset<0 || (iOffset+n)>p->nByte ){
+  if( n<0 || iOffset<0 || ((sqlite3_int64)iOffset+n)>p->nByte ){
     /* Request is out of range. Return a transient error. */
     rc = SQLITE_ERROR;
   }else if( v==0 ){

test/collate8.test

@@ -13,7 +13,9 @@
 # focus of this script is making sure collations pass through the
 # unary + operator.
 #
-# $Id: collate8.test,v 1.2 2008/08/25 12:14:09 drh Exp $
+# 2015-02-09:  Added tests to make sure COLLATE passes through function
+# calls.  Ticket [ca0d20b6cdddec5e81b8d66f89c46a5583b5f6f6].
+#
 
 set testdir [file dirname $argv0]
 source $testdir/tester.tcl
@@ -122,4 +124,34 @@ do_test collate8-2.8 {
   }
 } {abc}
 
+# Make sure the COLLATE operator perculates up through function calls
+# and other Expr structures that use the Expr.x.pList field.
+#
+do_execsql_test collate8-3.1 {
+  SELECT 'abc'==('ABC'||'') COLLATE nocase;
+  SELECT 'abc'==('ABC'||'' COLLATE nocase);
+  SELECT 'abc'==('ABC'||('' COLLATE nocase));
+  SELECT 'abc'==('ABC'||upper('' COLLATE nocase));
+} {1 1 1 1}
+do_execsql_test collate8-3.2 {
+  SELECT 'abc'==('ABC'||max('' COLLATE nocase,'' COLLATE binary));
+} {1}
+
+# The COLLATE binary is on the left and so takes precedence
+do_execsql_test collate8-3.3 {
+  SELECT 'abc'==('ABC'||max('' COLLATE binary,'' COLLATE nocase));
+} {0}
+
+do_execsql_test collate8-3.4 {
+  SELECT 'abc'==('ABC'||CASE WHEN 1-1=2 THEN '' COLLATE nocase
+                                        ELSE '' COLLATE binary END);
+  SELECT 'abc'==('ABC'||CASE WHEN 1+1=2 THEN '' COLLATE nocase
+                                        ELSE '' COLLATE binary END);
+} {1 1}
+do_execsql_test collate8-3.5 {
+  SELECT 'abc'==('ABC'||CASE WHEN 1=2 THEN '' COLLATE binary
+                                      ELSE '' COLLATE nocase END);
+} {0}
+
+
 finish_test

test/incrblob2.test

@@ -324,12 +324,34 @@ do_test incrblob2-6.2 {
   sqlite3_blob_read $rdHandle 0 2
 } {AB}
 
+do_test incrblob2-6.2b {
+  set rc [catch {
+    # Prior to 2015-02-07, the following caused a segfault due to
+    # integer overflow.
+    sqlite3_blob_read $rdHandle 2147483647 2147483647
+  } errmsg]
+  lappend rc $errmsg
+} {1 SQLITE_ERROR}
+
 do_test incrblob2-6.3 {
   set wrHandle [db incrblob t1 data 1]
   sqlite3_blob_write $wrHandle 0 ZZZZZZZZZZ
   sqlite3_blob_read $rdHandle 2 4
 } {ZZZZ}
 
+do_test incrblob2-6.3b {
+  set rc [catch {
+    # Prior to 2015-02-07, the following caused a segfault due to
+    # integer overflow.
+    sqlite3_blob_write $wrHandle 2147483647 YYYYYYYYYYYYYYYYYY
+  } errmsg]
+  lappend rc $errmsg
+} {1 SQLITE_ERROR}
+do_test incrblob2-6.3c {
+  sqlite3_blob_read $rdHandle 2 4
+} {ZZZZ}
+
+
 do_test incrblob2-6.4 {
   close $wrHandle
   close $rdHandle

test/select6.test

@@ -557,5 +557,61 @@ do_catchsql_test 10.8 {
   )
 } $err
 
+# 2015-02-09 Ticket [2f7170d73bf9abf80339187aa3677dce3dbcd5ca]
+# "misuse of aggregate" error if aggregate column from FROM
+# subquery is used in correlated subquery 
+#
+do_execsql_test 11.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(w INT, x INT);
+  INSERT INTO t1(w,x)
+   VALUES(1,10),(2,20),(3,30),
+         (2,21),(3,31),
+         (3,32);
+  CREATE INDEX t1wx ON t1(w,x);
+
+  DROP TABLE IF EXISTS t2;
+  CREATE TABLE t2(w INT, y VARCHAR(8));
+  INSERT INTO t2(w,y) VALUES(1,'one'),(2,'two'),(3,'three'),(4,'four');
+  CREATE INDEX t2wy ON t2(w,y);
+
+  SELECT cnt, xyz, (SELECT y FROM t2 WHERE w=cnt), '|'
+    FROM (SELECT count(*) AS cnt, w AS xyz FROM t1 GROUP BY 2)
+   ORDER BY cnt, xyz;
+} {1 1 one | 2 2 two | 3 3 three |}
+do_execsql_test 11.2 {
+  SELECT cnt, xyz, lower((SELECT y FROM t2 WHERE w=cnt)), '|'
+    FROM (SELECT count(*) AS cnt, w AS xyz FROM t1 GROUP BY 2)
+   ORDER BY cnt, xyz;
+} {1 1 one | 2 2 two | 3 3 three |}
+do_execsql_test 11.3 {
+  SELECT cnt, xyz, '|'
+    FROM (SELECT count(*) AS cnt, w AS xyz FROM t1 GROUP BY 2)
+   WHERE (SELECT y FROM t2 WHERE w=cnt)!='two'
+   ORDER BY cnt, xyz;
+} {1 1 | 3 3 |}
+do_execsql_test 11.4 {
+  SELECT cnt, xyz, '|'
+    FROM (SELECT count(*) AS cnt, w AS xyz FROM t1 GROUP BY 2)
+   ORDER BY lower((SELECT y FROM t2 WHERE w=cnt));
+} {1 1 | 3 3 | 2 2 |}
+do_execsql_test 11.5 {
+  SELECT cnt, xyz, 
+         CASE WHEN (SELECT y FROM t2 WHERE w=cnt)=='two'
+              THEN 'aaa' ELSE 'bbb'
+          END, '|'
+    FROM (SELECT count(*) AS cnt, w AS xyz FROM t1 GROUP BY 2)
+   ORDER BY +cnt;
+} {1 1 bbb | 2 2 aaa | 3 3 bbb |}
+
+do_execsql_test 11.100 {
+  DROP TABLE t1;
+  DROP TABLE t2;
+  CREATE TABLE t1(x);
+  CREATE TABLE t2(y, z);
+  SELECT ( SELECT y FROM t2 WHERE z = cnt )
+    FROM ( SELECT count(*) AS cnt FROM t1 );
+} {{}}
+
 
 finish_test

tool/spaceanal.tcl

@@ -4,6 +4,24 @@
 #
 
 if {[catch {
+
+# Argument $tname is the name of a table within the database opened by
+# database handle [db]. Return true if it is a WITHOUT ROWID table, or
+# false otherwise.
+#
+proc is_without_rowid {tname} {
+  set t [string map {' ''} $tname]
+  db eval "PRAGMA index_list = '$t'" o {
+    if {$o(origin) == "pk"} {
+      set n $o(name)
+      if {0==[db one { SELECT count(*) FROM sqlite_master WHERE name=$n }]} {
+        return 1
+      }
+    }
+  }
+  return 0
+}
+
 # Get the name of the database to analyze
 #
 proc usage {} {
@@ -167,20 +185,21 @@ set sql { SELECT name, tbl_name FROM sqlite_master WHERE rootpage>0 }
 foreach {name tblname} [concat sqlite_master sqlite_master [db eval $sql]] {
 
   set is_index [expr {$name!=$tblname}]
+  set idx_btree [expr {$is_index || [is_without_rowid $name]}]
   db eval {
     SELECT 
       sum(ncell) AS nentry,
-      sum(isleaf(pagetype, $is_index) * ncell) AS leaf_entries,
+      sum(isleaf(pagetype, $idx_btree) * ncell) AS leaf_entries,
       sum(payload) AS payload,
-      sum(isoverflow(pagetype, $is_index) * payload) AS ovfl_payload,
+      sum(isoverflow(pagetype, $idx_btree) * payload) AS ovfl_payload,
       sum(path LIKE '%+000000') AS ovfl_cnt,
       max(mx_payload) AS mx_payload,
-      sum(isinternal(pagetype, $is_index)) AS int_pages,
+      sum(isinternal(pagetype, $idx_btree)) AS int_pages,
-      sum(isleaf(pagetype, $is_index)) AS leaf_pages,
+      sum(isleaf(pagetype, $idx_btree)) AS leaf_pages,
-      sum(isoverflow(pagetype, $is_index)) AS ovfl_pages,
+      sum(isoverflow(pagetype, $idx_btree)) AS ovfl_pages,
-      sum(isinternal(pagetype, $is_index) * unused) AS int_unused,
+      sum(isinternal(pagetype, $idx_btree) * unused) AS int_unused,
-      sum(isleaf(pagetype, $is_index) * unused) AS leaf_unused,
+      sum(isleaf(pagetype, $idx_btree) * unused) AS leaf_unused,
-      sum(isoverflow(pagetype, $is_index) * unused) AS ovfl_unused,
+      sum(isoverflow(pagetype, $idx_btree) * unused) AS ovfl_unused,
       sum(pgsize) AS compressed_size
     FROM temp.dbstat WHERE name = $name
   } break