Add support for common table expressions (WITH clauses).

FossilOrigin-Name: 0171e3bb4f663a9414b0e8b64c87b5d0683855b5

manifest

@@ -1,5 +1,5 @@
-C Always\suse\savailable\sindices\sto\soptimize\sLIKE\soperators\seven\sif\sthe\spattern\nof\sthe\sLIKE\soperator\shas\sa\sCOLLATE\smodifier.\s\sThis\sfixes\san\sineffiency\sthat\nwas\sintroduced\sinto\s3.7.15\sby\scheck-in\s[8542e6180d4]\son\s2012-12-08.
+C Add\ssupport\sfor\scommon\stable\sexpressions\s(WITH\sclauses).
-D 2014-01-16T15:31:41.806
+D 2014-01-17T15:15:10.270
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -166,16 +166,16 @@ F src/auth.c 523da7fb4979469955d822ff9298352d6b31de34
 F src/backup.c a729e63cf5cd1829507cb7b8e89f99b95141bb53
 F src/bitvec.c 19a4ba637bd85f8f63fc8c9bae5ade9fb05ec1cb
 F src/btmutex.c 976f45a12e37293e32cae0281b15a21d48a8aaa7
-F src/btree.c 11e29ef8cf16a42925fde036bcffbeffd9cc82df
+F src/btree.c c15e1722696b66c4029c487acfb830b0985bf142
 F src/btree.h a61ddebc78c66795a2b93181321a116746302cc9
 F src/btreeInt.h f038e818bfadf75afbd09819ed93c26a333d39e0
-F src/build.c 8c56d91447770a746b16d08a6510109c161dbc1a
+F src/build.c 7e6c275ab1731510d6f793d0f88373ab3e858e69
 F src/callback.c 174e3c8656bc29f91d710ab61550d16eea34be98
 F src/complete.c dc1d136c0feee03c2f7550bafc0d29075e36deac
 F src/ctime.c 77779efbe78dd678d84bfb4fc2e87b6b6ad8dccd
 F src/date.c 593c744b2623971e45affd0bde347631bdfa4625
 F src/delete.c 91e1321021db5dc266360531b8b6550009d771ff
-F src/expr.c 4115ad67088cdd55f4fa0ef3ddd22cb8da8f9c94
+F src/expr.c e239763d8b43356fa1f46f1cf41d62a076f7f72e
 F src/fault.c 160a0c015b6c2629d3899ed2daf63d75754a32bb
 F src/fkey.c 2ab0f5384b70594468ef3ac5c7ed8ca24bfd17d5
 F src/func.c 6325ac2ec10833ccf4d5c36d323709221d37ea19
@@ -183,7 +183,7 @@ F src/global.c 1d7bb7ea8254ae6a68ed9bfaf65fcb3d1690b486
 F src/hash.c ac3470bbf1ca4ae4e306a8ecb0fdf1731810ffe4
 F src/hash.h 8890a25af81fb85a9ad7790d32eedab4b994da22
 F src/hwtime.h d32741c8f4df852c7d959236615444e2b1063b08
-F src/insert.c 5ddb48c7f1cb399993744f23f03948989ce1790e
+F src/insert.c cb4c8ad02b6feb95d34614d94a3c68e0116fbf07
 F src/journal.c b4124532212b6952f42eb2c12fa3c25701d8ba8d
 F src/legacy.c 0df0b1550b9cc1f58229644735e317ac89131f12
 F src/lempar.c cdf0a000315332fc9b50b62f3b5e22e080a0952b
@@ -209,7 +209,7 @@ F src/os_unix.c 3a4dcb554d3c915075766162f28c3fd4cdb75968
 F src/os_win.c 1b21af72c5fa6f9e519a5fcab33e80d182b1aedb
 F src/pager.c efa923693e958696eee69b205a20bfbc402c8480
 F src/pager.h ffd5607f7b3e4590b415b007a4382f693334d428
-F src/parse.y 3c5384533a8bfce5abd256cc9cb2c38bec05ad61
+F src/parse.y 475896cb883bbf4782e98abda42efbbdcbdb75f5
 F src/pcache.c f8043b433a57aba85384a531e3937a804432a346
 F src/pcache.h a5e4f5d9f5d592051d91212c5949517971ae6222
 F src/pcache1.c 57fee9a9a617218f5037afbbe49b09da65bde56b
@@ -219,16 +219,16 @@ F src/printf.c 85d07756e45d7496d19439dcae3e6e9e0090f269
 F src/random.c d10c1f85b6709ca97278428fd5db5bbb9c74eece
 F src/resolve.c 7eda9097b29fcf3d2b42fdc17d1de672134e09b6
 F src/rowset.c 64655f1a627c9c212d9ab497899e7424a34222e0
-F src/select.c 996d8b88603edbd478aaa70b75d535a3ddea933d
+F src/select.c 231079b8b07ea6f3cf4663e9c0ef2315abe2236c
 F src/shell.c 9f3bc02a658b8f61d2cbe60cfc482f660c1c6c48
-F src/sqlite.h.in d94a8b89522f526ba711182ee161e06f8669bcc9
+F src/sqlite.h.in eed7f7d66a60daaa7b4a597dcd9bad87aad9611b
 F src/sqlite3.rc 11094cc6a157a028b301a9f06b3d03089ea37c3e
 F src/sqlite3ext.h 886f5a34de171002ad46fae8c36a7d8051c190fc
-F src/sqliteInt.h 0c65967bb807dee3c9eef2bbd17f880eeb28ea30
+F src/sqliteInt.h 9600eeb486c274fbdb815d040e4a7f262b7317e1
 F src/sqliteLimit.h 164b0e6749d31e0daa1a4589a169d31c0dec7b3d
 F src/status.c 7ac05a5c7017d0b9f0b4bcd701228b784f987158
 F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
-F src/tclsqlite.c c43379f77f90399802b0e215faa71c0adc3a4d2e
+F src/tclsqlite.c 46073db71011b6542fde1f234c56a076d5ff23f9
 F src/test1.c db16ba651453b15001c7f2838c446284dde4ecaf
 F src/test2.c 7355101c085304b90024f2261e056cdff13c6c35
 F src/test3.c 1c0e5d6f080b8e33c1ce8b3078e7013fdbcd560c
@@ -242,7 +242,7 @@ F src/test_async.c 21e11293a2f72080eda70e1124e9102044531cd8
 F src/test_autoext.c dea8a01a7153b9adc97bd26161e4226329546e12
 F src/test_backup.c 3875e899222b651e18b662f86e0e50daa946344e
 F src/test_btree.c 5b89601dcb42a33ba8b820a6b763cc9cb48bac16
-F src/test_config.c 10d0e00dd6315879a6d9fac20bd063c7bbbfb8f8
+F src/test_config.c 0336e0bdbe541b4af89d7e3dd0656e8e6b51e585
 F src/test_demovfs.c 69b2085076654ebc18014cbc6386f04409c959a9
 F src/test_devsym.c e7498904e72ba7491d142d5c83b476c4e76993bc
 F src/test_fs.c ced436e3d4b8e4681328409b8081051ce614e28f
@@ -274,13 +274,13 @@ F src/test_thread.c 1e133a40b50e9c035b00174035b846e7eef481cb
 F src/test_vfs.c e72f555ef7a59080f898fcf1a233deb9eb704ea9
 F src/test_vfstrace.c 3a0ab304682fecbceb689e7d9b904211fde11d78
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
-F src/tokenize.c 5d04a1b7d1fe7e18556a869788f5d3e132a586b6
+F src/tokenize.c 6da2de6e12218ccb0aea5184b56727d011f4bee7
 F src/trigger.c 5c1c0b899ac0ce284763dcb8fdbaa38ecf15ef98
 F src/update.c c2706a6eb232a96345c35b7e1e75a188e26812bb
 F src/utf.c 6fc6c88d50448c469c5c196acf21617a24f90269
 F src/util.c e71f19b272f05c8695cf747b4bac1732685f9e5c
 F src/vacuum.c 3728d74919d4fb1356f9e9a13e27773db60b7179
-F src/vdbe.c b110887e415b5d2af58c2374c4dfdcf774c5d46c
+F src/vdbe.c ccc8594e89751966022642464ec2b5c5fa7840a2
 F src/vdbe.h e6c4c610fcabad4fa80ebb1efc6822a9367e2b26
 F src/vdbeInt.h 42db251e9f863401ff847b90d5fe1614c89a6a56
 F src/vdbeapi.c ce4e68ea4842cc6081046f533d088dcf01d247ad
@@ -292,8 +292,8 @@ F src/vdbetrace.c 6f52bc0c51e144b7efdcfb2a8f771167a8816767
 F src/vtab.c 21b932841e51ebd7d075e2d0ad1415dce8d2d5fd
 F src/wal.c 7dc3966ef98b74422267e7e6e46e07ff6c6eb1b4
 F src/wal.h df01efe09c5cb8c8e391ff1715cca294f89668a4
-F src/walker.c e9e593d5bb798c3e67fc3893dfe7055c9e7d8d74
+F src/walker.c 11edb74d587bc87b33ca96a5173e3ec1b8389e45
-F src/where.c 81cec50fe73633144b0730de477e141c53485862
+F src/where.c 5e11de480a94e6ff8f9922e3a04a31b56d5f33b5
 F src/whereInt.h 96a75c61f1d2b9d4a8e4bb17d89deb0cf7cba358
 F test/8_3_names.test ebbb5cd36741350040fd28b432ceadf495be25b2
 F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@@ -327,7 +327,7 @@ F test/attach2.test 0ec5defa340363de6cd50fd595046465e9aaba2d
 F test/attach3.test 359eb65d00102cdfcef6fa4e81dc1648f8f80b27
 F test/attach4.test 53bf502f17647c6d6c5add46dda6bac8b6f4665c
 F test/attachmalloc.test 3a4bfca9545bfe906a8d2e622de10fbac5b711b0
-F test/auth.test 9bea29041871807d9f289ee679d05d3ed103642f
+F test/auth.test 5bdf154eb28c0e4bbc0473f335858c0d96171768
 F test/auth2.test c3b415b76c033bedb81292118fb7c01f5f10cbcd
 F test/auth3.test a4755e6a2a2fea547ffe63c874eb569e60a28eb5
 F test/autoinc.test c58912526998a39e11f66b533e23cfabea7f25b7
@@ -1091,6 +1091,9 @@ F test/wild001.test bca33f499866f04c24510d74baf1e578d4e44b1c
 F test/win32heap.test ea19770974795cff26e11575e12d422dbd16893c
 F test/win32lock.test 7a6bd73a5dcdee39b5bb93e92395e1773a194361
 F test/win32longpath.test 169c75a3b2e43481f4a62122510210c67b08f26d
+F test/with1.test 90490c75e98e1914d84b7cef9e636b48917a020f
+F test/with2.test 790c4b7ab3f4eb6984a3bbdae8d4ab429ebe9259
+F test/withM.test 52448ce23e1c2ecba79d10e130ee49ce9f9a2a7a
 F test/without_rowid1.test aaa26da19d543cd8d3d2d0e686dfa255556c15c8
 F test/without_rowid2.test af260339f79d13cb220288b67cd287fbcf81ad99
 F test/without_rowid3.test eac3d5c8a1924725b58503a368f2cbd24fd6c8a0
@@ -1113,7 +1116,7 @@ F tool/lemon.c 07aba6270d5a5016ba8107b09e431eea4ecdc123
 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
 F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75
 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383
-F tool/mkkeywordhash.c 189d76644e373c7d0864c628deb8ce7b4f403591
+F tool/mkkeywordhash.c c9e05e4a7bcab8fab9f583d5b321fb72f565ad97
 F tool/mkopts.tcl 66ac10d240cc6e86abd37dc908d50382f84ff46e
 F tool/mkpragmatab.tcl 78a77b2c554d534c6f2dc903130186ed15715460
 F tool/mkspeedsql.tcl a1a334d288f7adfe6e996f2e712becf076745c97
@@ -1148,7 +1151,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P f61a70589ac7e05008a362bd9d5b7bde5d07a758
+P 16bd54783a3f5531c55564ddefdada657c078eb0 6a549187ed8b5ed50daefa676ff666ae2ed43346
-R c3393c71f3ebd7292d430bc7115b905d
+R 3149ee3ed72aa52234b32eaa7e696309
-U drh
+U dan
-Z 57df34b30b12b159eb99c0c8be54bec6
+Z e17f48449fd6868168f4a2f61958f3af

manifest.uuid

@@ -1 +1 @@
-16bd54783a3f5531c55564ddefdada657c078eb0
+0171e3bb4f663a9414b0e8b64c87b5d0683855b5

src/btree.c

@@ -7350,6 +7350,7 @@ static int clearDatabasePage(
   int rc;
   unsigned char *pCell;
   int i;
+  int hdr;
 
   assert( sqlite3_mutex_held(pBt->mutex) );
   if( pgno>btreePagecount(pBt) ){
@@ -7358,6 +7359,7 @@ static int clearDatabasePage(
 
   rc = getAndInitPage(pBt, pgno, &pPage, 0);
   if( rc ) return rc;
+  hdr = pPage->hdrOffset;
   for(i=0; i<pPage->nCell; i++){
     pCell = findCell(pPage, i);
     if( !pPage->leaf ){
@@ -7368,7 +7370,7 @@ static int clearDatabasePage(
     if( rc ) goto cleardatabasepage_out;
   }
   if( !pPage->leaf ){
-    rc = clearDatabasePage(pBt, get4byte(&pPage->aData[8]), 1, pnChange);
+    rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange);
     if( rc ) goto cleardatabasepage_out;
   }else if( pnChange ){
     assert( pPage->intKey );
@@ -7377,7 +7379,7 @@ static int clearDatabasePage(
   if( freePageFlag ){
     freePage(pPage, &rc);
   }else if( (rc = sqlite3PagerWrite(pPage->pDbPage))==0 ){
-    zeroPage(pPage, pPage->aData[0] | PTF_LEAF);
+    zeroPage(pPage, pPage->aData[hdr] | PTF_LEAF);
   }
 
 cleardatabasepage_out:

src/build.c

@@ -4198,3 +4198,73 @@ KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){
   }
   return sqlite3KeyInfoRef(pIdx->pKeyInfo);
 }
+
+#ifndef SQLITE_OMIT_CTE
+/* 
+** This routine is invoked once per CTE by the parser while parsing a 
+** WITH clause. 
+*/
+With *sqlite3WithAdd(
+  Parse *pParse,          /* Parsing context */
+  With *pWith,            /* Existing WITH clause, or NULL */
+  Token *pName,           /* Name of the common-table */
+  ExprList *pArglist,     /* Optional column name list for the table */
+  Select *pQuery          /* Query used to initialize the table */
+){
+  sqlite3 *db = pParse->db;
+  With *pNew;
+  char *zName;
+
+  /* Check that the CTE name is unique within this WITH clause. If
+  ** not, store an error in the Parse structure. */
+  zName = sqlite3NameFromToken(pParse->db, pName);
+  if( zName && pWith ){
+    int i;
+    for(i=0; i<pWith->nCte; i++){
+      if( sqlite3StrICmp(zName, pWith->a[i].zName)==0 ){
+        sqlite3ErrorMsg(pParse, "duplicate WITH table name: %s", zName);
+      }
+    }
+  }
+
+  if( pWith ){
+    int nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte);
+    pNew = sqlite3DbRealloc(db, pWith, nByte);
+  }else{
+    pNew = sqlite3DbMallocZero(db, sizeof(*pWith));
+  }
+  assert( zName!=0 || pNew==0 );
+  assert( db->mallocFailed==0 || pNew==0 );
+
+  if( pNew==0 ){
+    sqlite3ExprListDelete(db, pArglist);
+    sqlite3SelectDelete(db, pQuery);
+    sqlite3DbFree(db, zName);
+    pNew = pWith;
+  }else{
+    pNew->a[pNew->nCte].pSelect = pQuery;
+    pNew->a[pNew->nCte].pCols = pArglist;
+    pNew->a[pNew->nCte].zName = zName;
+    pNew->a[pNew->nCte].zErr = 0;
+    pNew->nCte++;
+  }
+
+  return pNew;
+}
+
+/*
+** Free the contents of the With object passed as the second argument.
+*/
+void sqlite3WithDelete(sqlite3 *db, With *pWith){
+  if( pWith ){
+    int i;
+    for(i=0; i<pWith->nCte; i++){
+      struct Cte *pCte = &pWith->a[i];
+      sqlite3ExprListDelete(db, pCte->pCols);
+      sqlite3SelectDelete(db, pCte->pSelect);
+      sqlite3DbFree(db, pCte->zName);
+    }
+    sqlite3DbFree(db, pWith);
+  }
+}
+#endif /* !defined(SQLITE_OMIT_CTE) */

src/expr.c

@@ -895,6 +895,33 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int flags, u8 **pzBuffer){
   return pNew;
 }
 
+/*
+** Create and return a deep copy of the object passed as the second 
+** argument. If an OOM condition is encountered, NULL is returned
+** and the db->mallocFailed flag set.
+*/
+#ifndef SQLITE_OMIT_CTE
+static With *withDup(sqlite3 *db, With *p){
+  With *pRet = 0;
+  if( p ){
+    int nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1);
+    pRet = sqlite3DbMallocZero(db, nByte);
+    if( pRet ){
+      int i;
+      pRet->nCte = p->nCte;
+      for(i=0; i<p->nCte; i++){
+        pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0);
+        pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0);
+        pRet->a[i].zName = sqlite3DbStrDup(db, p->a[i].zName);
+      }
+    }
+  }
+  return pRet;
+}
+#else
+# define withDup(x,y) 0
+#endif
+
 /*
 ** The following group of routines make deep copies of expressions,
 ** expression lists, ID lists, and select statements.  The copies can
@@ -975,6 +1002,7 @@ SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){
     pNewItem->regReturn = pOldItem->regReturn;
     pNewItem->isCorrelated = pOldItem->isCorrelated;
     pNewItem->viaCoroutine = pOldItem->viaCoroutine;
+    pNewItem->isRecursive = pOldItem->isRecursive;
     pNewItem->zIndex = sqlite3DbStrDup(db, pOldItem->zIndex);
     pNewItem->notIndexed = pOldItem->notIndexed;
     pNewItem->pIndex = pOldItem->pIndex;
@@ -1036,6 +1064,7 @@ Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){
   pNew->addrOpenEphm[0] = -1;
   pNew->addrOpenEphm[1] = -1;
   pNew->addrOpenEphm[2] = -1;
+  pNew->pWith = withDup(db, p->pWith);
   return pNew;
 }
 #else
@@ -1555,9 +1584,11 @@ int sqlite3FindInIndex(Parse *pParse, Expr *pX, int *prNotFound){
     iCol = (i16)pExpr->iColumn;
    
     /* Code an OP_VerifyCookie and OP_TableLock for <table>. */
-    iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
+    if( pTab->pSchema ){
-    sqlite3CodeVerifySchema(pParse, iDb);
+      iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
-    sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
+      sqlite3CodeVerifySchema(pParse, iDb);
+      sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
+    }
 
     /* This function is only called from two places. In both cases the vdbe
     ** has already been allocated. So assume sqlite3GetVdbe() is always

src/insert.c

@@ -667,7 +667,8 @@ void sqlite3Insert(
   **
   ** This is the 2nd template.
   */
-  if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){
+  if( pColumn==0 && pParse->pWith==0 
+   && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){
     assert( !pTrigger );
     assert( pList==0 );
     goto insert_end;

src/parse.y

@@ -204,8 +204,8 @@ columnid(A) ::= nm(X). {
   ABORT ACTION AFTER ANALYZE ASC ATTACH BEFORE BEGIN BY CASCADE CAST COLUMNKW
   CONFLICT DATABASE DEFERRED DESC DETACH EACH END EXCLUSIVE EXPLAIN FAIL FOR
   IGNORE IMMEDIATE INITIALLY INSTEAD LIKE_KW MATCH NO PLAN
-  QUERY KEY OF OFFSET PRAGMA RAISE RELEASE REPLACE RESTRICT ROW ROLLBACK
+  QUERY KEY OF OFFSET PRAGMA RAISE RECURSIVE RELEASE REPLACE RESTRICT ROW
-  SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITHOUT
+  ROLLBACK SAVEPOINT TEMP TRIGGER VACUUM VIEW VIRTUAL WITH WITHOUT
 %ifdef SQLITE_OMIT_COMPOUND_SELECT
   EXCEPT INTERSECT UNION
 %endif SQLITE_OMIT_COMPOUND_SELECT
@@ -407,12 +407,23 @@ cmd ::= select(X).  {
 
 %type select {Select*}
 %destructor select {sqlite3SelectDelete(pParse->db, $$);}
+%type selectnowith {Select*}
+%destructor selectnowith {sqlite3SelectDelete(pParse->db, $$);}
 %type oneselect {Select*}
 %destructor oneselect {sqlite3SelectDelete(pParse->db, $$);}
 
-select(A) ::= oneselect(X).                      {A = X;}
+select(A) ::= with(W) selectnowith(X). { 
+  if( X ){
+    X->pWith = W; 
+  }else{
+    sqlite3WithDelete(pParse->db, W);
+  }
+  A = X; 
+}
+
+selectnowith(A) ::= oneselect(X).                      {A = X;}
 %ifndef SQLITE_OMIT_COMPOUND_SELECT
-select(A) ::= select(X) multiselect_op(Y) oneselect(Z).  {
+selectnowith(A) ::= selectnowith(X) multiselect_op(Y) oneselect(Z).  {
   if( Z ){
     Z->op = (u8)Y;
     Z->pPrior = X;
@@ -648,15 +659,17 @@ limit_opt(A) ::= LIMIT expr(X) COMMA expr(Y).
 /////////////////////////// The DELETE statement /////////////////////////////
 //
 %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W) 
+cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W) 
         orderby_opt(O) limit_opt(L). {
+  sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "DELETE");
   sqlite3DeleteFrom(pParse,X,W);
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= DELETE FROM fullname(X) indexed_opt(I) where_opt(W). {
+cmd ::= with(C) DELETE FROM fullname(X) indexed_opt(I) where_opt(W). {
+  sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3DeleteFrom(pParse,X,W);
 }
@@ -671,8 +684,9 @@ where_opt(A) ::= WHERE expr(X).       {A = X.pExpr;}
 ////////////////////////// The UPDATE command ////////////////////////////////
 //
 %ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W)
+cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
-        orderby_opt(O) limit_opt(L).  {
+        where_opt(W) orderby_opt(O) limit_opt(L).  {
+  sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
   W = sqlite3LimitWhere(pParse, X, W, O, L.pLimit, L.pOffset, "UPDATE");
@@ -680,8 +694,9 @@ cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y) where_opt(W)
 }
 %endif
 %ifndef SQLITE_ENABLE_UPDATE_DELETE_LIMIT
-cmd ::= UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
+cmd ::= with(C) UPDATE orconf(R) fullname(X) indexed_opt(I) SET setlist(Y)
         where_opt(W).  {
+  sqlite3WithPush(pParse, C, 1);
   sqlite3SrcListIndexedBy(pParse, X, &I);
   sqlite3ExprListCheckLength(pParse,Y,"set list"); 
   sqlite3Update(pParse,X,Y,W,R);
@@ -702,10 +717,15 @@ setlist(A) ::= nm(X) EQ expr(Y). {
 
 ////////////////////////// The INSERT command /////////////////////////////////
 //
-cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S).
+cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) select(S). {
-            {sqlite3Insert(pParse, X, S, F, R);}
+  sqlite3WithPush(pParse, W, 1);
-cmd ::= insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES.
+  sqlite3Insert(pParse, X, S, F, R);
-            {sqlite3Insert(pParse, X, 0, F, R);}
+}
+cmd ::= with(W) insert_cmd(R) INTO fullname(X) inscollist_opt(F) DEFAULT VALUES.
+{
+  sqlite3WithPush(pParse, W, 1);
+  sqlite3Insert(pParse, X, 0, F, R);
+}
 
 %type insert_cmd {u8}
 insert_cmd(A) ::= INSERT orconf(R).   {A = R;}
@@ -851,10 +871,8 @@ expr(A) ::= expr(X) STAR|SLASH|REM(OP) expr(Y).
                                         {spanBinaryExpr(&A,pParse,@OP,&X,&Y);}
 expr(A) ::= expr(X) CONCAT(OP) expr(Y). {spanBinaryExpr(&A,pParse,@OP,&X,&Y);}
 %type likeop {struct LikeOp}
-likeop(A) ::= LIKE_KW(X).     {A.eOperator = X; A.bNot = 0;}
+likeop(A) ::= LIKE_KW|MATCH(X).     {A.eOperator = X; A.bNot = 0;}
-likeop(A) ::= NOT LIKE_KW(X). {A.eOperator = X; A.bNot = 1;}
+likeop(A) ::= NOT LIKE_KW|MATCH(X). {A.eOperator = X; A.bNot = 1;}
-likeop(A) ::= MATCH(X).       {A.eOperator = X; A.bNot = 0;}
-likeop(A) ::= NOT MATCH(X).   {A.eOperator = X; A.bNot = 1;}
 expr(A) ::= expr(X) likeop(OP) expr(Y).  [LIKE_KW]  {
   ExprList *pList;
   pList = sqlite3ExprListAppend(pParse,0, Y.pExpr);
@@ -1364,3 +1382,23 @@ anylist ::= .
 anylist ::= anylist LP anylist RP.
 anylist ::= anylist ANY.
 %endif  SQLITE_OMIT_VIRTUALTABLE
+
+
+//////////////////////// COMMON TABLE EXPRESSIONS ////////////////////////////
+%type with {With*}
+%type wqlist {With*}
+%destructor with {sqlite3WithDelete(pParse->db, $$);}
+%destructor wqlist {sqlite3WithDelete(pParse->db, $$);}
+
+with(A) ::= . {A = 0;}
+%ifndef SQLITE_OMIT_CTE
+with(A) ::= WITH wqlist(W).              { A = W; }
+with(A) ::= WITH RECURSIVE wqlist(W).    { A = W; }
+
+wqlist(A) ::= nm(X) idxlist_opt(Y) AS LP select(Z) RP. {
+  A = sqlite3WithAdd(pParse, 0, &X, Y, Z);
+}
+wqlist(A) ::= wqlist(W) COMMA nm(X) idxlist_opt(Y) AS LP select(Z) RP. {
+  A = sqlite3WithAdd(pParse, W, &X, Y, Z);
+}
+%endif  SQLITE_OMIT_CTE

src/select.c

@@ -29,6 +29,7 @@ static void clearSelect(sqlite3 *db, Select *p){
   sqlite3SelectDelete(db, p->pPrior);
   sqlite3ExprDelete(db, p->pLimit);
   sqlite3ExprDelete(db, p->pOffset);
+  sqlite3WithDelete(db, p->pWith);
 }
 
 /*
@@ -690,12 +691,26 @@ static void selectInnerLoop(
 
     /* Store the result as data using a unique key.
     */
+    case SRT_DistTable:
     case SRT_Table:
     case SRT_EphemTab: {
       int r1 = sqlite3GetTempReg(pParse);
       testcase( eDest==SRT_Table );
       testcase( eDest==SRT_EphemTab );
       sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nColumn, r1);
+#ifndef SQLITE_OMIT_CTE
+      if( eDest==SRT_DistTable ){
+        /* If the destination is DistTable, then cursor (iParm+1) is open
+        ** on an ephemeral index. If the current row is already present
+        ** in the index, do not write it to the output. If not, add the
+        ** current row to the index and proceed with writing it to the
+        ** output table as well.  */
+        int addr = sqlite3VdbeCurrentAddr(v) + 4;
+        sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0);
+        sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1);
+        assert( pOrderBy==0 );
+      }
+#endif
       if( pOrderBy ){
         pushOntoSorter(pParse, pOrderBy, p, r1);
       }else{
@@ -1202,7 +1217,7 @@ static const char *columnTypeImpl(
           sNC.pParse = pNC->pParse;
           zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol, &estWidth); 
         }
-      }else if( ALWAYS(pTab->pSchema) ){
+      }else if( pTab->pSchema ){
         /* A real table */
         assert( !pS );
         if( iCol<0 ) iCol = pTab->iPKey;
@@ -1729,6 +1744,7 @@ static int multiSelect(
   ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT.
   */
   assert( p && p->pPrior );  /* Calling function guarantees this much */
+  assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION );
   db = pParse->db;
   pPrior = p->pPrior;
   assert( pPrior->pRightmost!=pPrior );
@@ -1774,11 +1790,91 @@ static int multiSelect(
     goto multi_select_end;
   }
 
+#ifndef SQLITE_OMIT_CTE
+  if( p->selFlags & SF_Recursive ){
+    SrcList *pSrc = p->pSrc;
+    int nCol = p->pEList->nExpr;
+    int addrNext;
+    int addrSwap;
+    int iCont, iBreak;
+    int tmp1;                     /* Intermediate table */
+    int tmp2;                     /* Next intermediate table */
+    int tmp3 = 0;                 /* To ensure unique results if UNION */
+    int eDest = SRT_Table;
+    SelectDest tmp2dest;
+    int i;
+
+    /* Check that there is no ORDER BY or LIMIT clause. Neither of these 
+    ** are supported on recursive queries.  */
+    assert( p->pOffset==0 || p->pLimit );
+    if( p->pOrderBy || p->pLimit ){
+      sqlite3ErrorMsg(pParse, "%s in a recursive query is not allowed",
+          p->pOrderBy ? "ORDER BY" : "LIMIT"
+      );
+      goto multi_select_end;
+    }
+
+    if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ){
+      goto multi_select_end;
+    }
+    iBreak = sqlite3VdbeMakeLabel(v);
+    iCont = sqlite3VdbeMakeLabel(v);
+
+    for(i=0; ALWAYS(i<pSrc->nSrc); i++){
+      if( pSrc->a[i].isRecursive ){
+        tmp1 = pSrc->a[i].iCursor;
+        break;
+      }
+    }
+
+    tmp2 = pParse->nTab++;
+    if( p->op==TK_UNION ){
+      eDest = SRT_DistTable;
+      tmp3 = pParse->nTab++;
+    }
+    sqlite3SelectDestInit(&tmp2dest, eDest, tmp2);
+
+    sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp1, nCol);
+    sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp2, nCol);
+    if( tmp3 ){
+      p->addrOpenEphm[0] = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tmp3, 0);
+      p->selFlags |= SF_UsesEphemeral;
+    }
+
+    /* Store the results of the initial SELECT in tmp2. */
+    rc = sqlite3Select(pParse, pPrior, &tmp2dest);
+    if( rc ) goto multi_select_end;
+
+    /* Clear tmp1. Then switch the contents of tmp1 and tmp2. Then return 
+    ** the contents of tmp1 to the caller. Or, if tmp1 is empty at this
+    ** point, the recursive query has finished - jump to address iBreak.  */
+    addrSwap = sqlite3VdbeAddOp2(v, OP_SwapCursors, tmp1, tmp2);
+    sqlite3VdbeAddOp2(v, OP_Rewind, tmp1, iBreak);
+    addrNext = sqlite3VdbeCurrentAddr(v);
+    selectInnerLoop(pParse, p, p->pEList, tmp1, p->pEList->nExpr,
+        0, 0, &dest, iCont, iBreak);
+    sqlite3VdbeResolveLabel(v, iCont);
+    sqlite3VdbeAddOp2(v, OP_Next, tmp1, addrNext);
+
+    /* Execute the recursive SELECT. Store the results in tmp2. While this
+    ** SELECT is running, the contents of tmp1 are read by recursive 
+    ** references to the current CTE.  */
+    p->pPrior = 0;
+    rc = sqlite3Select(pParse, p, &tmp2dest);
+    assert( p->pPrior==0 );
+    p->pPrior = pPrior;
+    if( rc ) goto multi_select_end;
+
+    sqlite3VdbeAddOp2(v, OP_Goto, 0, addrSwap);
+    sqlite3VdbeResolveLabel(v, iBreak);
+  }else
+#endif
+
   /* Compound SELECTs that have an ORDER BY clause are handled separately.
   */
   if( p->pOrderBy ){
     return multiSelectOrderBy(pParse, p, pDest);
-  }
+  }else
 
   /* Generate code for the left and right SELECT statements.
   */
@@ -2842,6 +2938,14 @@ static void substSelect(
 **  (21)  The subquery does not use LIMIT or the outer query is not
 **        DISTINCT.  (See ticket [752e1646fc]).
 **
+**  (22)  The subquery is not a recursive CTE.
+**
+**  (23)  The parent is not a recursive CTE, or the sub-query is not a
+**        compound query. This restriction is because transforming the
+**        parent to a compound query confuses the code that handles
+**        recursive queries in multiSelect().
+**
+**
 ** In this routine, the "p" parameter is a pointer to the outer query.
 ** The subquery is p->pSrc->a[iFrom].  isAgg is true if the outer query
 ** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates.
@@ -2913,6 +3017,8 @@ static int flattenSubquery(
   if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){
      return 0;         /* Restriction (21) */
   }
+  if( pSub->selFlags & SF_Recursive ) return 0;          /* Restriction (22)  */
+  if( (p->selFlags & SF_Recursive) && pSub->pPrior ) return 0;       /* (23)  */
 
   /* OBSOLETE COMMENT 1:
   ** Restriction 3:  If the subquery is a join, make sure the subquery is 
@@ -3394,6 +3500,183 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){
   return WRC_Continue;
 }
 
+#ifndef SQLITE_OMIT_CTE
+/*
+** Argument pWith (which may be NULL) points to a linked list of nested 
+** WITH contexts, from inner to outermost. If the table identified by 
+** FROM clause element pItem is really a common-table-expression (CTE) 
+** then return a pointer to the CTE definition for that table. Otherwise
+** return NULL.
+*/
+static struct Cte *searchWith(With *pWith, struct SrcList_item *pItem){
+  const char *zName;
+  if( pItem->zDatabase==0 && (zName = pItem->zName)!=0 ){
+    With *p;
+    for(p=pWith; p; p=p->pOuter){
+      int i;
+      for(i=0; i<p->nCte; i++){
+        if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){
+          return &p->a[i];
+        }
+      }
+    }
+  }
+  return 0;
+}
+
+/* The code generator maintains a stack of active WITH clauses
+** with the inner-most WITH clause being at the top of the stack.
+**
+** This routine pushes the WITH clause passed as the second argument
+** onto the top of the stack. If argument bFree is true, then this
+** WITH clause will never be popped from the stack. In this case it
+** should be freed along with the Parse object. In other cases, when
+** bFree==0, the With object will be freed along with the SELECT 
+** statement with which it is associated.
+*/
+void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){
+  assert( bFree==0 || pParse->pWith==0 );
+  if( pWith ){
+    pWith->pOuter = pParse->pWith;
+    pParse->pWith = pWith;
+    pParse->bFreeWith = bFree;
+  }
+}
+
+/*
+** This function checks if argument pFrom refers to a CTE declared by 
+** a WITH clause on the stack currently maintained by the parser. And,
+** if currently processing a CTE expression, if it is a recursive
+** reference to the current CTE.
+**
+** If pFrom falls into either of the two categories above, pFrom->pTab
+** and other fields are populated accordingly. The caller should check
+** (pFrom->pTab!=0) to determine whether or not a successful match
+** was found.
+**
+** Whether or not a match is found, SQLITE_OK is returned if no error
+** occurs. If an error does occur, an error message is stored in the
+** parser and some error code other than SQLITE_OK returned.
+*/
+static int withExpand(
+  Walker *pWalker, 
+  struct SrcList_item *pFrom
+){
+  Table *pTab;
+  Parse *pParse = pWalker->pParse;
+  sqlite3 *db = pParse->db;
+  struct Cte *pCte;
+
+  assert( pFrom->pTab==0 );
+
+  pCte = searchWith(pParse->pWith, pFrom);
+  if( pCte ){
+    ExprList *pEList;
+    Select *pSel;
+    Select *pLeft;                /* Left-most SELECT statement */
+    int bMayRecursive;            /* True if compound joined by UNION [ALL] */
+
+    /* If pCte->zErr is non-NULL at this point, then this is an illegal
+    ** recursive reference to CTE pCte. Leave an error in pParse and return
+    ** early. If pCte->zErr is NULL, then this is not a recursive reference.
+    ** In this case, proceed.  */
+    if( pCte->zErr ){
+      sqlite3ErrorMsg(pParse, pCte->zErr, pCte->zName);
+      return WRC_Abort;
+    }
+
+    pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table));
+    if( pTab==0 ) return WRC_Abort;
+    pTab->nRef = 1;
+    pTab->zName = sqlite3DbStrDup(db, pCte->zName);
+    pTab->iPKey = -1;
+    pTab->nRowEst = 1048576;
+    pTab->tabFlags |= TF_Ephemeral;
+    pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0);
+    if( db->mallocFailed ) return SQLITE_NOMEM;
+    assert( pFrom->pSelect );
+
+    /* Check if this is a recursive CTE. */
+    pSel = pFrom->pSelect;
+    bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION );
+    if( bMayRecursive ){
+      int i;
+      SrcList *pSrc = pFrom->pSelect->pSrc;
+      for(i=0; i<pSrc->nSrc; i++){
+        struct SrcList_item *pItem = &pSrc->a[i];
+        if( pItem->zDatabase==0 
+         && pItem->zName!=0 
+         && 0==sqlite3StrICmp(pItem->zName, pCte->zName)
+          ){
+          pItem->pTab = pTab;
+          pItem->isRecursive = 1;
+          pTab->nRef++;
+          pSel->selFlags |= SF_Recursive;
+        }
+      }
+    }
+
+    /* Only one recursive reference is permitted. */ 
+    if( pTab->nRef>2 ){
+      sqlite3ErrorMsg(
+          pParse, "multiple references to recursive table: %s", pCte->zName
+      );
+      return WRC_Abort;
+    }
+    assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 ));
+
+    pCte->zErr = "circular reference: %s";
+    sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel);
+
+    for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior);
+    pEList = pLeft->pEList;
+    if( pCte->pCols ){
+      if( pEList->nExpr!=pCte->pCols->nExpr ){
+        sqlite3ErrorMsg(pParse, "table %s has %d values for %d columns",
+            pCte->zName, pEList->nExpr, pCte->pCols->nExpr
+        );
+        return WRC_Abort;
+      }
+      pEList = pCte->pCols;
+    }
+    selectColumnsFromExprList(pParse, pEList, &pTab->nCol, &pTab->aCol);
+
+    if( bMayRecursive ){
+      if( pSel->selFlags & SF_Recursive ){
+        pCte->zErr = "multiple recursive references: %s";
+      }else{
+        pCte->zErr = "recursive reference in a subquery: %s";
+      }
+      sqlite3WalkSelect(pWalker, pSel);
+    }
+    pCte->zErr = 0;
+  }
+
+  return SQLITE_OK;
+}
+#endif
+
+#ifndef SQLITE_OMIT_CTE
+/*
+** If the SELECT passed as the second argument has an associated WITH 
+** clause, pop it from the stack stored as part of the Parse object.
+**
+** This function is used as the xSelectCallback2() callback by
+** sqlite3SelectExpand() when walking a SELECT tree to resolve table
+** names and other FROM clause elements. 
+*/
+static void selectPopWith(Walker *pWalker, Select *p){
+  Parse *pParse = pWalker->pParse;
+  if( p->pWith ){
+    assert( pParse->pWith==p->pWith );
+    pParse->pWith = p->pWith->pOuter;
+  }
+  return WRC_Continue;
+}
+#else
+#define selectPopWith 0
+#endif
+
 /*
 ** This routine is a Walker callback for "expanding" a SELECT statement.
 ** "Expanding" means to do the following:
@@ -3437,6 +3720,7 @@ static int selectExpander(Walker *pWalker, Select *p){
   }
   pTabList = p->pSrc;
   pEList = p->pEList;
+  sqlite3WithPush(pParse, p->pWith, 0);
 
   /* Make sure cursor numbers have been assigned to all entries in
   ** the FROM clause of the SELECT statement.
@@ -3449,12 +3733,21 @@ static int selectExpander(Walker *pWalker, Select *p){
   */
   for(i=0, pFrom=pTabList->a; i<pTabList->nSrc; i++, pFrom++){
     Table *pTab;
+    assert( pFrom->isRecursive==0 || pFrom->pTab );
+    if( pFrom->isRecursive ) continue;
     if( pFrom->pTab!=0 ){
       /* This statement has already been prepared.  There is no need
       ** to go further. */
       assert( i==0 );
+#ifndef SQLITE_OMIT_CTE
+      selectPopWith(pWalker, p);
+#endif
       return WRC_Prune;
     }
+#ifndef SQLITE_OMIT_CTE
+    if( withExpand(pWalker, pFrom) ) return WRC_Abort;
+    if( pFrom->pTab ) {} else
+#endif
     if( pFrom->zName==0 ){
 #ifndef SQLITE_OMIT_SUBQUERY
       Select *pSel = pFrom->pSelect;
@@ -3717,6 +4010,7 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){
     sqlite3WalkSelect(&w, pSelect);
   }
   w.xSelectCallback = selectExpander;
+  w.xSelectCallback2 = selectPopWith;
   sqlite3WalkSelect(&w, pSelect);
 }
 
@@ -3735,7 +4029,7 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){
 ** at that point because identifiers had not yet been resolved.  This
 ** routine is called after identifier resolution.
 */
-static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){
+static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){
   Parse *pParse;
   int i;
   SrcList *pTabList;
@@ -3751,13 +4045,13 @@ static int selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){
       if( ALWAYS(pTab!=0) && (pTab->tabFlags & TF_Ephemeral)!=0 ){
         /* A sub-query in the FROM clause of a SELECT */
         Select *pSel = pFrom->pSelect;
-        assert( pSel );
+        if( pSel ){
-        while( pSel->pPrior ) pSel = pSel->pPrior;
+          while( pSel->pPrior ) pSel = pSel->pPrior;
-        selectAddColumnTypeAndCollation(pParse, pTab, pSel);
+          selectAddColumnTypeAndCollation(pParse, pTab, pSel);
+        }
       }
     }
   }
-  return WRC_Continue;
 }
 #endif
 
@@ -3773,10 +4067,9 @@ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){
 #ifndef SQLITE_OMIT_SUBQUERY
   Walker w;
   memset(&w, 0, sizeof(w));
-  w.xSelectCallback = selectAddSubqueryTypeInfo;
+  w.xSelectCallback2 = selectAddSubqueryTypeInfo;
   w.xExprCallback = exprWalkNoop;
   w.pParse = pParse;
-  w.bSelectDepthFirst = 1;
   sqlite3WalkSelect(&w, pSelect);
 #endif
 }

src/sqlite.h.in

@@ -2561,6 +2561,7 @@ int sqlite3_set_authorizer(
 #define SQLITE_FUNCTION             31   /* NULL            Function Name   */
 #define SQLITE_SAVEPOINT            32   /* Operation       Savepoint Name  */
 #define SQLITE_COPY                  0   /* No longer used */
+#define SQLITE_RECURSIVE            33   /* NULL            NULL            */
 
 /*
 ** CAPI3REF: Tracing And Profiling Functions

src/sqliteInt.h

@@ -760,6 +760,7 @@ typedef struct VTable VTable;
 typedef struct VtabCtx VtabCtx;
 typedef struct Walker Walker;
 typedef struct WhereInfo WhereInfo;
+typedef struct With With;
 
 /*
 ** Defer sourcing vdbe.h and btree.h until after the "u8" and 
@@ -1428,7 +1429,7 @@ struct Table {
 };
 
 /*
-** Allowed values for Tabe.tabFlags.
+** Allowed values for Table.tabFlags.
 */
 #define TF_Readonly        0x01    /* Read-only system table */
 #define TF_Ephemeral       0x02    /* An ephemeral table */
@@ -1436,6 +1437,7 @@ struct Table {
 #define TF_Autoincrement   0x08    /* Integer primary key is autoincrement */
 #define TF_Virtual         0x10    /* Is a virtual table */
 #define TF_WithoutRowid    0x20    /* No rowid used. PRIMARY KEY is the key */
+#define TF_Recursive       0x40    /* Recursive reference within CTE */
 
 
 /*
@@ -2017,6 +2019,7 @@ struct SrcList {
     unsigned notIndexed :1;    /* True if there is a NOT INDEXED clause */
     unsigned isCorrelated :1;  /* True if sub-query is correlated */
     unsigned viaCoroutine :1;  /* Implemented as a co-routine */
+    unsigned isRecursive :1;   /* True for recursive reference in WITH */
 #ifndef SQLITE_OMIT_EXPLAIN
     u8 iSelectId;     /* If pSelect!=0, the id of the sub-select in EQP */
 #endif
@@ -2143,6 +2146,7 @@ struct Select {
   Select *pRightmost;    /* Right-most select in a compound select statement */
   Expr *pLimit;          /* LIMIT expression. NULL means not used. */
   Expr *pOffset;         /* OFFSET expression. NULL means not used. */
+  With *pWith;           /* WITH clause attached to this select. Or NULL. */
 };
 
 /*
@@ -2160,6 +2164,7 @@ struct Select {
 #define SF_Materialize     0x0100  /* Force materialization of views */
 #define SF_NestedFrom      0x0200  /* Part of a parenthesized FROM clause */
 #define SF_MaybeConvert    0x0400  /* Need convertCompoundSelectToSubquery() */
+#define SF_Recursive       0x0800  /* The recursive part of a recursive CTE */
 
 
 /*
@@ -2180,6 +2185,7 @@ struct Select {
 #define SRT_Table        8  /* Store result as data with an automatic rowid */
 #define SRT_EphemTab     9  /* Create transient tab and store like SRT_Table */
 #define SRT_Coroutine   10  /* Generate a single row of result */
+#define SRT_DistTable   11  /* Like SRT_TABLE, but unique results only */
 
 /*
 ** An instance of this object describes where to put of the results of
@@ -2364,6 +2370,8 @@ struct Parse {
 #endif
   Table *pZombieTab;        /* List of Table objects to delete after code gen */
   TriggerPrg *pTriggerPrg;  /* Linked list of coded triggers */
+  With *pWith;              /* Current WITH clause, or NULL */
+  u8 bFreeWith;             /* True if pWith should be freed with parser */
 };
 
 /*
@@ -2605,9 +2613,9 @@ struct Sqlite3Config {
 struct Walker {
   int (*xExprCallback)(Walker*, Expr*);     /* Callback for expressions */
   int (*xSelectCallback)(Walker*,Select*);  /* Callback for SELECTs */
+  void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */
   Parse *pParse;                            /* Parser context.  */
   int walkerDepth;                          /* Number of subqueries */
-  u8 bSelectDepthFirst;                     /* Do subqueries first */
   union {                                   /* Extra data for callback */
     NameContext *pNC;                          /* Naming context */
     int i;                                     /* Integer value */
@@ -2631,6 +2639,21 @@ int sqlite3WalkSelectFrom(Walker*, Select*);
 #define WRC_Prune       1   /* Omit children but continue walking siblings */
 #define WRC_Abort       2   /* Abandon the tree walk */
 
+/*
+** An instance of this structure represents a set of one or more CTEs
+** (common table expressions) created by a single WITH clause.
+*/
+struct With {
+  int nCte;                       /* Number of CTEs in the WITH clause */
+  With *pOuter;                   /* Containing WITH clause, or NULL */
+  struct Cte {                    /* For each CTE in the WITH clause.... */
+    char *zName;                    /* Name of this CTE */
+    ExprList *pCols;                /* List of explicit column names, or NULL */
+    Select *pSelect;                /* The definition of this CTE */
+    const char *zErr;               /* Error message for circular references */
+  } a[1];
+};
+
 /*
 ** Assuming zIn points to the first byte of a UTF-8 character,
 ** advance zIn to point to the first byte of the next UTF-8 character.
@@ -3329,6 +3352,14 @@ const char *sqlite3JournalModename(int);
   int sqlite3Checkpoint(sqlite3*, int, int, int*, int*);
   int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int);
 #endif
+#ifndef SQLITE_OMIT_CTE
+  With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*);
+  void sqlite3WithDelete(sqlite3*,With*);
+  void sqlite3WithPush(Parse*, With*, u8);
+#else
+#define sqlite3WithPush(x,y,z)
+#define sqlite3WithDelete(x,y)
+#endif
 
 /* Declarations for functions in fkey.c. All of these are replaced by
 ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign

src/tclsqlite.c

@@ -914,6 +914,7 @@ static int auth_callback(
     case SQLITE_DROP_VTABLE       : zCode="SQLITE_DROP_VTABLE"; break;
     case SQLITE_FUNCTION          : zCode="SQLITE_FUNCTION"; break;
     case SQLITE_SAVEPOINT         : zCode="SQLITE_SAVEPOINT"; break;
+    case SQLITE_RECURSIVE         : zCode="SQLITE_RECURSIVE"; break;
     default                       : zCode="????"; break;
   }
   Tcl_DStringInit(&str);

src/test_config.c

@@ -225,6 +225,12 @@ static void set_options(Tcl_Interp *interp){
   Tcl_SetVar2(interp, "sqlite_options", "check", "1", TCL_GLOBAL_ONLY);
 #endif
 
+#ifdef SQLITE_OMIT_CTE
+  Tcl_SetVar2(interp, "sqlite_options", "cte", "0", TCL_GLOBAL_ONLY);
+#else
+  Tcl_SetVar2(interp, "sqlite_options", "cte", "1", TCL_GLOBAL_ONLY);
+#endif
+
 #ifdef SQLITE_ENABLE_COLUMN_METADATA
   Tcl_SetVar2(interp, "sqlite_options", "columnmetadata", "1", TCL_GLOBAL_ONLY);
 #else

src/tokenize.c

@@ -494,6 +494,7 @@ abort_parse:
     sqlite3DeleteTable(db, pParse->pNewTable);
   }
 
+  if( pParse->bFreeWith ) sqlite3WithDelete(db, pParse->pWith);
   sqlite3DeleteTrigger(db, pParse->pNewTrigger);
   for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]);
   sqlite3DbFree(db, pParse->azVar);

src/vdbe.c

@@ -3369,6 +3369,32 @@ case OP_OpenEphemeral: {
   break;
 }
 
+#ifndef SQLITE_OMIT_CTE
+/* Opcode: SwapCursors P1 P2 * * *
+**
+** Parameters P1 and P2 are both cursors opened by the OpenEphemeral
+** opcode. This opcode deletes the contents of epheremal table P1,
+** then renames P2 to P1 and P1 to P2. In other words, following this
+** opcode cursor P2 is open on an empty table and P1 is open on the
+** table that was initially accessed by P2.
+*/
+case OP_SwapCursors: {
+  Mem tmp;
+  VdbeCursor *pTmp;
+
+  tmp = p->aMem[p->nMem - pOp->p1];
+  p->aMem[p->nMem - pOp->p1] = p->aMem[p->nMem - pOp->p2];
+  p->aMem[p->nMem - pOp->p2] = tmp;
+
+  pTmp = p->apCsr[pOp->p1];
+  p->apCsr[pOp->p1] = p->apCsr[pOp->p2];
+  p->apCsr[pOp->p2] = pTmp;
+
+  rc = sqlite3BtreeClearTable(pTmp->pBt, MASTER_ROOT + !pTmp->isTable, 0);
+  break;
+}
+#endif /* ifndef SQLITE_OMIT_CTE */
+
 /* Opcode: SorterOpen P1 * * P4 *
 **
 ** This opcode works like OP_OpenEphemeral except that it opens

src/walker.c

@@ -113,9 +113,12 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){
 /*
 ** Call sqlite3WalkExpr() for every expression in Select statement p.
 ** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and
-** on the compound select chain, p->pPrior.  Invoke the xSelectCallback()
+** on the compound select chain, p->pPrior. 
-** either before or after the walk of expressions and FROM clause, depending
+**
-** on whether pWalker->bSelectDepthFirst is false or true, respectively.
+** If it is not NULL, the xSelectCallback() callback is invoked before
+** the walk of the expressions and FROM clause. The xSelectCallback2()
+** method, if it is not NULL, is invoked following the walk of the 
+** expressions and FROM clause.
 **
 ** Return WRC_Continue under normal conditions.  Return WRC_Abort if
 ** there is an abort request.
@@ -125,11 +128,13 @@ int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){
 */
 int sqlite3WalkSelect(Walker *pWalker, Select *p){
   int rc;
-  if( p==0 || pWalker->xSelectCallback==0 ) return WRC_Continue;
+  if( p==0 || (pWalker->xSelectCallback==0 && pWalker->xSelectCallback2==0) ){
+    return WRC_Continue;
+  }
   rc = WRC_Continue;
   pWalker->walkerDepth++;
   while( p ){
-    if( !pWalker->bSelectDepthFirst ){
+    if( pWalker->xSelectCallback ){
        rc = pWalker->xSelectCallback(pWalker, p);
        if( rc ) break;
     }
@@ -139,12 +144,8 @@ int sqlite3WalkSelect(Walker *pWalker, Select *p){
       pWalker->walkerDepth--;
       return WRC_Abort;
     }
-    if( pWalker->bSelectDepthFirst ){
+    if( pWalker->xSelectCallback2 ){
-      rc = pWalker->xSelectCallback(pWalker, p);
+      pWalker->xSelectCallback2(pWalker, p);
-      /* Depth-first search is currently only used for
-      ** selectAddSubqueryTypeInfo() and that routine always returns
-      ** WRC_Continue (0).  So the following branch is never taken. */
-      if( NEVER(rc) ) break;
     }
     p = p->pPrior;
   }

src/where.c

@@ -4196,6 +4196,7 @@ static int whereLoopAddBtree(
    && !pSrc->notIndexed
    && HasRowid(pTab)
    && !pSrc->isCorrelated
+   && !pSrc->isRecursive
   ){
     /* Generate auto-index WhereLoops */
     WhereTerm *pTerm;

test/auth.test

@@ -2080,6 +2080,42 @@ ifcapable {altertable} {
   execsql {DROP TABLE t5}
 } ;# ifcapable altertable
 
+ifcapable {cte} {
+  do_test auth-1.310 {
+    proc auth {code arg1 arg2 arg3 arg4} {
+      if {$code=="SQLITE_RECURSIVE"} {
+        return SQLITE_DENY
+      }
+      return SQLITE_OK
+    }
+    db eval {
+       DROP TABLE IF EXISTS t1;
+       CREATE TABLE t1(a,b);
+       INSERT INTO t1 VALUES(1,2),(3,4),(5,6);
+    }
+  } {}
+  do_catchsql_test auth-1.311 {
+    WITH
+       auth1311(x,y) AS (SELECT a+b, b-a FROM t1)
+    SELECT * FROM auth1311 ORDER BY x;
+  } {0 {3 1 7 1 11 1}}
+  do_catchsql_test auth-1.312 {
+    WITH RECURSIVE
+       auth1312(x,y) AS (SELECT a+b, b-a FROM t1)
+    SELECT x, y FROM auth1312 ORDER BY x;
+  } {0 {3 1 7 1 11 1}}
+  do_catchsql_test auth-1.313 {
+    WITH RECURSIVE
+       auth1313(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM auth1313 WHERE x<5)
+    SELECT * FROM t1;
+  } {0 {1 2 3 4 5 6}}
+  do_catchsql_test auth-1.314 {
+    WITH RECURSIVE
+       auth1314(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM auth1314 WHERE x<5)
+    SELECT * FROM t1 LEFT JOIN auth1314;
+  } {1 {not authorized}}
+} ;# ifcapable cte
+
 do_test auth-2.1 {
   proc auth {code arg1 arg2 arg3 arg4} {
     if {$code=="SQLITE_READ" && $arg1=="t3" && $arg2=="x"} {

test/with1.test

@@ -0,0 +1,329 @@
+# 2014 January 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the WITH clause.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix with1
+
+ifcapable {!cte} {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x INTEGER, y INTEGER);
+  WITH x(a) AS ( SELECT * FROM t1) SELECT 10
+} {10}
+
+do_execsql_test 1.1 {
+  SELECT * FROM ( WITH x AS ( SELECT * FROM t1) SELECT 10 );
+} {10}
+
+do_execsql_test 1.2 {
+  WITH x(a) AS ( SELECT * FROM t1) INSERT INTO t1 VALUES(1,2);
+} {}
+
+do_execsql_test 1.3 {
+  WITH x(a) AS ( SELECT * FROM t1) DELETE FROM t1;
+} {}
+
+do_execsql_test 1.4 {
+  WITH x(a) AS ( SELECT * FROM t1) UPDATE t1 SET x = y;
+} {}
+ 
+#--------------------------------------------------------------------------
+
+do_execsql_test 2.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(1);
+  INSERT INTO t1 VALUES(2);
+  WITH tmp AS ( SELECT * FROM t1 ) SELECT x FROM tmp;
+} {1 2}
+
+do_execsql_test 2.2 {
+  WITH tmp(a) AS ( SELECT * FROM t1 ) SELECT a FROM tmp;
+} {1 2}
+
+do_execsql_test 2.3 {
+  SELECT * FROM (
+    WITH tmp(a) AS ( SELECT * FROM t1 ) SELECT a FROM tmp
+  );
+} {1 2}
+
+do_execsql_test 2.4 {
+  WITH tmp1(a) AS ( SELECT * FROM t1 ),
+       tmp2(x) AS ( SELECT * FROM tmp1)
+  SELECT * FROM tmp2;
+} {1 2}
+
+do_execsql_test 2.5 {
+  WITH tmp2(x) AS ( SELECT * FROM tmp1),
+       tmp1(a) AS ( SELECT * FROM t1 )
+  SELECT * FROM tmp2;
+} {1 2}
+
+#-------------------------------------------------------------------------
+do_catchsql_test 3.1 {
+  WITH tmp2(x) AS ( SELECT * FROM tmp1 ),
+       tmp1(a) AS ( SELECT * FROM tmp2 )
+  SELECT * FROM tmp1;
+} {1 {circular reference: tmp1}}
+
+do_catchsql_test 3.2 {
+  CREATE TABLE t2(x INTEGER);
+  WITH tmp(a) AS (SELECT * FROM t1),
+       tmp(a) AS (SELECT * FROM t1)
+  SELECT * FROM tmp;
+} {1 {duplicate WITH table name: tmp}}
+
+do_execsql_test 3.3 {
+  CREATE TABLE t3(x);
+  CREATE TABLE t4(x);
+
+  INSERT INTO t3 VALUES('T3');
+  INSERT INTO t4 VALUES('T4');
+
+  WITH t3(a) AS (SELECT * FROM t4)
+  SELECT * FROM t3;
+} {T4}
+
+do_execsql_test 3.4 {
+  WITH tmp  AS ( SELECT * FROM t3 ),
+       tmp2 AS ( WITH tmp AS ( SELECT * FROM t4 ) SELECT * FROM tmp )
+  SELECT * FROM tmp2;
+} {T4}
+
+do_execsql_test 3.5 {
+  WITH tmp  AS ( SELECT * FROM t3 ),
+       tmp2 AS ( WITH xxxx AS ( SELECT * FROM t4 ) SELECT * FROM tmp )
+  SELECT * FROM tmp2;
+} {T3}
+
+do_catchsql_test 3.6 {
+  WITH tmp AS ( SELECT * FROM t3 ),
+  SELECT * FROM tmp;
+} {1 {near "SELECT": syntax error}}
+
+#-------------------------------------------------------------------------
+do_execsql_test 4.1 {
+  DROP TABLE IF EXISTS t1;
+  CREATE TABLE t1(x);
+  INSERT INTO t1 VALUES(1);
+  INSERT INTO t1 VALUES(2);
+  INSERT INTO t1 VALUES(3);
+  INSERT INTO t1 VALUES(4);
+
+  WITH dset AS ( SELECT 2 UNION ALL SELECT 4 )
+  DELETE FROM t1 WHERE x IN dset;
+  SELECT * FROM t1;
+} {1 3}
+
+do_execsql_test 4.2 {
+  WITH iset AS ( SELECT 2 UNION ALL SELECT 4 )
+  INSERT INTO t1 SELECT * FROM iset;
+  SELECT * FROM t1;
+} {1 3 2 4}
+
+do_execsql_test 4.3 {
+  WITH uset(a, b) AS ( SELECT 2, 8 UNION ALL SELECT 4, 9 )
+  UPDATE t1 SET x = COALESCE( (SELECT b FROM uset WHERE a=x), x );
+  SELECT * FROM t1;
+} {1 3 8 9}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 5.1 {
+  WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i)
+  SELECT x FROM i LIMIT 10;
+} {1 2 3 4 5 6 7 8 9 10}
+
+do_catchsql_test 5.2 {
+  WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i ORDER BY 1)
+  SELECT x FROM i LIMIT 10;
+} {1 {ORDER BY in a recursive query is not allowed}}
+
+do_catchsql_test 5.3 {
+  WITH i(x) AS ( VALUES(1) UNION ALL SELECT x+1 FROM i LIMIT 10 )
+  SELECT x FROM i LIMIT 10;
+} {1 {LIMIT in a recursive query is not allowed}}
+
+do_execsql_test 5.4 {
+  WITH i(x) AS ( VALUES(1) UNION ALL SELECT (x+1)%10 FROM i)
+  SELECT x FROM i LIMIT 20;
+} {1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0}
+
+do_execsql_test 5.5 {
+  WITH i(x) AS ( VALUES(1) UNION SELECT (x+1)%10 FROM i)
+  SELECT x FROM i LIMIT 20;
+} {1 2 3 4 5 6 7 8 9 0}
+
+do_catchsql_test 5.6.1 {
+  WITH i(x, y) AS ( VALUES(1) )
+  SELECT * FROM i;
+} {1 {table i has 1 values for 2 columns}}
+
+do_catchsql_test 5.6.2 {
+  WITH i(x) AS ( VALUES(1,2) )
+  SELECT * FROM i;
+} {1 {table i has 2 values for 1 columns}}
+
+do_catchsql_test 5.6.3 {
+  CREATE TABLE t5(a, b);
+  WITH i(x) AS ( SELECT * FROM t5 )
+  SELECT * FROM i;
+} {1 {table i has 2 values for 1 columns}}
+
+do_catchsql_test 5.6.4 {
+  WITH i(x) AS ( SELECT 1, 2 UNION ALL SELECT 1 )
+  SELECT * FROM i;
+} {1 {table i has 2 values for 1 columns}}
+
+do_catchsql_test 5.6.5 {
+  WITH i(x) AS ( SELECT 1 UNION ALL SELECT 1, 2 )
+  SELECT * FROM i;
+} {1 {SELECTs to the left and right of UNION ALL do not have the same number of result columns}}
+
+do_catchsql_test 5.6.6 {
+  WITH i(x) AS ( SELECT 1 UNION ALL SELECT x+1, x*2 FROM i )
+  SELECT * FROM i;
+} {1 {SELECTs to the left and right of UNION ALL do not have the same number of result columns}}
+
+do_catchsql_test 5.6.7 {
+  WITH i(x) AS ( SELECT 1, 2 UNION SELECT x+1 FROM i )
+  SELECT * FROM i;
+} {1 {table i has 2 values for 1 columns}}
+
+#-------------------------------------------------------------------------
+#
+do_execsql_test 6.1 {
+  CREATE TABLE f(
+      id INTEGER PRIMARY KEY, parentid REFERENCES f, name TEXT
+  );
+
+  INSERT INTO f VALUES(0, NULL, '');
+  INSERT INTO f VALUES(1, 0, 'bin');
+    INSERT INTO f VALUES(2, 1, 'true');
+    INSERT INTO f VALUES(3, 1, 'false');
+    INSERT INTO f VALUES(4, 1, 'ls');
+    INSERT INTO f VALUES(5, 1, 'grep');
+  INSERT INTO f VALUES(6, 0, 'etc');
+    INSERT INTO f VALUES(7, 6, 'rc.d');
+      INSERT INTO f VALUES(8, 7, 'rc.apache');
+      INSERT INTO f VALUES(9, 7, 'rc.samba');
+  INSERT INTO f VALUES(10, 0, 'home');
+    INSERT INTO f VALUES(11, 10, 'dan');
+      INSERT INTO f VALUES(12, 11, 'public_html');
+        INSERT INTO f VALUES(13, 12, 'index.html');
+          INSERT INTO f VALUES(14, 13, 'logo.gif');
+}
+
+do_execsql_test 6.2 {
+  WITH flat(fid, fpath) AS (
+    SELECT id, '' FROM f WHERE parentid IS NULL
+    UNION ALL
+    SELECT id, fpath || '/' || name FROM f, flat WHERE parentid=fid
+  )
+  SELECT fpath FROM flat WHERE fpath!='' ORDER BY 1;
+} {
+  /bin 
+  /bin/false /bin/grep /bin/ls /bin/true 
+  /etc 
+  /etc/rc.d 
+  /etc/rc.d/rc.apache /etc/rc.d/rc.samba 
+  /home 
+  /home/dan 
+  /home/dan/public_html 
+  /home/dan/public_html/index.html 
+  /home/dan/public_html/index.html/logo.gif
+}
+
+do_execsql_test 6.3 {
+  WITH flat(fid, fpath) AS (
+    SELECT id, '' FROM f WHERE parentid IS NULL
+    UNION ALL
+    SELECT id, fpath || '/' || name FROM f, flat WHERE parentid=fid
+  )
+  SELECT count(*) FROM flat;
+} {15}
+
+do_execsql_test 6.4 {
+  WITH x(i) AS (
+    SELECT 1
+    UNION ALL
+    SELECT i+1 FROM x WHERE i<10
+  )
+  SELECT count(*) FROM x
+} {10}
+
+
+#-------------------------------------------------------------------------
+
+do_execsql_test 7.1 {
+  CREATE TABLE tree(i, p);
+  INSERT INTO tree VALUES(1, NULL);
+  INSERT INTO tree VALUES(2, 1);
+  INSERT INTO tree VALUES(3, 1);
+  INSERT INTO tree VALUES(4, 2);
+  INSERT INTO tree VALUES(5, 4);
+}
+
+do_execsql_test 7.2 {
+  WITH t(id, path) AS (
+    SELECT i, '' FROM tree WHERE p IS NULL
+    UNION ALL
+    SELECT i, path || '/' || i FROM tree, t WHERE p = id
+  ) 
+  SELECT path FROM t;
+} {{} /2 /3 /2/4 /2/4/5}
+
+do_execsql_test 7.3 {
+  WITH t(id) AS (
+    VALUES(2)
+    UNION ALL
+    SELECT i FROM tree, t WHERE p = id
+  ) 
+  SELECT id FROM t;
+} {2 4 5}
+
+do_catchsql_test 7.4 {
+  WITH t(id) AS (
+    VALUES(2)
+    UNION ALL
+    SELECT i FROM tree WHERE p IN (SELECT id FROM t)
+  ) 
+  SELECT id FROM t;
+} {1 {recursive reference in a subquery: t}}
+
+do_catchsql_test 7.5 {
+  WITH t(id) AS (
+    VALUES(2)
+    UNION ALL
+    SELECT i FROM tree, t WHERE p = id AND p IN (SELECT id FROM t)
+  ) 
+  SELECT id FROM t;
+} {1 {multiple recursive references: t}}
+
+do_catchsql_test 7.6 {
+  WITH t(id) AS (
+    SELECT i FROM tree WHERE 2 IN (SELECT id FROM t)
+    UNION ALL
+    SELECT i FROM tree, t WHERE p = id
+  ) 
+  SELECT id FROM t;
+} {1 {circular reference: t}}
+
+
+
+finish_test

test/with2.test

@@ -0,0 +1,56 @@
+# 2014 January 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the WITH clause.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set ::testprefix with2
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(a);
+  INSERT INTO t1 VALUES(1);
+  INSERT INTO t1 VALUES(2);
+}
+
+do_execsql_test 1.1 {
+  WITH x1 AS (SELECT * FROM t1)
+  SELECT sum(a) FROM x1;
+} {3}
+
+do_execsql_test 1.2 {
+  WITH x1 AS (SELECT * FROM t1)
+  SELECT (SELECT sum(a) FROM x1);
+} {3}
+
+do_execsql_test 1.3 {
+  WITH x1 AS (SELECT * FROM t1)
+  SELECT (SELECT sum(a) FROM x1);
+} {3}
+
+do_execsql_test 1.4 {
+  CREATE TABLE t2(i);
+  INSERT INTO t2 VALUES(2);
+  INSERT INTO t2 VALUES(3);
+  INSERT INTO t2 VALUES(5);
+
+  WITH x1   AS (SELECT i FROM t2),
+       i(a) AS (
+         SELECT min(i)-1 FROM x1 UNION SELECT a+1 FROM i WHERE a<10
+       )
+  SELECT a FROM i WHERE a NOT IN x1
+} {1 4 6 7 8 9 10}
+
+finish_test
+
+
+

test/withM.test

@@ -0,0 +1,61 @@
+# 2014 January 11
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library.  The
+# focus of this file is testing the WITH clause.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/malloc_common.tcl
+set ::testprefix withM
+
+ifcapable {!cte} {
+  finish_test
+  return
+}
+
+do_execsql_test 1.0 {
+  CREATE TABLE t1(x INTEGER, y INTEGER);
+  INSERT INTO t1 VALUES(123, 456);
+}
+
+do_faultsim_test withM-1.1 -prep {
+  sqlite3 db test.db
+} -body {
+  execsql { 
+    WITH tmp AS ( SELECT * FROM t1 )
+    SELECT * FROM tmp;
+  }
+} -test {
+  faultsim_test_result {0 {123 456}}
+  db close
+}
+
+do_faultsim_test withM-1.2 -prep {
+  sqlite3 db test.db
+} -body {
+  execsql { 
+    WITH w1 AS ( SELECT * FROM t1 ),
+         w2 AS ( 
+           WITH w3 AS ( SELECT * FROM w1 )
+           SELECT * FROM w3
+         )
+    SELECT * FROM w2;
+  }
+} -test {
+  faultsim_test_result {0 {123 456}}
+  db close
+}
+
+finish_test
+
+
+

tool/mkkeywordhash.c

@@ -138,6 +138,11 @@ struct Keyword {
 #else
 #  define AUTOVACUUM 0x00020000
 #endif
+#ifdef SQLITE_OMIT_CTE
+#  define CTE        0
+#else
+#  define CTE        0x00040000
+#endif
 
 /*
 ** These are the keywords
@@ -234,6 +239,7 @@ static Keyword aKeywordTable[] = {
   { "PRIMARY",          "TK_PRIMARY",      ALWAYS                 },
   { "QUERY",            "TK_QUERY",        EXPLAIN                },
   { "RAISE",            "TK_RAISE",        TRIGGER                },
+  { "RECURSIVE",        "TK_RECURSIVE",    CTE                    },
   { "REFERENCES",       "TK_REFERENCES",   FKEY                   },
   { "REGEXP",           "TK_LIKE_KW",      ALWAYS                 },
   { "REINDEX",          "TK_REINDEX",      REINDEX                },
@@ -262,6 +268,7 @@ static Keyword aKeywordTable[] = {
   { "VALUES",           "TK_VALUES",       ALWAYS                 },
   { "VIEW",             "TK_VIEW",         VIEW                   },
   { "VIRTUAL",          "TK_VIRTUAL",      VTAB                   },
+  { "WITH",             "TK_WITH",         CTE                    },
   { "WITHOUT",          "TK_WITHOUT",      ALWAYS                 },
   { "WHEN",             "TK_WHEN",         ALWAYS                 },
   { "WHERE",            "TK_WHERE",        ALWAYS                 },