Merge all changes from the weak-fallback branch except those related to the

weak-fallback feature itself.

FossilOrigin-Name: aad718fb1afacf1c921966796ab1e149207c31b16409c5672f0371f4fb9d6565

manifest

@@ -1,5 +1,5 @@
-C Add\smissing\sVdbeCoverage()\sand\sVdbeCoverageNeverTaken()\smacros\sto\swindow.c.
+C Merge\sall\schanges\sfrom\sthe\sweak-fallback\sbranch\sexcept\sthose\srelated\sto\sthe\nweak-fallback\sfeature\sitself.
-D 2018-06-27T20:24:59.489
+D 2018-06-30T19:12:36.648
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 0a3a6c81e6fcb969ff9106e882f0a08547014ba463cb6beca4c4efaecc924ee6
@@ -484,7 +484,7 @@ F src/os_win.c ac29c25cde4cfb4adacc59cdec4aa45698ca0e29164ea127859585ccd9faa354
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 1bb6a57fa0465296a4d6109a1a64610a0e7adde1f3acf3ef539a9d972908ce8f
 F src/pager.h c571b064df842ec8f2e90855dead9acf4cbe0d1b2c05afe0ef0d0145f7fd0388
-F src/parse.y 4661ac1382c18bc67e1680ea3e44a6835ebc770667fe32ee1720c38454304165
+F src/parse.y 9b57f1d0d3d7578ab2917e07ff5d8def4b0aac571113dd7b7cb8108e7194d025
 F src/pcache.c 135ef0bc6fb2e3b7178d49ab5c9176254c8a691832c1bceb1156b2fbdd0869bd
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -493,14 +493,14 @@ F src/pragma.h bb83728944b42f6d409c77f5838a8edbdb0fe83046c5496ffc9602b40340a324
 F src/prepare.c e966ecc97c3671ff0e96227c8c877b83f2d33ea371ee190bbf1698b36b5605c0
 F src/printf.c 7f6f3cba8e0c49c19e30a1ff4e9aeda6e06814dcbad4b664a69e1b6cb6e7e365
 F src/random.c 80f5d666f23feb3e6665a6ce04c7197212a88384
-F src/resolve.c 43b97be1b1bfbe7aa6a0e8608f5a022eac8cd2d727bdf4d0287371ba32218e7b
+F src/resolve.c d8359c59c683fc33324c3a0abe0d393304861f05dd2e0d47e5974a501bcf4682
 F src/rowset.c 7b7e7e479212e65b723bf40128c7b36dc5afdfac
 F src/select.c 82aa95aa1cfeadef5ecae1126afff1efca64c49050bbf7116425a8a1d4fc5e6c
 F src/shell.c.in 8578421c5fb2a972461b2a996f7173646e55e0dbd2a2eee30c8f5dc7d3dbadfd
 F src/sqlite.h.in 243e0b1d302c237372c06002fe84f731fd22587eccd142d15de834a5296514ff
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 9887b27e69c01e79c2cbe74ef73bf01af5b5703d6a7f0a4371e386d7249cb1c7
-F src/sqliteInt.h 74728a0b20e9d2173470186a8ca1e3229c79abb19d1ac4eedc4562e8905d798d
+F src/sqliteInt.h c14c850bf0c4dd3860cc5092f140ae9faf93b77737b3febf38454088e1ab3988
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -558,7 +558,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9
 F src/test_window.c add59ee68568868129516999f30a68e8ab2afd276e272aba4f633c9fc52c1bb1
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
-F src/tokenize.c b42aa952b05df665ed10d54f02c467159cd9fcb8cbbe5ff3c9cd74781fa2803c
+F src/tokenize.c 0e3e0462f7da08bf95b3da1dca4c01cd2c3ca1d988ed0f9d2f66334a975e4020
 F src/treeview.c 2c5c4bc0a443401db5fd621542150452ddf5055d38edd4eef868bc2b6bfb0260
 F src/trigger.c 4ace6d1d5ba9a89822deb287317f33c810440526eafe185c2d8a48c31df1e995
 F src/update.c 46dc24c6158446aaab45caee09b6d99327cb479268b83ffeb5b701823da3b67b
@@ -1626,6 +1626,7 @@ F test/window3.test 47884c240d0d5234ad9c6da65452cfec1bfa69ec6f9c4158cd9750c3d88d
 F test/window4.tcl 7cec7e578aa9f78b7265bff8d552cda17a1d8d89f0449d0e74970a527b8846f5
 F test/window4.test dcd8767869988e0d23d56bc3f8b46ec116de23127b81b5f66fd48d5529072ed1
 F test/window5.test a4835b96d30eb3b81a1dbc683e333e57a614645eb6f2ae476a7ed2addf0b0f1f
+F test/window6.test fb4e464993630c3d6023e906aa6705667062600a8341085b7fd8c201f4c2f411
 F test/windowfault.test 97d5fc404308edb579a5a183e294ed874c844ecf01f0a28ba46df3141ebaee1f
 F test/with1.test 58475190cd8caaeebea8cfeb2a264ec97a0c492b8ffe9ad20cefbb23df462f96
 F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1ab
@@ -1663,7 +1664,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c 33892e2a243865f73e6c6e7cecce3c6eb4bb95db4a3d9d86d146c8064feb92fd
-F tool/lempar.c bf7db78e7213f1d51516710483eab506fd52bf632c7abfb3e2e9b885c90c03e1
+F tool/lempar.c 6f64bc81160495f93d0f2ae27c787cb7cf4d5c7f5d2abf3823bc878ded6d6293
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1743,7 +1744,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P 336de43a47e206fe7629072e5b8c80d4ede17ead8ef4dcf5d8da6833ff22d2f9
+P 4383cb68a1241768ddb3f180f8cbb9ea1638f8806210b10aac1384b8a7ecdca2 12d819e1c17d8036900352b0989c4bfcbc34193c3735bb9af7ab051f0f129d3d
-R b7bd98ed0e43c8f2d4cb4f0d7b13d574
+R 545d939ac542e77e69d38cfd02ddd127
 U dan
-Z 38db85555459c758350756078d19d70e
+Z 6125a1e4d6297855bc74fb4c9cdd182a

manifest.uuid

@@ -1 +1 @@
-4383cb68a1241768ddb3f180f8cbb9ea1638f8806210b10aac1384b8a7ecdca2
+aad718fb1afacf1c921966796ab1e149207c31b16409c5672f0371f4fb9d6565

src/parse.y

@@ -216,6 +216,9 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}
 %ifdef SQLITE_OMIT_COMPOUND_SELECT
   EXCEPT INTERSECT UNION
 %endif SQLITE_OMIT_COMPOUND_SELECT
+%ifndef SQLITE_OMIT_WINDOWFUNC
+  CURRENT FOLLOWING PARTITION PRECEDING RANGE UNBOUNDED
+%endif SQLITE_OMIT_WINDOWFUNC
   REINDEX RENAME CTIME_KW IF
   .
 %wildcard ANY.
@@ -1042,102 +1045,6 @@ term(A) ::= CTIME_KW(OP). {
   A = sqlite3ExprFunction(pParse, 0, &OP);
 }
 
-%ifndef SQLITE_OMIT_WINDOWFUNC
-
-%type windowdefn_opt {Window*}
-%destructor windowdefn_opt {sqlite3WindowDelete(pParse->db, $$);}
-windowdefn_opt(A) ::= . { A = 0; }
-windowdefn_opt(A) ::= WINDOW windowdefn_list(B). { A = B; }
-
-%type windowdefn_list {Window*}
-%destructor windowdefn_list {sqlite3WindowDelete(pParse->db, $$);}
-windowdefn_list(A) ::= windowdefn(Z). { A = Z; }
-windowdefn_list(A) ::= windowdefn_list(Y) COMMA windowdefn(Z). {
-  if( Z ) Z->pNextWin = Y;
-  A = Z;
-}
-
-%type windowdefn {Window*}
-%destructor windowdefn {sqlite3WindowDelete(pParse->db, $$);}
-windowdefn(A) ::= nm(X) AS window(Y). {
-  if( Y ){
-    Y->zName = sqlite3DbStrNDup(pParse->db, X.z, X.n);
-  }
-  A = Y;
-}
-
-%type over_opt {Window*}
-%destructor over_opt {sqlite3WindowDelete(pParse->db, $$);}
-
-%type window {Window*}
-%destructor window {sqlite3WindowDelete(pParse->db, $$);}
-
-%type frame_opt {Window*}
-%destructor frame_opt {sqlite3WindowDelete(pParse->db, $$);}
-
-%type window_or_nm {Window*}
-%destructor window_or_nm {
-sqlite3WindowDelete(pParse->db, $$);}
-
-%type part_opt {ExprList*}
-%destructor part_opt {sqlite3ExprListDelete(pParse->db, $$);}
-
-%type filter_opt {Expr*}
-%destructor filter_opt {sqlite3ExprDelete(pParse->db, $$);}
-
-%type range_or_rows {int}
-
-%type frame_bound {struct FrameBound}
-%destructor frame_bound {sqlite3ExprDelete(pParse->db, $$.pExpr);}
-
-over_opt(A) ::= . { A = 0; }
-over_opt(A) ::= filter_opt(W) OVER window_or_nm(Z). {
-  A = Z;
-  if( A ) A->pFilter = W;
-}
-
-window_or_nm(A) ::= window(Z). {A = Z;}
-window_or_nm(A) ::= nm(Z). {
-  A = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window));
-  if( A ){
-    A->zName = sqlite3DbStrNDup(pParse->db, Z.z, Z.n);
-  }
-}
-
-window(A) ::= LP part_opt(X) orderby_opt(Y) frame_opt(Z) RP. {
-  A = Z;
-  if( A ){
-    A->pPartition = X;
-    A->pOrderBy = Y;
-  }
-}
-
-part_opt(A) ::= PARTITION BY exprlist(X). { A = X; }
-part_opt(A) ::= .                         { A = 0; }
-filter_opt(A) ::= .                            { A = 0; }
-filter_opt(A) ::= FILTER LP WHERE expr(X) RP.  { A = X; }
-
-frame_opt(A) ::= .                             { 
-  A = sqlite3WindowAlloc(pParse, TK_RANGE, TK_UNBOUNDED, 0, TK_CURRENT, 0);
-}
-frame_opt(A) ::= range_or_rows(X) frame_bound(Y). { 
-  A = sqlite3WindowAlloc(pParse, X, Y.eType, Y.pExpr, TK_CURRENT, 0);
-}
-frame_opt(A) ::= range_or_rows(X) BETWEEN frame_bound(Y) AND frame_bound(Z). { 
-  A = sqlite3WindowAlloc(pParse, X, Y.eType, Y.pExpr, Z.eType, Z.pExpr);
-}
-
-range_or_rows(A) ::= RANGE.   { A = TK_RANGE; }
-range_or_rows(A) ::= ROWS.    { A = TK_ROWS;  }
-
-frame_bound(A) ::= UNBOUNDED PRECEDING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; }
-frame_bound(A) ::= expr(X) PRECEDING.   { A.eType = TK_PRECEDING; A.pExpr = X; }
-frame_bound(A) ::= CURRENT ROW.         { A.eType = TK_CURRENT  ; A.pExpr = 0; }
-frame_bound(A) ::= expr(X) FOLLOWING.   { A.eType = TK_FOLLOWING; A.pExpr = X; }
-frame_bound(A) ::= UNBOUNDED FOLLOWING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; }
-
-%endif // SQLITE_OMIT_WINDOWFUNC
-
 expr(A) ::= LP nexprlist(X) COMMA expr(Y) RP. {
   ExprList *pList = sqlite3ExprListAppend(pParse, X, Y);
   A = sqlite3PExpr(pParse, TK_VECTOR, 0, 0);
@@ -1689,3 +1596,104 @@ wqlist(A) ::= wqlist(A) COMMA nm(X) eidlist_opt(Y) AS LP select(Z) RP. {
   A = sqlite3WithAdd(pParse, A, &X, Y, Z);
 }
 %endif  SQLITE_OMIT_CTE
+
+//////////////////////// WINDOW FUNCTION EXPRESSIONS /////////////////////////
+// These must be at the end of this file. Specifically, the rules that
+// introduce tokens WINDOW, OVER and FILTER must appear last. This causes 
+// the integer values assigned to these tokens to be larger than all other 
+// tokens that may be output by the tokenizer except TK_SPACE and TK_ILLEGAL.
+//
+%ifndef SQLITE_OMIT_WINDOWFUNC
+%type windowdefn_list {Window*}
+%destructor windowdefn_list {sqlite3WindowDelete(pParse->db, $$);}
+windowdefn_list(A) ::= windowdefn(Z). { A = Z; }
+windowdefn_list(A) ::= windowdefn_list(Y) COMMA windowdefn(Z). {
+  if( Z ) Z->pNextWin = Y;
+  A = Z;
+}
+
+%type windowdefn {Window*}
+%destructor windowdefn {sqlite3WindowDelete(pParse->db, $$);}
+windowdefn(A) ::= nm(X) AS window(Y). {
+  if( Y ){
+    Y->zName = sqlite3DbStrNDup(pParse->db, X.z, X.n);
+  }
+  A = Y;
+}
+
+%type window {Window*}
+%destructor window {sqlite3WindowDelete(pParse->db, $$);}
+
+%type frame_opt {Window*}
+%destructor frame_opt {sqlite3WindowDelete(pParse->db, $$);}
+
+%type window_or_nm {Window*}
+%destructor window_or_nm {
+sqlite3WindowDelete(pParse->db, $$);}
+
+%type part_opt {ExprList*}
+%destructor part_opt {sqlite3ExprListDelete(pParse->db, $$);}
+
+%type filter_opt {Expr*}
+%destructor filter_opt {sqlite3ExprDelete(pParse->db, $$);}
+
+%type range_or_rows {int}
+
+%type frame_bound {struct FrameBound}
+%destructor frame_bound {sqlite3ExprDelete(pParse->db, $$.pExpr);}
+
+window_or_nm(A) ::= window(Z). {A = Z;}
+window_or_nm(A) ::= nm(Z). {
+  A = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window));
+  if( A ){
+    A->zName = sqlite3DbStrNDup(pParse->db, Z.z, Z.n);
+  }
+}
+
+window(A) ::= LP part_opt(X) orderby_opt(Y) frame_opt(Z) RP. {
+  A = Z;
+  if( A ){
+    A->pPartition = X;
+    A->pOrderBy = Y;
+  }
+}
+
+part_opt(A) ::= PARTITION BY exprlist(X). { A = X; }
+part_opt(A) ::= .                         { A = 0; }
+
+frame_opt(A) ::= .                             { 
+  A = sqlite3WindowAlloc(pParse, TK_RANGE, TK_UNBOUNDED, 0, TK_CURRENT, 0);
+}
+frame_opt(A) ::= range_or_rows(X) frame_bound(Y). { 
+  A = sqlite3WindowAlloc(pParse, X, Y.eType, Y.pExpr, TK_CURRENT, 0);
+}
+frame_opt(A) ::= range_or_rows(X) BETWEEN frame_bound(Y) AND frame_bound(Z). { 
+  A = sqlite3WindowAlloc(pParse, X, Y.eType, Y.pExpr, Z.eType, Z.pExpr);
+}
+
+range_or_rows(A) ::= RANGE.   { A = TK_RANGE; }
+range_or_rows(A) ::= ROWS.    { A = TK_ROWS;  }
+
+frame_bound(A) ::= UNBOUNDED PRECEDING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; }
+frame_bound(A) ::= expr(X) PRECEDING.   { A.eType = TK_PRECEDING; A.pExpr = X; }
+frame_bound(A) ::= CURRENT ROW.         { A.eType = TK_CURRENT  ; A.pExpr = 0; }
+frame_bound(A) ::= expr(X) FOLLOWING.   { A.eType = TK_FOLLOWING; A.pExpr = X; }
+frame_bound(A) ::= UNBOUNDED FOLLOWING. { A.eType = TK_UNBOUNDED; A.pExpr = 0; }
+
+%type windowdefn_opt {Window*}
+%destructor windowdefn_opt {sqlite3WindowDelete(pParse->db, $$);}
+windowdefn_opt(A) ::= . { A = 0; }
+windowdefn_opt(A) ::= WINDOW windowdefn_list(B). { A = B; }
+
+%type over_opt {Window*}
+%destructor over_opt {sqlite3WindowDelete(pParse->db, $$);}
+over_opt(A) ::= . { A = 0; }
+over_opt(A) ::= filter_opt(W) OVER window_or_nm(Z). {
+  A = Z;
+  if( A ) A->pFilter = W;
+}
+
+filter_opt(A) ::= .                            { A = 0; }
+filter_opt(A) ::= FILTER LP WHERE expr(X) RP.  { A = X; }
+%endif // SQLITE_OMIT_WINDOWFUNC
+

src/resolve.c

@@ -798,7 +798,11 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){
         pNC->nErr++;
       }
       if( is_agg ){
+#ifndef SQLITE_OMIT_WINDOWFUNC
         pNC->ncFlags &= ~(pExpr->pWin ? NC_AllowWin : NC_AllowAgg);
+#else
+        pNC->ncFlags &= ~NC_AllowAgg;
+#endif
       }
       sqlite3WalkExprList(pWalker, pList);
       if( is_agg ){

src/sqliteInt.h

@@ -4299,6 +4299,7 @@ char sqlite3IndexColumnAffinity(sqlite3*, Index*, int);
   void sqlite3ParserFree(void*, void(*)(void*));
 #endif
 void sqlite3Parser(void*, int, Token);
+int sqlite3ParserFallback(int);
 #ifdef YYTRACKMAXSTACKDEPTH
   int sqlite3ParserStackPeak(void*);
 #endif

src/tokenize.c

@@ -188,6 +188,82 @@ const char sqlite3IsEbcdicIdChar[] = {
 int sqlite3IsIdChar(u8 c){ return IdChar(c); }
 #endif
 
+#ifndef SQLITE_OMIT_WINDOWFUNC
+/*
+** Return the id of the next token in string (*pz). Before returning, set
+** (*pz) to point to the byte following the parsed token.
+*/
+static int getToken(const unsigned char **pz){
+  const unsigned char *z = *pz;
+  int t;                          /* Token type to return */
+  do {
+    z += sqlite3GetToken(z, &t);
+  }while( t==TK_SPACE );
+  if( t==TK_ID 
+   || t==TK_STRING 
+   || t==TK_JOIN_KW 
+   || t==TK_WINDOW 
+   || t==TK_OVER 
+   || sqlite3ParserFallback(t)==TK_ID 
+  ){
+    t = TK_ID;
+  }
+  *pz = z;
+  return t;
+}
+
+/*
+** The following three functions are called immediately after the tokenizer
+** reads the keywords WINDOW, OVER and FILTER, respectively, to determine
+** whether the token should be treated as a keyword or an SQL identifier.
+** This cannot be handled by the usual lemon %fallback method, due to
+** the ambiguity in some constructions. e.g.
+**
+**   SELECT sum(x) OVER ...
+**
+** In the above, "OVER" might be a keyword, or it might be an alias for the 
+** sum(x) expression. If a "%fallback ID OVER" directive were added to 
+** grammar, then SQLite would always treat "OVER" as an alias, making it
+** impossible to call a window-function without a FILTER clause.
+**
+** WINDOW is treated as a keyword if:
+**
+**   * the following token is an identifier, or a keyword that can fallback
+**     to being an identifier, and
+**   * the token after than one is TK_AS.
+**
+** OVER is a keyword if:
+**
+**   * the previous token was TK_RP, and
+**   * the next token is either TK_LP or an identifier.
+**
+** FILTER is a keyword if:
+**
+**   * the previous token was TK_RP, and
+**   * the next token is TK_LP.
+*/
+static int analyzeWindowKeyword(const unsigned char *z){
+  int t;
+  t = getToken(&z);
+  if( t!=TK_ID ) return TK_ID;
+  t = getToken(&z);
+  if( t!=TK_AS ) return TK_ID;
+  return TK_WINDOW;
+}
+static int analyzeOverKeyword(const unsigned char *z, int lastToken){
+  if( lastToken==TK_RP ){
+    int t = getToken(&z);
+    if( t==TK_LP || t==TK_ID ) return TK_OVER;
+  }
+  return TK_ID;
+}
+static int analyzeFilterKeyword(const unsigned char *z, int lastToken){
+  if( lastToken==TK_RP && getToken(&z)==TK_LP ){
+    return TK_FILTER;
+  }
+  return TK_ID;
+}
+#endif // SQLITE_OMIT_WINDOWFUNC
 
 /*
 ** Return the length (in bytes) of the token that begins at z[0]. 
@@ -509,44 +585,62 @@ int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
   assert( pParse->nVar==0 );
   assert( pParse->pVList==0 );
   while( 1 ){
-    if( zSql[0]!=0 ){
+    n = sqlite3GetToken((u8*)zSql, &tokenType);
-      n = sqlite3GetToken((u8*)zSql, &tokenType);
+    mxSqlLen -= n;
-      mxSqlLen -= n;
+    if( mxSqlLen<0 ){
-      if( mxSqlLen<0 ){
+      pParse->rc = SQLITE_TOOBIG;
-        pParse->rc = SQLITE_TOOBIG;
+      break;
-        break;
-      }
-    }else{
-      /* Upon reaching the end of input, call the parser two more times
-      ** with tokens TK_SEMI and 0, in that order. */
-      if( lastTokenParsed==TK_SEMI ){
-        tokenType = 0;
-      }else if( lastTokenParsed==0 ){
-        break;
-      }else{
-        tokenType = TK_SEMI;
-      }
-      n = 0;
     }
+#ifndef SQLITE_OMIT_WINDOWFUNC
+    if( tokenType>=TK_WINDOW ){
+      assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER
+           || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW 
+      );
+#else
     if( tokenType>=TK_SPACE ){
       assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL );
+#endif // SQLITE_OMIT_WINDOWFUNC
       if( db->u1.isInterrupted ){
         pParse->rc = SQLITE_INTERRUPT;
         break;
       }
-      if( tokenType==TK_ILLEGAL ){
+      if( tokenType==TK_SPACE ){
+        zSql += n;
+        continue;
+      }
+      if( zSql[0]==0 ){
+        /* Upon reaching the end of input, call the parser two more times
+        ** with tokens TK_SEMI and 0, in that order. */
+        if( lastTokenParsed==TK_SEMI ){
+          tokenType = 0;
+        }else if( lastTokenParsed==0 ){
+          break;
+        }else{
+          tokenType = TK_SEMI;
+        }
+        n = 0;
+#ifndef SQLITE_OMIT_WINDOWFUNC
+      }else if( tokenType==TK_WINDOW ){
+        assert( n==6 );
+        tokenType = analyzeWindowKeyword((const u8*)&zSql[6]);
+      }else if( tokenType==TK_OVER ){
+        assert( n==4 );
+        tokenType = analyzeOverKeyword((const u8*)&zSql[4], lastTokenParsed);
+      }else if( tokenType==TK_FILTER ){
+        assert( n==6 );
+        tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed);
+#endif // SQLITE_OMIT_WINDOWFUNC
+      }else{
         sqlite3ErrorMsg(pParse, "unrecognized token: \"%.*s\"", n, zSql);
         break;
       }
-      zSql += n;
-    }else{
-      pParse->sLastToken.z = zSql;
-      pParse->sLastToken.n = n;
-      sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
-      lastTokenParsed = tokenType;
-      zSql += n;
-      if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
     }
+    pParse->sLastToken.z = zSql;
+    pParse->sLastToken.n = n;
+    sqlite3Parser(pEngine, tokenType, pParse->sLastToken);
+    lastTokenParsed = tokenType;
+    zSql += n;
+    if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break;
   }
   assert( nErr==0 );
 #ifdef YYTRACKMAXSTACKDEPTH

test/window6.test

@@ -0,0 +1,143 @@
+# 2018 May 8
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library. Specifically,
+# it tests the sqlite3_create_window_function() API.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix window6
+
+ifcapable !windowfunc {
+  finish_test
+  return
+}
+
+set setup {
+  CREATE TABLE %t1(%x, %y %typename);
+  INSERT INTO %t1 VALUES(1, 'a');
+  INSERT INTO %t1 VALUES(2, 'b');
+  INSERT INTO %t1 VALUES(3, 'c');
+  INSERT INTO %t1 VALUES(4, 'd');
+  INSERT INTO %t1 VALUES(5, 'e');
+}
+
+foreach {tn vars} {
+  1 {}
+  2 { set A(%t1) over }
+  3 { set A(%x)  over }
+  4 { 
+    set A(%alias)   over 
+    set A(%x)       following 
+    set A(%y)       over 
+  }
+  5 { 
+    set A(%t1)      over
+    set A(%x)       following 
+    set A(%y)       preceding 
+    set A(%w)       current 
+    set A(%alias)   filter
+    set A(%typename)  window
+  }
+
+  6 { 
+    set A(%x)       window 
+  }
+} {
+  set A(%t1)    t1
+  set A(%x)     x
+  set A(%y)     y
+  set A(%w)     w
+  set A(%alias) alias
+  set A(%typename) integer
+  eval $vars
+
+  set MAP [array get A]
+  set setup_sql [string map $MAP $setup]
+  reset_db
+  execsql $setup_sql
+
+  do_execsql_test 1.$tn.1 [string map $MAP {
+    SELECT group_concat(%x, '.') OVER (ORDER BY %y) FROM %t1
+  }] {1 1.2 1.2.3 1.2.3.4 1.2.3.4.5}
+
+  do_execsql_test 1.$tn.2 [string map $MAP {
+    SELECT sum(%x) OVER %w FROM %t1 WINDOW %w AS (ORDER BY %y)
+  }] {1 3 6 10 15}
+
+  do_execsql_test 1.$tn.3 [string map $MAP {
+    SELECT sum(%alias.%x) OVER %w FROM %t1 %alias WINDOW %w AS (ORDER BY %y)
+  }] {1 3 6 10 15}
+
+  do_execsql_test 1.$tn.4 [string map $MAP {
+    SELECT sum(%x) %alias FROM %t1
+  }] {15}
+}
+
+
+proc winproc {args} { return "window: $args" }
+db func window winproc
+do_execsql_test 2.0 {
+  SELECT window('hello world');
+} {{window: {hello world}}}
+
+proc wincmp {a b} { string compare $b $a }
+db collate window wincmp
+do_execsql_test 3.0 {
+  CREATE TABLE window(x COLLATE window);
+  INSERT INTO window VALUES('bob'), ('alice'), ('cate');
+  SELECT * FROM window ORDER BY x COLLATE window;
+} {cate bob alice}
+do_execsql_test 3.1 {
+  DROP TABLE window;
+  CREATE TABLE x1(x);
+  INSERT INTO x1 VALUES('bob'), ('alice'), ('cate');
+  CREATE INDEX window ON x1(x COLLATE window);
+  SELECT * FROM x1 ORDER BY x COLLATE window;
+} {cate bob alice}
+
+
+do_execsql_test 4.0 { CREATE TABLE t4(x, y); }
+
+# do_execsql_test 4.1 { PRAGMA parser_trace = 1 }
+do_execsql_test 4.1 { 
+  SELECT * FROM t4 window, t4;
+}
+
+#-------------------------------------------------------------------------
+reset_db
+
+do_execsql_test 5.0 {
+  CREATE TABLE over(x, over);
+  CREATE TABLE window(x, window);
+  INSERT INTO over VALUES(1, 2), (3, 4), (5, 6);
+  INSERT INTO window VALUES(1, 2), (3, 4), (5, 6);
+  SELECT sum(x) over FROM over
+} {9}
+
+do_execsql_test 5.1 {
+  SELECT sum(x) over over FROM over WINDOW over AS ()
+} {9 9 9}
+
+do_execsql_test 5.2 {
+  SELECT sum(over) over over over FROM over over WINDOW over AS (ORDER BY over)
+} {2 6 12}
+
+do_execsql_test 5.3 {
+  SELECT sum(over) over over over FROM over over WINDOW over AS (ORDER BY over);
+} {2 6 12}
+
+do_execsql_test 5.4 {
+  SELECT sum(window) OVER window window FROM window window window window AS (ORDER BY window);
+} {2 6 12}
+
+finish_test
+

tool/lempar.c

@@ -90,6 +90,7 @@
 /************* Begin control #defines *****************************************/
 %%
 /************* End control #defines *******************************************/
+#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])))
 
 /* Define the yytestcase() macro to be a no-op if is not already defined
 ** otherwise.
@@ -519,11 +520,11 @@ static YYACTIONTYPE yy_find_shift_action(
   do{
     i = yy_shift_ofst[stateno];
     assert( i>=0 );
-    assert( i+YYNTOKEN<=(int)sizeof(yy_lookahead)/sizeof(yy_lookahead[0]) );
+    /* assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); */
     assert( iLookAhead!=YYNOCODE );
     assert( iLookAhead < YYNTOKEN );
     i += iLookAhead;
-    if( yy_lookahead[i]!=iLookAhead ){
+    if( i>=YY_NLOOKAHEAD || yy_lookahead[i]!=iLookAhead ){
 #ifdef YYFALLBACK
       YYCODETYPE iFallback;            /* Fallback token */
       if( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0])
@@ -1057,3 +1058,16 @@ void Parse(
 #endif
   return;
 }
+
+/*
+** Return the fallback token corresponding to canonical token iToken, or
+** 0 if iToken has no fallback.
+*/
+int ParseFallback(int iToken){
+#ifdef YYFALLBACK
+  if( iToken<sizeof(yyFallback)/sizeof(yyFallback[0]) ){
+    return yyFallback[iToken];
+  }
+#endif
+  return 0;
+}