Instead of using a lemon %fallback directive, have the tokenizer try to figure

out whether an instance of "WINDOW" should be TK_WINDOW or TK_ID. FossilOrigin-Name: 022079cb0d67be5ac0a50dd9a4d41ee55ce8df681ecd0a544170d75fc8649978
2025-12-24 14:17:58 +03:00 · 2018-06-29 17:44:52 +00:00
parent 04a8e0cd54
commit 59ff42516c
7 changed files with 208 additions and 23 deletions
--- a/19
+++ b/19
@@ -1,5 +1,5 @@
-C Modifications\sto\sparse.y\sto\sbetter\ssupport\sbackwards\scompatibility\sfor\sthe\n"window"\skeyword.
-D 2018-06-28T20:05:04.235
+C Instead\sof\susing\sa\slemon\s%fallback\sdirective,\shave\sthe\stokenizer\stry\sto\sfigure\nout\swhether\san\sinstance\sof\s"WINDOW"\sshould\sbe\sTK_WINDOW\sor\sTK_ID.
+D 2018-06-29T17:44:52.692
 F .fossil-settings/empty-dirs dbb81e8fc0401ac46a1491ab34a7f2c7c0452f2f06b54ebb845d024ca8283ef1
 F .fossil-settings/ignore-glob 35175cdfcf539b2318cb04a9901442804be81cd677d8b889fcc9149c21f239ea
 F Makefile.in 0a3a6c81e6fcb969ff9106e882f0a08547014ba463cb6beca4c4efaecc924ee6
@@ -484,7 +484,7 @@ F src/os_win.c ac29c25cde4cfb4adacc59cdec4aa45698ca0e29164ea127859585ccd9faa354
 F src/os_win.h 7b073010f1451abe501be30d12f6bc599824944a
 F src/pager.c 1bb6a57fa0465296a4d6109a1a64610a0e7adde1f3acf3ef539a9d972908ce8f
 F src/pager.h c571b064df842ec8f2e90855dead9acf4cbe0d1b2c05afe0ef0d0145f7fd0388
-F src/parse.y 9b5eb0e61e615832647a7a77f9e675f1da536a133426327e6812316d83f275e7
+F src/parse.y 216e0e953df076ef89668614e5aab8b489e0c2c24f7ff17cf42e7bea802793a9
 F src/pcache.c 135ef0bc6fb2e3b7178d49ab5c9176254c8a691832c1bceb1156b2fbdd0869bd
 F src/pcache.h 072f94d29281cffd99e46c1539849f248c4b56ae7684c1f36626797fee375170
 F src/pcache1.c 716975564c15eb6679e97f734cec1bfd6c16ac3d4010f05f1f8e509fc7d19880
@@ -500,7 +500,7 @@ F src/shell.c.in 8578421c5fb2a972461b2a996f7173646e55e0dbd2a2eee30c8f5dc7d3dbadf
 F src/sqlite.h.in 243e0b1d302c237372c06002fe84f731fd22587eccd142d15de834a5296514ff
 F src/sqlite3.rc 5121c9e10c3964d5755191c80dd1180c122fc3a8
 F src/sqlite3ext.h 9887b27e69c01e79c2cbe74ef73bf01af5b5703d6a7f0a4371e386d7249cb1c7
-F src/sqliteInt.h 74728a0b20e9d2173470186a8ca1e3229c79abb19d1ac4eedc4562e8905d798d
+F src/sqliteInt.h c14c850bf0c4dd3860cc5092f140ae9faf93b77737b3febf38454088e1ab3988
 F src/sqliteLimit.h 1513bfb7b20378aa0041e7022d04acb73525de35b80b252f1b83fedb4de6a76b
 F src/status.c 46e7aec11f79dad50965a5ca5fa9de009f7d6bde08be2156f1538a0a296d4d0e
 F src/table.c b46ad567748f24a326d9de40e5b9659f96ffff34
@@ -558,7 +558,7 @@ F src/test_windirent.h 90dfbe95442c9762357fe128dc7ae3dc199d006de93eb33ba3972e0a9
 F src/test_window.c add59ee68568868129516999f30a68e8ab2afd276e272aba4f633c9fc52c1bb1
 F src/test_wsd.c 41cadfd9d97fe8e3e4e44f61a4a8ccd6f7ca8fe9
 F src/threads.c 4ae07fa022a3dc7c5beb373cf744a85d3c5c6c3c
-F src/tokenize.c b42aa952b05df665ed10d54f02c467159cd9fcb8cbbe5ff3c9cd74781fa2803c
+F src/tokenize.c e25caaeb2ad912a378e528771d843badbaee560f2eb8c25dbface05a32c2e4cf
 F src/treeview.c 2c5c4bc0a443401db5fd621542150452ddf5055d38edd4eef868bc2b6bfb0260
 F src/trigger.c 4ace6d1d5ba9a89822deb287317f33c810440526eafe185c2d8a48c31df1e995
 F src/update.c 46dc24c6158446aaab45caee09b6d99327cb479268b83ffeb5b701823da3b67b
@@ -1626,6 +1626,7 @@ F test/window3.test 47884c240d0d5234ad9c6da65452cfec1bfa69ec6f9c4158cd9750c3d88d
 F test/window4.tcl 7cec7e578aa9f78b7265bff8d552cda17a1d8d89f0449d0e74970a527b8846f5
 F test/window4.test dcd8767869988e0d23d56bc3f8b46ec116de23127b81b5f66fd48d5529072ed1
 F test/window5.test a4835b96d30eb3b81a1dbc683e333e57a614645eb6f2ae476a7ed2addf0b0f1f
+F test/window6.test 97bd18d5cccd612b8a93be4017a7a26c2968a1c868a458d0d66d8d8cc62e33f8
 F test/windowfault.test 97d5fc404308edb579a5a183e294ed874c844ecf01f0a28ba46df3141ebaee1f
 F test/with1.test 58475190cd8caaeebea8cfeb2a264ec97a0c492b8ffe9ad20cefbb23df462f96
 F test/with2.test e0030e2f0267a910d6c0e4f46f2dfe941c1cc0d4f659ba69b3597728e7e8f1ab
@@ -1663,7 +1664,7 @@ F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
 F tool/kvtest-speed.sh 4761a9c4b3530907562314d7757995787f7aef8f
 F tool/lemon.c 93a321bd82b5c73191e95b2f27203048bf4d054fdab4a257b7a6c9541f31828d
-F tool/lempar.c c7371bad027fd516b7eaed36d03353972df35163672637cddd00f472889932f9
+F tool/lempar.c afc03d26f33db130c0ca7d5942000aea7eedd638455845ca76f792818896df19
 F tool/libvers.c caafc3b689638a1d88d44bc5f526c2278760d9b9
 F tool/loadfts.c c3c64e4d5e90e8ba41159232c2189dba4be7b862
 F tool/logest.c 11346aa019e2e77a00902aa7d0cabd27bd2e8cca
@@ -1743,7 +1744,7 @@ F vsixtest/vsixtest.tcl 6a9a6ab600c25a91a7acc6293828957a386a8a93
 F vsixtest/vsixtest.vcxproj.data 2ed517e100c66dc455b492e1a33350c1b20fbcdc
 F vsixtest/vsixtest.vcxproj.filters 37e51ffedcdb064aad6ff33b6148725226cd608e
 F vsixtest/vsixtest_TemporaryKey.pfx e5b1b036facdb453873e7084e1cae9102ccc67a0
-P c41d7079fb63831bd2f009b64cbf5a423e8119e6b841bfd9bd5a2d85fba3b142
-R 4e74da2117f8f8cceb7bb5b15336dc74
+P 7c4b879bdb10f30260f0fb28fcb559ab0949865d10f4ff2cf13834a326198998
+R 53ad10aee784d2508443a129bf20d430
 U dan
-Z e15fe89d6d9f69693116ca5d610ea195
+Z a766b00041899c4982e4087a15ea2a82
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-7c4b879bdb10f30260f0fb28fcb559ab0949865d10f4ff2cf13834a326198998
+022079cb0d67be5ac0a50dd9a4d41ee55ce8df681ecd0a544170d75fc8649978
--- a/src/parse.y
+++ b/src/parse.y
@@ -217,8 +217,8 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}
  EXCEPT INTERSECT UNION
 %endif SQLITE_OMIT_COMPOUND_SELECT
 %ifndef SQLITE_OMIT_WINDOWFUNC
-  CURRENT FILTER FOLLOWING OVER PARTITION 
-  PRECEDING RANGE UNBOUNDED 
+  CURRENT FILTER FOLLOWING OVER PARTITION
+  PRECEDING RANGE UNBOUNDED
 %endif SQLITE_OMIT_WINDOWFUNC
  REINDEX RENAME CTIME_KW IF
  .
@@ -257,7 +257,7 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}

 // And "ids" is an identifer-or-string.
 //
-%token_class ids ID|STRING.
+%token_class ids  ID|STRING.

 // The name of a column or table can be any of the following:
 //
@@ -265,7 +265,6 @@ columnname(A) ::= nm(A) typetoken(Y). {sqlite3AddColumn(pParse,&A,&Y);}
 nm(A) ::= id(A).
 nm(A) ::= STRING(A).
 nm(A) ::= JOIN_KW(A).
-nm(A) ::= WINDOW(A).

 // A typetoken is really zero or more tokens that form a type name such
 // as can be found after the column name in a CREATE TABLE statement.
@@ -281,8 +280,8 @@ typetoken(A) ::= typename(A) LP signed COMMA signed RP(Y). {
  A.n = (int)(&Y.z[Y.n] - A.z);
 }
 %type typename {Token}
-typename(A) ::= nm(A).
-typename(A) ::= typename(A) nm(Y). {A.n=Y.n+(int)(Y.z-A.z);}
+typename(A) ::= ids(A).
+typename(A) ::= typename(A) ids(Y). {A.n=Y.n+(int)(Y.z-A.z);}
 signed ::= plus_num.
 signed ::= minus_num.

@@ -343,7 +342,7 @@ ccons ::= CHECK LP expr(X) RP.   {sqlite3AddCheckConstraint(pParse,X);}
 ccons ::= REFERENCES nm(T) eidlist_opt(TA) refargs(R).
                                 {sqlite3CreateForeignKey(pParse,0,&T,TA,R);}
 ccons ::= defer_subclause(D).    {sqlite3DeferForeignKey(pParse,D);}
-ccons ::= COLLATE nm(C).        {sqlite3AddCollateType(pParse, &C);}
+ccons ::= COLLATE ids(C).        {sqlite3AddCollateType(pParse, &C);}

 // The optional AUTOINCREMENT keyword
 %type autoinc {int}
@@ -989,7 +988,6 @@ expr(A) ::= nm(X) DOT nm(Y) DOT nm(Z). {
 }
 term(A) ::= NULL|FLOAT|BLOB(X). {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/}
 term(A) ::= STRING(X).          {A=tokenExpr(pParse,@X,X); /*A-overwrites-X*/}
-term(A) ::= WINDOW(X).          {A=tokenExpr(pParse,TK_ID,X);/*A-overwrites-X*/}
 term(A) ::= INTEGER(X). {
  A = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &X, 1);
 }
@@ -1013,7 +1011,7 @@ expr(A) ::= VARIABLE(X).     {
    }
  }
 }
-expr(A) ::= expr(A) COLLATE nm(C). {
+expr(A) ::= expr(A) COLLATE ids(C). {
  A = sqlite3ExprAddCollateToken(pParse, A, &C, 1);
 }
 %ifndef SQLITE_OMIT_CAST
@@ -1022,7 +1020,7 @@ expr(A) ::= CAST LP expr(E) AS typetoken(T) RP. {
  sqlite3ExprAttachSubtrees(pParse->db, A, E, 0);
 }
 %endif  SQLITE_OMIT_CAST
-expr(A) ::= nm(X) LP distinct(D) exprlist(Y) RP 
+expr(A) ::= id(X) LP distinct(D) exprlist(Y) RP 
 %ifndef SQLITE_OMIT_WINDOWFUNC
  over_opt(Z)
 %endif
@@ -1036,7 +1034,7 @@ expr(A) ::= nm(X) LP distinct(D) exprlist(Y) RP
    A->flags |= EP_Distinct;
  }
 }
-expr(A) ::= nm(X) LP STAR RP
+expr(A) ::= id(X) LP STAR RP
 %ifndef SQLITE_OMIT_WINDOWFUNC
  over_opt(Z)
 %endif
@@ -1445,7 +1443,7 @@ eidlist(A) ::= nm(Y) collate(C) sortorder(Z). {

 %type collate {int}
 collate(C) ::= .              {C = 0;}
-collate(C) ::= COLLATE nm.    {C = 1;}
+collate(C) ::= COLLATE ids.   {C = 1;}


 ///////////////////////////// The DROP INDEX command /////////////////////////
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -4299,6 +4299,7 @@ char sqlite3IndexColumnAffinity(sqlite3*, Index*, int);
  void sqlite3ParserFree(void*, void(*)(void*));
 #endif
 void sqlite3Parser(void*, int, Token);
+int sqlite3ParserFallback(int);
 #ifdef YYTRACKMAXSTACKDEPTH
  int sqlite3ParserStackPeak(void*);
 #endif
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -188,6 +188,55 @@ const char sqlite3IsEbcdicIdChar[] = {
 int sqlite3IsIdChar(u8 c){ return IdChar(c); }
 #endif

+/*
+** Return the id of the next token in string (*pz). Before returning, set
+** (*pz) to point to the byte following the parsed token.
+**
+** This function assumes that any keywords that start with "w" are 
+** actually TK_ID.
+*/
+static int windowGetToken(const unsigned char **pz){
+  int ret;
+  const unsigned char *z = *pz;
+  if( z[0]=='w' || z[0]=='W' ){
+    do { z++; }while( IdChar(z[0]) );
+    ret = TK_ID;
+  }else{
+    z += sqlite3GetToken(z, &ret);
+  }
+  *pz = z;
+  return ret;
+}
+
+/*
+** The tokenizer has just parsed the keyword WINDOW. In this case the token
+** may really be the keyword (TK_WINDOW), or may be an identifier (TK_ID).
+** This function determines which it is by inspecting the next two tokens
+** in the input stream. Specifically, the token is TK_WINDOW if the following
+** two tokens are:
+**
+**   * TK_ID, or something else that can be used as a window name, and
+**   * TK_AS.
+**
+** Instead of using sqlite3GetToken() to parse tokens directly, this function
+** uses windowGetToken(). This is to avoid recursion if the input is similar
+** to "window window window window".
+*/
+static void analyzeWindowKeyword(const unsigned char *z, int *tokenType){
+  int t;
+  assert( *tokenType==TK_WINDOW );
+  while( (t = windowGetToken(&z))==TK_SPACE );
+  if( t!=TK_ID && t!=TK_STRING 
+   && t!=TK_JOIN_KW && sqlite3ParserFallback(t)!=TK_ID 
+  ){
+    *tokenType = TK_ID;
+  }else{
+    while( (t = windowGetToken(&z))==TK_SPACE );
+    if( t!=TK_AS ){
+      *tokenType = TK_ID;
+    }
+  }
+}

 /*
 ** Return the length (in bytes) of the token that begins at z[0]. 
@@ -433,7 +482,12 @@ int sqlite3GetToken(const unsigned char *z, int *tokenType){
        break;
      }
      *tokenType = TK_ID;
-      return keywordCode((char*)z, i, tokenType);
+      keywordCode((char*)z, i, tokenType);
+      if( *tokenType==TK_WINDOW ){
+        assert( i==6 );
+        analyzeWindowKeyword(&z[6], tokenType);
+      }
+      return i;
    }
    case CC_X: {
 #ifndef SQLITE_OMIT_BLOB_LITERAL
--- a/test/window6.test
+++ b/test/window6.test
@@ -0,0 +1,117 @@
+# 2018 May 8
+#
+# The author disclaims copyright to this source code.  In place of
+# a legal notice, here is a blessing:
+#
+#    May you do good and not evil.
+#    May you find forgiveness for yourself and forgive others.
+#    May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library. Specifically,
+# it tests the sqlite3_create_window_function() API.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+set testprefix window6
+
+ifcapable !windowfunc {
+  finish_test
+  return
+}
+
+set setup {
+  CREATE TABLE %t1(%a, %b %typename);
+  INSERT INTO %t1 VALUES(1, 'a');
+  INSERT INTO %t1 VALUES(2, 'b');
+  INSERT INTO %t1 VALUES(3, 'c');
+  INSERT INTO %t1 VALUES(4, 'd');
+  INSERT INTO %t1 VALUES(5, 'e');
+}
+
+foreach {tn vars} {
+  1 {}
+  2 { set A(%t1) over }
+  3 { set A(%a)  over }
+  4 { 
+    set A(%alias)   over 
+    set A(%a)       following 
+    set A(%b)       over 
+  }
+  5 { 
+    set A(%t1)      over 
+    set A(%a)       following 
+    set A(%b)       preceding 
+    set A(%w)       current 
+    set A(%alias)   filter 
+    set A(%typename)  window 
+  }
+
+  6 { 
+    set A(%a)       window 
+  }
+} {
+  set A(%t1)    t1
+  set A(%a)     a
+  set A(%b)     b
+  set A(%w)     w
+  set A(%alias) alias
+  set A(%typename) integer
+  eval $vars
+
+  set MAP [array get A]
+  set setup_sql [string map $MAP $setup]
+  reset_db
+  execsql $setup_sql
+
+  do_execsql_test 1.$tn.1 [string map $MAP {
+    SELECT group_concat(%a, '.') OVER (ORDER BY %b) FROM %t1
+  }] {1 1.2 1.2.3 1.2.3.4 1.2.3.4.5}
+
+  do_execsql_test 1.$tn.2 [string map $MAP {
+    SELECT sum(%a) OVER %w FROM %t1 WINDOW %w AS (ORDER BY %b)
+  }] {1 3 6 10 15}
+
+  do_execsql_test 1.$tn.3 [string map $MAP {
+    SELECT sum(%alias.%a) OVER %w FROM %t1 %alias WINDOW %w AS (ORDER BY %b)
+  }] {1 3 6 10 15}
+
+  do_execsql_test 1.$tn.4 [string map $MAP {
+    SELECT sum(%a) %alias FROM %t1
+  }] {15}
+}
+
+
+proc winproc {args} { return "window: $args" }
+db func window winproc
+do_execsql_test 2.0 {
+  SELECT window('hello world');
+} {{window: {hello world}}}
+
+proc wincmp {a b} { string compare $b $a }
+db collate window wincmp
+do_execsql_test 3.0 {
+  CREATE TABLE window(x COLLATE window);
+  INSERT INTO window VALUES('bob'), ('alice'), ('cate');
+  SELECT * FROM window ORDER BY x COLLATE window;
+} {cate bob alice}
+do_execsql_test 3.1 {
+  DROP TABLE window;
+  CREATE TABLE x1(x);
+  INSERT INTO x1 VALUES('bob'), ('alice'), ('cate');
+  CREATE INDEX window ON x1(x COLLATE window);
+  SELECT * FROM x1 ORDER BY x COLLATE window;
+} {cate bob alice}
+
+
+do_execsql_test 4.0 { CREATE TABLE t4(x, y); }
+
+# do_execsql_test 4.1 { PRAGMA parser_trace = 1 }
+do_execsql_test 4.1 { 
+  SELECT * FROM t4 window, t4;
+}
+
+
+finish_test
+
--- a/tool/lempar.c
+++ b/tool/lempar.c
@@ -1068,3 +1068,17 @@ void Parse(
 #endif
  return;
 }
+
+/*
+** Return the fallback token corresponding to canonical token iToken, or
+** 0 if iToken has no fallback.
+*/
+int ParseFallback(int iToken){
+#ifdef YYFALLBACK
+  if( iToken<sizeof(yyFallback)/sizeof(yyFallback[0]) ){
+    return yyFallback[iToken];
+  }
+#endif
+  return 0;
+}
+