Added the %fallback directive to the lemon parser generator and used this

in the parser to make the parse tables much smaller.  This reduced the size
of the library by 15K. (CVS 605)

FossilOrigin-Name: 7ac5bd293cbb2bf252f31f1571f7efac7e77280a

manifest

@@ -1,5 +1,5 @@
-C Fix\sa\scompiler\swarning.\s(CVS\s604)
+C Added\sthe\s%fallback\sdirective\sto\sthe\slemon\sparser\sgenerator\sand\sused\sthis\nin\sthe\sparser\sto\smake\sthe\sparse\stables\smuch\ssmaller.\s\sThis\sreduced\sthe\ssize\nof\sthe\slibrary\sby\s15K.\s(CVS\s605)
-D 2002-06-02T18:22:06
+D 2002-06-06T18:54:40
 F Makefile.in 6291a33b87d2a395aafd7646ee1ed562c6f2c28c
 F Makefile.template 4e11752e0b5c7a043ca50af4296ec562857ba495
 F README a4c0ba11354ef6ba0776b400d057c59da47a4cc0
@@ -27,21 +27,21 @@ F src/expr.c 8ce9c22655735ff62b1e33ab11ad9d44c4ab99c6
 F src/func.c 061a520a122da7e4f9dcac15697bb996aac7d5df
 F src/hash.c 6a6236b89c8c060c65dabd300a1c8ce7c10edb72
 F src/hash.h dca065dda89d4575f3176e75e9a3dc0f4b4fb8b9
-F src/insert.c 24b4e146319bada6f82a1d5eae6b38b3065d132f
+F src/insert.c 4b0bd94296fea46ef1b2ed8bfd05e12a38ce2c90
 F src/main.c 6e53c49a390fabd5fecce9e3b128c61c85208000
 F src/md5.c 0ae1f3e2cac92d06fc6246d1b4b8f61a2fe66d3b
 F src/os.c 9cc40c5384baba4a85e160e67807645ca98ba3cc
 F src/os.h 4a361fccfbc4e7609b3e1557f604f94c1e96ad10
 F src/pager.c 1e41053c949cea1f09d8dafada5fe8f90785e650
 F src/pager.h 6fddfddd3b73aa8abc081b973886320e3c614f0e
-F src/parse.y 68c0ab3d6bc938d1edcd087a18f28246c763076a
+F src/parse.y e8c65150e8f581da6cc0bc5a87063ed7a2b28564
 F src/printf.c d8032ee18b860c812eeff596c9bebfdacb7930fd
 F src/random.c 19e8e00fe0df32a742f115773f57651be327cabe
-F src/select.c ad9061b4735ccd79fc073415979882cd5c424c71
+F src/select.c 0293ec0190d9a991725579a5e9c3af2fb6c1b592
 F src/shell.c 1d22fe870ee852cfb975fd000dbe3973713d0a15
 F src/shell.tcl 27ecbd63dd88396ad16d81ab44f73e6c0ea9d20e
 F src/sqlite.h.in 0038faa6d642de06b91143ee65a131bd831d020b
-F src/sqliteInt.h 8ec47ae045cf8525c2bcc1a650853d814ca7675c
+F src/sqliteInt.h 3fd61a32c101b10aea610de8e7d931744657712f
 F src/table.c eed2098c9b577aa17f8abe89313a9c4413f57d63
 F src/tclsqlite.c 9300c9606a38bc0c75d6c0bc8a6197ab979353d1
 F src/test1.c 09d95048b66ce6dcd2bae90f443589043d7d631e
@@ -108,8 +108,8 @@ F test/update.test a0aa0bf83e6fad8407d0e4ad25ebb09b513f5bf4
 F test/vacuum.test 059871b312eb910bbe49dafde1d01490cc2c6bbe
 F test/view.test b9851e9142de5e5831fdf18f125cbe1256cb550a
 F test/where.test 1d85a7eba93e7acc0a971c6d9daead0e49cb023a
-F tool/lemon.c 77d026f58d7715543786d457cf9432f9103e3f62
+F tool/lemon.c 459cb2bb3738a1ad5cb0ad8b805587a88a885d95
-F tool/lempar.c ee508b94607f74d591d60eda5c8014db4e144de5
+F tool/lempar.c 73a991cc3017fb34804250fa901488b5147b3717
 F tool/memleak.awk 296dfbce7a9ca499b95ce04e30334e64a50052e0
 F tool/opNames.awk 5ba1f48aa854ee3b7c3d2b54233665bc3e649ea2
 F tool/opcodeDoc.awk b3a2a3d5d3075b8bd90b7afe24283efdd586659c
@@ -136,7 +136,7 @@ F www/speed.tcl da8afcc1d3ccc5696cfb388a68982bc3d9f7f00f
 F www/sqlite.tcl 8b5884354cb615049aed83039f8dfe1552a44279
 F www/tclsqlite.tcl 1db15abeb446aad0caf0b95b8b9579720e4ea331
 F www/vdbe.tcl 2013852c27a02a091d39a766bc87cff329f21218
-P 6fdcee3c99e994ef3ab83a0cc57344cdb16210df
+P 637ee587b5438c54ba2d8bd8fc15e584abb70946
-R 84cb93cd645bc6a21c3f645c617f9a5e
+R d5040ca10de549792ac65f5e92bf037a
 U drh
-Z 6dc156e6736d2538f2d30b5501e72ea4
+Z 67f180b15375cb80ef3d22765a7c3016

manifest.uuid

@@ -1 +1 @@
-637ee587b5438c54ba2d8bd8fc15e584abb70946
+7ac5bd293cbb2bf252f31f1571f7efac7e77280a

src/insert.c

@@ -12,7 +12,7 @@
 ** This file contains C code routines that are called by the parser
 ** to handle INSERT statements in SQLite.
 **
-** $Id: insert.c,v 1.59 2002/05/26 20:54:33 drh Exp $
+** $Id: insert.c,v 1.60 2002/06/06 18:54:40 drh Exp $
 */
 #include "sqliteInt.h"
 
@@ -551,8 +551,13 @@ void sqliteGenerateConstraintChecks(
 
   /* Test all CHECK constraints
   */
+  /**** TBD ****/
 
-  /* Test all UNIQUE constraints.  Add index records as we go.
+  /* If we have an INTEGER PRIMARY KEY, make sure the primary key
+  ** of the new record does not previously exist.  Except, if this
+  ** is an UPDATE and the primary key is not changing, that is OK.
+  ** Also, if the conflict resolution policy is REPLACE, then we
+  ** can skip this test.
   */
   if( (recnoChng || !isUpdate) && pTab->iPKey>=0 ){
     onError = pTab->keyConf;
@@ -593,6 +598,11 @@ void sqliteGenerateConstraintChecks(
       }
     }
   }
+
+  /* Test all UNIQUE constraints by creating entries for each UNIQUE
+  ** index and making sure that duplicate entries do not already exist.
+  ** Add the new records to the indices as we go.
+  */
   extra = 0;
   for(extra=(-1), iCur=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, iCur++){
     if( aIdxUsed && aIdxUsed[iCur]==0 ) continue;
@@ -642,7 +652,9 @@ void sqliteGenerateConstraintChecks(
       default: assert(0);
     }
     contAddr = sqliteVdbeCurrentAddr(v);
+#if NULL_DISTINCT_FOR_UNIQUE
     sqliteVdbeChangeP2(v, jumpInst1, contAddr);
+#endif
     sqliteVdbeChangeP2(v, jumpInst2, contAddr);
   }
 }

src/parse.y

@@ -14,7 +14,7 @@
 ** the parser.  Lemon will also generate a header file containing
 ** numeric codes for all of the tokens.
 **
-** @(#) $Id: parse.y,v 1.70 2002/06/02 18:19:00 drh Exp $
+** @(#) $Id: parse.y,v 1.71 2002/06/06 18:54:40 drh Exp $
 */
 %token_prefix TK_
 %token_type {Token}
@@ -110,44 +110,51 @@ columnid ::= ids(X).                {sqliteAddColumn(pParse,&X);}
 // keywords.  Any non-standard keyword can also be an identifier.
 //
 %type id {Token}
-id(A) ::= ABORT(X).      {A = X;}
+//id(A) ::= ABORT(X).      {A = X;}
-id(A) ::= AFTER(X).      {A = X;}
+//id(A) ::= AFTER(X).      {A = X;}
-id(A) ::= ASC(X).        {A = X;}
+//id(A) ::= ASC(X).        {A = X;}
-id(A) ::= BEFORE(X).     {A = X;}
+//id(A) ::= BEFORE(X).     {A = X;}
-id(A) ::= BEGIN(X).      {A = X;}
+//id(A) ::= BEGIN(X).      {A = X;}
-id(A) ::= CASCADE(X).    {A = X;}
+//id(A) ::= CASCADE(X).    {A = X;}
-id(A) ::= CLUSTER(X).    {A = X;}
+//id(A) ::= CLUSTER(X).    {A = X;}
-id(A) ::= CONFLICT(X).   {A = X;}
+//id(A) ::= CONFLICT(X).   {A = X;}
-id(A) ::= COPY(X).       {A = X;}
+//id(A) ::= COPY(X).       {A = X;}
-id(A) ::= DEFERRED(X).   {A = X;}
+//id(A) ::= DEFERRED(X).   {A = X;}
-id(A) ::= DELIMITERS(X). {A = X;}
+//id(A) ::= DELIMITERS(X). {A = X;}
-id(A) ::= DESC(X).       {A = X;}
+//id(A) ::= DESC(X).       {A = X;}
-id(A) ::= EACH(X).       {A = X;}
+//id(A) ::= EACH(X).       {A = X;}
-id(A) ::= END(X).        {A = X;}
+//id(A) ::= END(X).        {A = X;}
-id(A) ::= EXPLAIN(X).    {A = X;}
+//id(A) ::= EXPLAIN(X).    {A = X;}
-id(A) ::= FAIL(X).       {A = X;}
+//id(A) ::= FAIL(X).       {A = X;}
-id(A) ::= FOR(X).        {A = X;}
+//id(A) ::= FOR(X).        {A = X;}
-id(A) ::= FULL(X).       {A = X;}
+//id(A) ::= FULL(X).       {A = X;}
 id(A) ::= ID(X).         {A = X;}
-id(A) ::= IGNORE(X).     {A = X;}
+//id(A) ::= IGNORE(X).     {A = X;}
-id(A) ::= IMMEDATE(X).   {A = X;}
+//id(A) ::= IMMEDATE(X).   {A = X;}
-id(A) ::= INITIALLY(X).  {A = X;}
+//id(A) ::= INITIALLY(X).  {A = X;}
-id(A) ::= INSTEAD(X).    {A = X;}
+//id(A) ::= INSTEAD(X).    {A = X;}
-id(A) ::= MATCH(X).      {A = X;}
+//id(A) ::= MATCH(X).      {A = X;}
-id(A) ::= JOIN(X).       {A = X;}
+//id(A) ::= JOIN(X).       {A = X;}
-id(A) ::= KEY(X).        {A = X;}
+//id(A) ::= KEY(X).        {A = X;}
-id(A) ::= OF(X).         {A = X;}
+//id(A) ::= OF(X).         {A = X;}
-id(A) ::= OFFSET(X).     {A = X;}
+//id(A) ::= OFFSET(X).     {A = X;}
-id(A) ::= PARTIAL(X).    {A = X;}
+//id(A) ::= PARTIAL(X).    {A = X;}
-id(A) ::= PRAGMA(X).     {A = X;}
+//id(A) ::= PRAGMA(X).     {A = X;}
-id(A) ::= REPLACE(X).    {A = X;}
+//id(A) ::= REPLACE(X).    {A = X;}
-id(A) ::= RESTRICT(X).   {A = X;}
+//id(A) ::= RESTRICT(X).   {A = X;}
-id(A) ::= ROW(X).        {A = X;}
+//id(A) ::= ROW(X).        {A = X;}
-id(A) ::= STATEMENT(X).  {A = X;}
+//id(A) ::= STATEMENT(X).  {A = X;}
-id(A) ::= TEMP(X).       {A = X;}
+//id(A) ::= TEMP(X).       {A = X;}
-id(A) ::= TRIGGER(X).    {A = X;}
+//id(A) ::= TRIGGER(X).    {A = X;}
-id(A) ::= VACUUM(X).     {A = X;}
+//id(A) ::= VACUUM(X).     {A = X;}
-id(A) ::= VIEW(X).       {A = X;}
+//id(A) ::= VIEW(X).       {A = X;}
+
+%fallback ID 
+  ABORT AFTER ASC BEFORE BEGIN CASCADE CLUSTER CONFLICT
+  COPY DEFERRED DELIMITERS DESC EACH END EXPLAIN FAIL FOR
+  FULL IGNORE IMMEDIATE INITIALLY INSTEAD MATCH JOIN KEY
+  OF OFFSET PARTIAL PRAGMA REPLACE RESTRICT ROW STATEMENT
+  TEMP TRIGGER VACUUM VIEW.
 
 // And "ids" is an identifer-or-string.
 //

src/select.c

@@ -12,7 +12,7 @@
 ** This file contains C code routines that are called by the parser
 ** to handle SELECT statements in SQLite.
 **
-** $Id: select.c,v 1.90 2002/06/02 16:09:02 drh Exp $
+** $Id: select.c,v 1.91 2002/06/06 18:54:40 drh Exp $
 */
 #include "sqliteInt.h"
 
@@ -326,12 +326,9 @@ static int selectInnerLoop(
   ** part of the result.
   */
   if( distinct>=0 && pEList && pEList->nExpr>0 ){
-    /* For the purposes of the DISTINCT keyword to a SELECT, NULLs
+#if NULL_ALWAYS_DISTINCT
-    ** are indistinct.  This was confirmed by experiment in Oracle
+    sqliteVdbeAddOp(v, OP_IsNull, -pEList->nExpr, sqliteVdbeCurrentAddr(v)+7);
-    ** and PostgreSQL.  It seems contradictory, but it appears to be
+#endif
-    ** true.
-    ** sqliteVdbeAddOp(v, OP_IsNull, -pEList->nExpr,sqliteVdbeCurrentAddr(v)+7);
-    */
     sqliteVdbeAddOp(v, OP_MakeKey, pEList->nExpr, 1);
     sqliteVdbeAddOp(v, OP_Distinct, distinct, sqliteVdbeCurrentAddr(v)+3);
     sqliteVdbeAddOp(v, OP_Pop, pEList->nExpr+1, 0);
@@ -363,7 +360,7 @@ static int selectInnerLoop(
   ** table iParm.
   */
   if( eDest==SRT_Union ){
-    sqliteVdbeAddOp(v, OP_MakeRecord, nColumn, 0);
+    sqliteVdbeAddOp(v, OP_MakeRecord, nColumn, NULL_ALWAYS_DISTINCT);
     sqliteVdbeAddOp(v, OP_String, 0, 0);
     sqliteVdbeAddOp(v, OP_PutStrKey, iParm, 0);
   }else 
@@ -382,7 +379,8 @@ static int selectInnerLoop(
   ** the temporary table iParm.
   */
   if( eDest==SRT_Except ){
-    int addr = sqliteVdbeAddOp(v, OP_MakeRecord, nColumn, 0);
+    int addr;
+    addr = sqliteVdbeAddOp(v, OP_MakeRecord, nColumn, NULL_ALWAYS_DISTINCT);
     sqliteVdbeAddOp(v, OP_NotFound, iParm, addr+3);
     sqliteVdbeAddOp(v, OP_Delete, iParm, 0);
   }else 

src/sqliteInt.h

@@ -11,7 +11,7 @@
 *************************************************************************
 ** Internal interface definitions for SQLite.
 **
-** @(#) $Id: sqliteInt.h,v 1.119 2002/06/02 16:09:02 drh Exp $
+** @(#) $Id: sqliteInt.h,v 1.120 2002/06/06 18:54:41 drh Exp $
 */
 #include "sqlite.h"
 #include "hash.h"
@@ -30,6 +30,32 @@
 #define MAX_PAGES   2000
 #define TEMP_PAGES   500
 
+/*
+** If the following macro is set to 1, then NULL values are considered
+** distinct for the SELECT DISTINCT statement and for UNION or EXCEPT
+** compound queries.  No other SQL database engine (among those tested) 
+** works this way except for OCELOT.  But the SQL92 spec implies that
+** this is how things should work.
+**
+** If the following macro is set to 0, then NULLs are indistinct for
+** SELECT DISTINCT and for UNION.
+*/
+#define NULL_ALWAYS_DISTINCT 0
+
+/*
+** If the following macro is set to 1, then NULL values are considered
+** distinct when determining whether or not two entries are the same
+** in a UNIQUE index.  This is the way PostgreSQL, Oracle, DB2, MySQL,
+** OCELOT, and Firebird all work.  The SQL92 spec explicitly says this
+** is the way things are suppose to work.
+**
+** If the following macro is set to 0, the NULLs are indistinct for
+** a UNIQUE index.  In this mode, you can only have a single NULL entry
+** for a column declared UNIQUE.  This is the way Informix and SQL Server
+** work.
+*/
+#define NULL_DISTINCT_FOR_UNIQUE 1
+
 /*
 ** Integers of known sizes.  These typedefs might change for architectures
 ** where the sizes very.  Preprocessor macros are available so that the
@@ -214,7 +240,7 @@ struct sqlite {
 ** points to a linked list of these structures.
 */
 struct FuncDef {
-  void (*xFunc)(sqlite_func*,int,const char**);   /* Regular function */
+  void (*xFunc)(sqlite_func*,int,const char**);  /* Regular function */
   void (*xStep)(sqlite_func*,int,const char**);  /* Aggregate function step */
   void (*xFinalize)(sqlite_func*);           /* Aggregate function finializer */
   int nArg;                                  /* Number of arguments */
@@ -284,7 +310,7 @@ struct Table {
 /*
 ** SQLite supports 5 different ways to resolve a contraint
 ** error.  ROLLBACK processing means that a constraint violation
-** causes the operation in proces to fail and for the current transaction
+** causes the operation in process to fail and for the current transaction
 ** to be rolled back.  ABORT processing means the operation in process
 ** fails and any prior changes from that one operation are backed out,
 ** but the transaction is not rolled back.  FAIL processing means that

tool/lemon.c

@@ -129,6 +129,7 @@ struct symbol {
     NONTERMINAL
   } type;                  /* Symbols are all either TERMINALS or NTs */
   struct rule *rule;       /* Linked list of rules of this (if an NT) */
+  struct symbol *fallback; /* fallback token in case this token doesn't parse */
   int prec;                /* Precedence if defined (-1 otherwise) */
   enum e_assoc {
     LEFT,
@@ -269,6 +270,7 @@ struct lemon {
   int nconflict;           /* Number of parsing conflicts */
   int tablesize;           /* Size of the parse tables */
   int basisflag;           /* Print only basis configurations */
+  int has_fallback;        /* True if any %fallback is seen in the grammer */
   char *argv0;             /* Name of the program */
 };
 
@@ -1203,6 +1205,7 @@ char **argv;
   lem.argv0 = argv[0];
   lem.filename = OptArg(0);
   lem.basisflag = basisflag;
+  lem.has_fallback = 0;
   lem.nconflict = 0;
   lem.name = lem.include = lem.arg = lem.tokentype = lem.start = 0;
   lem.vartype = 0;
@@ -1722,8 +1725,10 @@ struct pstate {
     RESYNC_AFTER_RULE_ERROR,
     RESYNC_AFTER_DECL_ERROR,
     WAITING_FOR_DESTRUCTOR_SYMBOL,
-    WAITING_FOR_DATATYPE_SYMBOL
+    WAITING_FOR_DATATYPE_SYMBOL,
+    WAITING_FOR_FALLBACK_ID
   } state;                   /* The state of the parser */
+  struct symbol *fallback;   /* The fallback token */
   struct symbol *lhs;        /* Left-hand side of current rule */
   char *lhsalias;            /* Alias for the LHS */
   int nrhs;                  /* Number of right-hand side symbols seen */
@@ -2001,6 +2006,9 @@ to follow the previous rule.");
           psp->state = WAITING_FOR_DESTRUCTOR_SYMBOL;
 	}else if( strcmp(x,"type")==0 ){
           psp->state = WAITING_FOR_DATATYPE_SYMBOL;
+        }else if( strcmp(x,"fallback")==0 ){
+          psp->fallback = 0;
+          psp->state = WAITING_FOR_FALLBACK_ID;
         }else{
           ErrorMsg(psp->filename,psp->tokenlineno,
             "Unknown declaration keyword: \"%%%s\".",x);
@@ -2080,6 +2088,27 @@ to follow the previous rule.");
         psp->state = RESYNC_AFTER_DECL_ERROR;
       }
       break;
+    case WAITING_FOR_FALLBACK_ID:
+      if( x[0]=='.' ){
+        psp->state = WAITING_FOR_DECL_OR_RULE;
+      }else if( !isupper(x[0]) ){
+        ErrorMsg(psp->filename, psp->tokenlineno,
+          "%%fallback argument \"%s\" should be a token", x);
+        psp->errorcnt++;
+      }else{
+        struct symbol *sp = Symbol_new(x);
+        if( psp->fallback==0 ){
+          psp->fallback = sp;
+        }else if( sp->fallback ){
+          ErrorMsg(psp->filename, psp->tokenlineno,
+            "More than one fallback assigned to token %s", x);
+          psp->errorcnt++;
+        }else{
+          sp->fallback = psp->fallback;
+          psp->gp->has_fallback = 1;
+        }
+      }
+      break;
     case RESYNC_AFTER_RULE_ERROR:
 /*      if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE;
 **      break; */
@@ -2952,7 +2981,7 @@ int mhflag;     /* Output in makeheaders format if true */
   struct state *stp;
   struct action *ap;
   struct rule *rp;
-  int i;
+  int i, j;
   int tablecnt;
   char *name;
 
@@ -3037,6 +3066,9 @@ int mhflag;     /* Output in makeheaders format if true */
   fprintf(out,"#define YYNRULE %d\n",lemp->nrule);  lineno++;
   fprintf(out,"#define YYERRORSYMBOL %d\n",lemp->errsym->index);  lineno++;
   fprintf(out,"#define YYERRSYMDT yy%d\n",lemp->errsym->dtnum);  lineno++;
+  if( lemp->has_fallback ){
+    fprintf(out,"#define YYFALLBACK 1\n");  lineno++;
+  }
   tplt_xfer(lemp->name,in,out,&lineno);
 
   /* Generate the action table.
@@ -3146,7 +3178,24 @@ int mhflag;     /* Output in makeheaders format if true */
   }
   tplt_xfer(lemp->name,in,out,&lineno);
 
-  /* Generate a table containing the symbolic name of every symbol */
+  /* Generate the table of fallback tokens.
+  */
+  if( lemp->has_fallback ){
+    for(i=0; i<lemp->nterminal; i++){
+      struct symbol *p = lemp->symbols[i];
+      if( p->fallback==0 ){
+        fprintf(out, "    0,  /* %10s => nothing */\n", p->name);
+      }else{
+        fprintf(out, "  %3d,  /* %10s => %s */\n", p->fallback->index,
+          p->name, p->fallback->name);
+      }
+      lineno++;
+    }
+  }
+  tplt_xfer(lemp->name, in, out, &lineno);
+
+  /* Generate a table containing the symbolic name of every symbol
+  */
   for(i=0; i<lemp->nsymbol; i++){
     sprintf(line,"\"%s\",",lemp->symbols[i]->name);
     fprintf(out,"  %-15s",line);
@@ -3155,9 +3204,22 @@ int mhflag;     /* Output in makeheaders format if true */
   if( (i&3)!=0 ){ fprintf(out,"\n"); lineno++; }
   tplt_xfer(lemp->name,in,out,&lineno);
 
+  /* Generate a table containing a text string that describes every
+  ** rule in the rule set of the grammer.  This information is used
+  ** when tracing REDUCE actions.
+  */
+  for(i=0, rp=lemp->rule; rp; rp=rp->next, i++){
+    assert( rp->index==i );
+    fprintf(out," /* %3d */ \"%s ::=", i, rp->lhs->name);
+    for(j=0; j<rp->nrhs; j++) fprintf(out," %s",rp->rhs[j]->name);
+    fprintf(out,"\",\n"); lineno++;
+  }
+  tplt_xfer(lemp->name,in,out,&lineno);
+
   /* Generate code which executes every time a symbol is popped from
   ** the stack while processing errors or while destroying the parser. 
-  ** (In other words, generate the %destructor actions) */
+  ** (In other words, generate the %destructor actions)
+  */
   if( lemp->tokendest ){
     for(i=0; i<lemp->nsymbol; i++){
       struct symbol *sp = lemp->symbols[i];
@@ -3210,9 +3272,6 @@ int mhflag;     /* Output in makeheaders format if true */
   /* Generate code which execution during each REDUCE action */
   for(rp=lemp->rule; rp; rp=rp->next){
     fprintf(out,"      case %d:\n",rp->index); lineno++;
-    fprintf(out,"        YYTRACE(\"%s ::=",rp->lhs->name);
-    for(i=0; i<rp->nrhs; i++) fprintf(out," %s",rp->rhs[i]->name);
-    fprintf(out,"\")\n"); lineno++;
     emit_code(out,rp,lemp,&lineno);
     fprintf(out,"        break;\n"); lineno++;
   }
@@ -3562,6 +3621,7 @@ char *x;
     sp->name = Strsafe(x);
     sp->type = isupper(*x) ? TERMINAL : NONTERMINAL;
     sp->rule = 0;
+    sp->fallback = 0;
     sp->prec = -1;
     sp->assoc = UNK;
     sp->firstset = 0;

tool/lempar.c

@@ -31,6 +31,9 @@
 **                       to no legal terminal or nonterminal number.  This
 **                       number is used to fill in empty slots of the hash 
 **                       table.
+**    YYFALLBACK         If defined, this indicates that one or more tokens
+**                       have fall-back values which should be used if the
+**                       original value of the token will not parse.
 **    YYACTIONTYPE       is the data type used for storing terminal
 **                       and nonterminal numbers.  "unsigned char" is
 **                       used if there are fewer than 250 rules and
@@ -105,6 +108,22 @@ static const yyStateEntry yyStateTable[] = {
 %%
 };
 
+/* The next table maps tokens into fallback tokens.  If a construct
+** like the following:
+** 
+**      %fallback ID X Y Z.
+**
+** appears in the grammer, then ID becomes a fallback token for X, Y,
+** and Z.  Whenever one of the tokens X, Y, or Z is input to the parser
+** but it does not parse, the type of the token is changed to ID and
+** the parse is retried before an error is thrown.
+*/
+#ifdef YYFALLBACK
+static const YYCODETYPE yyFallback[] = {
+%%
+};
+#endif /* YYFALLBACK */
+
 /* The following structure represents a single element of the
 ** parser's stack.  Information stored includes:
 **
@@ -141,7 +160,9 @@ typedef struct yyParser yyParser;
 #include <stdio.h>
 static FILE *yyTraceFILE = 0;
 static char *yyTracePrompt = 0;
+#endif /* NDEBUG */
 
+#ifndef NDEBUG
 /* 
 ** Turn parser tracing on by giving a stream to which to write the trace
 ** and a prompt to preface each trace message.  Tracing is turned off
@@ -165,17 +186,23 @@ void ParseTrace(FILE *TraceFILE, char *zTracePrompt){
   if( yyTraceFILE==0 ) yyTracePrompt = 0;
   else if( yyTracePrompt==0 ) yyTraceFILE = 0;
 }
+#endif /* NDEBUG */
 
+#ifndef NDEBUG
 /* For tracing shifts, the names of all terminals and nonterminals
 ** are required.  The following table supplies these names */
 static const char *yyTokenName[] = { 
 %%
 };
-#define YYTRACE(X) if( yyTraceFILE ) fprintf(yyTraceFILE,"%sReduce [%s].\n",yyTracePrompt,X);
+#endif /* NDEBUG */
-#else
-#define YYTRACE(X)
-#endif
 
+#ifndef NDEBUG
+/* For tracing reduce actions, the names of all rules are required.
+*/
+static const char *yyRuleName[] = {
+%%
+};
+#endif /* NDEBUG */
 
 /*
 ** This function returns the symbolic name associated with a token
@@ -297,6 +324,7 @@ static int yy_find_parser_action(
 ){
   const yyStateEntry *pState;   /* Appropriate entry in the state table */
   const yyActionEntry *pAction; /* Action appropriate for the look-ahead */
+  int iFallback;                /* Fallback token */
  
   /* if( pParser->yyidx<0 ) return YY_NO_ACTION;  */
   pState = &yyStateTable[pParser->yytop->stateno];
@@ -306,9 +334,21 @@ static int yy_find_parser_action(
     pAction = &pState->hashtbl[iLookAhead % pState->nEntry];
     while( 1 ){
       if( pAction->lookahead==iLookAhead ) return pAction->action;
-      if( pAction->next==0 ) return pState->actionDefault;
+      if( pAction->next==0 ) break;
       pAction = &pState->hashtbl[pAction->next-1];
     }
+#ifdef YYFALLBACK
+    if( iLookAhead<sizeof(yyFallback)/sizeof(yyFallback[0])
+           && (iFallback = yyFallback[iLookAhead])!=0 ){
+#ifndef NDEBUG
+      if( yyTraceFILE ){
+        fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n",
+           yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]);
+      }
+#endif
+      return yy_find_parser_action(pParser, iFallback);
+    }
+#endif
   }else if( pState->hashtbl->lookahead!=YYNOCODE ){
     return YY_NO_ACTION;
   }
@@ -384,11 +424,18 @@ static void yy_reduce(
   int yysize;                     /* Amount to pop the stack */
   ParseARG_FETCH;
   yymsp = yypParser->yytop;
+#ifndef NDEBUG
+  if( yyTraceFILE && yyruleno>=0 
+        && yyruleno<sizeof(yyRuleName)/sizeof(yyRuleName[0]) ){
+    fprintf(yyTraceFILE, "%sReduce [%s].\n", yyTracePrompt,
+      yyRuleName[yyruleno]);
+  }
+#endif /* NDEBUG */
+
   switch( yyruleno ){
   /* Beginning here are the reduction cases.  A typical example
   ** follows:
   **   case 0:
-  **     YYTRACE("<text of the rule>");
   **  #line <lineno> <grammarfile>
   **     { ... }           // User supplied code
   **  #line <lineno> <thisfile>