In LEMON, fix a bug in the text formatter introduced by the previous

commit.  Also add the new "%token_class" directive for defining symbolic
names that stand any one of a collection of tokens.

FossilOrigin-Name: da7890ca6b1d8e511377a469047120220e8c3b2d

manifest

@@ -1,5 +1,5 @@
-C Do\snot\suse\ssprintf(),\sstrcpy()\sor\sstrcat()\sin\sthe\simplementation\sof\sthe\nlemon\sparser\sgenerator\stool,\sto\savoid\scompiler\swarnings\sin\sOpenBSD.
+C In\sLEMON,\sfix\sa\sbug\sin\sthe\stext\sformatter\sintroduced\sby\sthe\sprevious\ncommit.\s\sAlso\sadd\sthe\snew\s"%token_class"\sdirective\sfor\sdefining\ssymbolic\nnames\sthat\sstand\sany\sone\sof\sa\scollection\sof\stokens.
-D 2014-01-10T23:21:00.243
+D 2014-01-11T03:06:18.172
 F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
 F Makefile.in 2ef13430cd359f7b361bb863504e227b25cc7f81
 F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -1109,7 +1109,7 @@ F tool/fragck.tcl 5265a95126abcf6ab357f7efa544787e5963f439
 F tool/genfkey.README cf68fddd4643bbe3ff8e31b8b6d8b0a1b85e20f4
 F tool/genfkey.test 4196a8928b78f51d54ef58e99e99401ab2f0a7e5
 F tool/getlock.c f4c39b651370156cae979501a7b156bdba50e7ce
-F tool/lemon.c 04f60c891f6c07643221b068bbc2621be46c039a
+F tool/lemon.c 4a3d4a579c5dff6a42785e97d1f2b59789f3b8dd
 F tool/lempar.c 01ca97f87610d1dac6d8cd96ab109ab1130e76dc
 F tool/logest.c 7ad625cac3d54012b27d468b7af6612f78b9ba75
 F tool/mkautoconfamal.sh f8d8dbf7d62f409ebed5134998bf5b51d7266383
@@ -1148,7 +1148,7 @@ F tool/vdbe-compress.tcl 0cf56e9263a152b84da86e75a5c0cdcdb7a47891
 F tool/warnings-clang.sh f6aa929dc20ef1f856af04a730772f59283631d4
 F tool/warnings.sh d1a6de74685f360ab718efda6265994b99bbea01
 F tool/win/sqlite.vsix 030f3eeaf2cb811a3692ab9c14d021a75ce41fff
-P 7f1e7ae313c7625ef2623d78883dce776eecca30
+P e43c522dde01e134f1adc94f534d2b3eda74afc2
-R 153a0201d8a4a180b2a188d3abd169a6
+R 498b2eca2a4d9bdf3c22bba6f4386191
 U drh
-Z c49dfd6bd62bb08d757d02ffce1c49c4
+Z 303f39885598d1865b7eb401105ea9ea

manifest.uuid

@@ -1 +1 @@
-e43c522dde01e134f1adc94f534d2b3eda74afc2
+da7890ca6b1d8e511377a469047120220e8c3b2d

tool/lemon.c

@@ -71,12 +71,15 @@ static void lemon_addtext(
   char *zBuf,           /* The buffer to which text is added */
   int *pnUsed,          /* Slots of the buffer used so far */
   const char *zIn,      /* Text to add */
-  int nIn               /* Bytes of text to add.  -1 to use strlen() */
+  int nIn,              /* Bytes of text to add.  -1 to use strlen() */
+  int iWidth            /* Field width.  Negative to left justify */
 ){
   if( nIn<0 ) for(nIn=0; zIn[nIn]; nIn++){}
+  while( iWidth>nIn ){ zBuf[*(pnUsed++)] = ' '; iWidth--; }
   if( nIn==0 ) return;
   memcpy(&zBuf[*pnUsed], zIn, nIn);
   *pnUsed += nIn;
+  while( (-iWidth)>nIn ){ zBuf[*(pnUsed++)] = ' '; iWidth++; }
   zBuf[*pnUsed] = 0;
 }
 static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){
@@ -87,15 +90,22 @@ static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){
   str[0] = 0;
   for(i=j=0; (c = zFormat[i])!=0; i++){
     if( c=='%' ){
-      lemon_addtext(str, &nUsed, &zFormat[j], i-j);
+      int iWidth = 0;
+      lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0);
       c = zFormat[++i];
+      if( isdigit(c) || (c=='-' && isdigit(zFormat[i+1])) ){
+        if( c=='-' ) i++;
+        while( isdigit(zFormat[i]) ) iWidth = iWidth*10 + zFormat[i++] - '0';
+        if( c=='-' ) iWidth = -iWidth;
+        c = zFormat[i];
+      }
       if( c=='d' ){
         int v = va_arg(ap, int);
         if( v<0 ){
-          lemon_addtext(str, &nUsed, "-", 1);
+          lemon_addtext(str, &nUsed, "-", 1, iWidth);
           v = -v;
         }else if( v==0 ){
-          lemon_addtext(str, &nUsed, "0", 1);
+          lemon_addtext(str, &nUsed, "0", 1, iWidth);
         }
         k = 0;
         while( v>0 ){
@@ -103,17 +113,17 @@ static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){
           zTemp[sizeof(zTemp)-k] = (v%10) + '0';
           v /= 10;
         }
-        lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k);
+        lemon_addtext(str, &nUsed, &zTemp[sizeof(zTemp)-k], k, iWidth);
       }else if( c=='s' ){
         z = va_arg(ap, const char*);
-        lemon_addtext(str, &nUsed, z, -1);
+        lemon_addtext(str, &nUsed, z, -1, iWidth);
       }else if( c=='.' && memcmp(&zFormat[i], ".*s", 3)==0 ){
         i += 2;
         k = va_arg(ap, int);
         z = va_arg(ap, const char*);
-        lemon_addtext(str, &nUsed, z, k);
+        lemon_addtext(str, &nUsed, z, k, iWidth);
       }else if( c=='%' ){
-        lemon_addtext(str, &nUsed, "%", 1);
+        lemon_addtext(str, &nUsed, "%", 1, 0);
       }else{
         fprintf(stderr, "illegal format\n");
         exit(1);
@@ -121,7 +131,7 @@ static int lemon_vsprintf(char *str, const char *zFormat, va_list ap){
       j = i+1;
     }
   }
-  lemon_addtext(str, &nUsed, &zFormat[j], i-j);
+  lemon_addtext(str, &nUsed, &zFormat[j], i-j, 0);
   return nUsed;
 }
 static int lemon_sprintf(char *str, const char *format, ...){
@@ -1538,12 +1548,15 @@ int main(int argc, char **argv)
   }
 
   /* Count and index the symbols of the grammar */
-  lem.nsymbol = Symbol_count();
   Symbol_new("{default}");
+  lem.nsymbol = Symbol_count();
   lem.symbols = Symbol_arrayof();
-  for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i;
+  for(i=0; i<lem.nsymbol; i++) lem.symbols[i]->index = i;
-  qsort(lem.symbols,lem.nsymbol+1,sizeof(struct symbol*), Symbolcmpp);
+  qsort(lem.symbols,lem.nsymbol,sizeof(struct symbol*), Symbolcmpp);
-  for(i=0; i<=lem.nsymbol; i++) lem.symbols[i]->index = i;
+  for(i=0; i<lem.nsymbol; i++) lem.symbols[i]->index = i;
+  while( lem.symbols[i-1]->type==MULTITERMINAL ){ i--; }
+  assert( strcmp(lem.symbols[i-1]->name,"{default}")==0 );
+  lem.nsymbol = i - 1;
   for(i=1; isupper(lem.symbols[i]->name[0]); i++);
   lem.nterminal = i;
 
@@ -2031,7 +2044,9 @@ enum e_state {
   WAITING_FOR_DESTRUCTOR_SYMBOL,
   WAITING_FOR_DATATYPE_SYMBOL,
   WAITING_FOR_FALLBACK_ID,
-  WAITING_FOR_WILDCARD_ID
+  WAITING_FOR_WILDCARD_ID,
+  WAITING_FOR_CLASS_ID,
+  WAITING_FOR_CLASS_TOKEN
 };
 struct pstate {
   char *filename;       /* Name of the input file */
@@ -2041,6 +2056,7 @@ struct pstate {
   struct lemon *gp;     /* Global state vector */
   enum e_state state;        /* The state of the parser */
   struct symbol *fallback;   /* The fallback token */
+  struct symbol *tkclass;    /* Token class symbol */
   struct symbol *lhs;        /* Left-hand side of current rule */
   const char *lhsalias;      /* Alias for the LHS */
   int nrhs;                  /* Number of right-hand side symbols seen */
@@ -2345,6 +2361,8 @@ to follow the previous rule.");
           psp->state = WAITING_FOR_FALLBACK_ID;
         }else if( strcmp(x,"wildcard")==0 ){
           psp->state = WAITING_FOR_WILDCARD_ID;
+        }else if( strcmp(x,"token_class")==0 ){
+          psp->state = WAITING_FOR_CLASS_ID;
         }else{
           ErrorMsg(psp->filename,psp->tokenlineno,
             "Unknown declaration keyword: \"%%%s\".",x);
@@ -2513,6 +2531,40 @@ to follow the previous rule.");
         }
       }
       break;
+    case WAITING_FOR_CLASS_ID:
+      if( !islower(x[0]) ){
+        ErrorMsg(psp->filename, psp->tokenlineno,
+          "%%token_class must be followed by an identifier: ", x);
+        psp->errorcnt++;
+        psp->state = RESYNC_AFTER_DECL_ERROR;
+     }else if( Symbol_find(x) ){
+        ErrorMsg(psp->filename, psp->tokenlineno,
+          "Symbol \"%s\" already used", x);
+        psp->errorcnt++;
+        psp->state = RESYNC_AFTER_DECL_ERROR;
+      }else{
+        psp->tkclass = Symbol_new(x);
+        psp->tkclass->type = MULTITERMINAL;
+        psp->state = WAITING_FOR_CLASS_TOKEN;
+      }
+      break;
+    case WAITING_FOR_CLASS_TOKEN:
+      if( x[0]=='.' ){
+        psp->state = WAITING_FOR_DECL_OR_RULE;
+      }else if( isupper(x[0]) || ((x[0]=='|' || x[0]=='/') && isupper(x[1])) ){
+        struct symbol *msp = psp->tkclass;
+        msp->nsubsym++;
+        msp->subsym = (struct symbol **) realloc(msp->subsym,
+          sizeof(struct symbol*)*msp->nsubsym);
+        if( !isupper(x[0]) ) x++;
+        msp->subsym[msp->nsubsym-1] = Symbol_new(x);
+      }else{
+        ErrorMsg(psp->filename, psp->tokenlineno,
+          "%%token_class argument \"%s\" should be a token", x);
+        psp->errorcnt++;
+        psp->state = RESYNC_AFTER_DECL_ERROR;
+      }
+      break;
     case RESYNC_AFTER_RULE_ERROR:
 /*      if( x[0]=='.' ) psp->state = WAITING_FOR_DECL_OR_RULE;
 **      break; */
@@ -2867,11 +2919,13 @@ void Reprint(struct lemon *lemp)
     printf(" ::=");
     for(i=0; i<rp->nrhs; i++){
       sp = rp->rhs[i];
-      printf(" %s", sp->name);
       if( sp->type==MULTITERMINAL ){
+        printf(" %s", sp->subsym[0]->name);
         for(j=1; j<sp->nsubsym; j++){
           printf("|%s", sp->subsym[j]->name);
         }
+      }else{
+        printf(" %s", sp->name);
       }
       /* if( rp->rhsalias[i] ) printf("(%s)",rp->rhsalias[i]); */
     }
@@ -2893,11 +2947,13 @@ void ConfigPrint(FILE *fp, struct config *cfp)
     if( i==cfp->dot ) fprintf(fp," *");
     if( i==rp->nrhs ) break;
     sp = rp->rhs[i];
-    fprintf(fp," %s", sp->name);
     if( sp->type==MULTITERMINAL ){
+      fprintf(fp," %s", sp->subsym[0]->name);
       for(j=1; j<sp->nsubsym; j++){
         fprintf(fp,"|%s",sp->subsym[j]->name);
       }
+    }else{
+      fprintf(fp," %s", sp->name);
     }
   }
 }
@@ -3644,9 +3700,11 @@ static void writeRuleText(FILE *out, struct rule *rp){
   fprintf(out,"%s ::=", rp->lhs->name);
   for(j=0; j<rp->nrhs; j++){
     struct symbol *sp = rp->rhs[j];
-    fprintf(out," %s", sp->name);
+    if( sp->type!=MULTITERMINAL ){
-    if( sp->type==MULTITERMINAL ){
+      fprintf(out," %s", sp->name);
+    }else{
       int k;
+      fprintf(out," %s", sp->subsym[0]->name);
       for(k=1; k<sp->nsubsym; k++){
         fprintf(out,"|%s",sp->subsym[k]->name);
       }
@@ -4114,7 +4172,8 @@ void ReportHeader(struct lemon *lemp)
   if( in ){
     int nextChar;
     for(i=1; i<lemp->nterminal && fgets(line,LINESIZE,in); i++){
-      lemon_sprintf(pattern,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i);
+      lemon_sprintf(pattern,"#define %s%-30s %3d\n",
+                    prefix,lemp->symbols[i]->name,i);
       if( strcmp(line,pattern) ) break;
     }
     nextChar = fgetc(in);
@@ -4127,7 +4186,7 @@ void ReportHeader(struct lemon *lemp)
   out = file_open(lemp,".h","wb");
   if( out ){
     for(i=1; i<lemp->nterminal; i++){
-      fprintf(out,"#define %s%-30s %2d\n",prefix,lemp->symbols[i]->name,i);
+      fprintf(out,"#define %s%-30s %3d\n",prefix,lemp->symbols[i]->name,i);
     }
     fclose(out);  
   }
@@ -4497,11 +4556,15 @@ struct symbol *Symbol_new(const char *x)
   return sp;
 }
 
-/* Compare two symbols for working purposes
+/* Compare two symbols for sorting purposes.  Return negative,
+** zero, or positive if a is less then, equal to, or greater
+** than b.
 **
 ** Symbols that begin with upper case letters (terminals or tokens)
 ** must sort before symbols that begin with lower case letters
-** (non-terminals).  Other than that, the order does not matter.
+** (non-terminals).  And MULTITERMINAL symbols (created using the
+** %token_class directive) must sort at the very end. Other than
+** that, the order does not matter.
 **
 ** We find experimentally that leaving the symbols in their original
 ** order (the order they appeared in the grammar file) gives the
@@ -4509,12 +4572,11 @@ struct symbol *Symbol_new(const char *x)
 */
 int Symbolcmpp(const void *_a, const void *_b)
 {
-  const struct symbol **a = (const struct symbol **) _a;
+  const struct symbol *a = *(const struct symbol **) _a;
-  const struct symbol **b = (const struct symbol **) _b;
+  const struct symbol *b = *(const struct symbol **) _b;
-  int i1 = (**a).index + 10000000*((**a).name[0]>'Z');
+  int i1 = a->type==MULTITERMINAL ? 3 : a->name[0]>'Z' ? 2 : 1;
-  int i2 = (**b).index + 10000000*((**b).name[0]>'Z');
+  int i2 = b->type==MULTITERMINAL ? 3 : b->name[0]>'Z' ? 2 : 1;
-  assert( i1!=i2 || strcmp((**a).name,(**b).name)==0 );
+  return i1==i2 ? a->index - b->index : i1 - i2;
-  return i1-i2;
 }
 
 /* There is one instance of the following structure for each