Optimizations to the tokenizer. (CVS 815)

FossilOrigin-Name: 032b3daa1d3cf3e00a4a6ba0b09624f1aba6445c

manifest

@@ -1,5 +1,5 @@
-C Remove\sunnecessary\scode\sfrom\sthe\sVDBE.\s(CVS\s814)
+C Optimizations\sto\sthe\stokenizer.\s(CVS\s815)
-D 2003-01-06T23:54:06
+D 2003-01-07T01:44:38
 F Makefile.in 868c17a1ae1c07603d491274cc8f86c04acf2a1e
 F Makefile.linux-gcc b86a99c493a5bfb402d1d9178dcdc4bd4b32f906
 F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd
@@ -48,7 +48,7 @@ F src/test1.c a46e9f61915b32787c5d5a05a4b92e4dacc437d9
 F src/test2.c 03f05e984c8e2f2badc44644d42baf72b249096b
 F src/test3.c c12ea7f1c3fbbd58904e81e6cb10ad424e6fc728
 F src/threadtest.c d641a5219e718e18a1a80a50eb9bb549f451f42e
-F src/tokenize.c 75e3bb37305b64e118e709752066f494c4f93c30
+F src/tokenize.c 8eaa8f878aa8c6375b567504736a5047f367ce9b
 F src/trigger.c 5ba917fc226b96065108da28186c2efaec53e481
 F src/update.c 881e4c8e7c786545da4fd2d95da19252b2e31137
 F src/util.c e2d108842e02810d3d3242cac0e024b09cdb3c4a
@@ -152,7 +152,7 @@ F www/speed.tcl a20a792738475b68756ea7a19321600f23d1d803
 F www/sqlite.tcl ae3dcfb077e53833b59d4fcc94d8a12c50a44098
 F www/tclsqlite.tcl 1db15abeb446aad0caf0b95b8b9579720e4ea331
 F www/vdbe.tcl 2013852c27a02a091d39a766bc87cff329f21218
-P 5809132f5bf40bae2331f887e87fe5baecc15c46
+P b96ec281ff29aad8af340b30c6ff4e129ffeeefb
-R b3c7a57b7ec1a787dd9ebcc0f7db6f34
+R eca56f05aef2b5cf05bae2fe3b0da7b4
 U drh
-Z 5f8c0999fff5fe3e4b7d38d837b9f84a
+Z f083eac995258132bf28df3cbcdc9fef

manifest.uuid

@@ -1 +1 @@
-b96ec281ff29aad8af340b30c6ff4e129ffeeefb
+032b3daa1d3cf3e00a4a6ba0b09624f1aba6445c

src/tokenize.c

@@ -15,7 +15,7 @@
 ** individual tokens and sends those tokens one-by-one over to the
 ** parser for analysis.
 **
-** $Id: tokenize.c,v 1.51 2002/10/27 19:35:35 drh Exp $
+** $Id: tokenize.c,v 1.52 2003/01/07 01:44:38 drh Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -219,7 +219,7 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
   int i;
   switch( *z ){
     case ' ': case '\t': case '\n': case '\f': case '\r': {
-      for(i=1; z[i] && isspace(z[i]); i++){}
+      for(i=1; isspace(z[i]); i++){}
       *tokenType = TK_SPACE;
       return i;
     }
@@ -358,10 +358,10 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
     case '0': case '1': case '2': case '3': case '4':
     case '5': case '6': case '7': case '8': case '9': {
       *tokenType = TK_INTEGER;
-      for(i=1; z[i] && isdigit(z[i]); i++){}
+      for(i=1; isdigit(z[i]); i++){}
       if( z[i]=='.' ){
         i++;
-        while( z[i] && isdigit(z[i]) ){ i++; }
+        while( isdigit(z[i]) ){ i++; }
         *tokenType = TK_FLOAT;
       }
       if( (z[i]=='e' || z[i]=='E') &&
@@ -370,7 +370,7 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
            )
       ){
         i += 2;
-        while( z[i] && isdigit(z[i]) ){ i++; }
+        while( isdigit(z[i]) ){ i++; }
         *tokenType = TK_FLOAT;
       }else if( z[0]=='.' ){
         *tokenType = TK_FLOAT;
@@ -406,7 +406,6 @@ int sqliteRunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
   int nErr = 0;
   int i;
   void *pEngine;
-  int once = 1;
   sqlite *db = pParse->db;
   extern void *sqliteParserAlloc(void*(*)(int));
   extern void sqliteParserFree(void*, void(*)(void*));
@@ -420,33 +419,32 @@ int sqliteRunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
     sqliteSetString(pzErrMsg, "out of memory", 0);
     return 1;
   }
-  while( sqlite_malloc_failed==0 && nErr==0 && i>=0 && zSql[i]!=0 ){
+  pParse->sLastToken.dyn = 0;
+  while( sqlite_malloc_failed==0 && zSql[i]!=0 ){
     int tokenType;
     
-    if( (db->flags & SQLITE_Interrupt)!=0 ){
+    assert( i>=0 );
-      pParse->rc = SQLITE_INTERRUPT;
-      sqliteSetString(pzErrMsg, "interrupt", 0);
-      break;
-    }
     pParse->sLastToken.z = &zSql[i];
-    pParse->sLastToken.dyn = 0;
+    assert( pParse->sLastToken.dyn==0 );
     pParse->sLastToken.n = sqliteGetToken((unsigned char*)&zSql[i], &tokenType);
     i += pParse->sLastToken.n;
-    if( once ){
-      pParse->sFirstToken = pParse->sLastToken;
-      once = 0;
-    }
     switch( tokenType ){
       case TK_SPACE:
       case TK_COMMENT: {
+        if( (db->flags & SQLITE_Interrupt)!=0 ){
+          pParse->rc = SQLITE_INTERRUPT;
+          sqliteSetString(pzErrMsg, "interrupt", 0);
+          goto abort_parse;
+        }
         break;
       }
-      case TK_ILLEGAL:
+      case TK_ILLEGAL: {
         sqliteSetNString(pzErrMsg, "unrecognized token: \"", -1, 
            pParse->sLastToken.z, pParse->sLastToken.n, "\"", 1, 0);
         nErr++;
-        break;
+        goto abort_parse;cvs
-      default:
+      }
+      default: {
         sqliteParser(pEngine, tokenType, pParse->sLastToken, pParse);
         if( pParse->zErrMsg && pParse->sErrToken.z ){
           sqliteSetNString(pzErrMsg, "near \"", -1, 
@@ -457,14 +455,18 @@ int sqliteRunParser(Parse *pParse, const char *zSql, char **pzErrMsg){
           nErr++;
           sqliteFree(pParse->zErrMsg);
           pParse->zErrMsg = 0;
+          goto abort_parse;
         }else if( pParse->rc!=SQLITE_OK ){
           sqliteSetString(pzErrMsg, sqlite_error_string(pParse->rc), 0);
           nErr++;
+          goto abort_parse;
         }
         break;
       }
     }
-  if( zSql[i]==0 ){
+  }
+abort_parse:
+  if( zSql[i]==0 && nErr==0 ){
     sqliteParser(pEngine, TK_SEMI, pParse->sLastToken, pParse);
     sqliteParser(pEngine, 0, pParse->sLastToken, pParse);
     if( pParse->zErrMsg && pParse->sErrToken.z ){