From ba21256f98b56970db401c33ee855d79a96d4b2b Mon Sep 17 00:00:00 2001
From: drh <drh@noemail.net>
Date: Thu, 8 Jan 2004 02:17:31 +0000
Subject: [PATCH] Remove unused code and tighten existing code to make the
 library a little smaller. (CVS 1168)

FossilOrigin-Name: 34a6b7416c6c9bcdf301f5e7b072a0362a746105
---
 manifest       |  18 ++--
 manifest.uuid  |   2 +-
 src/date.c     |  67 ++++++-------
 src/hash.c     |   8 +-
 src/hash.h     |   4 +-
 src/tokenize.c | 248 ++++++++++++++++++++++++-------------------------
 6 files changed, 169 insertions(+), 178 deletions(-)

diff --git a/manifest b/manifest
index 80558853f0..b05a19d7d3 100644
--- a/manifest
+++ b/manifest
@@ -1,5 +1,5 @@
-C Rework\sthe\sfix\sto\sticket\s#461\sso\sthat\swe\sdo\snot\shave\sto\sdo\sredundant\stests\nof\sWHERE\sclause\sterms\slooking\sfor\sNULLs.\s\sSee\salso\scheck-in\s(1103).\s(CVS\s1167)
-D 2004-01-07T20:37:52
+C Remove\sunused\scode\sand\stighten\sexisting\scode\sto\smake\sthe\slibrary\sa\slittle\nsmaller.\s(CVS\s1168)
+D 2004-01-08T02:17:32
 F Makefile.in 0515ff9218ad8d5a8f6220f0494b8ef94c67013b
 F Makefile.linux-gcc b86a99c493a5bfb402d1d9178dcdc4bd4b32f906
 F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd
@@ -28,13 +28,13 @@ F src/btree.h 9b7c09f1e64274d7bb74a57bbfc63778f67b1048
 F src/btree_rb.c e4084b6a12270674b0cd7034655f55e6a2639c78
 F src/build.c a7493c433de5b552f9535d8fa7ed80aaf135491e
 F src/copy.c 9e47975ea96751c658bcf1a0c4f0bb7c6ee61e73
-F src/date.c eef0ccd9d0b1d9fe74ed2ae8e247578be3f49bc4
+F src/date.c bb89fdb9c89e367b9a728c58cb96e4823974a2c1
 F src/delete.c 0f81e6799c089487615d38e042a2de4d2d6192bc
 F src/encode.c 25ea901a9cefb3d93774afa4a06b57cb58acf544
 F src/expr.c abb40922fa9995aca06f999ed35d2061d44650d6
 F src/func.c 62cf8fae8147c0301d1c6a4a94fe0a78f7aa5b33
-F src/hash.c 058f077c1f36f266581aa16f907a3903abf64aa3
-F src/hash.h cd0433998bc1a3759d244e1637fe5a3c13b53bf8
+F src/hash.c 9b56ef3b291e25168f630d5643a4264ec011c70e
+F src/hash.h 3247573ab95b9dd90bcca0307a75d9a16da1ccc7
 F src/insert.c 01f66866f35c986eab4a57373ca689a3255ef2df
 F src/main.c 3dd3cae00bade294011da5a3cf9ff660a610c545
 F src/md5.c fe4f9c9c6f71dfc26af8da63e4d04489b1430565
@@ -58,7 +58,7 @@ F src/test2.c 5014337d8576b731cce5b5a14bec4f0daf432700
 F src/test3.c 30985ebdfaf3ee1462a9b0652d3efbdc8d9798f5
 F src/test4.c dcbbbb382626fd466a7c46907f74db35fc8bad64
 F src/threadtest.c d641a5219e718e18a1a80a50eb9bb549f451f42e
-F src/tokenize.c 5597df1fd76d6947e7da824856ac1910e2055729
+F src/tokenize.c 8c95dcd2620b18dc0db1cdc97f9e111d11e55fe0
 F src/trigger.c ce83e017b407d046e909d05373d7f8ee70f9f7f9
 F src/update.c 24260b4fda00c9726d27699a0561d53c0dccc397
 F src/util.c 64995b5949a5d377629ffd2598747bc771cade1e
@@ -179,7 +179,7 @@ F www/speed.tcl 2f6b1155b99d39adb185f900456d1d592c4832b3
 F www/sqlite.tcl 3c83b08cf9f18aa2d69453ff441a36c40e431604
 F www/tclsqlite.tcl b9271d44dcf147a93c98f8ecf28c927307abd6da
 F www/vdbe.tcl 9b9095d4495f37697fd1935d10e14c6015e80aa1
-P 03636c94a542b1f90a3acfbe65a9c2976872073f
-R 747698ac4eca9128ab63b6b98ff5a6c2
+P 5fd581787e88173f0303f870fc956ec9285cca4e
+R 42e0a61dd74faeac7023df4a5dc43247
 U drh
-Z af269badac40ad362e9b5738b29d6b42
+Z 17125e9495da03df5ae68d60c6a8dbf2
diff --git a/manifest.uuid b/manifest.uuid
index 42d2dbe1b5..9dd9f1351a 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-5fd581787e88173f0303f870fc956ec9285cca4e
\ No newline at end of file
+34a6b7416c6c9bcdf301f5e7b072a0362a746105
\ No newline at end of file
diff --git a/src/date.c b/src/date.c
index 4e5e46e0a8..4700cde15f 100644
--- a/src/date.c
+++ b/src/date.c
@@ -16,7 +16,7 @@
 ** sqliteRegisterDateTimeFunctions() found at the bottom of the file.
 ** All other code has file scope.
 **
-** $Id: date.c,v 1.7 2004/01/07 03:29:16 drh Exp $
+** $Id: date.c,v 1.8 2004/01/08 02:17:32 drh Exp $
 **
 ** NOTES:
 **
@@ -221,7 +221,7 @@ static void computeJD(DateTime *p){
     M = p->M;
     D = p->D;
   }else{
-    Y = 2000;
+    Y = 2000;  /* If no YMD specified, assume 2000-Jan-01 */
     M = 1;
     D = 1;
   }
@@ -368,6 +368,23 @@ static void computeHMS(DateTime *p){
   p->validHMS = 1;
 }
 
+/*
+** Compute both YMD and HMS
+*/
+static void computeYMD_HMS(DateTime *p){
+  computeYMD(p);
+  computeHMS(p);
+}
+
+/*
+** Clear the YMD and HMS and the TZ
+*/
+static void clearYMD_HMS_TZ(DateTime *p){
+  p->validYMD = 0;
+  p->validHMS = 0;
+  p->validTZ = 0;
+}
+
 /*
 ** Compute the difference (in days) between localtime and UTC (a.k.a. GMT)
 ** for the time value p where p is in UTC.
@@ -376,9 +393,8 @@ static double localtimeOffset(DateTime *p){
   DateTime x, y;
   time_t t;
   struct tm *pTm;
-  computeYMD(p);
-  computeHMS(p);
   x = *p;
+  computeYMD_HMS(&x);
   if( x.Y<1971 || x.Y>=2038 ){
     x.Y = 2000;
     x.M = 1;
@@ -408,9 +424,6 @@ static double localtimeOffset(DateTime *p){
   y.validJD = 0;
   y.validTZ = 0;
   computeJD(&y);
-  /* printf("x=%d-%02d-%02d %02d:%02d:%02d\n",x.Y,x.M,x.D,x.h,x.m,(int)x.s); */
-  /* printf("y=%d-%02d-%02d %02d:%02d:%02d\n",y.Y,y.M,y.D,y.h,y.m,(int)y.s); */
-  /* printf("diff=%.17g\n", y.rJD - x.rJD); */
   return y.rJD - x.rJD;
 }
 
@@ -454,9 +467,7 @@ static int parseModifier(const char *zMod, DateTime *p){
       if( strcmp(z, "localtime")==0 ){
         computeJD(p);
         p->rJD += localtimeOffset(p);
-        p->validYMD = 0;
-        p->validHMS = 0;
-        p->validTZ = 0;
+        clearYMD_HMS_TZ(p);
         rc = 0;
       }
       break;
@@ -470,22 +481,15 @@ static int parseModifier(const char *zMod, DateTime *p){
       */
       if( strcmp(z, "unixepoch")==0 && p->validJD ){
         p->rJD = p->rJD/86400.0 + 2440587.5;
-        p->validYMD = 0;
-        p->validHMS = 0;
-        p->validTZ = 0;
+        clearYMD_HMS_TZ(p);
         rc = 0;
       }else if( strcmp(z, "utc")==0 ){
         double c1;
         computeJD(p);
         c1 = localtimeOffset(p);
         p->rJD -= c1;
-        p->validYMD = 0;
-        p->validHMS = 0;
-        p->validTZ = 0;
+        clearYMD_HMS_TZ(p);
         p->rJD += c1 - localtimeOffset(p);
-        p->validYMD = 0;
-        p->validHMS = 0;
-        p->validTZ = 0;
         rc = 0;
       }
       break;
@@ -501,8 +505,7 @@ static int parseModifier(const char *zMod, DateTime *p){
       if( strncmp(z, "weekday ", 8)==0 && getValue(&z[8],&r)>0
                  && (n=r)==r && n>=0 && r<7 ){
         int Z;
-        computeYMD(p);
-        computeHMS(p);
+        computeYMD_HMS(p);
         p->validTZ = 0;
         p->validJD = 0;
         computeJD(p);
@@ -510,8 +513,7 @@ static int parseModifier(const char *zMod, DateTime *p){
         Z %= 7;
         if( Z>n ) Z -= 7;
         p->rJD += n - Z;
-        p->validYMD = 0;
-        p->validHMS = 0;
+        clearYMD_HMS_TZ(p);
         rc = 0;
       }
       break;
@@ -569,18 +571,14 @@ static int parseModifier(const char *zMod, DateTime *p){
       if( n==3 && strcmp(z,"day")==0 ){
         p->rJD += r;
       }else if( n==4 && strcmp(z,"hour")==0 ){
-        computeJD(p);
         p->rJD += r/24.0;
       }else if( n==6 && strcmp(z,"minute")==0 ){
-        computeJD(p);
         p->rJD += r/(24.0*60.0);
       }else if( n==6 && strcmp(z,"second")==0 ){
-        computeJD(p);
         p->rJD += r/(24.0*60.0*60.0);
       }else if( n==5 && strcmp(z,"month")==0 ){
         int x, y;
-        computeYMD(p);
-        computeHMS(p);
+        computeYMD_HMS(p);
         p->M += r;
         x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12;
         p->Y += x;
@@ -592,17 +590,14 @@ static int parseModifier(const char *zMod, DateTime *p){
           p->rJD += (r - y)*30.0;
         }
       }else if( n==4 && strcmp(z,"year")==0 ){
-        computeYMD(p);
-        computeHMS(p);
+        computeYMD_HMS(p);
         p->Y += r;
         p->validJD = 0;
         computeJD(p);
       }else{
         rc = 1;
       }
-      p->validYMD = 0;
-      p->validHMS = 0;
-      p->validTZ = 0;
+      clearYMD_HMS_TZ(p);
       break;
     }
     default: {
@@ -656,8 +651,7 @@ static void datetimeFunc(sqlite_func *context, int argc, const char **argv){
   DateTime x;
   if( isDate(argc, argv, &x)==0 ){
     char zBuf[100];
-    computeYMD(&x);
-    computeHMS(&x);
+    computeYMD_HMS(&x);
     sprintf(zBuf, "%04d-%02d-%02d %02d:%02d:%02d",x.Y, x.M, x.D, x.h, x.m,
            (int)(x.s));
     sqlite_set_result_string(context, zBuf, -1);
@@ -760,8 +754,7 @@ static void strftimeFunc(sqlite_func *context, int argc, const char **argv){
     if( z==0 ) return;
   }
   computeJD(&x);
-  computeYMD(&x);
-  computeHMS(&x);
+  computeYMD_HMS(&x);
   for(i=j=0; zFmt[i]; i++){
     if( zFmt[i]!='%' ){
       z[j++] = zFmt[i];
diff --git a/src/hash.c b/src/hash.c
index bb2291c6f3..c7feea8fd8 100644
--- a/src/hash.c
+++ b/src/hash.c
@@ -12,7 +12,7 @@
 ** This is the implementation of generic hash-tables
 ** used in SQLite.
 **
-** $Id: hash.c,v 1.10 2003/05/12 23:06:53 drh Exp $
+** $Id: hash.c,v 1.11 2004/01/08 02:17:33 drh Exp $
 */
 #include "sqliteInt.h"
 #include <assert.h>
@@ -75,6 +75,7 @@ static int intCompare(const void *pKey1, int n1, const void *pKey2, int n2){
   return n2 - n1;
 }
 
+#if 0 /* NOT USED */
 /*
 ** Hash and comparison functions when the mode is SQLITE_HASH_POINTER
 */
@@ -87,6 +88,7 @@ static int ptrCompare(const void *pKey1, int n1, const void *pKey2, int n2){
   if( pKey1<pKey2 ) return -1;
   return 1;
 }
+#endif
 
 /*
 ** Hash and comparison functions when the mode is SQLITE_HASH_STRING
@@ -130,7 +132,7 @@ static int binCompare(const void *pKey1, int n1, const void *pKey2, int n2){
 static int (*hashFunction(int keyClass))(const void*,int){
   switch( keyClass ){
     case SQLITE_HASH_INT:     return &intHash;
-    case SQLITE_HASH_POINTER: return &ptrHash;
+    /* case SQLITE_HASH_POINTER: return &ptrHash; // NOT USED */
     case SQLITE_HASH_STRING:  return &strHash;
     case SQLITE_HASH_BINARY:  return &binHash;;
     default: break;
@@ -147,7 +149,7 @@ static int (*hashFunction(int keyClass))(const void*,int){
 static int (*compareFunction(int keyClass))(const void*,int,const void*,int){
   switch( keyClass ){
     case SQLITE_HASH_INT:     return &intCompare;
-    case SQLITE_HASH_POINTER: return &ptrCompare;
+    /* case SQLITE_HASH_POINTER: return &ptrCompare; // NOT USED */
     case SQLITE_HASH_STRING:  return &strCompare;
     case SQLITE_HASH_BINARY:  return &binCompare;
     default: break;
diff --git a/src/hash.h b/src/hash.h
index bccfbebf6d..05d0d0079d 100644
--- a/src/hash.h
+++ b/src/hash.h
@@ -12,7 +12,7 @@
 ** This is the header file for the generic hash-table implemenation
 ** used in SQLite.
 **
-** $Id: hash.h,v 1.5 2002/06/08 23:25:09 drh Exp $
+** $Id: hash.h,v 1.6 2004/01/08 02:17:33 drh Exp $
 */
 #ifndef _SQLITE_HASH_H_
 #define _SQLITE_HASH_H_
@@ -71,7 +71,7 @@ struct HashElem {
 ** if the copyKey parameter to HashInit is 1.  
 */
 #define SQLITE_HASH_INT       1
-#define SQLITE_HASH_POINTER   2
+/* #define SQLITE_HASH_POINTER   2 // NOT USED */
 #define SQLITE_HASH_STRING    3
 #define SQLITE_HASH_BINARY    4
 
diff --git a/src/tokenize.c b/src/tokenize.c
index 010648fc6f..b2373b368c 100644
--- a/src/tokenize.c
+++ b/src/tokenize.c
@@ -15,7 +15,7 @@
 ** individual tokens and sends those tokens one-by-one over to the
 ** parser for analysis.
 **
-** $Id: tokenize.c,v 1.66 2003/12/23 02:17:35 drh Exp $
+** $Id: tokenize.c,v 1.67 2004/01/08 02:17:33 drh Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -29,122 +29,122 @@
 typedef struct Keyword Keyword;
 struct Keyword {
   char *zName;             /* The keyword name */
-  u16 len;                 /* Number of characters in the keyword */
-  u16 tokenType;           /* The token value for this keyword */
-  Keyword *pNext;          /* Next keyword with the same hash */
+  u8 tokenType;            /* Token value for this keyword */
+  u8 len;                  /* Length of this keyword */
+  u8 iNext;                /* Index in aKeywordTable[] of next with same hash */
 };
 
 /*
 ** These are the keywords
 */
 static Keyword aKeywordTable[] = {
-  { "ABORT",             0, TK_ABORT,            0 },
-  { "AFTER",             0, TK_AFTER,            0 },
-  { "ALL",               0, TK_ALL,              0 },
-  { "AND",               0, TK_AND,              0 },
-  { "AS",                0, TK_AS,               0 },
-  { "ASC",               0, TK_ASC,              0 },
-  { "ATTACH",            0, TK_ATTACH,           0 },
-  { "BEFORE",            0, TK_BEFORE,           0 },
-  { "BEGIN",             0, TK_BEGIN,            0 },
-  { "BETWEEN",           0, TK_BETWEEN,          0 },
-  { "BY",                0, TK_BY,               0 },
-  { "CASCADE",           0, TK_CASCADE,          0 },
-  { "CASE",              0, TK_CASE,             0 },
-  { "CHECK",             0, TK_CHECK,            0 },
-  { "CLUSTER",           0, TK_CLUSTER,          0 },
-  { "COLLATE",           0, TK_COLLATE,          0 },
-  { "COMMIT",            0, TK_COMMIT,           0 },
-  { "CONFLICT",          0, TK_CONFLICT,         0 },
-  { "CONSTRAINT",        0, TK_CONSTRAINT,       0 },
-  { "COPY",              0, TK_COPY,             0 },
-  { "CREATE",            0, TK_CREATE,           0 },
-  { "CROSS",             0, TK_JOIN_KW,          0 },
-  { "DATABASE",          0, TK_DATABASE,         0 },
-  { "DEFAULT",           0, TK_DEFAULT,          0 },
-  { "DEFERRED",          0, TK_DEFERRED,         0 },
-  { "DEFERRABLE",        0, TK_DEFERRABLE,       0 },
-  { "DELETE",            0, TK_DELETE,           0 },
-  { "DELIMITERS",        0, TK_DELIMITERS,       0 },
-  { "DESC",              0, TK_DESC,             0 },
-  { "DETACH",            0, TK_DETACH,           0 },
-  { "DISTINCT",          0, TK_DISTINCT,         0 },
-  { "DROP",              0, TK_DROP,             0 },
-  { "END",               0, TK_END,              0 },
-  { "EACH",              0, TK_EACH,             0 },
-  { "ELSE",              0, TK_ELSE,             0 },
-  { "EXCEPT",            0, TK_EXCEPT,           0 },
-  { "EXPLAIN",           0, TK_EXPLAIN,          0 },
-  { "FAIL",              0, TK_FAIL,             0 },
-  { "FOR",               0, TK_FOR,              0 },
-  { "FOREIGN",           0, TK_FOREIGN,          0 },
-  { "FROM",              0, TK_FROM,             0 },
-  { "FULL",              0, TK_JOIN_KW,          0 },
-  { "GLOB",              0, TK_GLOB,             0 },
-  { "GROUP",             0, TK_GROUP,            0 },
-  { "HAVING",            0, TK_HAVING,           0 },
-  { "IGNORE",            0, TK_IGNORE,           0 },
-  { "IMMEDIATE",         0, TK_IMMEDIATE,        0 },
-  { "IN",                0, TK_IN,               0 },
-  { "INDEX",             0, TK_INDEX,            0 },
-  { "INITIALLY",         0, TK_INITIALLY,        0 },
-  { "INNER",             0, TK_JOIN_KW,          0 },
-  { "INSERT",            0, TK_INSERT,           0 },
-  { "INSTEAD",           0, TK_INSTEAD,          0 },
-  { "INTERSECT",         0, TK_INTERSECT,        0 },
-  { "INTO",              0, TK_INTO,             0 },
-  { "IS",                0, TK_IS,               0 },
-  { "ISNULL",            0, TK_ISNULL,           0 },
-  { "JOIN",              0, TK_JOIN,             0 },
-  { "KEY",               0, TK_KEY,              0 },
-  { "LEFT",              0, TK_JOIN_KW,          0 },
-  { "LIKE",              0, TK_LIKE,             0 },
-  { "LIMIT",             0, TK_LIMIT,            0 },
-  { "MATCH",             0, TK_MATCH,            0 },
-  { "NATURAL",           0, TK_JOIN_KW,          0 },
-  { "NOT",               0, TK_NOT,              0 },
-  { "NOTNULL",           0, TK_NOTNULL,          0 },
-  { "NULL",              0, TK_NULL,             0 },
-  { "OF",                0, TK_OF,               0 },
-  { "OFFSET",            0, TK_OFFSET,           0 },
-  { "ON",                0, TK_ON,               0 },
-  { "OR",                0, TK_OR,               0 },
-  { "ORDER",             0, TK_ORDER,            0 },
-  { "OUTER",             0, TK_JOIN_KW,          0 },
-  { "PRAGMA",            0, TK_PRAGMA,           0 },
-  { "PRIMARY",           0, TK_PRIMARY,          0 },
-  { "RAISE",             0, TK_RAISE,            0 },
-  { "REFERENCES",        0, TK_REFERENCES,       0 },
-  { "REPLACE",           0, TK_REPLACE,          0 },
-  { "RESTRICT",          0, TK_RESTRICT,         0 },
-  { "RIGHT",             0, TK_JOIN_KW,          0 },
-  { "ROLLBACK",          0, TK_ROLLBACK,         0 },
-  { "ROW",               0, TK_ROW,              0 },
-  { "SELECT",            0, TK_SELECT,           0 },
-  { "SET",               0, TK_SET,              0 },
-  { "STATEMENT",         0, TK_STATEMENT,        0 },
-  { "TABLE",             0, TK_TABLE,            0 },
-  { "TEMP",              0, TK_TEMP,             0 },
-  { "TEMPORARY",         0, TK_TEMP,             0 },
-  { "THEN",              0, TK_THEN,             0 },
-  { "TRANSACTION",       0, TK_TRANSACTION,      0 },
-  { "TRIGGER",           0, TK_TRIGGER,          0 },
-  { "UNION",             0, TK_UNION,            0 },
-  { "UNIQUE",            0, TK_UNIQUE,           0 },
-  { "UPDATE",            0, TK_UPDATE,           0 },
-  { "USING",             0, TK_USING,            0 },
-  { "VACUUM",            0, TK_VACUUM,           0 },
-  { "VALUES",            0, TK_VALUES,           0 },
-  { "VIEW",              0, TK_VIEW,             0 },
-  { "WHEN",              0, TK_WHEN,             0 },
-  { "WHERE",             0, TK_WHERE,            0 },
+  { "ABORT",             TK_ABORT,        },
+  { "AFTER",             TK_AFTER,        },
+  { "ALL",               TK_ALL,          },
+  { "AND",               TK_AND,          },
+  { "AS",                TK_AS,           },
+  { "ASC",               TK_ASC,          },
+  { "ATTACH",            TK_ATTACH,       },
+  { "BEFORE",            TK_BEFORE,       },
+  { "BEGIN",             TK_BEGIN,        },
+  { "BETWEEN",           TK_BETWEEN,      },
+  { "BY",                TK_BY,           },
+  { "CASCADE",           TK_CASCADE,      },
+  { "CASE",              TK_CASE,         },
+  { "CHECK",             TK_CHECK,        },
+  { "CLUSTER",           TK_CLUSTER,      },
+  { "COLLATE",           TK_COLLATE,      },
+  { "COMMIT",            TK_COMMIT,       },
+  { "CONFLICT",          TK_CONFLICT,     },
+  { "CONSTRAINT",        TK_CONSTRAINT,   },
+  { "COPY",              TK_COPY,         },
+  { "CREATE",            TK_CREATE,       },
+  { "CROSS",             TK_JOIN_KW,      },
+  { "DATABASE",          TK_DATABASE,     },
+  { "DEFAULT",           TK_DEFAULT,      },
+  { "DEFERRED",          TK_DEFERRED,     },
+  { "DEFERRABLE",        TK_DEFERRABLE,   },
+  { "DELETE",            TK_DELETE,       },
+  { "DELIMITERS",        TK_DELIMITERS,   },
+  { "DESC",              TK_DESC,         },
+  { "DETACH",            TK_DETACH,       },
+  { "DISTINCT",          TK_DISTINCT,     },
+  { "DROP",              TK_DROP,         },
+  { "END",               TK_END,          },
+  { "EACH",              TK_EACH,         },
+  { "ELSE",              TK_ELSE,         },
+  { "EXCEPT",            TK_EXCEPT,       },
+  { "EXPLAIN",           TK_EXPLAIN,      },
+  { "FAIL",              TK_FAIL,         },
+  { "FOR",               TK_FOR,          },
+  { "FOREIGN",           TK_FOREIGN,      },
+  { "FROM",              TK_FROM,         },
+  { "FULL",              TK_JOIN_KW,      },
+  { "GLOB",              TK_GLOB,         },
+  { "GROUP",             TK_GROUP,        },
+  { "HAVING",            TK_HAVING,       },
+  { "IGNORE",            TK_IGNORE,       },
+  { "IMMEDIATE",         TK_IMMEDIATE,    },
+  { "IN",                TK_IN,           },
+  { "INDEX",             TK_INDEX,        },
+  { "INITIALLY",         TK_INITIALLY,    },
+  { "INNER",             TK_JOIN_KW,      },
+  { "INSERT",            TK_INSERT,       },
+  { "INSTEAD",           TK_INSTEAD,      },
+  { "INTERSECT",         TK_INTERSECT,    },
+  { "INTO",              TK_INTO,         },
+  { "IS",                TK_IS,           },
+  { "ISNULL",            TK_ISNULL,       },
+  { "JOIN",              TK_JOIN,         },
+  { "KEY",               TK_KEY,          },
+  { "LEFT",              TK_JOIN_KW,      },
+  { "LIKE",              TK_LIKE,         },
+  { "LIMIT",             TK_LIMIT,        },
+  { "MATCH",             TK_MATCH,        },
+  { "NATURAL",           TK_JOIN_KW,      },
+  { "NOT",               TK_NOT,          },
+  { "NOTNULL",           TK_NOTNULL,      },
+  { "NULL",              TK_NULL,         },
+  { "OF",                TK_OF,           },
+  { "OFFSET",            TK_OFFSET,       },
+  { "ON",                TK_ON,           },
+  { "OR",                TK_OR,           },
+  { "ORDER",             TK_ORDER,        },
+  { "OUTER",             TK_JOIN_KW,      },
+  { "PRAGMA",            TK_PRAGMA,       },
+  { "PRIMARY",           TK_PRIMARY,      },
+  { "RAISE",             TK_RAISE,        },
+  { "REFERENCES",        TK_REFERENCES,   },
+  { "REPLACE",           TK_REPLACE,      },
+  { "RESTRICT",          TK_RESTRICT,     },
+  { "RIGHT",             TK_JOIN_KW,      },
+  { "ROLLBACK",          TK_ROLLBACK,     },
+  { "ROW",               TK_ROW,          },
+  { "SELECT",            TK_SELECT,       },
+  { "SET",               TK_SET,          },
+  { "STATEMENT",         TK_STATEMENT,    },
+  { "TABLE",             TK_TABLE,        },
+  { "TEMP",              TK_TEMP,         },
+  { "TEMPORARY",         TK_TEMP,         },
+  { "THEN",              TK_THEN,         },
+  { "TRANSACTION",       TK_TRANSACTION,  },
+  { "TRIGGER",           TK_TRIGGER,      },
+  { "UNION",             TK_UNION,        },
+  { "UNIQUE",            TK_UNIQUE,       },
+  { "UPDATE",            TK_UPDATE,       },
+  { "USING",             TK_USING,        },
+  { "VACUUM",            TK_VACUUM,       },
+  { "VALUES",            TK_VALUES,       },
+  { "VIEW",              TK_VIEW,         },
+  { "WHEN",              TK_WHEN,         },
+  { "WHERE",             TK_WHERE,        },
 };
 
 /*
 ** This is the hash table
 */
-#define KEY_HASH_SIZE 71
-static Keyword *apHashTable[KEY_HASH_SIZE];
+#define KEY_HASH_SIZE 101
+static u8 aiHashTable[KEY_HASH_SIZE];
 
 
 /*
@@ -153,29 +153,29 @@ static Keyword *apHashTable[KEY_HASH_SIZE];
 ** returned.  If the input is not a keyword, TK_ID is returned.
 */
 int sqliteKeywordCode(const char *z, int n){
-  int h;
+  int h, i;
   Keyword *p;
   static char needInit = 1;
   if( needInit ){
     /* Initialize the keyword hash table */
     sqliteOsEnterMutex();
     if( needInit ){
-      int i;
-      int n;
-      n = sizeof(aKeywordTable)/sizeof(aKeywordTable[0]);
-      for(i=0; i<n; i++){
+      int nk;
+      nk = sizeof(aKeywordTable)/sizeof(aKeywordTable[0]);
+      for(i=0; i<nk; i++){
         aKeywordTable[i].len = strlen(aKeywordTable[i].zName);
         h = sqliteHashNoCase(aKeywordTable[i].zName, aKeywordTable[i].len);
         h %= KEY_HASH_SIZE;
-        aKeywordTable[i].pNext = apHashTable[h];
-        apHashTable[h] = &aKeywordTable[i];
+        aKeywordTable[i].iNext = aiHashTable[h];
+        aiHashTable[h] = i+1;
       }
       needInit = 0;
     }
     sqliteOsLeaveMutex();
   }
   h = sqliteHashNoCase(z, n) % KEY_HASH_SIZE;
-  for(p=apHashTable[h]; p; p=p->pNext){
+  for(i=aiHashTable[h]; i; i=p->iNext){
+    p = &aKeywordTable[i-1];
     if( p->len==n && sqliteStrNICmp(p->zName, z, n)==0 ){
       return p->tokenType;
     }
@@ -185,8 +185,12 @@ int sqliteKeywordCode(const char *z, int n){
 
 
 /*
-** If X is a character that can be used in an identifier then
-** isIdChar[X] will be 1.  Otherwise isIdChar[X] will be 0.
+** If X is a character that can be used in an identifier and
+** X&0x80==0 then isIdChar[X] will be 1.  If X&0x80==0x80 then
+** X is always an identifier character.  (Hence all UTF-8
+** characters can be part of an identifier).  isIdChar[X] will
+** be 0 for every character in the lower 128 ASCII characters
+** that cannot be used as part of an identifier.
 **
 ** In this implementation, an identifier can be a string of
 ** alphabetic characters, digits, and "_" plus any character
@@ -204,14 +208,6 @@ static const char isIdChar[] = {
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,  /* 5x */
     0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 6x */
     1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0,  /* 7x */
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 8x */
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* 9x */
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ax */
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Bx */
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Cx */
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Dx */
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Ex */
-    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,  /* Fx */
 };
 
 
@@ -380,10 +376,10 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
       return 1;
     }
     default: {
-      if( !isIdChar[*z] ){
+      if( (*z&0x80)==0 && !isIdChar[*z] ){
         break;
       }
-      for(i=1; isIdChar[z[i]]; i++){}
+      for(i=1; (z[i]&0x80)!=0 || isIdChar[z[i]]; i++){}
       *tokenType = sqliteKeywordCode((char*)z, i);
       return i;
     }