Various bugfixes. 68 Test cases still fail. (CVS 1471)

FossilOrigin-Name: 67a140cf78d99e38ccd94751c4f8ead1a2b96859

src/tokenize.c

@@ -15,7 +15,7 @@
 ** individual tokens and sends those tokens one-by-one over to the
 ** parser for analysis.
 **
-** $Id: tokenize.c,v 1.70 2004/05/10 10:34:53 danielk1977 Exp $
+** $Id: tokenize.c,v 1.71 2004/05/27 09:28:43 danielk1977 Exp $
 */
 #include "sqliteInt.h"
 #include "os.h"
@@ -375,6 +375,20 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
       *tokenType = TK_VARIABLE;
       return 1;
     }
+    case 'x': case 'X': {
+      if( z[1]=='\'' || z[1]=='"' ){
+        int delim = z[0];
+        for(i=1; z[i]; i++){
+          if( z[i]==delim ){
+            break;
+          }
+        }
+        if( z[i] ) i++;
+        *tokenType = TK_BLOB;
+        return i;
+      }
+      /* Otherwise fall through to the next case */
+    }
     default: {
       if( (*z&0x80)==0 && !isIdChar[*z] ){
         break;