mirror of
https://github.com/sqlite/sqlite.git
synced 2025-08-18 20:22:13 +03:00
Optimizations in the tokenizer. (CVS 1985)
FossilOrigin-Name: 26898c57cb2419d4200803f79fdd821c3093cba2
This commit is contained in:
drh
12
manifest
12
manifest
@@ -1,5 +1,5 @@
|
||||
C Code\scleanup:\sget\srid\sof\sthe\ssqlite3SetNString\sutility\sfunction.\s(CVS\s1984)
|
||||
D 2004-09-25T14:39:18
|
||||
C Optimizations\sin\sthe\stokenizer.\s(CVS\s1985)
|
||||
D 2004-09-25T15:25:26
|
||||
F Makefile.in abdeb5bd9d017822691884935c320037c33f6ee6
|
||||
F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457
|
||||
F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd
|
||||
@@ -68,7 +68,7 @@ F src/test2.c 0f3e0ad7b675a6f3323211ab4ea95490855654c3
|
||||
F src/test3.c 5b5b0f3d11b097399c1054fff73d8f3711092301
|
||||
F src/test4.c 7c6b9fc33dd1f3f93c7f1ee6e5e6d016afa6c1df
|
||||
F src/test5.c b001fa7f1b9e2dc5c2331de62fc641b5ab2bd7a1
|
||||
F src/tokenize.c 85a5b0de4635bc96ec9860274449448ff0d0726d
|
||||
F src/tokenize.c d4619367d9ba17c6bd4e018fb7e91015ae8562aa
|
||||
F src/trigger.c d1f770ee37a80391dd6d0948ee821b0272f99ae7
|
||||
F src/update.c 7157084216c4b02a23cdb23eb6d246aa9034fa4d
|
||||
F src/utf.c f4f83acd73389090e32d6589d307fc55d794c7ed
|
||||
@@ -247,7 +247,7 @@ F www/tclsqlite.tcl 560ecd6a916b320e59f2917317398f3d59b7cc25
|
||||
F www/vdbe.tcl 59288db1ac5c0616296b26dce071c36cb611dfe9
|
||||
F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0
|
||||
F www/whentouse.tcl a8335bce47cc2fddb07f19052cb0cb4d9129a8e4
|
||||
P cb631a135da92fd689ee40a8b7e2695a510eb765
|
||||
R 0e57f8bf788b1fab54d791e8a0ff38cb
|
||||
P 9ef4c24a9acc2128891303de1ffd2ef4509d779c
|
||||
R 5ed5520ba8ef2bfa197e6a975f384e30
|
||||
U drh
|
||||
Z 4e1f4e3ae25eddb22e2a4ce1e3a8ec1c
|
||||
Z 610f3581cf05ef77bbf5335c4f0a0443
|
||||
|
||||
@@ -1 +1 @@
|
||||
9ef4c24a9acc2128891303de1ffd2ef4509d779c
|
||||
26898c57cb2419d4200803f79fdd821c3093cba2
|
||||
@@ -15,7 +15,7 @@
|
||||
** individual tokens and sends those tokens one-by-one over to the
|
||||
** parser for analysis.
|
||||
**
|
||||
** $Id: tokenize.c,v 1.88 2004/09/25 14:39:19 drh Exp $
|
||||
** $Id: tokenize.c,v 1.89 2004/09/25 15:25:26 drh Exp $
|
||||
*/
|
||||
#include "sqliteInt.h"
|
||||
#include "os.h"
|
||||
@@ -52,7 +52,6 @@ static Keyword aKeywordTable[] = {
|
||||
{ "CASCADE", TK_CASCADE, },
|
||||
{ "CASE", TK_CASE, },
|
||||
{ "CHECK", TK_CHECK, },
|
||||
{ "CLUSTER", TK_CLUSTER, },
|
||||
{ "COLLATE", TK_COLLATE, },
|
||||
{ "COMMIT", TK_COMMIT, },
|
||||
{ "CONFLICT", TK_CONFLICT, },
|
||||
@@ -160,11 +159,11 @@ int sqlite3KeywordCode(const char *z, int n){
|
||||
if( needInit ){
|
||||
int nk;
|
||||
nk = sizeof(aKeywordTable)/sizeof(aKeywordTable[0]);
|
||||
for(i=0; i<nk; i++){
|
||||
aKeywordTable[i].len = strlen(aKeywordTable[i].zName);
|
||||
h = sqlite3HashNoCase(aKeywordTable[i].zName, aKeywordTable[i].len);
|
||||
h %= KEY_HASH_SIZE;
|
||||
aKeywordTable[i].iNext = aiHashTable[h];
|
||||
for(i=0, p=aKeywordTable; i<nk; i++, p++){
|
||||
const char *zName = p->zName;
|
||||
int len = p->len = strlen(zName);
|
||||
h = sqlite3HashNoCase(zName, len) % KEY_HASH_SIZE;
|
||||
p->iNext = aiHashTable[h];
|
||||
aiHashTable[h] = i+1;
|
||||
}
|
||||
needInit = 0;
|
||||
@@ -198,9 +197,6 @@ int sqlite3KeywordCode(const char *z, int n){
|
||||
*/
|
||||
static const char isIdChar[] = {
|
||||
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
|
||||
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
|
||||
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
|
||||
@@ -208,13 +204,14 @@ static const char isIdChar[] = {
|
||||
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
|
||||
};
|
||||
|
||||
#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x2f && isIdChar[c-0x30]))
|
||||
|
||||
/*
|
||||
** Return the length of the token that begins at z[0].
|
||||
** Store the token type in *tokenType before returning.
|
||||
*/
|
||||
static int sqliteGetToken(const unsigned char *z, int *tokenType){
|
||||
int i;
|
||||
int i, c;
|
||||
switch( *z ){
|
||||
case ' ': case '\t': case '\n': case '\f': case '\r': {
|
||||
for(i=1; isspace(z[i]); i++){}
|
||||
@@ -223,7 +220,7 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
|
||||
}
|
||||
case '-': {
|
||||
if( z[1]=='-' ){
|
||||
for(i=2; z[i] && z[i]!='\n'; i++){}
|
||||
for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
|
||||
*tokenType = TK_COMMENT;
|
||||
return i;
|
||||
}
|
||||
@@ -255,8 +252,8 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
|
||||
*tokenType = TK_SLASH;
|
||||
return 1;
|
||||
}
|
||||
for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){}
|
||||
if( z[i] ) i++;
|
||||
for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
|
||||
if( c ) i++;
|
||||
*tokenType = TK_COMMENT;
|
||||
return i;
|
||||
}
|
||||
@@ -269,13 +266,13 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
|
||||
return 1 + (z[1]=='=');
|
||||
}
|
||||
case '<': {
|
||||
if( z[1]=='=' ){
|
||||
if( (c=z[1])=='=' ){
|
||||
*tokenType = TK_LE;
|
||||
return 2;
|
||||
}else if( z[1]=='>' ){
|
||||
}else if( c=='>' ){
|
||||
*tokenType = TK_NE;
|
||||
return 2;
|
||||
}else if( z[1]=='<' ){
|
||||
}else if( c=='<' ){
|
||||
*tokenType = TK_LSHIFT;
|
||||
return 2;
|
||||
}else{
|
||||
@@ -284,10 +281,10 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
|
||||
}
|
||||
}
|
||||
case '>': {
|
||||
if( z[1]=='=' ){
|
||||
if( (c=z[1])=='=' ){
|
||||
*tokenType = TK_GE;
|
||||
return 2;
|
||||
}else if( z[1]=='>' ){
|
||||
}else if( c=='>' ){
|
||||
*tokenType = TK_RSHIFT;
|
||||
return 2;
|
||||
}else{
|
||||
@@ -327,8 +324,8 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
|
||||
}
|
||||
case '\'': case '"': {
|
||||
int delim = z[0];
|
||||
for(i=1; z[i]; i++){
|
||||
if( z[i]==delim ){
|
||||
for(i=1; (c=z[i])!=0; i++){
|
||||
if( c==delim ){
|
||||
if( z[i+1]==delim ){
|
||||
i++;
|
||||
}else{
|
||||
@@ -336,7 +333,7 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
|
||||
}
|
||||
}
|
||||
}
|
||||
if( z[i] ) i++;
|
||||
if( c ) i++;
|
||||
*tokenType = TK_STRING;
|
||||
return i;
|
||||
}
|
||||
@@ -365,7 +362,7 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
|
||||
return i;
|
||||
}
|
||||
case '[': {
|
||||
for(i=1; z[i] && z[i-1]!=']'; i++){}
|
||||
for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
|
||||
*tokenType = TK_ID;
|
||||
return i;
|
||||
}
|
||||
@@ -375,12 +372,11 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
|
||||
return i;
|
||||
}
|
||||
case ':': {
|
||||
for(i=1; (z[i]&0x80)!=0 || isIdChar[z[i]]; i++){}
|
||||
for(i=1; IdChar(z[i]); i++){}
|
||||
*tokenType = i>1 ? TK_VARIABLE : TK_ILLEGAL;
|
||||
return i;
|
||||
}
|
||||
case '$': {
|
||||
int c;
|
||||
*tokenType = TK_VARIABLE;
|
||||
if( z[1]=='{' ){
|
||||
int nBrace = 1;
|
||||
@@ -418,29 +414,29 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
|
||||
return i;
|
||||
}
|
||||
case 'x': case 'X': {
|
||||
if( z[1]=='\'' || z[1]=='"' ){
|
||||
int delim = z[1];
|
||||
if( (c=z[1])=='\'' || c=='"' ){
|
||||
int delim = c;
|
||||
*tokenType = TK_BLOB;
|
||||
for(i=2; z[i]; i++){
|
||||
if( z[i]==delim ){
|
||||
for(i=2; (c=z[i])!=0; i++){
|
||||
if( c==delim ){
|
||||
if( i%2 ) *tokenType = TK_ILLEGAL;
|
||||
break;
|
||||
}
|
||||
if( !isxdigit(z[i]) ){
|
||||
if( !isxdigit(c) ){
|
||||
*tokenType = TK_ILLEGAL;
|
||||
return i;
|
||||
}
|
||||
}
|
||||
if( z[i] ) i++;
|
||||
if( c ) i++;
|
||||
return i;
|
||||
}
|
||||
/* Otherwise fall through to the next case */
|
||||
}
|
||||
default: {
|
||||
if( (*z&0x80)==0 && !isIdChar[*z] ){
|
||||
if( !IdChar(*z) ){
|
||||
break;
|
||||
}
|
||||
for(i=1; (z[i]&0x80)!=0 || isIdChar[z[i]]; i++){}
|
||||
for(i=1; IdChar(z[i]); i++){}
|
||||
*tokenType = sqlite3KeywordCode((char*)z, i);
|
||||
return i;
|
||||
}
|
||||
@@ -692,10 +688,11 @@ int sqlite3_complete(const char *zSql){
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
if( isIdChar[(u8)*zSql] ){
|
||||
int c;
|
||||
if( IdChar((u8)*zSql) ){
|
||||
/* Keywords and unquoted identifiers */
|
||||
int nId;
|
||||
for(nId=1; isIdChar[(u8)zSql[nId]]; nId++){}
|
||||
for(nId=1; IdChar(zSql[nId]); nId++){}
|
||||
switch( *zSql ){
|
||||
case 'c': case 'C': {
|
||||
if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){
|
||||
|
||||
Reference in New Issue
Block a user