database/sqlite
1
0
Fork 0
mirror of https://github.com/sqlite/sqlite.git synced 2025-08-18 20:22:13 +03:00
Code Activity

Optimizations in the tokenizer. (CVS 1985)

Browse Source 
FossilOrigin-Name: 26898c57cb2419d4200803f79fdd821c3093cba2
This commit is contained in:
drh
2004-09-25 15:25:26 +00:00
parent ae29ffbe23
commit aa756b0978
3 changed files with 39 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
manifest
View File

@@ -1,5 +1,5 @@
C Code\scleanup:\sget\srid\sof\sthe\ssqlite3SetNString\sutility\sfunction.\s(CVS\s1984) C Optimizations\sin\sthe\stokenizer.\s(CVS\s1985)
D 2004-09-25T14:39:18 D 2004-09-25T15:25:26
F Makefile.in abdeb5bd9d017822691884935c320037c33f6ee6 F Makefile.in abdeb5bd9d017822691884935c320037c33f6ee6
F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457 F Makefile.linux-gcc a9e5a0d309fa7c38e7c14d3ecf7690879d3a5457
F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd F README f1de682fbbd94899d50aca13d387d1b3fd3be2dd
@@ -68,7 +68,7 @@ F src/test2.c 0f3e0ad7b675a6f3323211ab4ea95490855654c3
F src/test3.c 5b5b0f3d11b097399c1054fff73d8f3711092301 F src/test3.c 5b5b0f3d11b097399c1054fff73d8f3711092301
F src/test4.c 7c6b9fc33dd1f3f93c7f1ee6e5e6d016afa6c1df F src/test4.c 7c6b9fc33dd1f3f93c7f1ee6e5e6d016afa6c1df
F src/test5.c b001fa7f1b9e2dc5c2331de62fc641b5ab2bd7a1 F src/test5.c b001fa7f1b9e2dc5c2331de62fc641b5ab2bd7a1
F src/tokenize.c 85a5b0de4635bc96ec9860274449448ff0d0726d F src/tokenize.c d4619367d9ba17c6bd4e018fb7e91015ae8562aa
F src/trigger.c d1f770ee37a80391dd6d0948ee821b0272f99ae7 F src/trigger.c d1f770ee37a80391dd6d0948ee821b0272f99ae7
F src/update.c 7157084216c4b02a23cdb23eb6d246aa9034fa4d F src/update.c 7157084216c4b02a23cdb23eb6d246aa9034fa4d
F src/utf.c f4f83acd73389090e32d6589d307fc55d794c7ed F src/utf.c f4f83acd73389090e32d6589d307fc55d794c7ed
@@ -247,7 +247,7 @@ F www/tclsqlite.tcl 560ecd6a916b320e59f2917317398f3d59b7cc25
F www/vdbe.tcl 59288db1ac5c0616296b26dce071c36cb611dfe9 F www/vdbe.tcl 59288db1ac5c0616296b26dce071c36cb611dfe9
F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0 F www/version3.tcl 092a01f5ef430d2c4acc0ae558d74c4bb89638a0
F www/whentouse.tcl a8335bce47cc2fddb07f19052cb0cb4d9129a8e4 F www/whentouse.tcl a8335bce47cc2fddb07f19052cb0cb4d9129a8e4
P cb631a135da92fd689ee40a8b7e2695a510eb765 P 9ef4c24a9acc2128891303de1ffd2ef4509d779c
R 0e57f8bf788b1fab54d791e8a0ff38cb R 5ed5520ba8ef2bfa197e6a975f384e30
U drh U drh
Z 4e1f4e3ae25eddb22e2a4ce1e3a8ec1c Z 610f3581cf05ef77bbf5335c4f0a0443

2
manifest.uuid
View File

@@ -1 +1 @@
9ef4c24a9acc2128891303de1ffd2ef4509d779c 26898c57cb2419d4200803f79fdd821c3093cba2

67
src/tokenize.c
View File

@@ -15,7 +15,7 @@
** individual tokens and sends those tokens one-by-one over to the ** individual tokens and sends those tokens one-by-one over to the
** parser for analysis. ** parser for analysis.
** **
** $Id: tokenize.c,v 1.88 2004/09/25 14:39:19 drh Exp $ ** $Id: tokenize.c,v 1.89 2004/09/25 15:25:26 drh Exp $
*/ */
#include "sqliteInt.h" #include "sqliteInt.h"
#include "os.h" #include "os.h"
@@ -52,7 +52,6 @@ static Keyword aKeywordTable[] = {
{ "CASCADE", TK_CASCADE, }, { "CASCADE", TK_CASCADE, },
{ "CASE", TK_CASE, }, { "CASE", TK_CASE, },
{ "CHECK", TK_CHECK, }, { "CHECK", TK_CHECK, },
{ "CLUSTER", TK_CLUSTER, },
{ "COLLATE", TK_COLLATE, }, { "COLLATE", TK_COLLATE, },
{ "COMMIT", TK_COMMIT, }, { "COMMIT", TK_COMMIT, },
{ "CONFLICT", TK_CONFLICT, }, { "CONFLICT", TK_CONFLICT, },
@@ -160,11 +159,11 @@ int sqlite3KeywordCode(const char *z, int n){
if( needInit ){ if( needInit ){
int nk; int nk;
nk = sizeof(aKeywordTable)/sizeof(aKeywordTable[0]); nk = sizeof(aKeywordTable)/sizeof(aKeywordTable[0]);
for(i=0; i<nk; i++){ for(i=0, p=aKeywordTable; i<nk; i++, p++){
aKeywordTable[i].len = strlen(aKeywordTable[i].zName); const char *zName = p->zName;
h = sqlite3HashNoCase(aKeywordTable[i].zName, aKeywordTable[i].len); int len = p->len = strlen(zName);
h %= KEY_HASH_SIZE; h = sqlite3HashNoCase(zName, len) % KEY_HASH_SIZE;
aKeywordTable[i].iNext = aiHashTable[h]; p->iNext = aiHashTable[h];
aiHashTable[h] = i+1; aiHashTable[h] = i+1;
} }
needInit = 0; needInit = 0;
@@ -198,9 +197,6 @@ int sqlite3KeywordCode(const char *z, int n){
*/ */
static const char isIdChar[] = { static const char isIdChar[] = {
/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */
0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */
@@ -208,13 +204,14 @@ static const char isIdChar[] = {
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */
}; };
#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x2f && isIdChar[c-0x30]))
/* /*
** Return the length of the token that begins at z[0]. ** Return the length of the token that begins at z[0].
** Store the token type in *tokenType before returning. ** Store the token type in *tokenType before returning.
*/ */
static int sqliteGetToken(const unsigned char *z, int *tokenType){ static int sqliteGetToken(const unsigned char *z, int *tokenType){
int i; int i, c;
switch( *z ){ switch( *z ){
case ' ': case '\t': case '\n': case '\f': case '\r': { case ' ': case '\t': case '\n': case '\f': case '\r': {
for(i=1; isspace(z[i]); i++){} for(i=1; isspace(z[i]); i++){}
@@ -223,7 +220,7 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
} }
case '-': { case '-': {
if( z[1]=='-' ){ if( z[1]=='-' ){
for(i=2; z[i] && z[i]!='\n'; i++){} for(i=2; (c=z[i])!=0 && c!='\n'; i++){}
*tokenType = TK_COMMENT; *tokenType = TK_COMMENT;
return i; return i;
} }
@@ -255,8 +252,8 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
*tokenType = TK_SLASH; *tokenType = TK_SLASH;
return 1; return 1;
} }
for(i=3; z[i] && (z[i]!='/' || z[i-1]!='*'); i++){} for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){}
if( z[i] ) i++; if( c ) i++;
*tokenType = TK_COMMENT; *tokenType = TK_COMMENT;
return i; return i;
} }
@@ -269,13 +266,13 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
return 1 + (z[1]=='='); return 1 + (z[1]=='=');
} }
case '<': { case '<': {
if( z[1]=='=' ){ if( (c=z[1])=='=' ){
*tokenType = TK_LE; *tokenType = TK_LE;
return 2; return 2;
}else if( z[1]=='>' ){ }else if( c=='>' ){
*tokenType = TK_NE; *tokenType = TK_NE;
return 2; return 2;
}else if( z[1]=='<' ){ }else if( c=='<' ){
*tokenType = TK_LSHIFT; *tokenType = TK_LSHIFT;
return 2; return 2;
}else{ }else{
@@ -284,10 +281,10 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
} }
} }
case '>': { case '>': {
if( z[1]=='=' ){ if( (c=z[1])=='=' ){
*tokenType = TK_GE; *tokenType = TK_GE;
return 2; return 2;
}else if( z[1]=='>' ){ }else if( c=='>' ){
*tokenType = TK_RSHIFT; *tokenType = TK_RSHIFT;
return 2; return 2;
}else{ }else{
@@ -327,8 +324,8 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
} }
case '\'': case '"': { case '\'': case '"': {
int delim = z[0]; int delim = z[0];
for(i=1; z[i]; i++){ for(i=1; (c=z[i])!=0; i++){
if( z[i]==delim ){ if( c==delim ){
if( z[i+1]==delim ){ if( z[i+1]==delim ){
i++; i++;
}else{ }else{
@@ -336,7 +333,7 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
} }
} }
} }
if( z[i] ) i++; if( c ) i++;
*tokenType = TK_STRING; *tokenType = TK_STRING;
return i; return i;
} }
@@ -365,7 +362,7 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
return i; return i;
} }
case '[': { case '[': {
for(i=1; z[i] && z[i-1]!=']'; i++){} for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){}
*tokenType = TK_ID; *tokenType = TK_ID;
return i; return i;
} }
@@ -375,12 +372,11 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
return i; return i;
} }
case ':': { case ':': {
for(i=1; (z[i]&0x80)!=0 || isIdChar[z[i]]; i++){} for(i=1; IdChar(z[i]); i++){}
*tokenType = i>1 ? TK_VARIABLE : TK_ILLEGAL; *tokenType = i>1 ? TK_VARIABLE : TK_ILLEGAL;
return i; return i;
} }
case '$': { case '$': {
int c;
*tokenType = TK_VARIABLE; *tokenType = TK_VARIABLE;
if( z[1]=='{' ){ if( z[1]=='{' ){
int nBrace = 1; int nBrace = 1;
@@ -418,29 +414,29 @@ static int sqliteGetToken(const unsigned char *z, int *tokenType){
return i; return i;
} }
case 'x': case 'X': { case 'x': case 'X': {
if( z[1]=='\'' || z[1]=='"' ){ if( (c=z[1])=='\'' || c=='"' ){
int delim = z[1]; int delim = c;
*tokenType = TK_BLOB; *tokenType = TK_BLOB;
for(i=2; z[i]; i++){ for(i=2; (c=z[i])!=0; i++){
if( z[i]==delim ){ if( c==delim ){
if( i%2 ) *tokenType = TK_ILLEGAL; if( i%2 ) *tokenType = TK_ILLEGAL;
break; break;
} }
if( !isxdigit(z[i]) ){ if( !isxdigit(c) ){
*tokenType = TK_ILLEGAL; *tokenType = TK_ILLEGAL;
return i; return i;
} }
} }
if( z[i] ) i++; if( c ) i++;
return i; return i;
} }
/* Otherwise fall through to the next case */ /* Otherwise fall through to the next case */
} }
default: { default: {
if( (*z&0x80)==0 && !isIdChar[*z] ){ if( !IdChar(*z) ){
break; break;
} }
for(i=1; (z[i]&0x80)!=0 || isIdChar[z[i]]; i++){} for(i=1; IdChar(z[i]); i++){}
*tokenType = sqlite3KeywordCode((char*)z, i); *tokenType = sqlite3KeywordCode((char*)z, i);
return i; return i;
} }
@@ -692,10 +688,11 @@ int sqlite3_complete(const char *zSql){
break; break;
} }
default: { default: {
if( isIdChar[(u8)*zSql] ){ int c;
if( IdChar((u8)*zSql) ){
/* Keywords and unquoted identifiers */ /* Keywords and unquoted identifiers */
int nId; int nId;
for(nId=1; isIdChar[(u8)zSql[nId]]; nId++){} for(nId=1; IdChar(zSql[nId]); nId++){}
switch( *zSql ){ switch( *zSql ){
case 'c': case 'C': { case 'c': case 'C': {
if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){ if( nId==6 && sqlite3StrNICmp(zSql, "create", 6)==0 ){