database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-30 11:03:19 +03:00
Code Activity

Remove fixed-size literal buffer from ecpg's lexer (same

Browse Source 
fix recently applied to backend's lexer).  I see that YY_USES_REJECT
still gets defined for this lexer, which means it's going to have trouble
parsing really long tokens.  Not sure if it's worth doing anything about
that or not; I don't have the interest right now to understand why
ecpg's additions to the syntax cause this problem...
This commit is contained in:
Tom Lane
1999-10-22 23:14:50 +00:00
parent abceb20a52
commit 45c002460c
2 changed files with 107 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
src/interfaces/ecpg/preproc/Makefile
View File

@ -15,11 +15,19 @@ OBJ=preproc.o pgc.o type.o ecpg.o ecpg_keywords.o \
all:: ecpg all:: ecpg
# Rule that really do something.
ecpg: $(OBJ)
$(CC) -o ecpg $(OBJ) $(LEXLIB) $(LDFLAGS)
preproc.c preproc.h: preproc.y preproc.c preproc.h: preproc.y
$(YACC) $(YFLAGS) $< $(YACC) $(YFLAGS) $<
mv y.tab.c preproc.c mv y.tab.c preproc.c
mv y.tab.h preproc.h mv y.tab.h preproc.h
pgc.c: pgc.l
$(LEX) $<
mv lex.yy.c pgc.c
clean: clean:
rm -f *.o core a.out ecpg$(X) *~ *.output rm -f *.o core a.out ecpg$(X) *~ *.output
# And the garbage that might have been left behind by partial build: # And the garbage that might have been left behind by partial build:
@ -33,19 +41,9 @@ install: all
uninstall: uninstall:
rm -f $(BINDIR)/ecpg rm -f $(BINDIR)/ecpg
# Rule that really do something. preproc.o: preproc.h ../include/ecpgtype.h keywords.c c_keywords.c ecpg_keywords.c
ecpg: $(OBJ) type.o: ../include/ecpgtype.h
$(CC) -o ecpg $(OBJ) $(LEXLIB) $(LDFLAGS) pgc.o: ../include/ecpgtype.h keywords.c c_keywords.c ecpg_keywords.c preproc.h
pgc.c: pgc.l
$(LEX) $<
sed -e 's/#define YY_BUF_SIZE .*/#define YY_BUF_SIZE 65536/' \
<lex.yy.c >pgc.c
rm -f lex.yy.c
preproc.o : preproc.h ../include/ecpgtype.h keywords.c c_keywords.c ecpg_keywords.c
type.o : ../include/ecpgtype.h
pgc.o : ../include/ecpgtype.h keywords.c c_keywords.c ecpg_keywords.c preproc.h
keywords.o: ../include/ecpgtype.h preproc.h keywords.o: ../include/ecpgtype.h preproc.h
c_keywords.o: ../include/ecpgtype.h preproc.h c_keywords.o: ../include/ecpgtype.h preproc.h
ecpg_keywords.o: ../include/ecpgtype.h preproc.h ecpg_keywords.o: ../include/ecpgtype.h preproc.h

167
src/interfaces/ecpg/preproc/pgc.l
View File

@ -1,38 +1,61 @@
/* This is a modified version of src/backend/parser/scan.l */
%{ %{
/*-------------------------------------------------------------------------
*
* pgc.l
* lexical scanner for ecpg
*
* This is a modified version of src/backend/parser/scan.l
*
*
* Copyright (c) 1994, Regents of the University of California
*
*
* IDENTIFICATION
* $Header: /cvsroot/pgsql/src/interfaces/ecpg/preproc/pgc.l,v 1.45 1999/10/22 23:14:50 tgl Exp $
*
*-------------------------------------------------------------------------
*/
#include <ctype.h> #include <ctype.h>
#include <sys/types.h> #include <sys/types.h>
#include <limits.h> #include <limits.h>
#include <errno.h> #include <errno.h>
#include "postgres.h" #include "postgres.h"
#ifndef PATH_MAX #ifndef PATH_MAX
#include <sys/param.h> #include <sys/param.h>
#define PATH_MAX MAXPATHLEN #define PATH_MAX MAXPATHLEN
#endif #endif
#include "miscadmin.h" #include "miscadmin.h"
#include "nodes/pg_list.h"
#include "nodes/parsenodes.h" #include "nodes/parsenodes.h"
#include "nodes/pg_list.h"
#include "parser/gramparse.h" #include "parser/gramparse.h"
#include "parser/scansup.h" #include "parser/scansup.h"
#include "extern.h" #include "extern.h"
#include "preproc.h" #include "preproc.h"
#include "utils/builtins.h" #include "utils/builtins.h"
#ifdef YY_READ_BUF_SIZE
#undef YY_READ_BUF_SIZE
#endif
#define YY_READ_BUF_SIZE MAX_PARSE_BUFFER
/* some versions of lex define this as a macro */ /* some versions of lex define this as a macro */
#if defined(yywrap) #if defined(yywrap)
#undef yywrap #undef yywrap
#endif /* yywrap */ #endif /* yywrap */
extern YYSTYPE yylval; extern YYSTYPE yylval;
int llen;
char literal[MAX_PARSE_BUFFER]; /*
* literalbuf is used to accumulate literal values when multiple rules
* are needed to parse a single literal. Call startlit to reset buffer
* to empty, addlit to add text. Note that the buffer is permanently
* malloc'd to the largest size needed so far in the current run.
*/
static char *literalbuf = NULL; /* expandable buffer */
static int literallen; /* actual current length */
static int literalalloc; /* current allocated buffer size */
#define startlit() (literalbuf[0] = '\0', literallen = 0)
static void addlit(char *ytext, int yleng);
int before_comment; int before_comment;
struct _yy_buffer { YY_BUFFER_STATE buffer; struct _yy_buffer { YY_BUFFER_STATE buffer;
@ -142,16 +165,14 @@ self [,()\[\].;$\:\+\-\*\/\%\^\<\>\=\|]
op_and_self [\~\!\@\#\^\&\|\`\?\$\:\+\-\*\/\%\<\>\=] op_and_self [\~\!\@\#\^\&\|\`\?\$\:\+\-\*\/\%\<\>\=]
operator {op_and_self}+ operator {op_and_self}+
/* we do not allow unary minus in numbers. /* we no longer allow unary minus in numbers.
* instead we pass it verbatim to parser. there it gets * instead we pass it separately to parser. there it gets
* coerced via doNegate() -- Leon aug 20 1999 * coerced via doNegate() -- Leon aug 20 1999
*/ */
integer {digit}+ integer {digit}+
decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*)) decimal (({digit}*\.{digit}+)|({digit}+\.{digit}*))
real ((({digit}*\.{digit}+)|({digit}+\.{digit}*)|({digit}+))([Ee][-+]?{digit}+)) real ((({digit}*\.{digit}+)|({digit}+\.{digit}*)|({digit}+))([Ee][-+]?{digit}+))
/*
real (((({digit}*\.{digit}+)|({digit}+\.{digit}*))([Ee][-+]?{digit}+)?)|({digit}+[Ee][-+]?{digit}+))
*/
param \${integer} param \${integer}
@ -200,25 +221,21 @@ cppline {space}*#.*(\\{space}*\n)*\n*
<SQL>{xbstart} { <SQL>{xbstart} {
BEGIN(xb); BEGIN(xb);
llen = 0; startlit();
*literal = '\0';
} }
<xb>{xbstop} { <xb>{xbstop} {
char* endptr; char* endptr;
BEGIN(SQL); BEGIN(SQL);
errno = 0; errno = 0;
yylval.ival = strtol((char *)literal,&endptr,2); yylval.ival = strtol(literalbuf, &endptr, 2);
if (*endptr != '\0' || errno == ERANGE) if (*endptr != '\0' || errno == ERANGE)
yyerror("ERROR: Bad binary integer input!"); yyerror("ERROR: Bad binary integer input!");
return ICONST; return ICONST;
} }
<xh>{xhinside} | <xh>{xhinside} |
<xb>{xbinside} { <xb>{xbinside} {
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1)) addlit(yytext, yyleng);
yyerror("ERROR: quoted string parse buffer exceeded");
memcpy(literal+llen, yytext, yyleng+1);
llen += yyleng;
} }
<xh>{xhcat} | <xh>{xhcat} |
<xb>{xbcat} { <xb>{xbcat} {
@ -226,15 +243,14 @@ cppline {space}*#.*(\\{space}*\n)*\n*
<SQL>{xhstart} { <SQL>{xhstart} {
BEGIN(xh); BEGIN(xh);
llen = 0; startlit();
*literal = '\0';
} }
<xh>{xhstop} { <xh>{xhstop} {
char* endptr; char* endptr;
BEGIN(SQL); BEGIN(SQL);
errno = 0; errno = 0;
yylval.ival = strtol((char *)literal,&endptr,16); yylval.ival = strtol(literalbuf, &endptr, 16);
if (*endptr != '\0' || errno == ERANGE) if (*endptr != '\0' || errno == ERANGE)
yyerror("ERROR: Bad hexadecimal integer input"); yyerror("ERROR: Bad hexadecimal integer input");
return ICONST; return ICONST;
@ -242,21 +258,17 @@ cppline {space}*#.*(\\{space}*\n)*\n*
<SQL>{xqstart} { <SQL>{xqstart} {
BEGIN(xq); BEGIN(xq);
llen = 0; startlit();
*literal = '\0';
} }
<xq>{xqstop} { <xq>{xqstop} {
BEGIN(SQL); BEGIN(SQL);
yylval.str = mm_strdup(literal); yylval.str = mm_strdup(literalbuf);
return SCONST; return SCONST;
} }
<xq>{xqdouble} | <xq>{xqdouble} |
<xq>{xqinside} | <xq>{xqinside} |
<xq>{xqliteral} { <xq>{xqliteral} {
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1)) addlit(yytext, yyleng);
yyerror("ERROR: quoted string parse buffer exceeded");
memcpy(literal+llen, yytext, yyleng+1);
llen += yyleng;
} }
<xq>{xqcat} { <xq>{xqcat} {
} }
@ -264,35 +276,27 @@ cppline {space}*#.*(\\{space}*\n)*\n*
<SQL>{xdstart} { <SQL>{xdstart} {
BEGIN(xd); BEGIN(xd);
llen = 0; startlit();
*literal = '\0';
} }
<xd>{xdstop} { <xd>{xdstop} {
BEGIN(SQL); BEGIN(SQL);
yylval.str = mm_strdup(literal); yylval.str = mm_strdup(literalbuf);
return CSTRING; return CSTRING;
} }
<xd>{xdinside} { <xd>{xdinside} {
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1)) addlit(yytext, yyleng);
yyerror("ERROR: quoted string parse buffer exceeded");
memcpy(literal+llen, yytext, yyleng+1);
llen += yyleng;
} }
{xdstart} { {xdstart} {
BEGIN(xdc); BEGIN(xdc);
llen = 0; startlit();
*literal = '\0';
} }
<xdc>{xdstop} { <xdc>{xdstop} {
BEGIN(C); BEGIN(C);
yylval.str = mm_strdup(literal); yylval.str = mm_strdup(literalbuf);
return CSTRING; return CSTRING;
} }
<xdc>{xdcinside} { <xdc>{xdcinside} {
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1)) addlit(yytext, yyleng);
yyerror("ERROR: quoted string parse buffer exceeded");
memcpy(literal+llen, yytext, yyleng+1);
llen += yyleng;
} }
<SQL>{typecast} { return TYPECAST; } <SQL>{typecast} { return TYPECAST; }
<SQL>{self} { /* <SQL>{self} { /*
@ -324,24 +328,24 @@ cppline {space}*#.*(\\{space}*\n)*\n*
{ {
errno = 0; errno = 0;
yylval.str = mm_strdup((char*)yytext); yylval.str = mm_strdup((char*)yytext);
return SCONST; return SCONST;
} }
return ICONST; return ICONST;
} }
{decimal} { {decimal} {
char* endptr; char* endptr;
if (strlen((char *)yytext) <= 17) if (strlen((char *)yytext) <= 17)
{ {
errno = 0; errno = 0;
yylval.dval = strtod((char *)yytext,&endptr); yylval.dval = strtod((char *)yytext,&endptr);
if (*endptr != '\0' || errno == ERANGE) if (*endptr != '\0' || errno == ERANGE)
yyerror("ERROR: Bad float8 input"); yyerror("ERROR: Bad float8 input");
return FCONST; return FCONST;
} }
yylval.str = mm_strdup((char*)yytext); yylval.str = mm_strdup((char*)yytext);
return SCONST; return SCONST;
} }
<C,SQL>{real} { <C,SQL>{real} {
char* endptr; char* endptr;
@ -420,7 +424,7 @@ cppline {space}*#.*(\\{space}*\n)*\n*
{ {
errno = 0; errno = 0;
yylval.str = mm_strdup((char*)yytext); yylval.str = mm_strdup((char*)yytext);
return SCONST; return SCONST;
} }
return ICONST; return ICONST;
} }
@ -486,8 +490,7 @@ cppline {space}*#.*(\\{space}*\n)*\n*
<def_ident>{identifier} { <def_ident>{identifier} {
old = mm_strdup(yytext); old = mm_strdup(yytext);
BEGIN(def); BEGIN(def);
llen = 0; startlit();
*literal = '\0';
} }
<def>{space} /* eat the whitespace */ <def>{space} /* eat the whitespace */
<def>";" { <def>";" {
@ -498,8 +501,8 @@ cppline {space}*#.*(\\{space}*\n)*\n*
if (strcmp(old, ptr->old) == 0) if (strcmp(old, ptr->old) == 0)
{ {
free(ptr->new); free(ptr->new);
/* ptr->new = mm_strdup(scanstr(literal));*/ /* ptr->new = mm_strdup(scanstr(literalbuf));*/
ptr->new = mm_strdup(literal); ptr->new = mm_strdup(literalbuf);
} }
} }
if (ptr == NULL) if (ptr == NULL)
@ -508,8 +511,8 @@ cppline {space}*#.*(\\{space}*\n)*\n*
/* initial definition */ /* initial definition */
this->old = old; this->old = old;
/* this->new = mm_strdup(scanstr(literal));*/ /* this->new = mm_strdup(scanstr(literalbuf));*/
this->new = mm_strdup(literal); this->new = mm_strdup(literalbuf);
this->next = defines; this->next = defines;
defines = this; defines = this;
} }
@ -517,10 +520,7 @@ cppline {space}*#.*(\\{space}*\n)*\n*
BEGIN(C); BEGIN(C);
} }
<def>[^";"] { <def>[^";"] {
if ((llen+yyleng) > (MAX_PARSE_BUFFER - 1)) addlit(yytext, yyleng);
yyerror("ERROR: define statement parse buffer exceeded");
memcpy(literal+llen, yytext, yyleng+1);
llen += yyleng;
} }
<C>{exec}{space}{sql}{space}{include} { BEGIN(incl); } <C>{exec}{space}{sql}{space}{include} { BEGIN(incl); }
<incl>{space} /* eat the whitespace */ <incl>{space} /* eat the whitespace */
@ -602,9 +602,34 @@ void
lex_init(void) lex_init(void)
{ {
braces_open = 0; braces_open = 0;
/* initialize literal buffer to a reasonable but expansible size */
if (literalbuf == NULL)
{
literalalloc = 128;
literalbuf = (char *) malloc(literalalloc);
}
startlit();
BEGIN C; BEGIN C;
} }
static void
addlit(char *ytext, int yleng)
{
/* enlarge buffer if needed */
if ((literallen+yleng) >= literalalloc)
{
do {
literalalloc *= 2;
} while ((literallen+yleng) >= literalalloc);
literalbuf = (char *) realloc(literalbuf, literalalloc);
}
/* append data --- note we assume ytext is null-terminated */
memcpy(literalbuf+literallen, ytext, yleng+1);
literallen += yleng;
}
int yywrap(void) int yywrap(void)
{ {
return 1; return 1;