Scanner performance improvements

Use flex flags -CF.  Pass the to-be-scanned string around as StringInfo
type, to avoid querying the length repeatedly.  Clean up some code and
remove lex-compatibility cruft.  Escape backslash sequences inline.  Use
flex-provided yy_scan_buffer() function to set up input, rather than using
myinput().

src/backend/parser/Makefile

@@ -2,7 +2,7 @@
 #
 # Makefile for parser
 #
-# $Header: /cvsroot/pgsql/src/backend/parser/Makefile,v 1.36 2002/03/08 07:12:11 tgl Exp $
+# $Header: /cvsroot/pgsql/src/backend/parser/Makefile,v 1.37 2002/04/20 21:56:14 petere Exp $
 #
 #-------------------------------------------------------------------------
 
@@ -14,6 +14,8 @@ OBJS= analyze.o gram.o keywords.o parser.o parse_agg.o parse_clause.o \
       parse_expr.o parse_func.o parse_node.o parse_oper.o parse_relation.o \
       parse_type.o parse_coerce.o parse_target.o scan.o scansup.o
 
+FLEXFLAGS = -CF
+
 
 all: SUBSYS.o
 
@@ -42,7 +44,7 @@ endif
 
 $(srcdir)/scan.c: scan.l
 ifdef FLEX
-	$(FLEX) $(FLEXFLAGS) -Pbase_yy -o'$@' $<
+	$(FLEX) $(FLEXFLAGS) -o'$@' $<
 else
 	@$(missing) flex $< $@
 endif
@@ -59,13 +61,3 @@ clean:
 	rm -f SUBSYS.o $(OBJS)
 # And the garbage that might have been left behind by partial build:
 	@rm -f y.tab.c y.tab.h lex.yy.c
-
-
-# This is unusual:  We actually have to build some of the parts before
-# we know what the header file dependencies are.  
-dep depend: gram.c scan.c
-	$(CC) -MM $(CFLAGS) *.c >depend
-
-ifeq (depend,$(wildcard depend))
-include depend
-endif

src/backend/parser/parse_type.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/parser/parse_type.c,v 1.39 2002/03/30 01:02:41 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/parser/parse_type.c,v 1.40 2002/04/20 21:56:14 petere Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -473,17 +473,17 @@ typeidTypeRelid(Oid type_id)
 void
 parseTypeString(const char *str, Oid *type_id, int32 *typmod)
 {
-	char	   *buf;
+	StringInfoData buf;
 	List	   *raw_parsetree_list;
 	SelectStmt *stmt;
 	ResTarget  *restarget;
 	A_Const    *aconst;
 	TypeName   *typename;
 
-	buf = (char *) palloc(strlen(str) + 16);
+	initStringInfo(&buf);
-	sprintf(buf, "SELECT (NULL::%s)", str);
+	appendStringInfo(&buf, "SELECT (NULL::%s)", str);
 
-	raw_parsetree_list = parser(buf, NULL, 0);
+	raw_parsetree_list = parser(&buf, NULL, 0);
 
 	/*
 	 * Make sure we got back exactly what we expected and no more;
@@ -528,5 +528,5 @@ parseTypeString(const char *str, Oid *type_id, int32 *typmod)
 	*type_id = typenameTypeId(typename);
 	*typmod = typename->typmod;
 
-	pfree(buf);
+	pfree(buf.data);
 }

src/backend/parser/parser.c

@@ -14,7 +14,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/parser/parser.c,v 1.51 2001/11/05 17:46:26 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/parser/parser.c,v 1.52 2002/04/20 21:56:14 petere Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -28,12 +28,6 @@
 #include "parser/parse_expr.h"
 
 
-#if defined(FLEX_SCANNER)
-extern void DeleteBuffer(void);
-#endif   /* FLEX_SCANNER */
-
-char	   *parseString;		/* the char* which holds the string to be
-								 * parsed */
 List	   *parsetree;			/* result of parsing is left here */
 
 static int	lookahead_token;	/* one-token lookahead */
@@ -48,24 +42,20 @@ static bool have_lookahead;		/* lookahead_token set? */
  * Returns a list of raw (un-analyzed) parse trees.
  */
 List *
-parser(char *str, Oid *typev, int nargs)
+parser(StringInfo str, Oid *typev, int nargs)
 {
 	int			yyresult;
 
-	parseString = str;
 	parsetree = NIL;			/* in case parser forgets to set it */
 	have_lookahead = false;
 
-	scanner_init();
+	scanner_init(str);
 	parser_init(typev, nargs);
 	parse_expr_init();
 
 	yyresult = yyparse();
 
-#if defined(FLEX_SCANNER)
+	scanner_finish();
-	DeleteBuffer();
-#endif   /* FLEX_SCANNER */
-
 	clearerr(stdin);
 
 	if (yyresult)				/* error */

src/backend/parser/scan.l

@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/parser/scan.l,v 1.91 2002/03/06 06:09:56 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/parser/scan.l,v 1.92 2002/04/20 21:56:14 petere Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -17,9 +17,6 @@
 
 #include <ctype.h>
 #include <unistd.h>
-#ifndef __linux__
-#include <math.h>
-#endif
 #include <errno.h>
 
 #include "miscadmin.h"
@@ -28,45 +25,18 @@
 #include "parser/gramparse.h"
 #include "parser/keywords.h"
 #include "parser/parse.h"
-#include "parser/scansup.h"
 #include "utils/builtins.h"
 
 #ifdef MULTIBYTE
 #include "mb/pg_wchar.h"
 #endif
 
-extern char *parseString;
+/* No reason to constrain amount of data slurped */
-static char *parseCh;
-
-/* some versions of lex define this as a macro */
-#if defined(yywrap)
-#undef yywrap
-#endif /* yywrap */
-
-/* set up my input handler --- need one flavor for flex, one for lex */
-#if defined(FLEX_SCANNER)
-
-#define YY_NEVER_INTERACTIVE 1
-#define YY_NO_UNPUT
-static int myinput(char* buf, int max);
-#undef YY_INPUT
-#define YY_INPUT(buf,result,max) {result = myinput(buf,max);}
-
-/* No reason to constrain amount of data slurped per myinput() call. */
 #define YY_READ_BUF_SIZE 16777216
 
 /* Avoid exit() on fatal scanner errors (a bit ugly -- see yy_fatal_error) */
 #define fprintf(file, fmt, msg)  elog(FATAL, "%s", (msg))
 
-#else /* !FLEX_SCANNER */
-
-#undef input
-int input();
-#undef unput
-void unput(char);
-
-#endif /* FLEX_SCANNER */
-
 extern YYSTYPE yylval;
 
 static int		xcdepth = 0;	/* depth of nesting in slash-star comments */
@@ -83,8 +53,23 @@ static int		literalalloc;	/* current allocated buffer size */
 
 #define startlit()  (literalbuf[0] = '\0', literallen = 0)
 static void addlit(char *ytext, int yleng);
+static void addlitchar(unsigned char ychar);
+static char *litbufdup(void);
+
+/* Handles to the buffer that the lexer uses internally */
+static YY_BUFFER_STATE scanbufhandle;
+static char *scanbuf;
+
+unsigned char unescape_single_char(unsigned char c);
 
 %}
+
+%option 8bit
+%option never-interactive
+%option nounput
+%option noyywrap
+%option prefix="base_yy"
+
 /*
  * OK, here is a short description of lex/flex rules behavior.
  * The longest pattern which matches an input string is always chosen.
@@ -126,18 +111,17 @@ xhcat			{quote}{whitespace_with_newline}{quote}
 /* Extended quote
  * xqdouble implements SQL92 embedded quote
  * xqcat allows strings to cross input lines
- * Note: reduction of '' and \ sequences to output text is done in scanstr(),
- * not by rules here.  But we do get rid of xqcat sequences here.
  */
 quote			'
 xqstart			{quote}
 xqstop			{quote}
 xqdouble		{quote}{quote}
 xqinside		[^\\']+
-xqliteral		[\\](.|\n)
+xqescape		[\\][^0-7]
+xqoctesc		[\\][0-7]{1,3}
 xqcat			{quote}{whitespace_with_newline}{quote}
 
-/* Delimited quote
+/* Double quote
  * Allows embedded spaces and other special characters into identifiers.
  */
 dquote			\"
@@ -238,10 +222,7 @@ whitespace_with_newline	({horiz_whitespace}*{newline}{whitespace}*)
 
 other			.
 
-/* DO NOT PUT ANY COMMENTS IN THE FOLLOWING SECTION.
+/*
- * AT&T lex does not properly handle C-style comments in this second lex block.
- * So, put comments here. thomas - 1997-09-08
- *
  * Quoted strings must allow some special characters such as single-quote
  *  and newline.
  * Embedded single-quotes are implemented both in the SQL92-standard
@@ -285,14 +266,14 @@ other			.
 {xbitstart}		{
 					BEGIN(xbit);
 					startlit();
-					addlit("b", 1);
+					addlitchar('b');
 				}
 <xbit>{xbitstop}	{
 					BEGIN(INITIAL);
 					if (literalbuf[strspn(literalbuf + 1, "01") + 1] != '\0')
 						elog(ERROR, "invalid bit string input: '%s'",
 							 literalbuf);
-					yylval.str = pstrdup(literalbuf);
+					yylval.str = litbufdup();
 					return BITCONST;
 				}
 <xh>{xhinside}	|
@@ -335,14 +316,22 @@ other			.
 				}
 <xq>{xqstop}	{
 					BEGIN(INITIAL);
-					yylval.str = scanstr(literalbuf);
+					yylval.str = litbufdup();
 					return SCONST;
 				}
-<xq>{xqdouble}	|
+<xq>{xqdouble}  {
-<xq>{xqinside}	|
+					addlitchar('\'');
-<xq>{xqliteral} {
+				}
+<xq>{xqinside}  {
 					addlit(yytext, yyleng);
 				}
+<xq>{xqescape}  {
+					addlitchar(unescape_single_char(yytext[1]));
+				}
+<xq>{xqoctesc}  {
+					unsigned char c = strtoul(yytext+1, NULL, 8);
+					addlitchar(c);
+				}
 <xq>{xqcat}		{
 					/* ignore */
 				}
@@ -371,11 +360,11 @@ other			.
 						literalbuf[NAMEDATALEN-1] = '\0';
 #endif
 					}
-					yylval.str = pstrdup(literalbuf);
+					yylval.str = litbufdup();
 					return IDENT;
 				}
 <xd>{xddouble} {
-					addlit(yytext, yyleng-1);
+					addlitchar('"');
 				}
 <xd>{xdinside}	{
 					addlit(yytext, yyleng);
@@ -540,36 +529,44 @@ yyerror(const char *message)
 	elog(ERROR, "parser: %s at or near \"%s\"", message, yytext);
 }
 
-int
-yywrap(void)
-{
-	return(1);
-}
 
 /*
- scanner_init:
+ * Called before any actual parsing is done
-	called by postgres before any actual parsing is done
+ */
-*/
 void
-scanner_init(void)
+scanner_init(StringInfo str)
 {
-	/* it's important to set this to NULL
+	/*
-	   because input()/myinput() checks the non-nullness of parseCh
+	 * Might be left over after elog()
-	   to know when to pass the string to lex/flex */
+	 */
-	parseCh = NULL;
+	if (YY_CURRENT_BUFFER)
+		yy_delete_buffer(YY_CURRENT_BUFFER);
+
+	scanbuf = palloc(str->len + 2);
+	memcpy(scanbuf, str->data, str->len);
+	scanbuf[str->len] = scanbuf[str->len + 1] = YY_END_OF_BUFFER_CHAR;
+	scanbufhandle = yy_scan_buffer(scanbuf, str->len + 2);
 
 	/* initialize literal buffer to a reasonable but expansible size */
 	literalalloc = 128;
 	literalbuf = (char *) palloc(literalalloc);
 	startlit();
 
-#if defined(FLEX_SCANNER)
+	BEGIN(INITIAL);
-	if (YY_CURRENT_BUFFER)
-		yy_flush_buffer(YY_CURRENT_BUFFER);
-#endif /* FLEX_SCANNER */
-	BEGIN INITIAL;
 }
 
+
+/*
+ * Called after parsing is done to clean up after scanner_init()
+ */
+void
+scanner_finish(void)
+{
+	yy_delete_buffer(scanbufhandle);
+	pfree(scanbuf);
+}
+
+
 static void
 addlit(char *ytext, int yleng)
 {
@@ -587,54 +584,55 @@ addlit(char *ytext, int yleng)
 	literalbuf[literallen] = '\0';
 }
 
-#if !defined(FLEX_SCANNER)
 
-/* get lex input from a string instead of from stdin */
+static void
-int
+addlitchar(unsigned char ychar)
-input()
 {
-	if (parseCh == NULL)
+	/* enlarge buffer if needed */
-		parseCh = parseString;
+	if ((literallen+1) >= literalalloc)
-	if (*parseCh == '\0')
+	{
-		return(0);
+		literalalloc *= 2;
-	else
+		literalbuf = (char *) repalloc(literalbuf, literalalloc);
-		return(*parseCh++);
+	}
+	/* append new data, add trailing null */
+	literalbuf[literallen] = ychar;
+	literallen += 1;
+	literalbuf[literallen] = '\0';
 }
 
-/* undo lex input from a string instead of from stdin */
+
-void
+/*
-unput(char c)
+ * One might be tempted to write pstrdup(literalbuf) instead of this,
+ * but for long literals this is much faster because the length is
+ * already known.
+ */
+static char *
+litbufdup(void)
 {
-	if (parseCh == NULL)
+	char *new;
-		elog(FATAL, "Unput() failed.\n");
+
-	else if (c != 0)
+	new = palloc(literallen + 1);
-		*--parseCh = c;
+	memcpy(new, literalbuf, literallen+1);
+	return new;
 }
 
-#endif /* !defined(FLEX_SCANNER) */
 
-#ifdef FLEX_SCANNER
+unsigned char
-
+unescape_single_char(unsigned char c)
-/* input routine for flex to read input from a string instead of a file */
-static int
-myinput(char* buf, int max)
 {
-	int len;
+	switch (c)
-
+	{
-	if (parseCh == NULL)
+		case 'b':
-		parseCh = parseString;
+			return '\b';
-	len = strlen(parseCh);		/* remaining data available */
+		case 'f':
-	/* Note: this code used to think that flex wants a null-terminated
+			return '\f';
-	 * string.  It does NOT, and returning 1 less character than it asks
+		case 'n':
-	 * for will cause failure under the right boundary conditions.  So
+			return '\n';
-	 * shut up and fill the buffer to the limit, you hear?
+		case 'r':
-	 */
+			return '\r';
-	if (len > max)
+		case 't':
-		len = max;
+			return '\t';
-	if (len > 0)
+		default:
-		memcpy(buf, parseCh, len);
+			return c;
-	parseCh += len;
+	}
-	return len;
 }
-
-#endif /* FLEX_SCANNER */

src/backend/tcop/postgres.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/tcop/postgres.c,v 1.261 2002/04/18 20:01:09 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/tcop/postgres.c,v 1.262 2002/04/20 21:56:15 petere Exp $
  *
  * NOTES
  *	  this is the "main" module of the postgres backend and
@@ -120,7 +120,7 @@ int			XfuncMode = 0;
 static int	InteractiveBackend(StringInfo inBuf);
 static int	SocketBackend(StringInfo inBuf);
 static int	ReadCommand(StringInfo inBuf);
-static List *pg_parse_query(char *query_string, Oid *typev, int nargs);
+static List *pg_parse_query(StringInfo query_string, Oid *typev, int nargs);
 static List *pg_analyze_and_rewrite(Node *parsetree);
 static void start_xact_command(void);
 static void finish_xact_command(void);
@@ -330,11 +330,15 @@ pg_parse_and_rewrite(char *query_string,		/* string to execute */
 	List	   *raw_parsetree_list;
 	List	   *querytree_list;
 	List	   *list_item;
+	StringInfoData stri;
+
+	initStringInfo(&stri);
+	appendStringInfo(&stri, "%s", query_string);
 
 	/*
 	 * (1) parse the request string into a list of raw parse trees.
 	 */
-	raw_parsetree_list = pg_parse_query(query_string, typev, nargs);
+	raw_parsetree_list = pg_parse_query(&stri, typev, nargs);
 
 	/*
 	 * (2) Do parse analysis and rule rewrite.
@@ -365,12 +369,12 @@ pg_parse_and_rewrite(char *query_string,		/* string to execute */
  * commands are not processed any further than the raw parse stage.
  */
 static List *
-pg_parse_query(char *query_string, Oid *typev, int nargs)
+pg_parse_query(StringInfo query_string, Oid *typev, int nargs)
 {
 	List	   *raw_parsetree_list;
 
 	if (Debug_print_query)
-		elog(LOG, "query: %s", query_string);
+		elog(LOG, "query: %s", query_string->data);
 
 	if (Show_parser_stats)
 		ResetUsage();
@@ -549,7 +553,7 @@ pg_plan_query(Query *querytree)
  */
 
 void
-pg_exec_query_string(char *query_string,		/* string to execute */
+pg_exec_query_string(StringInfo query_string,		/* string to execute */
 					 CommandDest dest,	/* where results should go */
 					 MemoryContext parse_context)		/* context for
 														 * parsetrees */
@@ -559,7 +563,7 @@ pg_exec_query_string(char *query_string,		/* string to execute */
 	List	   *parsetree_list,
 			   *parsetree_item;
 
-	debug_query_string = query_string;	/* used by pgmonitor */
+	debug_query_string = query_string->data;	/* used by pgmonitor */
 
 	/*
 	 * Start up a transaction command.	All queries generated by the
@@ -725,7 +729,7 @@ pg_exec_query_string(char *query_string,		/* string to execute */
 				 * process utility functions (create, destroy, etc..)
 				 */
 				if (Debug_print_query)
-					elog(LOG, "ProcessUtility: %s", query_string);
+					elog(LOG, "ProcessUtility: %s", query_string->data);
 				else elog(DEBUG2, "ProcessUtility");
 
 				if (querytree->originalQuery)
@@ -1688,7 +1692,7 @@ PostgresMain(int argc, char *argv[], const char *username)
 	if (!IsUnderPostmaster)
 	{
 		puts("\nPOSTGRES backend interactive interface ");
-		puts("$Revision: 1.261 $ $Date: 2002/04/18 20:01:09 $\n");
+		puts("$Revision: 1.262 $ $Date: 2002/04/20 21:56:15 $\n");
 	}
 
 	/*
@@ -1913,7 +1917,7 @@ PostgresMain(int argc, char *argv[], const char *username)
 
 					pgstat_report_activity(parser_input->data);
 
-					pg_exec_query_string(parser_input->data,
+					pg_exec_query_string(parser_input,
 										 whereToSendOutput,
 										 QueryContext);
 

src/include/parser/gramparse.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $Id: gramparse.h,v 1.20 2002/04/09 20:35:55 tgl Exp $
+ * $Id: gramparse.h,v 1.21 2002/04/20 21:56:15 petere Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -15,11 +15,14 @@
 #ifndef GRAMPARSE_H
 #define GRAMPARSE_H
 
+#include "lib/stringinfo.h"
+
 /* from parser.c */
 extern int	yylex(void);
 
 /* from scan.l */
-extern void scanner_init(void);
+extern void scanner_init(StringInfo str);
+extern void scanner_finish(void);
 extern int	base_yylex(void);
 extern void yyerror(const char *message);
 

src/include/parser/parser.h

@@ -7,15 +7,16 @@
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $Id: parser.h,v 1.11 2001/11/05 17:46:35 momjian Exp $
+ * $Id: parser.h,v 1.12 2002/04/20 21:56:15 petere Exp $
  *
  *-------------------------------------------------------------------------
  */
 #ifndef PARSER_H
 #define PARSER_H
 
+#include "lib/stringinfo.h"
 #include "parser/parse_node.h"
 
-extern List *parser(char *str, Oid *typev, int nargs);
+extern List *parser(StringInfo str, Oid *typev, int nargs);
 
 #endif   /* PARSER_H */

src/include/tcop/tcopprot.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $Id: tcopprot.h,v 1.47 2001/11/10 23:51:14 tgl Exp $
+ * $Id: tcopprot.h,v 1.48 2002/04/20 21:56:15 petere Exp $
  *
  * OLD COMMENTS
  *	  This file was created so that other c files could get the two
@@ -22,6 +22,7 @@
 #include <setjmp.h>
 
 #include "executor/execdesc.h"
+#include "lib/stringinfo.h"
 #include "tcop/dest.h"
 
 
@@ -37,7 +38,7 @@ extern bool ShowPortNumber;
 extern List *pg_parse_and_rewrite(char *query_string,
 					 Oid *typev, int nargs);
 extern Plan *pg_plan_query(Query *querytree);
-extern void pg_exec_query_string(char *query_string,
+extern void pg_exec_query_string(StringInfo query_string,
 					 CommandDest dest,
 					 MemoryContext parse_context);
 #endif   /* BOOTSTRAP_INCLUDE */