Build all Flex files standalone

The proposed Meson build system will need a way to ignore certain
generated files in order to coexist with the autoconf build system,
and C files generated by Flex which are #include'd into .y files make
this more difficult. In similar vein to 72b1e3a21, arrange for all Flex
C files to compile to their own .o targets.

Reviewed by Andres Freund

Discussion: https://www.postgresql.org/message-id/20220810171935.7k5zgnjwqzalzmtm%40awork3.anarazel.de
Discussion: https://www.postgresql.org/message-id/CAFBsxsF8Gc2StS3haXofshHCzqNMRXiSxvQEYGwnFsTmsdwNeg@mail.gmail.com

src/backend/Makefile

@@ -174,10 +174,10 @@ utils/probes.o: utils/probes.d $(SUBDIROBJS)
 # Be sure that these files get removed by the maintainer-clean target
 distprep:
 	$(MAKE) -C parser	gram.c gram.h scan.c
-	$(MAKE) -C bootstrap	bootparse.c bootscanner.c
+	$(MAKE) -C bootstrap	bootparse.c bootparse.h bootscanner.c
 	$(MAKE) -C catalog	distprep
 	$(MAKE) -C nodes	distprep
-	$(MAKE) -C replication	repl_gram.c repl_scanner.c syncrep_gram.c syncrep_scanner.c
+	$(MAKE) -C replication	repl_gram.c repl_gram.h repl_scanner.c syncrep_gram.c syncrep_gram.h syncrep_scanner.c
 	$(MAKE) -C storage/lmgr	lwlocknames.h lwlocknames.c
 	$(MAKE) -C utils	distprep
 	$(MAKE) -C utils/adt	jsonpath_gram.c jsonpath_scan.c
@@ -292,13 +292,16 @@ maintainer-clean: distclean
 	$(MAKE) -C nodes $@
 	$(MAKE) -C utils $@
 	rm -f bootstrap/bootparse.c \
+	      bootstrap/bootparse.h \
 	      bootstrap/bootscanner.c \
 	      parser/gram.c \
 	      parser/gram.h \
 	      parser/scan.c \
 	      replication/repl_gram.c \
+	      replication/repl_gram.h \
 	      replication/repl_scanner.c \
 	      replication/syncrep_gram.c \
+	      replication/syncrep_gram.h \
 	      replication/syncrep_scanner.c \
 	      storage/lmgr/lwlocknames.c \
 	      storage/lmgr/lwlocknames.h \

src/backend/bootstrap/.gitignore

@@ -1,2 +1,3 @@
+/bootparse.h
 /bootparse.c
 /bootscanner.c

src/backend/bootstrap/Makefile

@@ -14,12 +14,19 @@ override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS)
 
 OBJS = \
 	bootparse.o \
+	bootscanner.o \
 	bootstrap.o
 
 include $(top_srcdir)/src/backend/common.mk
 
-# bootscanner is compiled as part of bootparse
-bootparse.o: bootscanner.c
+# See notes in src/backend/parser/Makefile about the following two rules
+bootparse.h: bootparse.c
+	touch $@
+
+bootparse.c: BISONFLAGS += -d
+
+# Force these dependencies to be known even without dependency info built:
+bootparse.o bootscanner.o: bootparse.h
 
 # bootparse.c and bootscanner.c are in the distribution tarball, so
 # they are not cleaned here.

src/backend/bootstrap/bootparse.y

@@ -488,5 +488,3 @@ boot_ident:
 		| XNULL			{ $$ = pstrdup($1); }
 		;
 %%
-
-#include "bootscanner.c"

src/backend/bootstrap/bootscanner.l

@@ -1,4 +1,4 @@
-%{
+%top{
 /*-------------------------------------------------------------------------
  *
  * bootscanner.l
@@ -15,11 +15,17 @@
  */
 #include "postgres.h"
 
+/*
+ * NB: include bootparse.h only AFTER including bootstrap.h, because bootstrap.h
+ * includes node definitions needed for YYSTYPE.
+ */
 #include "bootstrap/bootstrap.h"
+#include "bootparse.h"
 #include "utils/guc.h"
 
-/* Not needed now that this file is compiled as part of bootparse. */
-/* #include "bootparse.h" */
+}
+
+%{
 
 /* LCOV_EXCL_START */
 
@@ -52,7 +58,7 @@ id		[-A-Za-z0-9_]+
 sid		\'([^']|\'\')*\'
 
 /*
- * Keyword tokens return the keyword text (as a constant string) in yylval.kw,
+ * Keyword tokens return the keyword text (as a constant string) in boot_yylval.kw,
  * just in case that's needed because we want to treat the keyword as an
  * unreserved identifier.  Note that _null_ is not treated as a keyword
  * for this purpose; it's the one "reserved word" in the bootstrap syntax.
@@ -60,23 +66,23 @@ sid		\'([^']|\'\')*\'
  * Notice that all the keywords are case-sensitive, and for historical
  * reasons some must be upper case.
  *
- * String tokens return a palloc'd string in yylval.str.
+ * String tokens return a palloc'd string in boot_yylval.str.
  */
 
 %%
 
-open			{ yylval.kw = "open"; return OPEN; }
+open			{ boot_yylval.kw = "open"; return OPEN; }
 
-close			{ yylval.kw = "close"; return XCLOSE; }
+close			{ boot_yylval.kw = "close"; return XCLOSE; }
 
-create			{ yylval.kw = "create"; return XCREATE; }
+create			{ boot_yylval.kw = "create"; return XCREATE; }
 
-OID				{ yylval.kw = "OID"; return OBJ_ID; }
-bootstrap		{ yylval.kw = "bootstrap"; return XBOOTSTRAP; }
-shared_relation	{ yylval.kw = "shared_relation"; return XSHARED_RELATION; }
-rowtype_oid		{ yylval.kw = "rowtype_oid"; return XROWTYPE_OID; }
+OID				{ boot_yylval.kw = "OID"; return OBJ_ID; }
+bootstrap		{ boot_yylval.kw = "bootstrap"; return XBOOTSTRAP; }
+shared_relation	{ boot_yylval.kw = "shared_relation"; return XSHARED_RELATION; }
+rowtype_oid		{ boot_yylval.kw = "rowtype_oid"; return XROWTYPE_OID; }
 
-insert			{ yylval.kw = "insert"; return INSERT_TUPLE; }
+insert			{ boot_yylval.kw = "insert"; return INSERT_TUPLE; }
 
 _null_			{ return NULLVAL; }
 
@@ -90,25 +96,25 @@ _null_			{ return NULLVAL; }
 
 ^\#[^\n]*		;		/* drop everything after "#" for comments */
 
-declare			{ yylval.kw = "declare"; return XDECLARE; }
-build			{ yylval.kw = "build"; return XBUILD; }
-indices			{ yylval.kw = "indices"; return INDICES; }
-unique			{ yylval.kw = "unique"; return UNIQUE; }
-index			{ yylval.kw = "index"; return INDEX; }
-on				{ yylval.kw = "on"; return ON; }
-using			{ yylval.kw = "using"; return USING; }
-toast			{ yylval.kw = "toast"; return XTOAST; }
-FORCE			{ yylval.kw = "FORCE"; return XFORCE; }
-NOT				{ yylval.kw = "NOT"; return XNOT; }
-NULL			{ yylval.kw = "NULL"; return XNULL; }
+declare			{ boot_yylval.kw = "declare"; return XDECLARE; }
+build			{ boot_yylval.kw = "build"; return XBUILD; }
+indices			{ boot_yylval.kw = "indices"; return INDICES; }
+unique			{ boot_yylval.kw = "unique"; return UNIQUE; }
+index			{ boot_yylval.kw = "index"; return INDEX; }
+on				{ boot_yylval.kw = "on"; return ON; }
+using			{ boot_yylval.kw = "using"; return USING; }
+toast			{ boot_yylval.kw = "toast"; return XTOAST; }
+FORCE			{ boot_yylval.kw = "FORCE"; return XFORCE; }
+NOT				{ boot_yylval.kw = "NOT"; return XNOT; }
+NULL			{ boot_yylval.kw = "NULL"; return XNULL; }
 
 {id}			{
-					yylval.str = pstrdup(yytext);
+					boot_yylval.str = pstrdup(yytext);
 					return ID;
 				}
 {sid}			{
 					/* strip quotes and escapes */
-					yylval.str = DeescapeQuotedString(yytext);
+					boot_yylval.str = DeescapeQuotedString(yytext);
 					return ID;
 				}
 
@@ -121,7 +127,7 @@ NULL			{ yylval.kw = "NULL"; return XNULL; }
 /* LCOV_EXCL_STOP */
 
 void
-yyerror(const char *message)
+boot_yyerror(const char *message)
 {
 	elog(ERROR, "%s at line %d", message, yyline);
 }

src/backend/replication/.gitignore

@@ -1,4 +1,6 @@
+/repl_gram.h
 /repl_gram.c
 /repl_scanner.c
+/syncrep_gram.h
 /syncrep_gram.c
 /syncrep_scanner.c

src/backend/replication/Makefile

@@ -16,10 +16,12 @@ override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS)
 
 OBJS = \
 	repl_gram.o \
+	repl_scanner.o \
 	slot.o \
 	slotfuncs.o \
 	syncrep.o \
 	syncrep_gram.o \
+	syncrep_scanner.o \
 	walreceiver.o \
 	walreceiverfuncs.o \
 	walsender.o
@@ -28,11 +30,23 @@ SUBDIRS = logical
 
 include $(top_srcdir)/src/backend/common.mk
 
-# repl_scanner is compiled as part of repl_gram
-repl_gram.o: repl_scanner.c
+# See notes in src/backend/parser/Makefile about the following two rules
+repl_gram.h: repl_gram.c
+	touch $@
 
-# syncrep_scanner is compiled as part of syncrep_gram
-syncrep_gram.o: syncrep_scanner.c
+repl_gram.c: BISONFLAGS += -d
+
+# Force these dependencies to be known even without dependency info built:
+repl_gram.o repl_scanner.o: repl_gram.h
+
+# See notes in src/backend/parser/Makefile about the following two rules
+syncrep_gram.h: syncrep_gram.c
+	touch $@
+
+syncrep_gram.c: BISONFLAGS += -d
+
+# Force these dependencies to be known even without dependency info built:
+syncrep_gram.o syncrep_scanner.o: syncrep_gram.h
 
 # repl_gram.c, repl_scanner.c, syncrep_gram.c and syncrep_scanner.c
 # are in the distribution tarball, so they are not cleaned here.

src/backend/replication/repl_gram.y

@@ -416,5 +416,3 @@ ident_or_keyword:
 		;
 
 %%
-
-#include "repl_scanner.c"

src/backend/replication/repl_scanner.l

@@ -1,4 +1,4 @@
-%{
+%top{
 /*-------------------------------------------------------------------------
  *
  * repl_scanner.l
@@ -18,6 +18,15 @@
 #include "utils/builtins.h"
 #include "parser/scansup.h"
 
+/*
+ * NB: include repl_gram.h only AFTER including walsender_private.h, because
+ * walsender_private includes headers that define XLogRecPtr.
+ */
+#include "replication/walsender_private.h"
+#include "repl_gram.h"
+}
+
+%{
 /* Avoid exit() on fatal scanner errors (a bit ugly -- see yy_fatal_error) */
 #undef fprintf
 #define fprintf(file, fmt, msg)  fprintf_to_ereport(fmt, msg)
@@ -130,7 +139,7 @@ WAIT				{ return K_WAIT; }
 {space}+		{ /* do nothing */ }
 
 {digit}+		{
-					yylval.uintval = strtoul(yytext, NULL, 10);
+					replication_yylval.uintval = strtoul(yytext, NULL, 10);
 					return UCONST;
 				}
 
@@ -138,8 +147,8 @@ WAIT				{ return K_WAIT; }
 					uint32	hi,
 							lo;
 					if (sscanf(yytext, "%X/%X", &hi, &lo) != 2)
-						yyerror("invalid streaming start location");
-					yylval.recptr = ((uint64) hi) << 32 | lo;
+						replication_yyerror("invalid streaming start location");
+					replication_yylval.recptr = ((uint64) hi) << 32 | lo;
 					return RECPTR;
 				}
 
@@ -151,7 +160,7 @@ WAIT				{ return K_WAIT; }
 <xq>{quotestop}	{
 					yyless(1);
 					BEGIN(INITIAL);
-					yylval.str = litbufdup();
+					replication_yylval.str = litbufdup();
 					return SCONST;
 				}
 
@@ -173,9 +182,9 @@ WAIT				{ return K_WAIT; }
 
 					yyless(1);
 					BEGIN(INITIAL);
-					yylval.str = litbufdup();
-					len = strlen(yylval.str);
-					truncate_identifier(yylval.str, len, true);
+					replication_yylval.str = litbufdup();
+					len = strlen(replication_yylval.str);
+					truncate_identifier(replication_yylval.str, len, true);
 					return IDENT;
 				}
 
@@ -186,7 +195,7 @@ WAIT				{ return K_WAIT; }
 {identifier}	{
 					int			len = strlen(yytext);
 
-					yylval.str = downcase_truncate_identifier(yytext, len, true);
+					replication_yylval.str = downcase_truncate_identifier(yytext, len, true);
 					return IDENT;
 				}
 
@@ -195,7 +204,7 @@ WAIT				{ return K_WAIT; }
 					return yytext[0];
 				}
 
-<xq,xd><<EOF>>	{ yyerror("unterminated quoted string"); }
+<xq,xd><<EOF>>	{ replication_yyerror("unterminated quoted string"); }
 
 
 <<EOF>>			{
@@ -231,7 +240,7 @@ addlitchar(unsigned char ychar)
 }
 
 void
-yyerror(const char *message)
+replication_yyerror(const char *message)
 {
 	ereport(ERROR,
 			(errcode(ERRCODE_SYNTAX_ERROR),

src/backend/replication/syncrep_gram.y

@@ -112,5 +112,3 @@ create_syncrep_config(const char *num_sync, List *members, uint8 syncrep_method)
 
 	return config;
 }
-
-#include "syncrep_scanner.c"

src/backend/replication/syncrep_scanner.l

@@ -1,4 +1,4 @@
-%{
+%top{
 /*-------------------------------------------------------------------------
  *
  * syncrep_scanner.l
@@ -17,6 +17,15 @@
 
 #include "lib/stringinfo.h"
 
+/*
+ * NB: include syncrep_gram.h only AFTER including syncrep.h, because syncrep.h
+ * includes node definitions needed for YYSTYPE.
+ */
+#include "replication/syncrep.h"
+#include "syncrep_gram.h"
+}
+
+%{
 /* Avoid exit() on fatal scanner errors (a bit ugly -- see yy_fatal_error) */
 #undef fprintf
 #define fprintf(file, fmt, msg)  fprintf_to_ereport(fmt, msg)
@@ -82,28 +91,28 @@ xdinside		[^"]+
 				appendStringInfoString(&xdbuf, yytext);
 		}
 <xd>{xdstop} {
-				yylval.str = xdbuf.data;
+				syncrep_yylval.str = xdbuf.data;
 				xdbuf.data = NULL;
 				BEGIN(INITIAL);
 				return NAME;
 		}
 <xd><<EOF>> {
-				yyerror("unterminated quoted identifier");
+				syncrep_yyerror("unterminated quoted identifier");
 				return JUNK;
 		}
 
 {identifier} {
-				yylval.str = pstrdup(yytext);
+				syncrep_yylval.str = pstrdup(yytext);
 				return NAME;
 		}
 
 {digit}+	{
-				yylval.str = pstrdup(yytext);
+				syncrep_yylval.str = pstrdup(yytext);
 				return NUM;
 		}
 
 "*"		{
-				yylval.str = "*";
+				syncrep_yylval.str = "*";
 				return NAME;
 		}
 

src/backend/utils/adt/.gitignore

@@ -1,2 +1,3 @@
+/jsonpath_gram.h
 /jsonpath_gram.c
 /jsonpath_scan.c

src/backend/utils/adt/Makefile

@@ -57,6 +57,7 @@ OBJS = \
 	jsonpath.o \
 	jsonpath_exec.o \
 	jsonpath_gram.o \
+	jsonpath_scan.o \
 	like.o \
 	like_support.o \
 	lockfuncs.o \
@@ -119,11 +120,17 @@ OBJS = \
 	xid8funcs.o \
 	xml.o
 
+# See notes in src/backend/parser/Makefile about the following two rules
+jsonpath_gram.h: jsonpath_gram.c
+	touch $@
+
+jsonpath_gram.c: BISONFLAGS += -d
+
 jsonpath_scan.c: FLEXFLAGS = -CF -p -p
 jsonpath_scan.c: FLEX_NO_BACKUP=yes
 
-# jsonpath_scan is compiled as part of jsonpath_gram
-jsonpath_gram.o: jsonpath_scan.c
+# Force these dependencies to be known even without dependency info built:
+jsonpath_gram.o jsonpath_scan.o: jsonpath_gram.h
 
 # jsonpath_gram.c and jsonpath_scan.c are in the distribution tarball,
 # so they are not cleaned here.

src/backend/utils/adt/jsonpath_gram.y

@@ -18,26 +18,11 @@
 
 #include "catalog/pg_collation.h"
 #include "fmgr.h"
+#include "jsonpath_internal.h"
 #include "miscadmin.h"
 #include "nodes/pg_list.h"
 #include "regex/regex.h"
 #include "utils/builtins.h"
-#include "utils/jsonpath.h"
-
-/* struct JsonPathString is shared between scan and gram */
-typedef struct JsonPathString
-{
-	char	   *val;
-	int			len;
-	int			total;
-}			JsonPathString;
-
-union YYSTYPE;
-
-/* flex 2.5.4 doesn't bother with a decl for this */
-int	jsonpath_yylex(union YYSTYPE *yylval_param);
-int	jsonpath_yyparse(JsonPathParseResult **result);
-void jsonpath_yyerror(JsonPathParseResult **result, const char *message);
 
 static JsonPathParseItem *makeItemType(JsonPathItemType type);
 static JsonPathParseItem *makeItemString(JsonPathString *s);
@@ -593,13 +578,3 @@ jspConvertRegexFlags(uint32 xflags)
 
 	return cflags;
 }
-
-/*
- * jsonpath_scan.l is compiled as part of jsonpath_gram.y.  Currently, this is
- * unavoidable because jsonpath_gram does not create a .h file to export its
- * token symbols.  If these files ever grow large enough to be worth compiling
- * separately, that could be fixed; but for now it seems like useless
- * complication.
- */
-
-#include "jsonpath_scan.c"

src/backend/utils/adt/jsonpath_internal.h

@@ -0,0 +1,32 @@
+/*-------------------------------------------------------------------------
+ *
+ * jsonpath_internal.h
+ *     Private definitions for jsonpath scanner & parser
+ *
+ * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * src/backend/utils/adt/jsonpath_internal.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef JSONPATH_INTERNAL_H
+#define JSONPATH_INTERNAL_H
+
+/* struct JsonPathString is shared between scan and gram */
+typedef struct JsonPathString
+{
+	char	   *val;
+	int			len;
+	int			total;
+}			JsonPathString;
+
+#include "utils/jsonpath.h"
+#include "jsonpath_gram.h"
+
+extern int     jsonpath_yylex(YYSTYPE *yylval_param);
+extern int     jsonpath_yyparse(JsonPathParseResult **result);
+extern void jsonpath_yyerror(JsonPathParseResult **result, const char *message);
+
+#endif							/* JSONPATH_INTERNAL_H */

src/backend/utils/adt/jsonpath_scan.l

@@ -1,4 +1,4 @@
-%{
+%top{
 /*-------------------------------------------------------------------------
  *
  * jsonpath_scan.l
@@ -17,9 +17,18 @@
 
 #include "postgres.h"
 
+/*
+ * NB: include jsonpath_gram.h only AFTER including jsonpath_internal.h,
+ * because jsonpath_internal.h contains the declaration for JsonPathString.
+ */
+#include "jsonpath_internal.h"
+#include "jsonpath_gram.h"
+
 #include "mb/pg_wchar.h"
 #include "nodes/pg_list.h"
+}
 
+%{
 static JsonPathString scanstring;
 
 /* Handles to the buffer that the lexer uses internally */
@@ -142,9 +151,9 @@ hex_fail	\\x{hex_dig}{0,1}
 
 <xnq,xq,xvq>{hex_char}		{ parseHexChar(yytext); }
 
-<xnq,xq,xvq>{unicode}*{unicodefail}	{ yyerror(NULL, "invalid unicode sequence"); }
+<xnq,xq,xvq>{unicode}*{unicodefail}	{ jsonpath_yyerror(NULL, "invalid unicode sequence"); }
 
-<xnq,xq,xvq>{hex_fail}		{ yyerror(NULL, "invalid hex character sequence"); }
+<xnq,xq,xvq>{hex_fail}		{ jsonpath_yyerror(NULL, "invalid hex character sequence"); }
 
 <xnq,xq,xvq>{unicode}+\\	{
 								/* throw back the \\, and treat as unicode */
@@ -154,9 +163,9 @@ hex_fail	\\x{hex_dig}{0,1}
 
 <xnq,xq,xvq>\\.				{ addchar(false, yytext[1]); }
 
-<xnq,xq,xvq>\\				{ yyerror(NULL, "unexpected end after backslash"); }
+<xnq,xq,xvq>\\				{ jsonpath_yyerror(NULL, "unexpected end after backslash"); }
 
-<xq,xvq><<EOF>>				{ yyerror(NULL, "unexpected end of quoted string"); }
+<xq,xvq><<EOF>>				{ jsonpath_yyerror(NULL, "unexpected end of quoted string"); }
 
 <xq>\"							{
 									yylval->str = scanstring;
@@ -178,7 +187,7 @@ hex_fail	\\x{hex_dig}{0,1}
 
 <xc>\*							{ }
 
-<xc><<EOF>>						{ yyerror(NULL, "unexpected end of comment"); }
+<xc><<EOF>>						{ jsonpath_yyerror(NULL, "unexpected end of comment"); }
 
 \&\&							{ return AND_P; }
 
@@ -244,10 +253,10 @@ hex_fail	\\x{hex_dig}{0,1}
 									return INT_P;
 								}
 
-{realfail}						{ yyerror(NULL, "invalid numeric literal"); }
-{integer_junk}					{ yyerror(NULL, "trailing junk after numeric literal"); }
-{decimal_junk}					{ yyerror(NULL, "trailing junk after numeric literal"); }
-{real_junk}						{ yyerror(NULL, "trailing junk after numeric literal"); }
+{realfail}						{ jsonpath_yyerror(NULL, "invalid numeric literal"); }
+{integer_junk}					{ jsonpath_yyerror(NULL, "trailing junk after numeric literal"); }
+{decimal_junk}					{ jsonpath_yyerror(NULL, "trailing junk after numeric literal"); }
+{real_junk}						{ jsonpath_yyerror(NULL, "trailing junk after numeric literal"); }
 
 \"								{
 									addchar(true, '\0');

src/backend/utils/misc/Makefile

@@ -16,6 +16,7 @@ override CPPFLAGS := -I. -I$(srcdir) $(CPPFLAGS)
 
 OBJS = \
 	guc.o \
+	guc-file.o \
 	help_config.o \
 	pg_config.o \
 	pg_controldata.o \
@@ -37,10 +38,6 @@ endif
 
 include $(top_srcdir)/src/backend/common.mk
 
-# guc-file is compiled as part of guc
-guc.o: guc-file.c
-
 # Note: guc-file.c is not deleted by 'make clean',
 # since we want to ship it in distribution tarballs.
 clean:
-	@rm -f lex.yy.c

src/backend/utils/misc/guc-file.l

@@ -1,4 +1,4 @@
-/* -*-pgsql-c-*- */
+%top{
 /*
  * Scanner for the configuration file
  *
@@ -7,8 +7,6 @@
  * src/backend/utils/misc/guc-file.l
  */
 
-%{
-
 #include "postgres.h"
 
 #include <ctype.h>
@@ -19,8 +17,12 @@
 #include "mb/pg_wchar.h"
 #include "miscadmin.h"
 #include "storage/fd.h"
+#include <sys/stat.h>
+#include "utils/memutils.h"
+}
 
 
+%{
 /*
  * flex emits a yy_fatal_error() function that it calls in response to
  * critical errors like malloc failure, file I/O errors, and detection of

src/backend/utils/misc/guc.c

@@ -13333,5 +13333,3 @@ check_default_with_oids(bool *newval, void **extra, GucSource source)
 
 	return true;
 }
-
-#include "guc-file.c"