Add support for include_dir in config file.

This allows easily splitting configuration into many files, deployed in a
directory.

Magnus Hagander, Greg Smith, Selena Deckelmann, reviewed by Noah Misch.

doc/src/sgml/config.sgml

@@ -79,38 +79,6 @@ shared_buffers = 128MB
      value, write either two quotes (preferred) or backslash-quote.
     </para>
 
-    <para>
-     <indexterm>
-      <primary><literal>include</></primary>
-      <secondary>in configuration file</secondary>
-     </indexterm>
-     In addition to parameter settings, the <filename>postgresql.conf</>
-     file can contain <firstterm>include directives</>, which specify
-     another file to read and process as if it were inserted into the
-     configuration file at this point.  This feature allows a configuration
-     file to be divided into physically separate parts.
-     Include directives simply look like:
-<programlisting>
-include 'filename'
-</programlisting>
-     If the file name is not an absolute path, it is taken as relative to
-     the directory containing the referencing configuration file.
-     Inclusions can be nested.
-    </para>
-
-    <para>
-     <indexterm>
-      <primary><literal>include_if_exists</></primary>
-      <secondary>in configuration file</secondary>
-     </indexterm>
-     There is also an <literal>include_if_exists</> directive, which acts
-     the same as the <literal>include</> directive, except for the behavior
-     when the referenced file does not exist or cannot be read.  A regular
-     <literal>include</> will consider this an error condition, but
-     <literal>include_if_exists</> merely logs a message and continues
-     processing the referencing configuration file.
-    </para>
-
     <para>
      <indexterm>
       <primary>SIGHUP</primary>
@@ -213,6 +181,122 @@ SET ENABLE_SEQSCAN TO OFF;
     </para>
 
    </sect2>
+
+   <sect2 id="config-includes">
+    <title>Configuration File Includes</title>
+
+     <para>
+      <indexterm>
+       <primary><literal>include</></primary>
+       <secondary>in configuration file</secondary>
+       </indexterm>
+         In addition to parameter settings, the <filename>postgresql.conf</>
+         file can contain <firstterm>include directives</>, which specify
+         another file to read and process as if it were inserted into the
+         configuration file at this point.  This feature allows a configuration
+         file to be divided into physically separate parts.
+         Include directives simply look like:
+<programlisting>
+include 'filename'
+</programlisting>
+         If the file name is not an absolute path, it is taken as relative to
+         the directory containing the referencing configuration file.
+         Inclusions can be nested.
+     </para>
+
+     <para>
+      <indexterm>
+       <primary><literal>include_if_exists</></primary>
+       <secondary>in configuration file</secondary>
+      </indexterm>
+         There is also an <literal>include_if_exists</> directive, which acts
+         the same as the <literal>include</> directive, except for the behavior
+         when the referenced file does not exist or cannot be read.  A regular
+         <literal>include</> will consider this an error condition, but
+         <literal>include_if_exists</> merely logs a message and continues
+         processing the referencing configuration file.
+     </para>
+
+     <para>
+      <indexterm>
+       <primary><literal>include_dir</></primary>
+       <secondary>in configuration file</secondary>
+      </indexterm>
+        The <filename>postgresql.conf</> file can also contain
+        <firstterm>include_dir directives</>, which specify an entire directory
+        of configuration files to include.  It is used similarly:
+ <programlisting>
+ include_dir 'directory'
+ </programlisting>
+        Non-absolute directory names follow the same rules as single file include
+        directives:  they are relative to the directory containing the referencing
+        configuration file.  Within that directory, only non-directory files whose
+        names end with the suffix <literal>.conf</literal> will be included.  File
+        names that start with the <literal>.</literal> character are also excluded,
+        to prevent mistakes as they are hidden on some platforms.  Multiple files
+        within an include directory are processed in filename order. The filenames
+        are ordered by C locale rules, ie. numbers before letters, and uppercase
+        letters before lowercase ones.
+     </para>
+
+     <para>
+       Include files or directories can be used to logically separate portions
+       of the database configuration, rather than having a single large
+       <filename>postgresql.conf</> file.  Consider a company that has two
+       database servers, each with a different amount of memory.  There are likely
+       elements of the configuration both will share, for things such as logging.
+       But memory-related parameters on the server will vary between the two.  And
+       there might be server specific customizations, too.  One way to manage this
+       situation is to break the custom configuration changes for your site into
+       three files.  You could add this to the end of your
+       <filename>postgresql.conf</> file to include them:
+ <programlisting>
+ include 'shared.conf'
+ include 'memory.conf'
+ include 'server.conf'
+ </programlisting>
+       All systems would have the same <filename>shared.conf</>.  Each server
+       with a particular amount of memory could share the same
+       <filename>memory.conf</>; you might have one for all servers with 8GB of RAM,
+       another for those having 16GB.  And finally <filename>server.conf</> could
+       have truly server-specific configuration information in it.
+     </para>
+
+     <para>
+       Another possibility is to create a configuration file directory and
+       put this information into files there. For example, a <filename>conf.d</>
+       directory could be referenced at the end of<filename>postgresql.conf</>:
+ <screen>
+ include_dir 'conf.d'
+ </screen>
+       Then you could name the files in the <filename>conf.d</> directory like this:
+ <screen>
+ 00shared.conf
+ 01memory.conf
+ 02server.conf
+ </screen>
+       This shows a clear order in which these files will be loaded.  This is
+       important because only the last setting encountered when the server is
+       reading its configuration will be used.  Something set in
+       <filename>conf.d/02server.conf</> in this example would override a value
+       set in <filename>conf.d/01memory.conf</>.
+     </para>
+
+     <para>
+       You might instead use this configuration directory approach while naming
+       these files more descriptively:
+ <screen>
+ 00shared.conf
+ 01memory-8GB.conf
+ 02server-foo.conf
+ </screen>
+       This sort of arrangement gives a unique name for each configuration file
+       variation.  This can help eliminate ambiguity when several servers have
+       their configurations all stored in one place, such as in a version
+       control repository.  (Storing database configuration files under version
+       control is another good practice to consider).
+     </para>
+    </sect2>
    </sect1>
 
    <sect1 id="runtime-config-file-locations">

src/backend/utils/misc/guc-file.l

@@ -362,6 +362,39 @@ ProcessConfigFile(GucContext context)
 	}
 }
 
+/*
+ * Given a configuration file or directory location that may be a relative
+ * path, return an absolute one.  We consider the location to be relative to
+ * the directory holding the calling file.
+ */
+static char *
+AbsoluteConfigLocation(const char *location, const char *calling_file)
+{
+	char		abs_path[MAXPGPATH];
+
+	if (is_absolute_path(location))
+		return pstrdup(location);
+	else
+	{
+		if (calling_file != NULL)
+		{
+			strlcpy(abs_path, calling_file, sizeof(abs_path));
+			get_parent_directory(abs_path);
+			join_path_components(abs_path, abs_path, location);
+			canonicalize_path(abs_path);
+		}
+		else
+		{
+			/*
+			 * calling_file is NULL, we make an absolute path from $PGDATA
+			 */
+			join_path_components(abs_path, data_directory, location);
+			canonicalize_path(abs_path);
+		}
+		return pstrdup(abs_path);
+	}
+}
+
 /*
  * Read and parse a single configuration file.  This function recurses
  * to handle "include" directives.
@@ -378,7 +411,6 @@ ParseConfigFile(const char *config_file, const char *calling_file, bool strict,
 {
 	bool		OK = true;
 	FILE	   *fp;
-	char		abs_path[MAXPGPATH];
 
 	/*
 	 * Reject too-deep include nesting depth.  This is just a safety check
@@ -394,31 +426,7 @@ ParseConfigFile(const char *config_file, const char *calling_file, bool strict,
 		return false;
 	}
 
-	/*
+	config_file = AbsoluteConfigLocation(config_file,calling_file);
-	 * If config_file is a relative path, convert to absolute.  We consider
-	 * it to be relative to the directory holding the calling file.
-	 */
-	if (!is_absolute_path(config_file))
-	{
-		if (calling_file != NULL)
-		{
-			strlcpy(abs_path, calling_file, sizeof(abs_path));
-			get_parent_directory(abs_path);
-			join_path_components(abs_path, abs_path, config_file);
-			canonicalize_path(abs_path);
-			config_file = abs_path;
-		}
-		else
-		{
-			/*
-			 * calling_file is NULL, we make an absolute path from $PGDATA
-			 */
-			join_path_components(abs_path, data_directory, config_file);
-			canonicalize_path(abs_path);
-			config_file = abs_path;
-		}
-	}
-
 	fp = AllocateFile(config_file, "r");
 	if (!fp)
 	{
@@ -563,7 +571,22 @@ ParseConfigFp(FILE *fp, const char *config_file, int depth, int elevel,
 		}
 
 		/* OK, process the option name and value */
-		if (guc_name_compare(opt_name, "include_if_exists") == 0)
+		if (guc_name_compare(opt_name, "include_dir") == 0)
+		{
+			/*
+			 * An include_dir directive isn't a variable and should be
+			 * processed immediately.
+			 */
+			if (!ParseConfigDirectory(opt_value, config_file,
+								 depth + 1, elevel,
+								 head_p, tail_p))
+				OK = false;
+			yy_switch_to_buffer(lex_buffer);
+			ConfigFileLineno = save_ConfigFileLineno;
+			pfree(opt_name);
+			pfree(opt_value);
+		}
+		else if (guc_name_compare(opt_name, "include_if_exists") == 0)
 		{
 			/*
 			 * An include_if_exists directive isn't a variable and should be
@@ -665,6 +688,111 @@ cleanup:
 	return OK;
 }
 
+/*
+ * Read and parse all config files in a subdirectory in alphabetical order
+ */
+bool
+ParseConfigDirectory(const char *includedir,
+					 const char *calling_file,
+					 int depth, int elevel,
+					 ConfigVariable **head_p,
+					 ConfigVariable **tail_p)
+{
+	char	   *directory;
+	DIR		   *d;
+	struct dirent *de;
+	char	  **filenames = NULL;
+	int			num_filenames = 0;
+	int			size_filenames = 0;
+	bool		status;
+
+	directory = AbsoluteConfigLocation(includedir, calling_file);
+	d = AllocateDir(directory);
+	if (d == NULL)
+	{
+		ereport(elevel,
+				(errcode_for_file_access(),
+				 errmsg("could not open configuration directory \"%s\": %m",
+						directory)));
+		return false;
+	}
+
+	/*
+	 * Read the directory and put the filenames in an array, so we can sort
+	 * them prior to processing the contents.
+	 */
+	while ((de = ReadDir(d, directory)) != NULL)
+	{
+		struct stat st;
+		char	filename[MAXPGPATH];
+
+		/*
+		 * Only parse files with names ending in ".conf".  Explicitly reject
+		 * files starting with ".".  This excludes things like "." and "..",
+		 * as well as typical hidden files, backup files, and editor debris.
+		 */
+		if (strlen(de->d_name) < 6)
+			continue;
+		if (de->d_name[0] == '.')
+			continue;
+		if (strcmp(de->d_name + strlen(de->d_name) - 5, ".conf") != 0)
+			continue;
+
+		join_path_components(filename, directory, de->d_name);
+		canonicalize_path(filename);
+		if (stat(filename, &st) == 0)
+		{
+			if (!S_ISDIR(st.st_mode))
+			{
+				/* Add file to list, increasing its size in blocks of 32 */
+				if (num_filenames == size_filenames)
+				{
+					size_filenames += 32;
+					if (num_filenames == 0)
+						/* Must initialize, repalloc won't take NULL input */
+						filenames = palloc(size_filenames * sizeof(char *));
+					else
+						filenames = repalloc(filenames, size_filenames * sizeof(char *));
+				}
+				filenames[num_filenames] = pstrdup(filename);
+				num_filenames++;
+			}
+		}
+		else
+		{
+			/*
+			 * stat does not care about permissions, so the most likely reason
+			 * a file can't be accessed now is if it was removed between the
+			 * directory listing and now.
+			 */
+			ereport(elevel,
+					(errcode_for_file_access(),
+					 errmsg("could not stat file \"%s\": %m",
+							filename)));
+			return false;
+		}
+	}
+
+	if (num_filenames > 0)
+	{
+		int			i;
+		qsort(filenames, num_filenames, sizeof(char *), pg_qsort_strcmp);
+		for (i = 0; i < num_filenames; i++)
+		{
+			if (!ParseConfigFile(filenames[i], NULL, true,
+								 depth, elevel, head_p, tail_p))
+			{
+				status = false;
+				goto cleanup;
+			}
+		}
+	}
+	status = true;
+
+cleanup:
+	FreeDir(d);
+	return status;
+}
 
 /*
  * Free a list of ConfigVariables, including the names and the values

src/backend/utils/misc/postgresql.conf.sample

@@ -566,6 +566,17 @@
 #exit_on_error = off			# terminate session on any error?
 #restart_after_crash = on		# reinitialize after backend crash?
 
+#------------------------------------------------------------------------------
+# CONFIG FILE INCLUDES
+#------------------------------------------------------------------------------
+
+# These options allow settings to be loaded from files other than the
+# default postgresql.conf
+
+#include_dir = 'conf.d'			# include files ending in '.conf' from
+					# directory 'conf.d'
+#include_if_exists = 'exists.conf'	# include file only if it exists
+#include = 'special.conf'		# include file
 
 #------------------------------------------------------------------------------
 # CUSTOMIZED OPTIONS

src/include/utils/guc.h

@@ -116,6 +116,11 @@ extern bool ParseConfigFile(const char *config_file, const char *calling_file,
 extern bool ParseConfigFp(FILE *fp, const char *config_file,
 			  int depth, int elevel,
 			  ConfigVariable **head_p, ConfigVariable **tail_p);
+extern bool ParseConfigDirectory(const char *includedir,
+					 const char *calling_file,
+					 int depth, int elevel,
+					 ConfigVariable **head_p,
+					 ConfigVariable **tail_p);
 extern void FreeConfigVariables(ConfigVariable *list);
 
 /*