Add pg_basebackup tool for streaming base backups

This tool makes it possible to do the pg_start_backup/
copy files/pg_stop_backup step in a single command.

There are still some steps to be done before this is a
complete backup solution, such as the ability to stream
the required WAL logs, but it's still usable, and
could do with some buildfarm coverage.

In passing, make the checkpoint request optionally
fast instead of hardcoding it.

Magnus Hagander, reviewed by Fujii Masao and Dimitri Fontaine

src/backend/replication/basebackup.c

@@ -40,7 +40,7 @@ static void send_int8_string(StringInfoData *buf, int64 intval);
 static void SendBackupHeader(List *tablespaces);
 static void SendBackupDirectory(char *location, char *spcoid);
 static void base_backup_cleanup(int code, Datum arg);
-static void perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir);
+static void perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir, bool fastcheckpoint);
 
 typedef struct
 {
@@ -67,9 +67,9 @@ base_backup_cleanup(int code, Datum arg)
  * clobbered by longjmp" from stupider versions of gcc.
  */
 static void
-perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir)
+perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir, bool fastcheckpoint)
 {
-	do_pg_start_backup(backup_label, true);
+	do_pg_start_backup(backup_label, fastcheckpoint);
 
 	PG_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0);
 	{
@@ -135,7 +135,7 @@ perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir)
  * pg_stop_backup() for the user.
  */
 void
-SendBaseBackup(const char *backup_label, bool progress)
+SendBaseBackup(const char *backup_label, bool progress, bool fastcheckpoint)
 {
 	DIR		   *dir;
 	MemoryContext backup_context;
@@ -168,7 +168,7 @@ SendBaseBackup(const char *backup_label, bool progress)
 		ereport(ERROR,
 				(errmsg("unable to open directory pg_tblspc: %m")));
 
-	perform_base_backup(backup_label, progress, dir);
+	perform_base_backup(backup_label, progress, dir, fastcheckpoint);
 
 	FreeDir(dir);
 
@@ -333,7 +333,16 @@ sendDir(char *path, int basepathlen, bool sizeonly)
 		if (strcmp(pathbuf, "./pg_xlog") == 0)
 		{
 			if (!sizeonly)
+			{
+				/* If pg_xlog is a symlink, write it as a directory anyway */
+#ifndef WIN32
+				if (S_ISLNK(statbuf.st_mode))
+#else
+				if (pgwin32_is_junction(pathbuf))
+#endif
+					statbuf.st_mode = S_IFDIR | S_IRWXU;
 				_tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf);
+			}
 			size += 512;		/* Size of the header just added */
 			continue;			/* don't recurse into pg_xlog */
 		}

src/backend/replication/repl_gram.y

@@ -66,11 +66,12 @@ Node *replication_parse_result;
 %token K_IDENTIFY_SYSTEM
 %token K_LABEL
 %token K_PROGRESS
+%token K_FAST
 %token K_START_REPLICATION
 
 %type <node>	command
 %type <node>	base_backup start_replication identify_system
-%type <boolval>	opt_progress
+%type <boolval>	opt_progress opt_fast
 %type <str>     opt_label
 
 %%
@@ -102,15 +103,16 @@ identify_system:
 			;
 
 /*
- * BASE_BACKUP [LABEL <label>] [PROGRESS]
+ * BASE_BACKUP [LABEL <label>] [PROGRESS] [FAST]
  */
 base_backup:
-			K_BASE_BACKUP opt_label opt_progress
+			K_BASE_BACKUP opt_label opt_progress opt_fast
 				{
 					BaseBackupCmd *cmd = (BaseBackupCmd *) makeNode(BaseBackupCmd);
 
 					cmd->label = $2;
 					cmd->progress = $3;
+					cmd->fastcheckpoint = $4;
 
 					$$ = (Node *) cmd;
 				}
@@ -123,6 +125,9 @@ opt_label: K_LABEL SCONST { $$ = $2; }
 opt_progress: K_PROGRESS		{ $$ = true; }
 			| /* EMPTY */		{ $$ = false; }
 			;
+opt_fast: K_FAST		{ $$ = true; }
+			| /* EMPTY */		{ $$ = false; }
+			;
 
 /*
  * START_REPLICATION %X/%X

src/backend/replication/repl_scanner.l

@@ -57,6 +57,7 @@ quotestop		{quote}
 %%
 
 BASE_BACKUP			{ return K_BASE_BACKUP; }
+FAST			{ return K_FAST; }
 IDENTIFY_SYSTEM		{ return K_IDENTIFY_SYSTEM; }
 LABEL			{ return K_LABEL; }
 PROGRESS			{ return K_PROGRESS; }

src/backend/replication/walsender.c

@@ -402,7 +402,7 @@ HandleReplicationCommand(const char *cmd_string)
 			{
 				BaseBackupCmd *cmd = (BaseBackupCmd *) cmd_node;
 
-				SendBaseBackup(cmd->label, cmd->progress);
+				SendBaseBackup(cmd->label, cmd->progress, cmd->fastcheckpoint);
 
 				/* Send CommandComplete and ReadyForQuery messages */
 				EndCommand("SELECT", DestRemote);

src/bin/Makefile

@@ -14,7 +14,7 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS = initdb pg_ctl pg_dump \
-	psql scripts pg_config pg_controldata pg_resetxlog
+	psql scripts pg_config pg_controldata pg_resetxlog pg_basebackup
 ifeq ($(PORTNAME), win32)
 SUBDIRS+=pgevent
 endif

src/bin/pg_basebackup/Makefile

@@ -0,0 +1,38 @@
+#-------------------------------------------------------------------------
+#
+# Makefile for src/bin/pg_basebackup
+#
+# Portions Copyright (c) 1996-2011, PostgreSQL Global Development Group
+# Portions Copyright (c) 1994, Regents of the University of California
+#
+# src/bin/pg_basebackup/Makefile
+#
+#-------------------------------------------------------------------------
+
+PGFILEDESC = "pg_basebackup - takes a streaming base backup of a PostgreSQL instance"
+PGAPPICON=win32
+
+subdir = src/bin/pg_basebackup
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
+
+OBJS=	pg_basebackup.o $(WIN32RES)
+
+all: pg_basebackup
+
+pg_basebackup: $(OBJS) | submake-libpq submake-libpgport
+	$(CC) $(CFLAGS) $(OBJS) $(libpq_pgport) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+
+install: all installdirs
+	$(INSTALL_PROGRAM) pg_basebackup$(X) '$(DESTDIR)$(bindir)/pg_basebackup$(X)'
+
+installdirs:
+	$(MKDIR_P) '$(DESTDIR)$(bindir)'
+
+uninstall:
+	rm -f '$(DESTDIR)$(bindir)/pg_basebackup$(X)'
+
+clean distclean maintainer-clean:
+	rm -f pg_basebackup$(X) $(OBJS)

src/bin/pg_basebackup/nls.mk

@@ -0,0 +1,5 @@
+# src/bin/pg_basebackup/nls.mk
+CATALOG_NAME	:= pg_basebackup
+AVAIL_LANGUAGES	:= 
+GETTEXT_FILES	:= pg_basebackup.c
+GETTEXT_TRIGGERS:= _

src/include/replication/basebackup.h

@@ -12,6 +12,6 @@
 #ifndef _BASEBACKUP_H
 #define _BASEBACKUP_H
 
-extern void SendBaseBackup(const char *backup_label, bool progress);
+extern void SendBaseBackup(const char *backup_label, bool progress, bool fastcheckpoint);
 
 #endif   /* _BASEBACKUP_H */

src/include/replication/replnodes.h

@@ -47,6 +47,7 @@ typedef struct BaseBackupCmd
 	NodeTag		type;
 	char	   *label;
 	bool		progress;
+	bool		fastcheckpoint;
 }	BaseBackupCmd;
 
 

src/tools/msvc/Mkvcbuild.pm

@@ -275,6 +275,8 @@ sub mkvcbuild
     $initdb->AddLibrary('wsock32.lib');
     $initdb->AddLibrary('ws2_32.lib');
 
+    my $pgbasebackup = AddSimpleFrontend('pg_basebackup', 1);
+
     my $pgconfig = AddSimpleFrontend('pg_config');
 
     my $pgcontrol = AddSimpleFrontend('pg_controldata');