Add pg_basebackup tool for streaming base backups

This tool makes it possible to do the pg_start_backup/
copy files/pg_stop_backup step in a single command.

There are still some steps to be done before this is a
complete backup solution, such as the ability to stream
the required WAL logs, but it's still usable, and
could do with some buildfarm coverage.

In passing, make the checkpoint request optionally
fast instead of hardcoding it.

Magnus Hagander, reviewed by Fujii Masao and Dimitri Fontaine

doc/src/sgml/backup.sgml

@@ -812,6 +812,16 @@ SELECT pg_stop_backup();
   </orderedlist>
    </para>
 
+   <para>
+    You can also use the <xref linkend="app-pgbasebackup"> tool to take
+    the backup, instead of manually copying the files. This tool will take
+    care of the <function>pg_start_backup()</>, copy and
+    <function>pg_stop_backup()</> steps automatically, and transfers the
+    backup over a regular <productname>PostgreSQL</productname> connection
+    using the replication protocol, instead of requiring filesystem level
+    access.
+   </para>
+
    <para>
     Some file system backup tools emit warnings or errors
     if the files they are trying to copy change while the copy proceeds.

doc/src/sgml/config.sgml

@@ -1921,7 +1921,8 @@ SET ENABLE_SEQSCAN TO OFF;
        <listitem>
        <para>
         Specifies the maximum number of concurrent connections from standby
-        servers (i.e., the maximum number of simultaneously running WAL sender
+        servers or streaming base backup clients (i.e., the maximum number of
+        simultaneously running WAL sender
         processes). The default is zero. This parameter can only be set at
         server start. <varname>wal_level</> must be set to <literal>archive</>
         or <literal>hot_standby</> to allow connections from standby servers.

doc/src/sgml/protocol.sgml

@@ -1460,7 +1460,7 @@ The commands accepted in walsender mode are:
   </varlistentry>
 
   <varlistentry>
-    <term>BASE_BACKUP [<literal>LABEL</literal> <replaceable>'label'</replaceable>] [<literal>PROGRESS</literal>]</term>
+    <term>BASE_BACKUP [<literal>LABEL</literal> <replaceable>'label'</replaceable>] [<literal>PROGRESS</literal>] [<literal>FAST</literal>]</term>
     <listitem>
      <para>
       Instructs the server to start streaming a base backup.
@@ -1496,6 +1496,15 @@ The commands accepted in walsender mode are:
          </para>
         </listitem>
        </varlistentry>
+
+       <varlistentry>
+        <term><literal>FAST</></term>
+        <listitem>
+         <para>
+          Request a fast checkpoint.
+         </para>
+        </listitem>
+       </varlistentry>
       </variablelist>
      </para>
      <para>

doc/src/sgml/ref/allfiles.sgml

@@ -160,6 +160,7 @@ Complete list of usable sgml source files in this directory.
 <!entity dropuser           system "dropuser.sgml">
 <!entity ecpgRef            system "ecpg-ref.sgml">
 <!entity initdb             system "initdb.sgml">
+<!entity pgBasebackup       system "pg_basebackup.sgml">
 <!entity pgConfig           system "pg_config-ref.sgml">
 <!entity pgControldata      system "pg_controldata.sgml">
 <!entity pgCtl              system "pg_ctl-ref.sgml">

doc/src/sgml/reference.sgml

@@ -202,6 +202,7 @@
    &droplang;
    &dropuser;
    &ecpgRef;
+   &pgBasebackup;
    &pgConfig;
    &pgDump;
    &pgDumpall;

src/backend/replication/basebackup.c

@@ -40,7 +40,7 @@ static void send_int8_string(StringInfoData *buf, int64 intval);
 static void SendBackupHeader(List *tablespaces);
 static void SendBackupDirectory(char *location, char *spcoid);
 static void base_backup_cleanup(int code, Datum arg);
-static void perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir);
+static void perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir, bool fastcheckpoint);
 
 typedef struct
 {
@@ -67,9 +67,9 @@ base_backup_cleanup(int code, Datum arg)
  * clobbered by longjmp" from stupider versions of gcc.
  */
 static void
-perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir)
+perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir, bool fastcheckpoint)
 {
-	do_pg_start_backup(backup_label, true);
+	do_pg_start_backup(backup_label, fastcheckpoint);
 
 	PG_ENSURE_ERROR_CLEANUP(base_backup_cleanup, (Datum) 0);
 	{
@@ -135,7 +135,7 @@ perform_base_backup(const char *backup_label, bool progress, DIR *tblspcdir)
  * pg_stop_backup() for the user.
  */
 void
-SendBaseBackup(const char *backup_label, bool progress)
+SendBaseBackup(const char *backup_label, bool progress, bool fastcheckpoint)
 {
 	DIR		   *dir;
 	MemoryContext backup_context;
@@ -168,7 +168,7 @@ SendBaseBackup(const char *backup_label, bool progress)
 		ereport(ERROR,
 				(errmsg("unable to open directory pg_tblspc: %m")));
 
-	perform_base_backup(backup_label, progress, dir);
+	perform_base_backup(backup_label, progress, dir, fastcheckpoint);
 
 	FreeDir(dir);
 
@@ -333,7 +333,16 @@ sendDir(char *path, int basepathlen, bool sizeonly)
 		if (strcmp(pathbuf, "./pg_xlog") == 0)
 		{
 			if (!sizeonly)
+			{
+				/* If pg_xlog is a symlink, write it as a directory anyway */
+#ifndef WIN32
+				if (S_ISLNK(statbuf.st_mode))
+#else
+				if (pgwin32_is_junction(pathbuf))
+#endif
+					statbuf.st_mode = S_IFDIR | S_IRWXU;
 				_tarWriteHeader(pathbuf + basepathlen + 1, NULL, &statbuf);
+			}
 			size += 512;		/* Size of the header just added */
 			continue;			/* don't recurse into pg_xlog */
 		}

src/backend/replication/repl_gram.y

@@ -66,11 +66,12 @@ Node *replication_parse_result;
 %token K_IDENTIFY_SYSTEM
 %token K_LABEL
 %token K_PROGRESS
+%token K_FAST
 %token K_START_REPLICATION
 
 %type <node>	command
 %type <node>	base_backup start_replication identify_system
-%type <boolval>	opt_progress
+%type <boolval>	opt_progress opt_fast
 %type <str>     opt_label
 
 %%
@@ -102,15 +103,16 @@ identify_system:
 			;
 
 /*
- * BASE_BACKUP [LABEL <label>] [PROGRESS]
+ * BASE_BACKUP [LABEL <label>] [PROGRESS] [FAST]
  */
 base_backup:
-			K_BASE_BACKUP opt_label opt_progress
+			K_BASE_BACKUP opt_label opt_progress opt_fast
 				{
 					BaseBackupCmd *cmd = (BaseBackupCmd *) makeNode(BaseBackupCmd);
 
 					cmd->label = $2;
 					cmd->progress = $3;
+					cmd->fastcheckpoint = $4;
 
 					$$ = (Node *) cmd;
 				}
@@ -123,6 +125,9 @@ opt_label: K_LABEL SCONST { $$ = $2; }
 opt_progress: K_PROGRESS		{ $$ = true; }
 			| /* EMPTY */		{ $$ = false; }
 			;
+opt_fast: K_FAST		{ $$ = true; }
+			| /* EMPTY */		{ $$ = false; }
+			;
 
 /*
  * START_REPLICATION %X/%X

src/backend/replication/repl_scanner.l

@@ -57,6 +57,7 @@ quotestop		{quote}
 %%
 
 BASE_BACKUP			{ return K_BASE_BACKUP; }
+FAST			{ return K_FAST; }
 IDENTIFY_SYSTEM		{ return K_IDENTIFY_SYSTEM; }
 LABEL			{ return K_LABEL; }
 PROGRESS			{ return K_PROGRESS; }

src/backend/replication/walsender.c

@@ -402,7 +402,7 @@ HandleReplicationCommand(const char *cmd_string)
 			{
 				BaseBackupCmd *cmd = (BaseBackupCmd *) cmd_node;
 
-				SendBaseBackup(cmd->label, cmd->progress);
+				SendBaseBackup(cmd->label, cmd->progress, cmd->fastcheckpoint);
 
 				/* Send CommandComplete and ReadyForQuery messages */
 				EndCommand("SELECT", DestRemote);

src/bin/Makefile

@@ -14,7 +14,7 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 
 SUBDIRS = initdb pg_ctl pg_dump \
-	psql scripts pg_config pg_controldata pg_resetxlog
+	psql scripts pg_config pg_controldata pg_resetxlog pg_basebackup
 ifeq ($(PORTNAME), win32)
 SUBDIRS+=pgevent
 endif

src/bin/pg_basebackup/Makefile

+#-------------------------------------------------------------------------
+#
+# Makefile for src/bin/pg_basebackup
+#
+# Portions Copyright (c) 1996-2011, PostgreSQL Global Development Group
+# Portions Copyright (c) 1994, Regents of the University of California
+#
+# src/bin/pg_basebackup/Makefile
+#
+#-------------------------------------------------------------------------
+
+PGFILEDESC = "pg_basebackup - takes a streaming base backup of a PostgreSQL instance"
+PGAPPICON=win32
+
+subdir = src/bin/pg_basebackup
+top_builddir = ../../..
+include $(top_builddir)/src/Makefile.global
+
+override CPPFLAGS := -I$(libpq_srcdir) $(CPPFLAGS)
+
+OBJS=	pg_basebackup.o $(WIN32RES)
+
+all: pg_basebackup
+
+pg_basebackup: $(OBJS) | submake-libpq submake-libpgport
+	$(CC) $(CFLAGS) $(OBJS) $(libpq_pgport) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X)
+
+install: all installdirs
+	$(INSTALL_PROGRAM) pg_basebackup$(X) '$(DESTDIR)$(bindir)/pg_basebackup$(X)'
+
+installdirs:
+	$(MKDIR_P) '$(DESTDIR)$(bindir)'
+
+uninstall:
+	rm -f '$(DESTDIR)$(bindir)/pg_basebackup$(X)'
+
+clean distclean maintainer-clean:
+	rm -f pg_basebackup$(X) $(OBJS)

src/bin/pg_basebackup/nls.mk

+# src/bin/pg_basebackup/nls.mk
+CATALOG_NAME	:= pg_basebackup
+AVAIL_LANGUAGES	:= 
+GETTEXT_FILES	:= pg_basebackup.c
+GETTEXT_TRIGGERS:= _

src/include/replication/basebackup.h

@@ -12,6 +12,6 @@
 #ifndef _BASEBACKUP_H
 #define _BASEBACKUP_H
 
-extern void SendBaseBackup(const char *backup_label, bool progress);
+extern void SendBaseBackup(const char *backup_label, bool progress, bool fastcheckpoint);
 
 #endif   /* _BASEBACKUP_H */

src/include/replication/replnodes.h

@@ -47,6 +47,7 @@ typedef struct BaseBackupCmd
 	NodeTag		type;
 	char	   *label;
 	bool		progress;
+	bool		fastcheckpoint;
 }	BaseBackupCmd;
 
 

src/tools/msvc/Mkvcbuild.pm

@@ -275,6 +275,8 @@ sub mkvcbuild
     $initdb->AddLibrary('wsock32.lib');
     $initdb->AddLibrary('ws2_32.lib');
 
+    my $pgbasebackup = AddSimpleFrontend('pg_basebackup', 1);
+
     my $pgconfig = AddSimpleFrontend('pg_config');
 
     my $pgcontrol = AddSimpleFrontend('pg_controldata');