Exclude additional directories in pg_basebackup

The list of files and directories that pg_basebackup excludes from the
backup was somewhat incomplete and unorganized.  Change that with having
the exclusion driven from tables.  Clean up some code around it.  Also
document the exclusions in more detail so that users of pg_start_backup
can make use of it as well.

The contents of these directories are now excluded from the backup:
pg_dynshmem, pg_notify, pg_serial, pg_snapshots, pg_subtrans

Also fix a bug that a pg_repl_slot or pg_stat_tmp being a symlink would
cause a corrupt tar header to be created.  Now such symlinks are
included in the backup as empty directories.  Bug found by Ashutosh
Sharma <ashu.coek88@gmail.com>.

From: David Steele <david@pgmasters.net>
Reviewed-by: Michael Paquier <michael.paquier@gmail.com>

doc/src/sgml/backup.sgml

@@ -1089,6 +1089,22 @@ SELECT pg_stop_backup();
     the new master comes on line.
    </para>
 
+   <para>
+    The contents of the directories <filename>pg_dynshmem/</>,
+    <filename>pg_notify/</>, <filename>pg_serial/</>,
+    <filename>pg_snapshots/</>, <filename>pg_stat_tmp/</>,
+    and <filename>pg_subtrans/</> (but not the directories themselves) can be
+    omitted from the backup as they will be initialized on postmaster startup.
+    If <xref linkend="guc-stats-temp-directory"> is set and is under the data
+    directory then the contents of that directory can also be omitted.
+   </para>
+
+   <para>
+    Any file or directory beginning with <filename>pgsql_tmp</filename> can be
+    omitted from the backup.  These files are removed on postmaster start and
+    the directories will be recreated as needed.
+   </para>
+
    <para>
     The backup label
     file includes the label string you gave to <function>pg_start_backup</>,

doc/src/sgml/protocol.sgml

@@ -2069,7 +2069,9 @@ The commands accepted in walsender mode are:
        </listitem>
        <listitem>
         <para>
-         various temporary files created during the operation of the PostgreSQL server
+         Various temporary files and directories created during the operation
+         of the PostgreSQL server, such as any file or directory beginning
+         with <filename>pgsql_tmp</>.
         </para>
        </listitem>
        <listitem>
@@ -2082,13 +2084,18 @@ The commands accepted in walsender mode are:
        </listitem>
        <listitem>
         <para>
-         <filename>pg_replslot</> is copied as an empty directory.
+         <filename>pg_dynshmem</>, <filename>pg_notify</>,
+         <filename>pg_replslot</>, <filename>pg_serial</>,
+         <filename>pg_snapshots</>, <filename>pg_stat_tmp</>, and
+         <filename>pg_subtrans</> are copied as empty directories (even if
+         they are symbolic links).
         </para>
        </listitem>
        <listitem>
         <para>
          Files other than regular files and directories, such as symbolic
-         links and special device files, are skipped.  (Symbolic links
+         links (other than for the directories listed above) and special
+         device files, are skipped.  (Symbolic links
          in <filename>pg_tblspc</filename> are maintained.)
         </para>
        </listitem>

doc/src/sgml/ref/pg_basebackup.sgml

@@ -610,10 +610,12 @@ PostgreSQL documentation
   <para>
    The backup will include all files in the data directory and tablespaces,
    including the configuration files and any additional files placed in the
-   directory by third parties. But only regular files and directories are
-   copied.  Symbolic links (other than those used for tablespaces) and special
-   device files are skipped.  (See <xref linkend="protocol-replication"> for
-   the precise details.)
+   directory by third parties, except certain temporary files managed by
+   PostgreSQL.  But only regular files and directories are copied, except that
+   symbolic links used for tablespaces are preserved.  Symbolic links pointing
+   to certain directories known to PostgreSQL are copied as empty directories.
+   Other symbolic links and special device files are skipped.
+   See <xref linkend="protocol-replication"> for the precise details.
   </para>
 
   <para>

src/bin/pg_basebackup/t/010_pg_basebackup.pl

@@ -4,7 +4,7 @@ use Cwd;
 use Config;
 use PostgresNode;
 use TestLib;
-use Test::More tests => 54;
+use Test::More tests => 67;
 
 program_help_ok('pg_basebackup');
 program_version_ok('pg_basebackup');
@@ -55,15 +55,43 @@ print CONF "wal_level = replica\n";
 close CONF;
 $node->restart;
 
+# Write some files to test that they are not copied.
+foreach my $filename (qw(backup_label tablespace_map postgresql.auto.conf.tmp))
+{
+	open FILE, ">>$pgdata/$filename";
+	print FILE "DONOTCOPY";
+	close FILE;
+}
+
 $node->command_ok([ 'pg_basebackup', '-D', "$tempdir/backup" ],
 	'pg_basebackup runs');
 ok(-f "$tempdir/backup/PG_VERSION", 'backup was created');
 
+# Only archive_status directory should be copied in pg_xlog/.
 is_deeply(
 	[ sort(slurp_dir("$tempdir/backup/pg_xlog/")) ],
 	[ sort qw(. .. archive_status) ],
 	'no WAL files copied');
 
+# Contents of these directories should not be copied.
+foreach my $dirname (qw(pg_dynshmem pg_notify pg_replslot pg_serial pg_snapshots pg_stat_tmp pg_subtrans))
+{
+	is_deeply(
+		[ sort(slurp_dir("$tempdir/backup/$dirname/")) ],
+		[ sort qw(. ..) ],
+		"contents of $dirname/ not copied");
+}
+
+# These files should not be copied.
+foreach my $filename (qw(postgresql.auto.conf.tmp postmaster.opts postmaster.pid tablespace_map))
+{
+	ok(! -f "$tempdir/backup/$filename", "$filename not copied");
+}
+
+# Make sure existing backup_label was ignored.
+isnt(slurp_file("$tempdir/backup/backup_label"), 'DONOTCOPY',
+	 'existing backup_label not copied');
+
 $node->command_ok(
 	[   'pg_basebackup', '-D', "$tempdir/backup2", '--xlogdir',
 		"$tempdir/xlog2" ],
@@ -110,7 +138,17 @@ unlink "$pgdata/$superlongname";
 # skip on Windows.
 SKIP:
 {
-	skip "symlinks not supported on Windows", 10 if ($windows_os);
+	skip "symlinks not supported on Windows", 11 if ($windows_os);
+
+	# Move pg_replslot out of $pgdata and create a symlink to it.
+	$node->stop;
+
+	rename("$pgdata/pg_replslot", "$tempdir/pg_replslot")
+		or BAIL_OUT "could not move $pgdata/pg_replslot";
+	symlink("$tempdir/pg_replslot", "$pgdata/pg_replslot")
+		or BAIL_OUT "could not symlink to $pgdata/pg_replslot";
+
+	$node->start;
 
 	# Create a temporary directory in the system location and symlink it
 	# to our physical temp location.  That way we can use shorter names
@@ -148,6 +186,8 @@ SKIP:
 		"tablespace symlink was updated");
 	closedir $dh;
 
+	ok(-d "$tempdir/backup1/pg_replslot", 'pg_replslot symlink copied as directory');
+
 	mkdir "$tempdir/tbl=spc2";
 	$node->safe_psql('postgres', "DROP TABLE test1;");
 	$node->safe_psql('postgres', "DROP TABLESPACE tblspc1;");