Have pg_stop_backup() wait for all archive files to be sent, rather than

returing right away.  This guarantees that when pg_stop_backup()
returns, you have a valid backup.

Simon Riggs

doc/src/sgml/backup.sgml

-<!-- $PostgreSQL: pgsql/doc/src/sgml/backup.sgml,v 2.116 2008/03/28 15:00:28 heikki Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/backup.sgml,v 2.117 2008/04/05 01:34:05 momjian Exp $ -->
 
 <chapter id="backup">
  <title>Backup and Restore</title>
@@ -761,12 +761,19 @@ SELECT pg_stop_backup();
     <para>
      Once the WAL segment files used during the backup are archived, you are
      done.  The file identified by <function>pg_stop_backup</>'s result is
-     the last segment that needs to be archived to complete the backup.
-     Archival of these files will happen automatically, since you have
-     already configured <varname>archive_command</>. In many cases, this
-     happens fairly quickly, but you are advised to monitor your archival
-     system to ensure this has taken place so that you can be certain you
-     have a complete backup.
+     the last segment that is required to form a complete set of backup files. 
+     <function>pg_stop_backup</> does not return until the last segment has
+     been archived. 
+     Archiving of these files happens automatically since you have
+     already configured <varname>archive_command</>. In most cases this
+     happens quickly, but you are advised to monitor your archive
+     system to ensure there are no delays.
+     If the archive process has fallen behind
+     because of failures of the archive command, it will keep retrying
+     until the archive succeeds and the backup is complete.
+     If you wish to place a time limit on the execution of
+     <function>pg_stop_backup</>, set an appropriate
+     <varname>statement_timeout</varname> value.
     </para>
    </listitem>
   </orderedlist>
@@ -1044,7 +1051,7 @@ restore_command = 'cp /mnt/server/archivedir/%f %p'
    <note>
      <para>
       The stop point must be after the ending time of the base backup, i.e.,
-      the time of <function>pg_stop_backup</>.  You cannot use a base backup
+      the end time of <function>pg_stop_backup</>.  You cannot use a base backup
       to recover to a time when that backup was still going on.  (To
       recover to such a time, you must go back to your previous base backup
       and roll forward from there.)
@@ -1322,6 +1329,7 @@ tar -rf /var/lib/pgsql/backup.tar /var/lib/pgsql/archive/
       After the backup the switch file is removed. Archived WAL files are
       then added to the backup so that both base backup and all required
       WAL files are part of the same <application>tar</> file.
+      Please remember to add error handling to your backup scripts.
      </para>
     </sect3>
 

src/backend/access/transam/xlog.c

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2008, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.295 2008/03/25 22:42:42 tgl Exp $
+ * $PostgreSQL: pgsql/src/backend/access/transam/xlog.c,v 1.296 2008/04/05 01:34:06 momjian Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -382,7 +382,7 @@ static bool InRedo = false;
 
 static void XLogArchiveNotify(const char *xlog);
 static void XLogArchiveNotifySeg(uint32 log, uint32 seg);
-static bool XLogArchiveCheckDone(const char *xlog);
+static bool XLogArchiveCheckDone(const char *xlog, bool create_if_missing);
 static void XLogArchiveCleanup(const char *xlog);
 static void readRecoveryCommandFile(void);
 static void exitArchiveRecovery(TimeLineID endTLI,
@@ -1128,7 +1128,7 @@ XLogArchiveNotifySeg(uint32 log, uint32 seg)
  * create <XLOG>.ready fails, we'll retry during subsequent checkpoints.
  */
 static bool
-XLogArchiveCheckDone(const char *xlog)
+XLogArchiveCheckDone(const char *xlog, bool create_if_missing)
 {
 	char		archiveStatusPath[MAXPGPATH];
 	struct stat stat_buf;
@@ -1153,7 +1153,9 @@ XLogArchiveCheckDone(const char *xlog)
 		return true;
 
 	/* Retry creation of the .ready file */
-	XLogArchiveNotify(xlog);
+	if (create_if_missing)
+		XLogArchiveNotify(xlog);
+
 	return false;
 }
 
@@ -2704,7 +2706,7 @@ RemoveOldXlogFiles(uint32 log, uint32 seg, XLogRecPtr endptr)
 			strspn(xlde->d_name, "0123456789ABCDEF") == 24 &&
 			strcmp(xlde->d_name + 8, lastoff + 8) <= 0)
 		{
-			if (XLogArchiveCheckDone(xlde->d_name))
+			if (XLogArchiveCheckDone(xlde->d_name, true))
 			{
 				snprintf(path, MAXPGPATH, XLOGDIR "/%s", xlde->d_name);
 
@@ -2771,7 +2773,7 @@ CleanupBackupHistory(void)
 			strcmp(xlde->d_name + strlen(xlde->d_name) - strlen(".backup"),
 				   ".backup") == 0)
 		{
-			if (XLogArchiveCheckDone(xlde->d_name))
+			if (XLogArchiveCheckDone(xlde->d_name, true))
 			{
 				ereport(DEBUG2,
 				(errmsg("removing transaction log backup history file \"%s\"",
@@ -6556,6 +6558,8 @@ pg_stop_backup(PG_FUNCTION_ARGS)
 	FILE	   *fp;
 	char		ch;
 	int			ich;
+	int			seconds_before_warning;
+	int			waits = 0;
 
 	if (!superuser())
 		ereport(ERROR,
@@ -6659,6 +6663,39 @@ pg_stop_backup(PG_FUNCTION_ARGS)
 	 */
 	CleanupBackupHistory();
 
+	/*
+	 * Wait until the history file has been archived. We assume that the 
+	 * alphabetic sorting property of the WAL files ensures the last WAL
+	 * file is guaranteed archived by the time the history file is archived.
+	 *
+	 * We wait forever, since archive_command is supposed to work and
+	 * we assume the admin wanted his backup to work completely. If you 
+	 * don't wish to wait, you can SET statement_timeout = xx;
+	 *
+	 * If the status file is missing, we assume that is because it was
+	 * set to .ready before we slept, then while asleep it has been set
+	 * to .done and then removed by a concurrent checkpoint.
+	 */
+	BackupHistoryFileName(histfilepath, ThisTimeLineID, _logId, _logSeg,
+						  startpoint.xrecoff % XLogSegSize);
+
+	seconds_before_warning = 60;
+	waits = 0;
+
+	while (!XLogArchiveCheckDone(histfilepath, false))
+	{
+		CHECK_FOR_INTERRUPTS();
+
+		pg_usleep(1000000L);
+
+		if (++waits >= seconds_before_warning)
+		{
+			seconds_before_warning *= 2;     /* This wraps in >10 years... */
+			elog(WARNING, "pg_stop_backup() waiting for archive to complete " 
+							"(%d seconds delay)", waits);
+		}
+	}
+
 	/*
 	 * We're done.  As a convenience, return the ending WAL location.
 	 */