XLOG file archiving and point-in-time recovery. There are still some

loose ends and a glaring lack of documentation, but it basically works.

Simon Riggs with some editorialization by Tom Lane.

src/backend/access/nbtree/nbtsort.c

@@ -56,7 +56,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/nbtree/nbtsort.c,v 1.83 2004/07/11 18:01:45 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/nbtree/nbtsort.c,v 1.84 2004/07/19 02:47:03 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -64,6 +64,7 @@
 #include "postgres.h"
 
 #include "access/nbtree.h"
+#include "access/xlog.h"
 #include "miscadmin.h"
 #include "storage/smgr.h"
 #include "utils/tuplesort.h"
@@ -222,15 +223,9 @@ _bt_leafbuild(BTSpool *btspool, BTSpool *btspool2)
 	/*
 	 * We need to log index creation in WAL iff WAL archiving is enabled
 	 * AND it's not a temp index.
-	 *
-	 * XXX when WAL archiving is actually supported, this test will likely
-	 * need to change; and the hardwired extern is cruddy anyway ...
 	 */
-	{
-		extern char XLOG_archive_dir[];
+	wstate.btws_use_wal = XLogArchivingActive() && !wstate.index->rd_istemp;
 
-		wstate.btws_use_wal = XLOG_archive_dir[0] && !wstate.index->rd_istemp;
-	}
 	/* reserve the metapage */
 	wstate.btws_pages_alloced = BTREE_METAPAGE + 1;
 	wstate.btws_pages_written = 0;

src/backend/access/transam/recovery.conf.sample

+# -------------------------------
+# PostgreSQL recovery config file
+# -------------------------------
+#
+# Edit this file to provide the parameters that PostgreSQL
+# needs to perform an archive recovery of a database
+#
+# If "recovery.conf" is present in the PostgreSQL data directory, it is
+# read on postmaster startup.  After successful recovery, it is renamed
+# to "recovery.done" to ensure that we do not accidentally re-enter archive
+# recovery mode.
+#
+# This file consists of lines of the form:
+#
+#   name = 'value'
+#
+# (The quotes around the value are NOT optional, but the "=" is.)
+#
+# Comments are introduced with '#'.
+#
+# The complete list of option names and
+# allowed values can be found in the PostgreSQL documentation. The
+# commented-out settings shown below are sample values.
+#
+#---------------------------------------------------------------------------
+# REQUIRED PARAMETERS
+#---------------------------------------------------------------------------
+#
+# restore command
+#
+# specifies the shell command that is executed to copy log files
+# back from archival storage.  The command string may contain %f,
+# which is replaced by the name of the desired log file, and %p,
+# which is replaced by the absolute path to copy the log file to.
+#
+# It is important that the command return nonzero exit status on failure.
+# The command *will* be asked for log files that are not present in the
+# archive; it must return nonzero when so asked.
+#
+# NOTE that the basename of %p will be different from %f; do not
+# expect them to be interchangeable.
+#
+#
+#restore_command = 'cp /mnt/server/archivedir/%f %p'
+#
+#
+#---------------------------------------------------------------------------
+# OPTIONAL PARAMETERS
+#---------------------------------------------------------------------------
+#
+# By default, recovery will rollforward to the end of the WAL log.
+# If you want to stop rollforward before that point, you
+# MUST set a recovery target.
+#
+# You may set a recovery target either by transactionId, or
+# by timestamp. Recovery may either include or exclude the
+# records with the recovery target value (ie, stop either just
+# after or just before the given target).
+#
+#recovery_target_time = '2004-07-14 22:39:00'
+#
+# note: target time is interpreted by strptime() and must therefore be
+# given in your system's default timezone.
+#
+#recovery_target_xid = '11000'
+#
+# true or false
+#recovery_target_inclusive = 'true'
+#
+#---------------------------------------------------------------------------

src/backend/commands/tablecmds.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/commands/tablecmds.c,v 1.120 2004/07/17 17:28:29 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/commands/tablecmds.c,v 1.121 2004/07/19 02:47:06 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -5414,15 +5414,8 @@ copy_relation_data(Relation rel, SMgrRelation dst)
 	/*
 	 * We need to log the copied data in WAL iff WAL archiving is enabled
 	 * AND it's not a temp rel.
-	 *
-	 * XXX when WAL archiving is actually supported, this test will likely
-	 * need to change; and the hardwired extern is cruddy anyway ...
 	 */
-	{
-		extern char XLOG_archive_dir[];
-
-		use_wal = XLOG_archive_dir[0] && !rel->rd_istemp;
-	}
+	use_wal = XLogArchivingActive() && !rel->rd_istemp;
 
 	nblocks = RelationGetNumberOfBlocks(rel);
 	for (blkno = 0; blkno < nblocks; blkno++)

src/backend/postmaster/Makefile

@@ -4,7 +4,7 @@
 #    Makefile for src/backend/postmaster
 #
 # IDENTIFICATION
-#    $PostgreSQL: pgsql/src/backend/postmaster/Makefile,v 1.15 2004/05/29 22:48:19 tgl Exp $
+#    $PostgreSQL: pgsql/src/backend/postmaster/Makefile,v 1.16 2004/07/19 02:47:08 tgl Exp $
 #
 #-------------------------------------------------------------------------
 
@@ -12,7 +12,7 @@ subdir = src/backend/postmaster
 top_builddir = ../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = postmaster.o bgwriter.o pgstat.o
+OBJS = postmaster.o bgwriter.o pgstat.o pgarch.o
 
 all: SUBSYS.o
 

src/backend/postmaster/postmaster.c

@@ -37,7 +37,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/postmaster/postmaster.c,v 1.411 2004/07/12 19:14:56 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/postmaster/postmaster.c,v 1.412 2004/07/19 02:47:08 tgl Exp $
  *
  * NOTES
  *
@@ -104,6 +104,7 @@
 #include "miscadmin.h"
 #include "nodes/nodes.h"
 #include "postmaster/postmaster.h"
+#include "postmaster/pgarch.h"
 #include "storage/fd.h"
 #include "storage/ipc.h"
 #include "storage/pg_shmem.h"
@@ -198,6 +199,7 @@ char	   *preload_libraries_string = NULL;
 /* PIDs of special child processes; 0 when not running */
 static pid_t StartupPID = 0,
 			BgWriterPID = 0,
+			PgArchPID = 0,
 			PgStatPID = 0;
 
 /* Startup/shutdown state */
@@ -826,7 +828,8 @@ PostmasterMain(int argc, char *argv[])
 	 *
 	 * CAUTION: when changing this list, check for side-effects on the signal
 	 * handling setup of child processes.  See tcop/postgres.c,
-	 * bootstrap/bootstrap.c, postmaster/bgwriter.c, and postmaster/pgstat.c.
+	 * bootstrap/bootstrap.c, postmaster/bgwriter.c, postmaster/pgarch.c,
+	 * and postmaster/pgstat.c.
 	 */
 	pqinitmask();
 	PG_SETMASK(&BlockSig);
@@ -1217,6 +1220,11 @@ ServerLoop(void)
 				kill(BgWriterPID, SIGUSR2);
 		}
 
+		/* If we have lost the archiver, try to start a new one */
+		if (XLogArchivingActive() && PgArchPID == 0 && 
+            StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
+			PgArchPID = pgarch_start();
+ 
 		/* If we have lost the stats collector, try to start a new one */
 		if (PgStatPID == 0 &&
 			StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
@@ -1760,6 +1768,8 @@ SIGHUP_handler(SIGNAL_ARGS)
 		SignalChildren(SIGHUP);
 		if (BgWriterPID != 0)
 			kill(BgWriterPID, SIGHUP);
+		if (PgArchPID != 0)
+			kill(PgArchPID, SIGHUP);
 		/* PgStatPID does not currently need SIGHUP */
 		load_hba();
 		load_ident();
@@ -1818,6 +1828,9 @@ pmdie(SIGNAL_ARGS)
 			/* And tell it to shut down */
 			if (BgWriterPID != 0)
 				kill(BgWriterPID, SIGUSR2);
+			/* Tell pgarch to shut down too; nothing left for it to do */
+			if (PgArchPID != 0)
+				kill(PgArchPID, SIGQUIT);
 			/* Tell pgstat to shut down too; nothing left for it to do */
 			if (PgStatPID != 0)
 				kill(PgStatPID, SIGQUIT);
@@ -1862,6 +1875,9 @@ pmdie(SIGNAL_ARGS)
 			/* And tell it to shut down */
 			if (BgWriterPID != 0)
 				kill(BgWriterPID, SIGUSR2);
+			/* Tell pgarch to shut down too; nothing left for it to do */
+			if (PgArchPID != 0)
+				kill(PgArchPID, SIGQUIT);
 			/* Tell pgstat to shut down too; nothing left for it to do */
 			if (PgStatPID != 0)
 				kill(PgStatPID, SIGQUIT);
@@ -1880,6 +1896,8 @@ pmdie(SIGNAL_ARGS)
 				kill(StartupPID, SIGQUIT);
 			if (BgWriterPID != 0)
 				kill(BgWriterPID, SIGQUIT);
+			if (PgArchPID != 0)
+				kill(PgArchPID, SIGQUIT);
 			if (PgStatPID != 0)
 				kill(PgStatPID, SIGQUIT);
 			if (DLGetHead(BackendList))
@@ -1967,12 +1985,16 @@ reaper(SIGNAL_ARGS)
 
 			/*
 			 * Go to shutdown mode if a shutdown request was pending.
-			 * Otherwise, try to start the stats collector too.
+			 * Otherwise, try to start the archiver and stats collector too.
 			 */
 			if (Shutdown > NoShutdown && BgWriterPID != 0)
 				kill(BgWriterPID, SIGUSR2);
-			else if (PgStatPID == 0 && Shutdown == NoShutdown)
-				PgStatPID = pgstat_start();
+			else if (Shutdown == NoShutdown) {
+                    if (XLogArchivingActive() && PgArchPID == 0)
+        				PgArchPID = pgarch_start();
+                    if (PgStatPID == 0)
+        				PgStatPID = pgstat_start();
+            }
 
 			continue;
 		}
@@ -2004,6 +2026,23 @@ reaper(SIGNAL_ARGS)
 			continue;
 		}
 
+		/*
+		 * Was it the archiver?  If so, just try to start a new
+		 * one; no need to force reset of the rest of the system.  (If fail,
+		 * we'll try again in future cycles of the main loop.)
+		 */
+		if (PgArchPID != 0 && pid == PgArchPID)
+		{
+			PgArchPID = 0;
+			if (exitstatus != 0)
+				LogChildExit(LOG, gettext("archiver process"),
+							 pid, exitstatus);
+			if (XLogArchivingActive() &&
+				StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
+				PgArchPID = pgarch_start();
+			continue;
+		}
+
 		/*
 		 * Was it the statistics collector?  If so, just try to start a new
 		 * one; no need to force reset of the rest of the system.  (If fail,
@@ -2029,8 +2068,9 @@ reaper(SIGNAL_ARGS)
 	if (FatalError)
 	{
 		/*
-		 * Wait for all children exit, then reset shmem and
-		 * StartupDataBase.
+		 * Wait for all important children to exit, then reset shmem and
+		 * StartupDataBase.  (We can ignore the archiver and stats processes
+		 * here since they are not connected to shmem.)
 		 */
 		if (DLGetHead(BackendList) || StartupPID != 0 || BgWriterPID != 0)
 			goto reaper_done;
@@ -2191,6 +2231,17 @@ HandleChildCrash(int pid,
 		kill(BgWriterPID, (SendStop ? SIGSTOP : SIGQUIT));
 	}
 
+	/* Force a power-cycle of the pgarch process too */
+	/* (Shouldn't be necessary, but just for luck) */
+	if (PgArchPID != 0 && !FatalError)
+	{
+		ereport(DEBUG2,
+				(errmsg_internal("sending %s to process %d",
+								 "SIGQUIT",
+								 (int) PgArchPID)));
+		kill(PgArchPID, SIGQUIT);
+	}
+
 	/* Force a power-cycle of the pgstat processes too */
 	/* (Shouldn't be necessary, but just for luck) */
 	if (PgStatPID != 0 && !FatalError)
@@ -2873,6 +2924,16 @@ SubPostmasterMain(int argc, char *argv[])
 		BootstrapMain(argc - 2, argv + 2);
 		proc_exit(0);
 	}
+	if (strcmp(argv[1], "-forkarch") == 0)
+	{
+		/* Close the postmaster's sockets */
+		ClosePostmasterPorts();
+
+		/* Do not want to attach to shared memory */
+
+		PgArchiverMain(argc, argv);
+		proc_exit(0);
+	}
 	if (strcmp(argv[1], "-forkbuf") == 0)
 	{
 		/* Close the postmaster's sockets */
@@ -2951,6 +3012,18 @@ sigusr1_handler(SIGNAL_ARGS)
 		if (Shutdown <= SmartShutdown)
 			SignalChildren(SIGUSR1);
 	}
+ 
+	if (PgArchPID != 0 && Shutdown == NoShutdown)
+	{
+		if (CheckPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER))
+		{
+			/*
+			 * Send SIGUSR1 to archiver process, to wake it up and begin
+			 * archiving next transaction log file.
+			 */
+            kill(PgArchPID, SIGUSR1);
+		}
+    }
 
 	PG_SETMASK(&UnBlockSig);
 

src/backend/utils/misc/guc.c

@@ -10,7 +10,7 @@
  * Written by Peter Eisentraut <peter_e@gmx.net>.
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/utils/misc/guc.c,v 1.219 2004/07/12 02:22:51 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/utils/misc/guc.c,v 1.220 2004/07/19 02:47:10 tgl Exp $
  *
  *--------------------------------------------------------------------
  */
@@ -1371,6 +1371,15 @@ static struct config_real ConfigureNamesReal[] =
 
 static struct config_string ConfigureNamesString[] =
 {
+ 	{
+ 		{"archive_command", PGC_SIGHUP, WAL_SETTINGS,
+ 			gettext_noop("WAL archiving command."),
+ 			gettext_noop("The shell command that will be called to archive a WAL file.")
+ 		},
+ 		&XLogArchiveCommand,
+ 		"", NULL, NULL
+ 	},
+
 	{
 		{"client_encoding", PGC_USERSET, CLIENT_CONN_LOCALE,
 			gettext_noop("Sets the client's character set encoding."),

src/backend/utils/misc/postgresql.conf.sample

@@ -111,6 +111,16 @@
 #commit_delay = 0		# range 0-100000, in microseconds
 #commit_siblings = 5		# range 1-1000
 
+# - Archiving -
+
+#archive_command = ''		# command to use to archive a logfile segment
+
+# If archive_command is '' then archiving is disabled.  Otherwise, set it
+# to a command to copy a file to the proper place.  A simplistic example
+# is 'cp %p /mnt/server/archivedir/%f'.  Any %p in the string is replaced
+# by the absolute path of the file to archive, while any %f is replaced by
+# the file name only.  NOTE: it is important for the command to return
+# zero exit status if and only if it succeeded.
 
 #---------------------------------------------------------------------------
 # QUERY TUNING

src/bin/initdb/initdb.c

@@ -39,7 +39,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  * Portions taken from FreeBSD.
  *
- * $PostgreSQL: pgsql/src/bin/initdb/initdb.c,v 1.43 2004/07/14 17:55:10 petere Exp $
+ * $PostgreSQL: pgsql/src/bin/initdb/initdb.c,v 1.44 2004/07/19 02:47:12 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -145,7 +145,7 @@ static char *get_id(void);
 static char *get_encoding_id(char *encoding_name);
 static char *get_short_version(void);
 static int check_data_dir(void);
-static bool mkdatadir(char *subdir);
+static bool mkdatadir(const char *subdir);
 static void set_input(char **dest, char *filename);
 static void check_input(char *path);
 static void set_short_version(char *short_version, char *extrapath);
@@ -900,7 +900,7 @@ check_data_dir(void)
  * make the data directory (or one of its subdirectories if subdir is not NULL)
  */
 static bool
-mkdatadir(char *subdir)
+mkdatadir(const char *subdir)
 {
 	char	   *path;
 
@@ -2022,8 +2022,16 @@ main(int argc, char *argv[])
 	char	   *short_version;
 	char	   *pgdenv;			/* PGDATA value got from sent to
 								 * environment */
-	char	   *subdirs[] =
-	{"global", "pg_xlog", "pg_clog", "pg_subtrans", "base", "base/1", "pg_tblspc"};
+	static const char *subdirs[] = {
+		"global",
+		"pg_xlog",
+		"pg_xlog/archive_status",
+		"pg_clog",
+		"pg_subtrans",
+		"base",
+		"base/1",
+		"pg_tblspc"
+	};
 
 	progname = get_progname(argv[0]);
 	set_pglocale_pgservice(argv[0], "initdb");

src/include/access/xlog.h

@@ -6,7 +6,7 @@
  * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/xlog.h,v 1.52 2004/07/01 00:51:38 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/xlog.h,v 1.53 2004/07/19 02:47:13 tgl Exp $
  */
 #ifndef XLOG_H
 #define XLOG_H
@@ -208,9 +208,12 @@ extern XLogRecPtr ProcLastRecEnd;
 /* these variables are GUC parameters related to XLOG */
 extern int	CheckPointSegments;
 extern int	XLOGbuffers;
+extern char *XLogArchiveCommand;
 extern char *XLOG_sync_method;
 extern const char XLOG_sync_method_default[];
 
+#define XLogArchivingActive()	(XLogArchiveCommand[0] != '\0')
+
 #ifdef WAL_DEBUG
 extern bool	XLOG_DEBUG;
 #endif

src/include/postmaster/pgarch.h

+/*-------------------------------------------------------------------------
+ *
+ * pgarch.h
+ *	  Exports from postmaster/pgarch.c.
+ *
+ * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
+ * Portions Copyright (c) 1994, Regents of the University of California
+ *
+ * $PostgreSQL: pgsql/src/include/postmaster/pgarch.h,v 1.1 2004/07/19 02:47:16 tgl Exp $
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef _PGARCH_H
+#define _PGARCH_H
+
+/* ----------
+ * Functions called from postmaster
+ * ----------
+ */
+extern int	pgarch_start(void);
+
+#ifdef EXEC_BACKEND
+extern void PgArchiverMain(int argc, char *argv[]);
+#endif
+
+#endif   /* _PGARCH_H */

src/include/storage/pmsignal.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/storage/pmsignal.h,v 1.8 2004/05/29 22:48:23 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/storage/pmsignal.h,v 1.9 2004/07/19 02:47:15 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -24,6 +24,7 @@ typedef enum
 {
 	PMSIGNAL_PASSWORD_CHANGE,	/* pg_pwd file has changed */
 	PMSIGNAL_WAKEN_CHILDREN,	/* send a SIGUSR1 signal to all backends */
+	PMSIGNAL_WAKEN_ARCHIVER,	/* send a NOTIFY signal to xlog archiver */
 
 	NUM_PMSIGNALS				/* Must be last value of enum! */
 } PMSignalReason;