pgarch.c 14.3 KB
Newer Older
1 2 3 4 5 6
/*-------------------------------------------------------------------------
 *
 * pgarch.c
 *
 *	PostgreSQL WAL archiver
 *
Bruce Momjian's avatar
Bruce Momjian committed
7
 *	All functions relating to archiver are included here
8
 *
Bruce Momjian's avatar
Bruce Momjian committed
9
 *	- All functions executed by archiver process
10
 *
Bruce Momjian's avatar
Bruce Momjian committed
11 12 13
 *	- archiver is forked from postmaster, and the two
 *	processes then communicate using signals. All functions
 *	executed by postmaster are included in this file.
14
 *
Bruce Momjian's avatar
Bruce Momjian committed
15
 *	Initial author: Simon Riggs		simon@2ndquadrant.com
16
 *
17
 * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
18 19 20 21
 * Portions Copyright (c) 1994, Regents of the University of California
 *
 *
 * IDENTIFICATION
22
 *	  $PostgreSQL: pgsql/src/backend/postmaster/pgarch.c,v 1.30 2007/08/02 23:39:44 adunstan Exp $
23 24 25 26 27 28 29 30 31
 *
 *-------------------------------------------------------------------------
 */
#include "postgres.h"

#include <fcntl.h>
#include <signal.h>
#include <time.h>
#include <sys/time.h>
32
#include <sys/wait.h>
33 34
#include <unistd.h>

35
#include "access/xlog_internal.h"
36 37
#include "libpq/pqsignal.h"
#include "miscadmin.h"
38
#include "postmaster/fork_process.h"
39
#include "postmaster/pgarch.h"
40 41 42 43 44 45 46 47 48 49 50 51 52
#include "postmaster/postmaster.h"
#include "storage/fd.h"
#include "storage/ipc.h"
#include "storage/pg_shmem.h"
#include "storage/pmsignal.h"
#include "utils/guc.h"
#include "utils/ps_status.h"


/* ----------
 * Timer definitions.
 * ----------
 */
53 54 55 56 57
#define PGARCH_AUTOWAKE_INTERVAL 60		/* How often to force a poll of the
										 * archive status directory; in
										 * seconds. */
#define PGARCH_RESTART_INTERVAL 10		/* How often to attempt to restart a
										 * failed archiver; in seconds. */
58 59 60 61 62 63 64 65 66 67 68

/* ----------
 * Archiver control info.
 *
 * We expect that archivable files within pg_xlog will have names between
 * MIN_XFN_CHARS and MAX_XFN_CHARS in length, consisting only of characters
 * appearing in VALID_XFN_CHARS.  The status files in archive_status have
 * corresponding names with ".ready" or ".done" appended.
 * ----------
 */
#define MIN_XFN_CHARS	16
69
#define MAX_XFN_CHARS	40
Bruce Momjian's avatar
Bruce Momjian committed
70
#define VALID_XFN_CHARS "0123456789ABCDEF.history.backup"
71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133

#define NUM_ARCHIVE_RETRIES 3


/* ----------
 * Local data
 * ----------
 */
static time_t last_pgarch_start_time;

/*
 * Flags set by interrupt handlers for later service in the main loop.
 */
static volatile sig_atomic_t got_SIGHUP = false;
static volatile sig_atomic_t wakened = false;

/* ----------
 * Local function forward declarations
 * ----------
 */
#ifdef EXEC_BACKEND
static pid_t pgarch_forkexec(void);
#endif

NON_EXEC_STATIC void PgArchiverMain(int argc, char *argv[]);
static void pgarch_exit(SIGNAL_ARGS);
static void ArchSigHupHandler(SIGNAL_ARGS);
static void pgarch_waken(SIGNAL_ARGS);
static void pgarch_MainLoop(void);
static void pgarch_ArchiverCopyLoop(void);
static bool pgarch_archiveXlog(char *xlog);
static bool pgarch_readyXlog(char *xlog);
static void pgarch_archiveDone(char *xlog);


/* ------------------------------------------------------------
 * Public functions called from postmaster follow
 * ------------------------------------------------------------
 */

/*
 * pgarch_start
 *
 *	Called from postmaster at startup or after an existing archiver
 *	died.  Attempt to fire up a fresh archiver process.
 *
 *	Returns PID of child process, or 0 if fail.
 *
 *	Note: if fail, we will be called again from the postmaster main loop.
 */
int
pgarch_start(void)
{
	time_t		curtime;
	pid_t		pgArchPid;

	/*
	 * Do nothing if no archiver needed
	 */
	if (!XLogArchivingActive())
		return 0;

	/*
Bruce Momjian's avatar
Bruce Momjian committed
134
	 * Do nothing if too soon since last archiver start.  This is a safety
135 136 137
	 * valve to protect against continuous respawn attempts if the archiver is
	 * dying immediately at launch. Note that since we will be re-called from
	 * the postmaster main loop, we will get another chance later.
138 139 140 141
	 */
	curtime = time(NULL);
	if ((unsigned int) (curtime - last_pgarch_start_time) <
		(unsigned int) PGARCH_RESTART_INTERVAL)
Bruce Momjian's avatar
Bruce Momjian committed
142
		return 0;
143 144 145 146 147
	last_pgarch_start_time = curtime;

#ifdef EXEC_BACKEND
	switch ((pgArchPid = pgarch_forkexec()))
#else
148
	switch ((pgArchPid = fork_process()))
149 150 151 152 153 154 155 156 157 158 159
#endif
	{
		case -1:
			ereport(LOG,
					(errmsg("could not fork archiver: %m")));
			return 0;

#ifndef EXEC_BACKEND
		case 0:
			/* in postmaster child ... */
			/* Close the postmaster's sockets */
160
			ClosePostmasterPorts(false);
161

162 163 164
			/* Lose the postmaster's on-exit routines */
			on_exit_reset();

165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
			/* Drop our connection to postmaster's shared memory, as well */
			PGSharedMemoryDetach();

			PgArchiverMain(0, NULL);
			break;
#endif

		default:
			return (int) pgArchPid;
	}

	/* shouldn't get here */
	return 0;
}

/* ------------------------------------------------------------
 * Local functions called by archiver follow
 * ------------------------------------------------------------
 */


#ifdef EXEC_BACKEND

/*
 * pgarch_forkexec() -
 *
 * Format up the arglist for, then fork and exec, archive process
 */
static pid_t
pgarch_forkexec(void)
{
Bruce Momjian's avatar
Bruce Momjian committed
196 197
	char	   *av[10];
	int			ac = 0;
198 199 200

	av[ac++] = "postgres";

201
	av[ac++] = "--forkarch";
202 203 204 205 206 207 208 209

	av[ac++] = NULL;			/* filled in by postmaster_forkexec */

	av[ac] = NULL;
	Assert(ac < lengthof(av));

	return postmaster_forkexec(ac, av);
}
Bruce Momjian's avatar
Bruce Momjian committed
210
#endif   /* EXEC_BACKEND */
211 212 213 214 215 216 217 218 219 220 221


/*
 * PgArchiverMain
 *
 *	The argc/argv parameters are valid only in EXEC_BACKEND case.  However,
 *	since we don't use 'em, it hardly matters...
 */
NON_EXEC_STATIC void
PgArchiverMain(int argc, char *argv[])
{
Bruce Momjian's avatar
Bruce Momjian committed
222
	IsUnderPostmaster = true;	/* we are a postmaster subprocess now */
223

Bruce Momjian's avatar
Bruce Momjian committed
224
	MyProcPid = getpid();		/* reset MyProcPid */
225

226 227
	MyStartTime = time(NULL);   /* record Start Time for logging */

228 229 230 231 232 233 234 235 236
	/*
	 * If possible, make this process a group leader, so that the postmaster
	 * can signal any child processes too.
	 */
#ifdef HAVE_SETSID
	if (setsid() < 0)
		elog(FATAL, "setsid() failed: %m");
#endif

Bruce Momjian's avatar
Bruce Momjian committed
237 238
	/*
	 * Ignore all signals usually bound to some action in the postmaster,
239
	 * except for SIGHUP, SIGUSR1 and SIGQUIT.
Bruce Momjian's avatar
Bruce Momjian committed
240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
	 */
	pqsignal(SIGHUP, ArchSigHupHandler);
	pqsignal(SIGINT, SIG_IGN);
	pqsignal(SIGTERM, SIG_IGN);
	pqsignal(SIGQUIT, pgarch_exit);
	pqsignal(SIGALRM, SIG_IGN);
	pqsignal(SIGPIPE, SIG_IGN);
	pqsignal(SIGUSR1, pgarch_waken);
	pqsignal(SIGUSR2, SIG_IGN);
	pqsignal(SIGCHLD, SIG_DFL);
	pqsignal(SIGTTIN, SIG_DFL);
	pqsignal(SIGTTOU, SIG_DFL);
	pqsignal(SIGCONT, SIG_DFL);
	pqsignal(SIGWINCH, SIG_DFL);
	PG_SETMASK(&UnBlockSig);

	/*
	 * Identify myself via ps
	 */
259
	init_ps_display("archiver process", "", "", "");
Bruce Momjian's avatar
Bruce Momjian committed
260 261

	pgarch_MainLoop();
262

Bruce Momjian's avatar
Bruce Momjian committed
263
	exit(0);
264 265 266 267 268 269 270 271
}

/* SIGQUIT signal handler for archiver process */
static void
pgarch_exit(SIGNAL_ARGS)
{
	/*
	 * For now, we just nail the doors shut and get out of town.  It might
Bruce Momjian's avatar
Bruce Momjian committed
272 273
	 * seem cleaner to finish up any pending archive copies, but there's a
	 * nontrivial risk that init will kill us partway through.
274
	 */
Bruce Momjian's avatar
Bruce Momjian committed
275
	exit(0);
276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299
}

/* SIGHUP: set flag to re-read config file at next convenient time */
static void
ArchSigHupHandler(SIGNAL_ARGS)
{
	got_SIGHUP = true;
}

/* SIGUSR1 signal handler for archiver process */
static void
pgarch_waken(SIGNAL_ARGS)
{
	wakened = true;
}

/*
 * pgarch_MainLoop
 *
 * Main loop for archiver
 */
static void
pgarch_MainLoop(void)
{
Bruce Momjian's avatar
Bruce Momjian committed
300
	time_t		last_copy_time = 0;
301 302 303

	/*
	 * We run the copy loop immediately upon entry, in case there are
304 305 306
	 * unarchived files left over from a previous database run (or maybe the
	 * archiver died unexpectedly).  After that we wait for a signal or
	 * timeout before doing more.
307 308 309
	 */
	wakened = true;

Bruce Momjian's avatar
Bruce Momjian committed
310 311
	do
	{
312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
		/* Check for config update */
		if (got_SIGHUP)
		{
			got_SIGHUP = false;
			ProcessConfigFile(PGC_SIGHUP);
			if (!XLogArchivingActive())
				break;			/* user wants us to shut down */
		}

		/* Do what we're here for */
		if (wakened)
		{
			wakened = false;
			pgarch_ArchiverCopyLoop();
			last_copy_time = time(NULL);
		}

		/*
330
		 * There shouldn't be anything for the archiver to do except to wait
331 332 333 334 335 336 337
		 * for a signal ... however, the archiver exists to protect our data,
		 * so she wakes up occasionally to allow herself to be proactive.
		 *
		 * On some platforms, signals won't interrupt the sleep.  To ensure we
		 * respond reasonably promptly when someone signals us, break down the
		 * sleep into 1-second increments, and check for interrupts after each
		 * nap.
338
		 */
339
		while (!(wakened || got_SIGHUP))
340
		{
341
			time_t		curtime;
342

343
			pg_usleep(1000000L);
344 345 346 347 348
			curtime = time(NULL);
			if ((unsigned int) (curtime - last_copy_time) >=
				(unsigned int) PGARCH_AUTOWAKE_INTERVAL)
				wakened = true;
		}
Bruce Momjian's avatar
Bruce Momjian committed
349
	} while (PostmasterIsAlive(true));
350 351 352 353 354 355 356 357 358 359
}

/*
 * pgarch_ArchiverCopyLoop
 *
 * Archives all outstanding xlogs then returns
 */
static void
pgarch_ArchiverCopyLoop(void)
{
Bruce Momjian's avatar
Bruce Momjian committed
360 361 362 363
	char		xlog[MAX_XFN_CHARS + 1];

	/*
	 * loop through all xlogs with archive_status of .ready and archive
364 365 366
	 * them...mostly we expect this to be a single file, though it is possible
	 * some backend will add files onto the list of those that need archiving
	 * while we are still copying earlier archives
Bruce Momjian's avatar
Bruce Momjian committed
367
	 */
368
	while (pgarch_readyXlog(xlog))
369
	{
Bruce Momjian's avatar
Bruce Momjian committed
370
		int			failures = 0;
371 372 373

		for (;;)
		{
374 375 376 377
			/* Abandon processing if we notice our postmaster has died */
			if (!PostmasterIsAlive(true))
				return;

378 379 380 381 382 383 384 385 386 387 388
			if (pgarch_archiveXlog(xlog))
			{
				/* successful */
				pgarch_archiveDone(xlog);
				break;			/* out of inner retry loop */
			}
			else
			{
				if (++failures >= NUM_ARCHIVE_RETRIES)
				{
					ereport(WARNING,
389
							(errmsg("transaction log file \"%s\" could not be archived: too many failures",
390 391 392
									xlog)));
					return;		/* give up archiving for now */
				}
393
				pg_usleep(1000000L);	/* wait a bit before retrying */
394 395 396 397 398 399 400 401 402 403 404 405 406 407 408
			}
		}
	}
}

/*
 * pgarch_archiveXlog
 *
 * Invokes system(3) to copy one archive file to wherever it should go
 *
 * Returns true if successful
 */
static bool
pgarch_archiveXlog(char *xlog)
{
Bruce Momjian's avatar
Bruce Momjian committed
409 410 411 412
	char		xlogarchcmd[MAXPGPATH];
	char		pathname[MAXPGPATH];
	char	   *dp;
	char	   *endp;
413
	const char *sp;
Bruce Momjian's avatar
Bruce Momjian committed
414
	int			rc;
415

416
	snprintf(pathname, MAXPGPATH, XLOGDIR "/%s", xlog);
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431

	/*
	 * construct the command to be executed
	 */
	dp = xlogarchcmd;
	endp = xlogarchcmd + MAXPGPATH - 1;
	*endp = '\0';

	for (sp = XLogArchiveCommand; *sp; sp++)
	{
		if (*sp == '%')
		{
			switch (sp[1])
			{
				case 'p':
432
					/* %p: relative path of source file */
433
					sp++;
434
					strlcpy(dp, pathname, endp - dp);
435
					make_native_path(dp);
436 437 438 439 440
					dp += strlen(dp);
					break;
				case 'f':
					/* %f: filename of source file */
					sp++;
441
					strlcpy(dp, xlog, endp - dp);
442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465
					dp += strlen(dp);
					break;
				case '%':
					/* convert %% to a single % */
					sp++;
					if (dp < endp)
						*dp++ = *sp;
					break;
				default:
					/* otherwise treat the % as not special */
					if (dp < endp)
						*dp++ = *sp;
					break;
			}
		}
		else
		{
			if (dp < endp)
				*dp++ = *sp;
		}
	}
	*dp = '\0';

	ereport(DEBUG3,
Bruce Momjian's avatar
Bruce Momjian committed
466
			(errmsg_internal("executing archive command \"%s\"",
467
							 xlogarchcmd)));
Bruce Momjian's avatar
Bruce Momjian committed
468 469 470
	rc = system(xlogarchcmd);
	if (rc != 0)
	{
471 472 473 474 475 476 477 478 479 480 481 482 483
		/*
		 * If either the shell itself, or a called command, died on a signal,
		 * abort the archiver.  We do this because system() ignores SIGINT and
		 * SIGQUIT while waiting; so a signal is very likely something that
		 * should have interrupted us too.  If we overreact it's no big deal,
		 * the postmaster will just start the archiver again.
		 *
		 * Per the Single Unix Spec, shells report exit status > 128 when
		 * a called command died on a signal.
		 */
		bool	signaled = WIFSIGNALED(rc) || WEXITSTATUS(rc) > 128;

		ereport(signaled ? FATAL : LOG,
484 485
				(errmsg("archive command \"%s\" failed: return code %d",
						xlogarchcmd, rc)));
486

Bruce Momjian's avatar
Bruce Momjian committed
487 488
		return false;
	}
489
	ereport(LOG,
Bruce Momjian's avatar
Bruce Momjian committed
490
			(errmsg("archived transaction log file \"%s\"", xlog)));
491

Bruce Momjian's avatar
Bruce Momjian committed
492
	return true;
493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508
}

/*
 * pgarch_readyXlog
 *
 * Return name of the oldest xlog file that has not yet been archived.
 * No notification is set that file archiving is now in progress, so
 * this would need to be extended if multiple concurrent archival
 * tasks were created. If a failure occurs, we will completely
 * re-copy the file at the next available opportunity.
 *
 * It is important that we return the oldest, so that we archive xlogs
 * in order that they were written, for two reasons:
 * 1) to maintain the sequential chain of xlogs required for recovery
 * 2) because the oldest ones will sooner become candidates for
 * recycling at time of checkpoint
509 510 511 512 513 514
 *
 * NOTE: the "oldest" comparison will presently consider all segments of
 * a timeline with a smaller ID to be older than all segments of a timeline
 * with a larger ID; the net result being that past timelines are given
 * higher priority for archiving.  This seems okay, or at least not
 * obviously worth changing.
515 516 517 518 519
 */
static bool
pgarch_readyXlog(char *xlog)
{
	/*
520 521 522 523
	 * open xlog status directory and read through list of xlogs that have the
	 * .ready suffix, looking for earliest file. It is possible to optimise
	 * this code, though only a single file is expected on the vast majority
	 * of calls, so....
524
	 */
525
	char		XLogArchiveStatusDir[MAXPGPATH];
Bruce Momjian's avatar
Bruce Momjian committed
526 527 528 529
	char		newxlog[MAX_XFN_CHARS + 6 + 1];
	DIR		   *rldir;
	struct dirent *rlde;
	bool		found = false;
530

531
	snprintf(XLogArchiveStatusDir, MAXPGPATH, XLOGDIR "/archive_status");
532 533 534
	rldir = AllocateDir(XLogArchiveStatusDir);
	if (rldir == NULL)
		ereport(ERROR,
Bruce Momjian's avatar
Bruce Momjian committed
535
				(errcode_for_file_access(),
536 537
				 errmsg("could not open archive status directory \"%s\": %m",
						XLogArchiveStatusDir)));
538

539
	while ((rlde = ReadDir(rldir, XLogArchiveStatusDir)) != NULL)
540
	{
Bruce Momjian's avatar
Bruce Momjian committed
541
		int			basenamelen = (int) strlen(rlde->d_name) - 6;
542 543 544 545 546 547

		if (basenamelen >= MIN_XFN_CHARS &&
			basenamelen <= MAX_XFN_CHARS &&
			strspn(rlde->d_name, VALID_XFN_CHARS) >= basenamelen &&
			strcmp(rlde->d_name + basenamelen, ".ready") == 0)
		{
Bruce Momjian's avatar
Bruce Momjian committed
548 549 550 551 552 553 554 555 556 557
			if (!found)
			{
				strcpy(newxlog, rlde->d_name);
				found = true;
			}
			else
			{
				if (strcmp(rlde->d_name, newxlog) < 0)
					strcpy(newxlog, rlde->d_name);
			}
558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581
		}
	}
	FreeDir(rldir);

	if (found)
	{
		/* truncate off the .ready */
		newxlog[strlen(newxlog) - 6] = '\0';
		strcpy(xlog, newxlog);
	}
	return found;
}

/*
 * pgarch_archiveDone
 *
 * Emit notification that an xlog file has been successfully archived.
 * We do this by renaming the status file from NNN.ready to NNN.done.
 * Eventually, a checkpoint process will notice this and delete both the
 * NNN.done file and the xlog file itself.
 */
static void
pgarch_archiveDone(char *xlog)
{
Bruce Momjian's avatar
Bruce Momjian committed
582 583
	char		rlogready[MAXPGPATH];
	char		rlogdone[MAXPGPATH];
584

585 586
	StatusFilePath(rlogready, xlog, ".ready");
	StatusFilePath(rlogdone, xlog, ".done");
Bruce Momjian's avatar
Bruce Momjian committed
587 588
	if (rename(rlogready, rlogdone) < 0)
		ereport(WARNING,
589
				(errcode_for_file_access(),
590
				 errmsg("could not rename file \"%s\" to \"%s\": %m",
591
						rlogready, rlogdone)));
592
}