Revise postmaster startup/shutdown logic to eliminate the problem that a

constant flow of new connection requests could prevent the postmaster from
completing a shutdown or crash restart.  This is done by labeling child
processes that are "dead ends", that is, we know that they were launched only
to tell a client that it can't connect.  These processes are managed
separately so that they don't confuse us into thinking that we can't advance
to the next stage of a shutdown or restart sequence, until the very end
where we must wait for them to drain out so we can delete the shmem segment.
Per discussion of a misbehavior reported by Keaton Adams.

Since this code was baroque already, and my first attempt at fixing the
problem made it entirely impenetrable, I took the opportunity to rewrite it
in a state-machine style.  That eliminates some duplicated code sections and
hopefully makes everything a bit clearer.

src/backend/postmaster/postmaster.c

@@ -37,7 +37,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/postmaster/postmaster.c,v 1.539 2007/08/04 03:15:49 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/postmaster/postmaster.c,v 1.540 2007/08/09 01:18:43 tgl Exp $
  *
  * NOTES
  *
@@ -131,22 +131,31 @@
  *
  * "Special" children such as the startup, bgwriter and autovacuum launcher
  * tasks are not in this list.  Autovacuum worker processes are in it.
+ * Also, "dead_end" children are in it: these are children launched just
+ * for the purpose of sending a friendly rejection message to a would-be
+ * client.  We must track them because they are attached to shared memory,
+ * but we know they will never become live backends.
  */
 typedef struct bkend
 {
 	pid_t		pid;			/* process id of backend */
 	long		cancel_key;		/* cancel key for cancels for this backend */
 	bool		is_autovacuum;	/* is it an autovacuum process? */
+	bool		dead_end;		/* is it going to send an error and quit? */
 } Backend;
 
 static Dllist *BackendList;
 
 #ifdef EXEC_BACKEND
 /*
- * Number of entries in the backend table. Twice the number of backends,
- * plus five other subprocesses (stats, bgwriter, walwriter, autovac, logger).
+ * Number of entries in the shared-memory backend table.  This table is used
+ * only for sending cancels, and therefore only includes children we allow
+ * cancels on: regular backends and autovac workers.  In particular we exclude
+ * dead_end children, allowing the table to have a known maximum size, to wit
+ * the same too-many-children limit enforced by canAcceptConnections().
  */
-#define NUM_BACKENDARRAY_ELEMS (2*MaxBackends + 5)
+#define NUM_BACKENDARRAY_ELEMS (2*MaxBackends)
+
 static Backend *ShmemBackendArray;
 #endif
 
@@ -180,7 +189,7 @@ static char ExtraOptions[MAXPGPATH];
  * backend dumps core.	Normally, it kills all peers of the dead backend
  * and reinitializes shared memory.  By specifying -s or -n, we can have
  * the postmaster stop (rather than kill) peers and not reinitialize
- * shared data structures.
+ * shared data structures.  (Reinit is currently dead code, though.)
  */
 static bool Reinit = true;
 static int	SendStop = false;
@@ -216,10 +225,45 @@ static int	Shutdown = NoShutdown;
 
 static bool FatalError = false; /* T if recovering from backend crash */
 
+/*
+ * We use a simple state machine to control startup, shutdown, and
+ * crash recovery (which is rather like shutdown followed by startup).
+ *
+ * Normal child backends can only be launched when we are in PM_RUN state.
+ * In other states we handle connection requests by launching "dead_end"
+ * child processes, which will simply send the client an error message and
+ * quit.  (We track these in the BackendList so that we can know when they
+ * are all gone; this is important because they're still connected to shared
+ * memory, and would interfere with an attempt to destroy the shmem segment,
+ * possibly leading to SHMALL failure when we try to make a new one.)
+ * In PM_WAIT_DEAD_END state we are waiting for all the dead_end children
+ * to drain out of the system, and therefore stop accepting connection
+ * requests at all until the last existing child has quit (which hopefully
+ * will not be very long).
+ *
+ * Notice that this state variable does not distinguish *why* we entered
+ * PM_WAIT_BACKENDS or later states --- Shutdown and FatalError must be
+ * consulted to find that out.  FatalError is never true in PM_RUN state, nor
+ * in PM_SHUTDOWN state (because we don't enter that state when trying to
+ * recover from a crash).  It can be true in PM_STARTUP state, because we
+ * don't clear it until we've successfully recovered.
+ */
+typedef enum {
+	PM_INIT,					/* postmaster starting */
+	PM_STARTUP,					/* waiting for startup subprocess */
+	PM_RUN,						/* normal "database is alive" state */
+	PM_WAIT_BACKENDS,			/* waiting for live backends to exit */
+	PM_SHUTDOWN,				/* waiting for bgwriter to do shutdown ckpt */
+	PM_WAIT_DEAD_END,			/* waiting for dead_end children to exit */
+	PM_NO_CHILDREN				/* all important children have exited */
+} PMState;
+
+static PMState pmState = PM_INIT;
+
 bool		ClientAuthInProgress = false;		/* T during new-client
 												 * authentication */
 
-bool redirection_done = false; 
+bool redirection_done = false;	/* stderr redirected for syslogger? */
 
 /* received START_AUTOVAC_LAUNCHER signal */
 static volatile sig_atomic_t start_autovac_launcher = false;
@@ -262,6 +306,7 @@ static void CleanupBackend(int pid, int exitstatus);
 static void HandleChildCrash(int pid, int exitstatus, const char *procname);
 static void LogChildExit(int lev, const char *procname,
 			 int pid, int exitstatus);
+static void PostmasterStateMachine(void);
 static void BackendInitialize(Port *port);
 static int	BackendRun(Port *port);
 static void ExitPostmaster(int status);
@@ -275,8 +320,9 @@ static enum CAC_state canAcceptConnections(void);
 static long PostmasterRandom(void);
 static void RandomSalt(char *cryptSalt, char *md5Salt);
 static void signal_child(pid_t pid, int signal);
-static void SignalChildren(int signal);
 static void SignalSomeChildren(int signal, bool only_autovac);
+#define SignalChildren(sig)			SignalSomeChildren(sig, false)
+#define SignalAutovacWorkers(sig)	SignalSomeChildren(sig, true)
 static int	CountChildren(void);
 static bool CreateOptsFile(int argc, char *argv[], char *fullprogname);
 static pid_t StartChildProcess(AuxProcType type);
@@ -888,6 +934,7 @@ PostmasterMain(int argc, char *argv[])
 		ExitPostmaster(1);
 
 #ifdef EXEC_BACKEND
+	/* Write out nondefault GUC settings for child processes to use */
 	write_nondefault_variables(PGC_POSTMASTER);
 #endif
 
@@ -974,6 +1021,8 @@ PostmasterMain(int argc, char *argv[])
 	 * We're ready to rock and roll...
 	 */
 	StartupPID = StartupDataBase();
+	Assert(StartupPID != 0);
+	pmState = PM_STARTUP;
 
 	status = ServerLoop();
 
@@ -1078,7 +1127,6 @@ checkDataDir(void)
 static void
 reg_reply(DNSServiceRegistrationReplyErrorType errorCode, void *context)
 {
-
 }
 #endif   /* USE_BONJOUR */
 
@@ -1110,7 +1158,8 @@ pmdaemonize(void)
 
 	MyStartTime = time(NULL);
 
-/* GH: If there's no setsid(), we hopefully don't need silent mode.
+	/*
+	 * GH: If there's no setsid(), we hopefully don't need silent mode.
 	 * Until there's a better solution.
 	 */
 #ifdef HAVE_SETSID
@@ -1150,26 +1199,38 @@ ServerLoop(void)
 
 	for (;;)
 	{
-		Port	   *port;
 		fd_set		rmask;
-		struct timeval timeout;
 		int			selres;
-		int			i;
 
 		/*
-		 * Wait for something to happen.
+		 * Wait for a connection request to arrive.
 		 *
 		 * We wait at most one minute, to ensure that the other background
 		 * tasks handled below get done even when no requests are arriving.
+		 *
+		 * If we are in PM_WAIT_DEAD_END state, then we don't want to
+		 * accept any new connections, so we don't call select() at all;
+		 * just sleep for a little bit with signals unblocked.
 		 */
 		memcpy((char *) &rmask, (char *) &readmask, sizeof(fd_set));
 
+		PG_SETMASK(&UnBlockSig);
+
+		if (pmState == PM_WAIT_DEAD_END)
+		{
+			pg_usleep(100000L);		/* 100 msec seems reasonable */
+			selres = 0;
+		}
+		else
+		{
+			/* must set timeout each time; some OSes change it! */
+			struct timeval timeout;
+
 			timeout.tv_sec = 60;
 			timeout.tv_usec = 0;
 
-		PG_SETMASK(&UnBlockSig);
-
 			selres = select(nSockets, &rmask, NULL, NULL, &timeout);
+		}
 
 		/*
 		 * Block all signals until we wait again.  (This makes it safe for our
@@ -1177,6 +1238,7 @@ ServerLoop(void)
 		 */
 		PG_SETMASK(&BlockSig);
 
+		/* Now check the select() result */
 		if (selres < 0)
 		{
 			if (errno != EINTR && errno != EWOULDBLOCK)
@@ -1194,12 +1256,16 @@ ServerLoop(void)
 		 */
 		if (selres > 0)
 		{
+			int			i;
+
 			for (i = 0; i < MAXLISTEN; i++)
 			{
 				if (ListenSocket[i] == -1)
 					break;
 				if (FD_ISSET(ListenSocket[i], &rmask))
 				{
+					Port	   *port;
+
 					port = ConnCreate(ListenSocket[i]);
 					if (port)
 					{
@@ -1225,27 +1291,20 @@ ServerLoop(void)
 		 * state that prevents it, start one.  It doesn't matter if this
 		 * fails, we'll just try again later.
 		 */
-		if (BgWriterPID == 0 && StartupPID == 0 && !FatalError)
-		{
+		if (BgWriterPID == 0 && pmState == PM_RUN)
 			BgWriterPID = StartBackgroundWriter();
-			/* If shutdown is pending, set it going */
-			if (Shutdown > NoShutdown && BgWriterPID != 0)
-				signal_child(BgWriterPID, SIGUSR2);
-		}
 
 		/*
 		 * Likewise, if we have lost the walwriter process, try to start a
-		 * new one.  We don't need walwriter to complete a shutdown, so
-		 * don't start it if shutdown already initiated.
+		 * new one.
 		 */
-		if (WalWriterPID == 0 &&
-			StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
+		if (WalWriterPID == 0 && pmState == PM_RUN)
 			WalWriterPID = StartWalWriter();
 
 		/* If we have lost the autovacuum launcher, try to start a new one */
 		if (AutoVacPID == 0 &&
 			(AutoVacuumingActive() || start_autovac_launcher) &&
-			StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
+			pmState == PM_RUN)
 		{
 			AutoVacPID = StartAutoVacLauncher();
 			if (AutoVacPID != 0)
@@ -1253,13 +1312,11 @@ ServerLoop(void)
 		}
 
 		/* If we have lost the archiver, try to start a new one */
-		if (XLogArchivingActive() && PgArchPID == 0 &&
-			StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
+		if (XLogArchivingActive() && PgArchPID == 0 && pmState == PM_RUN)
 			PgArchPID = pgarch_start();
 
 		/* If we have lost the stats collector, try to start a new one */
-		if (PgStatPID == 0 &&
-			StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
+		if (PgStatPID == 0 && pmState == PM_RUN)
 			PgStatPID = pgstat_start();
 
 		/*
@@ -1285,7 +1342,7 @@ ServerLoop(void)
 static int
 initMasks(fd_set *rmask)
 {
-	int			nsocks = -1;
+	int			maxsock = -1;
 	int			i;
 
 	FD_ZERO(rmask);
@@ -1297,16 +1354,16 @@ initMasks(fd_set *rmask)
 		if (fd == -1)
 			break;
 		FD_SET(fd, rmask);
-		if (fd > nsocks)
-			nsocks = fd;
+		if (fd > maxsock)
+			maxsock = fd;
 	}
 
-	return nsocks + 1;
+	return maxsock + 1;
 }
 
 
 /*
- * Read the startup packet and do something according to it.
+ * Read a client's startup packet and do something according to it.
  *
  * Returns STATUS_OK or STATUS_ERROR, or might call ereport(FATAL) and
  * not return at all.
@@ -1594,7 +1651,6 @@ retry1:
 					 errmsg("sorry, too many clients already")));
 			break;
 		case CAC_OK:
-		default:
 			break;
 	}
 
@@ -1670,12 +1726,14 @@ static enum CAC_state
 canAcceptConnections(void)
 {
 	/* Can't start backends when in startup/shutdown/recovery state. */
+	if (pmState != PM_RUN)
+	{
 		if (Shutdown > NoShutdown)
-		return CAC_SHUTDOWN;
-	if (StartupPID)
-		return CAC_STARTUP;
-	if (FatalError)
-		return CAC_RECOVERY;
+			return CAC_SHUTDOWN;		/* shutdown is pending */
+		if (pmState == PM_STARTUP && !FatalError)
+			return CAC_STARTUP;			/* normal startup */
+		return CAC_RECOVERY;			/* else must be crash recovery */
+	}
 
 	/*
 	 * Don't start too many children.
@@ -1685,6 +1743,9 @@ canAcceptConnections(void)
 	 * backend might exit before the auth cycle is completed. The exact
 	 * MaxBackends limit is enforced when a new backend tries to join the
 	 * shared-inval backend array.
+	 *
+	 * In the EXEC_BACKEND case, the limit here must match the size of the
+	 * ShmemBackendArray, since all these processes will have cancel codes.
 	 */
 	if (CountChildren() >= 2 * MaxBackends)
 		return CAC_TOOMANY;
@@ -1895,36 +1956,24 @@ pmdie(SIGNAL_ARGS)
 			ereport(LOG,
 					(errmsg("received smart shutdown request")));
 
+			if (pmState == PM_RUN)
+			{
 				/* autovacuum workers are told to shut down immediately */
-			if (DLGetHead(BackendList))
-				SignalSomeChildren(SIGTERM, true);
+				SignalAutovacWorkers(SIGTERM);
 				/* and the autovac launcher too */
 				if (AutoVacPID != 0)
 					signal_child(AutoVacPID, SIGTERM);
 				/* and the walwriter too */
 				if (WalWriterPID != 0)
 					signal_child(WalWriterPID, SIGTERM);
-
-			if (DLGetHead(BackendList) || AutoVacPID != 0 || WalWriterPID != 0)
-				break;			/* let reaper() handle this */
+				pmState = PM_WAIT_BACKENDS;
+			}
 
 			/*
-			 * No children left. Begin shutdown of data base system.
+			 * Now wait for backends to exit.  If there are none,
+			 * PostmasterStateMachine will take the next step.
 			 */
-			if (StartupPID != 0 || FatalError)
-				break;			/* let reaper() handle this */
-			/* Start the bgwriter if not running */
-			if (BgWriterPID == 0)
-				BgWriterPID = StartBackgroundWriter();
-			/* And tell it to shut down */
-			if (BgWriterPID != 0)
-				signal_child(BgWriterPID, SIGUSR2);
-			/* Tell pgarch to shut down too; nothing left for it to do */
-			if (PgArchPID != 0)
-				signal_child(PgArchPID, SIGQUIT);
-			/* Tell pgstat to shut down too; nothing left for it to do */
-			if (PgStatPID != 0)
-				signal_child(PgStatPID, SIGQUIT);
+			PostmasterStateMachine();
 			break;
 
 		case SIGINT:
@@ -1941,48 +1990,28 @@ pmdie(SIGNAL_ARGS)
 			ereport(LOG,
 					(errmsg("received fast shutdown request")));
 
-			if (DLGetHead(BackendList) || AutoVacPID != 0 || WalWriterPID != 0)
-			{
-				if (!FatalError)
+			if (StartupPID != 0)
+				signal_child(StartupPID, SIGTERM);
+			if (pmState == PM_RUN)
 			{
 				ereport(LOG,
 						(errmsg("aborting any active transactions")));
+				/* shut down all backends and autovac workers */
 				SignalChildren(SIGTERM);
+				/* and the autovac launcher too */
 				if (AutoVacPID != 0)
 					signal_child(AutoVacPID, SIGTERM);
+				/* and the walwriter too */
 				if (WalWriterPID != 0)
 					signal_child(WalWriterPID, SIGTERM);
-					/* reaper() does the rest */
-				}
-				break;
+				pmState = PM_WAIT_BACKENDS;
 			}
 
 			/*
-			 * No children left. Begin shutdown of data base system.
-			 *
-			 * Note: if we previously got SIGTERM then we may send SIGUSR2 to
-			 * the bgwriter a second time here.  This should be harmless.
-			 * Ditto for the signals to the other special children.
+			 * Now wait for backends to exit.  If there are none,
+			 * PostmasterStateMachine will take the next step.
 			 */
-			if (StartupPID != 0)
-			{
-				signal_child(StartupPID, SIGTERM);
-				break;			/* let reaper() do the rest */
-			}
-			if (FatalError)
-				break;			/* let reaper() handle this case */
-			/* Start the bgwriter if not running */
-			if (BgWriterPID == 0)
-				BgWriterPID = StartBackgroundWriter();
-			/* And tell it to shut down */
-			if (BgWriterPID != 0)
-				signal_child(BgWriterPID, SIGUSR2);
-			/* Tell pgarch to shut down too; nothing left for it to do */
-			if (PgArchPID != 0)
-				signal_child(PgArchPID, SIGQUIT);
-			/* Tell pgstat to shut down too; nothing left for it to do */
-			if (PgStatPID != 0)
-				signal_child(PgStatPID, SIGQUIT);
+			PostmasterStateMachine();
 			break;
 
 		case SIGQUIT:
@@ -1995,6 +2024,7 @@ pmdie(SIGNAL_ARGS)
 			 */
 			ereport(LOG,
 					(errmsg("received immediate shutdown request")));
+			SignalChildren(SIGQUIT);
 			if (StartupPID != 0)
 				signal_child(StartupPID, SIGQUIT);
 			if (BgWriterPID != 0)
@@ -2007,8 +2037,6 @@ pmdie(SIGNAL_ARGS)
 				signal_child(PgArchPID, SIGQUIT);
 			if (PgStatPID != 0)
 				signal_child(PgStatPID, SIGQUIT);
-			if (DLGetHead(BackendList))
-				SignalChildren(SIGQUIT);
 			ExitPostmaster(0);
 			break;
 	}
@@ -2019,55 +2047,54 @@ pmdie(SIGNAL_ARGS)
 }
 
 /*
- * Reaper -- signal handler to cleanup after a backend (child) dies.
+ * Reaper -- signal handler to cleanup after a child process dies.
  */
 static void
 reaper(SIGNAL_ARGS)
 {
 	int			save_errno = errno;
+	int			pid;			/* process id of dead child process */
+	int			exitstatus;		/* its exit status */
 
+	/* These macros hide platform variations in getting child status */
 #ifdef HAVE_WAITPID
-	int			status;			/* backend exit status */
-#else
+	int			status;			/* child exit status */
+#define LOOPTEST()		((pid = waitpid(-1, &status, WNOHANG)) > 0)
+#define LOOPHEADER()	(exitstatus = status)
+#else   /* !HAVE_WAITPID */
 #ifndef WIN32
-	union wait	status;			/* backend exit status */
-#endif
-#endif
-	int			exitstatus;
-	int			pid;			/* process id of dead backend */
-
-	PG_SETMASK(&BlockSig);
-
-	ereport(DEBUG4,
-			(errmsg_internal("reaping dead processes")));
-#ifdef HAVE_WAITPID
-	while ((pid = waitpid(-1, &status, WNOHANG)) > 0)
-	{
-		exitstatus = status;
-#else
-#ifndef WIN32
-	while ((pid = wait3(&status, WNOHANG, NULL)) > 0)
-	{
-		exitstatus = status.w_status;
-#else
-	while ((pid = win32_waitpid(&exitstatus)) > 0)
-	{
+	union wait	status;			/* child exit status */
+#define LOOPTEST()		((pid = wait3(&status, WNOHANG, NULL)) > 0)
+#define LOOPHEADER()	(exitstatus = status.w_status)
+#else	/* WIN32 */
+#define LOOPTEST()		((pid = win32_waitpid(&exitstatus)) > 0)
 	/*
 	 * We need to do this here, and not in CleanupBackend, since this is
 	 * to be called on all children when we are done with them. Could move
 	 * to LogChildExit, but that seems like asking for future trouble...
 	 */
-		win32_RemoveChild(pid);
+#define LOOPHEADER()	(win32_RemoveChild(pid))
 #endif   /* WIN32 */
 #endif   /* HAVE_WAITPID */
 
+	PG_SETMASK(&BlockSig);
+
+	ereport(DEBUG4,
+			(errmsg_internal("reaping dead processes")));
+
+	while (LOOPTEST())
+	{
+		LOOPHEADER();
+
 		/*
 		 * Check if this child was a startup process.
 		 */
-		if (StartupPID != 0 && pid == StartupPID)
+		if (pid == StartupPID)
 		{
 			StartupPID = 0;
-			/* Note: FATAL exit of startup is treated as catastrophic */
+			Assert(pmState == PM_STARTUP);
+
+			/* FATAL exit of startup is treated as catastrophic */
 			if (!EXIT_STATUS_0(exitstatus))
 			{
 				LogChildExit(LOG, _("startup process"),
@@ -2083,6 +2110,21 @@ reaper(SIGNAL_ARGS)
 			 */
 			FatalError = false;
 
+			/*
+			 * Go to shutdown mode if a shutdown request was pending.
+			 */
+			if (Shutdown > NoShutdown)
+			{
+				pmState = PM_WAIT_BACKENDS;
+				/* PostmasterStateMachine logic does the rest */
+				continue;
+			}
+
+			/*
+			 * Otherwise, commence normal operations.
+			 */
+			pmState = PM_RUN;
+
 			/*
 			 * Load the flat authorization file into postmaster's cache. The
 			 * startup process has recomputed this from the database contents,
@@ -2098,26 +2140,21 @@ reaper(SIGNAL_ARGS)
 			BgWriterPID = StartBackgroundWriter();
 
 			/*
-			 * Go to shutdown mode if a shutdown request was pending.
-			 * Otherwise, try to start the other special children.
+			 * Likewise, start other special children as needed.  In a restart
+			 * situation, some of them may be alive already.
 			 */
-			if (Shutdown > NoShutdown && BgWriterPID != 0)
-				signal_child(BgWriterPID, SIGUSR2);
-			else if (Shutdown == NoShutdown)
-			{
 			if (WalWriterPID == 0)
 				WalWriterPID = StartWalWriter();
+			if (AutoVacuumingActive() && AutoVacPID == 0)
+				AutoVacPID = StartAutoVacLauncher();
 			if (XLogArchivingActive() && PgArchPID == 0)
 				PgArchPID = pgarch_start();
 			if (PgStatPID == 0)
 				PgStatPID = pgstat_start();
-				if (AutoVacuumingActive() && AutoVacPID == 0)
-					AutoVacPID = StartAutoVacLauncher();
 
 			/* at this point we are really open for business */
 			ereport(LOG,
 					(errmsg("database system is ready to accept connections")));
-			}
 
 			continue;
 		}
@@ -2125,50 +2162,34 @@ reaper(SIGNAL_ARGS)
 		/*
 		 * Was it the bgwriter?
 		 */
-		if (BgWriterPID != 0 && pid == BgWriterPID)
+		if (pid == BgWriterPID)
 		{
 			BgWriterPID = 0;
-			if (EXIT_STATUS_0(exitstatus) &&
-				Shutdown > NoShutdown && !FatalError &&
-				!DLGetHead(BackendList) &&
-				WalWriterPID == 0 && AutoVacPID == 0)
+			if (EXIT_STATUS_0(exitstatus) && pmState == PM_SHUTDOWN)
 			{
 				/*
-				 * Normal postmaster exit is here: we've seen normal exit of
-				 * the bgwriter after it's been told to shut down. We expect
-				 * that it wrote a shutdown checkpoint.  (If for some reason
-				 * it didn't, recovery will occur on next postmaster start.)
+				 * OK, we saw normal exit of the bgwriter after it's been
+				 * told to shut down.  We expect that it wrote a shutdown
+				 * checkpoint.  (If for some reason it didn't, recovery will
+				 * occur on next postmaster start.)
 				 *
-				 * Note: we do not wait around for exit of the archiver or
-				 * stats processes.  They've been sent SIGQUIT by this point,
-				 * and in any case contain logic to commit hara-kiri if they
-				 * notice the postmaster is gone.
+				 * At this point we should have no normal children left
+				 * (else we'd not be in PM_SHUTDOWN state) but we might have
+				 * dead_end children.
 				 */
-				ExitPostmaster(0);
+				Assert(Shutdown > NoShutdown);
+				pmState = PM_WAIT_DEAD_END;
 			}
-
+			else
+			{
 				/*
 				 * Any unexpected exit of the bgwriter (including FATAL exit)
 				 * is treated as a crash.
 				 */
 				HandleChildCrash(pid, exitstatus,
 								 _("background writer process"));
-
-			/*
-			 * If the bgwriter crashed while trying to write the shutdown
-			 * checkpoint, we may as well just stop here; any recovery
-			 * required will happen on next postmaster start.
-			 */
-			if (Shutdown > NoShutdown &&
-				!DLGetHead(BackendList) &&
-				WalWriterPID == 0 && AutoVacPID == 0)
-			{
-				ereport(LOG,
-						(errmsg("abnormal database system shutdown")));
-				ExitPostmaster(1);
 			}
 
-			/* Else, proceed as in normal crash recovery */
 			continue;
 		}
 
@@ -2177,7 +2198,7 @@ reaper(SIGNAL_ARGS)
 		 * start a new one at the next iteration of the postmaster's main loop,
 		 * if necessary.  Any other exit condition is treated as a crash.
 		 */
-		if (WalWriterPID != 0 && pid == WalWriterPID)
+		if (pid == WalWriterPID)
 		{
 			WalWriterPID = 0;
 			if (!EXIT_STATUS_0(exitstatus))
@@ -2191,7 +2212,7 @@ reaper(SIGNAL_ARGS)
 		 * start a new one at the next iteration of the postmaster's main loop,
 		 * if necessary.  Any other exit condition is treated as a crash.
 		 */
-		if (AutoVacPID != 0 && pid == AutoVacPID)
+		if (pid == AutoVacPID)
 		{
 			AutoVacPID = 0;
 			if (!EXIT_STATUS_0(exitstatus))
@@ -2205,14 +2226,13 @@ reaper(SIGNAL_ARGS)
 		 * to force reset of the rest of the system.  (If fail, we'll try
 		 * again in future cycles of the main loop.)
 		 */
-		if (PgArchPID != 0 && pid == PgArchPID)
+		if (pid == PgArchPID)
 		{
 			PgArchPID = 0;
 			if (!EXIT_STATUS_0(exitstatus))
 				LogChildExit(LOG, _("archiver process"),
 							 pid, exitstatus);
-			if (XLogArchivingActive() &&
-				StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
+			if (XLogArchivingActive() && pmState == PM_RUN)
 				PgArchPID = pgarch_start();
 			continue;
 		}
@@ -2222,19 +2242,19 @@ reaper(SIGNAL_ARGS)
 		 * one; no need to force reset of the rest of the system.  (If fail,
 		 * we'll try again in future cycles of the main loop.)
 		 */
-		if (PgStatPID != 0 && pid == PgStatPID)
+		if (pid == PgStatPID)
 		{
 			PgStatPID = 0;
 			if (!EXIT_STATUS_0(exitstatus))
 				LogChildExit(LOG, _("statistics collector process"),
 							 pid, exitstatus);
-			if (StartupPID == 0 && !FatalError && Shutdown == NoShutdown)
+			if (pmState == PM_RUN)
 				PgStatPID = pgstat_start();
 			continue;
 		}
 
-		/* Was it the system logger? try to start a new one */
-		if (SysLoggerPID != 0 && pid == SysLoggerPID)
+		/* Was it the system logger?  If so, try to start a new one */
+		if (pid == SysLoggerPID)
 		{
 			SysLoggerPID = 0;
 			/* for safety's sake, launch new logger *first* */
@@ -2251,48 +2271,13 @@ reaper(SIGNAL_ARGS)
 		CleanupBackend(pid, exitstatus);
 	}							/* loop over pending child-death reports */
 
-	if (FatalError)
-	{
 	/*
-		 * Wait for all important children to exit, then reset shmem and
-		 * StartupDataBase.  (We can ignore the archiver and stats processes
-		 * here since they are not connected to shmem.)
+	 * After cleaning out the SIGCHLD queue, see if we have any state changes
+	 * or actions to make.
 	 */
-		if (DLGetHead(BackendList) || StartupPID != 0 ||
-			BgWriterPID != 0 || WalWriterPID != 0 ||
-			AutoVacPID != 0)
-			goto reaper_done;
-		ereport(LOG,
-				(errmsg("all server processes terminated; reinitializing")));
-
-		shmem_exit(0);
-		reset_shared(PostPortNumber);
-
-		StartupPID = StartupDataBase();
-
-		goto reaper_done;
-	}
+	PostmasterStateMachine();
 
-	if (Shutdown > NoShutdown)
-	{
-		if (DLGetHead(BackendList) || StartupPID != 0 || AutoVacPID != 0 ||
-			WalWriterPID != 0)
-			goto reaper_done;
-		/* Start the bgwriter if not running */
-		if (BgWriterPID == 0)
-			BgWriterPID = StartBackgroundWriter();
-		/* And tell it to shut down */
-		if (BgWriterPID != 0)
-			signal_child(BgWriterPID, SIGUSR2);
-		/* Tell pgarch to shut down too; nothing left for it to do */
-		if (PgArchPID != 0)
-			signal_child(PgArchPID, SIGQUIT);
-		/* Tell pgstat to shut down too; nothing left for it to do */
-		if (PgStatPID != 0)
-			signal_child(PgStatPID, SIGQUIT);
-	}
-
-reaper_done:
+	/* Done with signal handler */
 	PG_SETMASK(&UnBlockSig);
 
 	errno = save_errno;
@@ -2330,12 +2315,13 @@ CleanupBackend(int pid,
 
 		if (bp->pid == pid)
 		{
-			DLRemove(curr);
-			free(bp);
-			DLFreeElem(curr);
 #ifdef EXEC_BACKEND
+			if (!bp->dead_end)
 				ShmemBackendArrayRemove(pid);
 #endif
+			DLRemove(curr);
+			free(bp);
+			DLFreeElem(curr);
 			break;
 		}
 	}
@@ -2376,12 +2362,13 @@ HandleChildCrash(int pid, int exitstatus, const char *procname)
 			/*
 			 * Found entry for freshly-dead backend, so remove it.
 			 */
-			DLRemove(curr);
-			free(bp);
-			DLFreeElem(curr);
 #ifdef EXEC_BACKEND
+			if (!bp->dead_end)
 				ShmemBackendArrayRemove(pid);
 #endif
+			DLRemove(curr);
+			free(bp);
+			DLFreeElem(curr);
 			/* Keep looping so we can signal remaining backends */
 		}
 		else
@@ -2394,6 +2381,9 @@ HandleChildCrash(int pid, int exitstatus, const char *procname)
 			 * and let the user know what's going on. But if SendStop is set
 			 * (-s on command line), then we send SIGSTOP instead, so that we
 			 * can get core dumps from all backends by hand.
+			 *
+			 * We could exclude dead_end children here, but at least in the
+			 * SIGSTOP case it seems better to include them.
 			 */
 			if (!FatalError)
 			{
@@ -2442,8 +2432,12 @@ HandleChildCrash(int pid, int exitstatus, const char *procname)
 		signal_child(AutoVacPID, (SendStop ? SIGSTOP : SIGQUIT));
 	}
 
-	/* Force a power-cycle of the pgarch process too */
-	/* (Shouldn't be necessary, but just for luck) */
+	/*
+	 * Force a power-cycle of the pgarch process too.  (This isn't absolutely
+	 * necessary, but it seems like a good idea for robustness, and it
+	 * simplifies the state-machine logic in the case where a shutdown
+	 * request arrives during crash processing.)
+	 */
 	if (PgArchPID != 0 && !FatalError)
 	{
 		ereport(DEBUG2,
@@ -2453,8 +2447,12 @@ HandleChildCrash(int pid, int exitstatus, const char *procname)
 		signal_child(PgArchPID, SIGQUIT);
 	}
 
-	/* Force a power-cycle of the pgstat process too */
-	/* (Shouldn't be necessary, but just for luck) */
+	/*
+	 * Force a power-cycle of the pgstat process too.  (This isn't absolutely
+	 * necessary, but it seems like a good idea for robustness, and it
+	 * simplifies the state-machine logic in the case where a shutdown
+	 * request arrives during crash processing.)
+	 */
 	if (PgStatPID != 0 && !FatalError)
 	{
 		ereport(DEBUG2,
@@ -2468,6 +2466,9 @@ HandleChildCrash(int pid, int exitstatus, const char *procname)
 	/* We do NOT restart the syslogger */
 
 	FatalError = true;
+	/* We now transit into a state of waiting for children to die */
+	if (pmState == PM_RUN || pmState == PM_SHUTDOWN)
+		pmState = PM_WAIT_BACKENDS;
 }
 
 /*
@@ -2523,6 +2524,151 @@ LogChildExit(int lev, const char *procname, int pid, int exitstatus)
 						procname, pid, exitstatus)));
 }
 
+/*
+ * Advance the postmaster's state machine and take actions as appropriate
+ *
+ * This is common code for pmdie() and reaper(), which receive the signals
+ * that might mean we need to change state.
+ */
+static void
+PostmasterStateMachine(void)
+{
+	/*
+	 * If we are in a state-machine state that implies waiting for backends
+	 * to exit, see if they're all gone, and change state if so.
+	 */
+	if (pmState == PM_WAIT_BACKENDS)
+	{
+		/*
+		 * PM_WAIT_BACKENDS state ends when we have no regular backends
+		 * (including autovac workers) and no walwriter or autovac launcher.
+		 * If we are doing crash recovery then we expect the bgwriter to
+		 * exit too, otherwise not.  The archiver, stats, and syslogger
+		 * processes are disregarded since they are not connected to shared
+		 * memory; we also disregard dead_end children here.
+		 */
+		if (CountChildren() == 0 &&
+			StartupPID == 0 &&
+			(BgWriterPID == 0 || !FatalError) &&
+			WalWriterPID == 0 &&
+			AutoVacPID == 0)
+		{
+			if (FatalError)
+			{
+				/*
+				 * Start waiting for dead_end children to die.  This state
+				 * change causes ServerLoop to stop creating new ones.
+				 */
+				pmState = PM_WAIT_DEAD_END;
+			}
+			else
+			{
+				/*
+				 * If we get here, we are proceeding with normal shutdown.
+				 * All the regular children are gone, and it's time to tell
+				 * the bgwriter to do a shutdown checkpoint.
+				 */
+				Assert(Shutdown > NoShutdown);
+				/* Start the bgwriter if not running */
+				if (BgWriterPID == 0)
+					BgWriterPID = StartBackgroundWriter();
+				/* And tell it to shut down */
+				if (BgWriterPID != 0)
+				{
+					signal_child(BgWriterPID, SIGUSR2);
+					pmState = PM_SHUTDOWN;
+				}
+				else
+				{
+					/*
+					 * If we failed to fork a bgwriter, just shut down.
+					 * Any required cleanup will happen at next restart.
+					 * We set FatalError so that an "abnormal shutdown"
+					 * message gets logged when we exit.
+					 */
+					FatalError = true;
+					pmState = PM_WAIT_DEAD_END;
+				}
+				/* Tell pgarch to shut down too; nothing left for it to do */
+				if (PgArchPID != 0)
+					signal_child(PgArchPID, SIGQUIT);
+				/* Tell pgstat to shut down too; nothing left for it to do */
+				if (PgStatPID != 0)
+					signal_child(PgStatPID, SIGQUIT);
+			}
+		}
+	}
+
+	if (pmState == PM_WAIT_DEAD_END)
+	{
+		/*
+		 * PM_WAIT_DEAD_END state ends when the BackendList is entirely
+		 * empty (ie, no dead_end children remain).
+		 */
+		if (!DLGetHead(BackendList))
+		{
+			/* These other guys should be dead already */
+			Assert(StartupPID == 0);
+			Assert(BgWriterPID == 0);
+			Assert(WalWriterPID == 0);
+			Assert(AutoVacPID == 0);
+			/* archiver, stats, and syslogger are not considered here */
+			pmState = PM_NO_CHILDREN;
+		}
+	}
+
+	/*
+	 * If we've been told to shut down, we exit as soon as there are no
+	 * remaining children.  If there was a crash, cleanup will occur at the
+	 * next startup.  (Before PostgreSQL 8.3, we tried to recover from the
+	 * crash before exiting, but that seems unwise if we are quitting because
+	 * we got SIGTERM from init --- there may well not be time for recovery
+	 * before init decides to SIGKILL us.)
+	 *
+	 * Note: we do not wait around for exit of the archiver or stats
+	 * processes.  They've been sent SIGQUIT by this point (either when we
+	 * entered PM_SHUTDOWN state, or when we set FatalError, and at least one
+	 * of those must have happened by now).  In any case they contain logic to
+	 * commit hara-kiri if they notice the postmaster is gone.  Since they
+	 * aren't connected to shared memory, they pose no problem for shutdown.
+	 * The syslogger is not considered either, since it's intended to survive
+	 * till the postmaster exits.
+	 */
+	if (Shutdown > NoShutdown && pmState == PM_NO_CHILDREN)
+	{
+		if (FatalError)
+		{
+			ereport(LOG, (errmsg("abnormal database system shutdown")));
+			ExitPostmaster(1);
+		}
+		else
+		{
+			/* Normal exit from the postmaster is here */
+			ExitPostmaster(0);
+		}
+	}
+
+	/*
+	 * If we need to recover from a crash, wait for all shmem-connected
+	 * children to exit, then reset shmem and StartupDataBase.  (We can ignore
+	 * the archiver and stats processes here since they are not connected to
+	 * shmem.)
+	 */
+	if (FatalError && pmState == PM_NO_CHILDREN)
+	{
+		ereport(LOG,
+				(errmsg("all server processes terminated; reinitializing")));
+
+		shmem_exit(0);
+		reset_shared(PostPortNumber);
+
+		StartupPID = StartupDataBase();
+		Assert(StartupPID != 0);
+		pmState = PM_STARTUP;
+	}
+}
+
+
 /*
  * Send a signal to a postmaster child process
  *
@@ -2561,19 +2707,9 @@ signal_child(pid_t pid, int signal)
 }
 
 /*
- * Send a signal to all backend children, including autovacuum workers (but NOT
- * special children).
- */
-static void
-SignalChildren(int signal)
-{
-	SignalSomeChildren(signal, false);
-}
-
-/*
- * Send a signal to all backend children, including autovacuum workers (but NOT
- * special children).  If only_autovac is TRUE, only the autovacuum worker
- * processes are signalled.
+ * Send a signal to all backend children, including autovacuum workers
+ * (but NOT special children; dead_end children are never signaled, either).
+ * If only_autovac is TRUE, only the autovacuum worker processes are signalled.
  */
 static void
 SignalSomeChildren(int signal, bool only_autovac)
@@ -2584,6 +2720,8 @@ SignalSomeChildren(int signal, bool only_autovac)
 	{
 		Backend    *bp = (Backend *) DLE_VAL(curr);
 
+		if (bp->dead_end)
+			continue;
 		if (only_autovac && !bp->is_autovacuum)
 			continue;
 
@@ -2688,8 +2826,10 @@ BackendStartup(Port *port)
 	bn->pid = pid;
 	bn->cancel_key = MyCancelKey;
 	bn->is_autovacuum = false;
+	bn->dead_end = (port->canAcceptConnections != CAC_OK);
 	DLAddHead(BackendList, DLNewElem(bn));
 #ifdef EXEC_BACKEND
+	if (!bn->dead_end)
 		ShmemBackendArrayAdd(bn);
 #endif
 
@@ -3647,7 +3787,7 @@ sigusr1_handler(SIGNAL_ARGS)
 	}
 
 	if (CheckPostmasterSignal(PMSIGNAL_WAKEN_ARCHIVER) &&
-		PgArchPID != 0 && Shutdown == NoShutdown)
+		PgArchPID != 0 && Shutdown <= SmartShutdown)
 	{
 		/*
 		 * Send SIGUSR1 to archiver process, to wake it up and begin archiving
@@ -3790,7 +3930,8 @@ PostmasterRandom(void)
 }
 
 /*
- * Count up number of child processes (regular backends only)
+ * Count up number of child processes (excluding special children and
+ * dead_end children)
  */
 static int
 CountChildren(void)
@@ -3799,7 +3940,12 @@ CountChildren(void)
 	int			cnt = 0;
 
 	for (curr = DLGetHead(BackendList); curr; curr = DLGetSucc(curr))
+	{
+		Backend    *bp = (Backend *) DLE_VAL(curr);
+
+		if (!bp->dead_end)
 			cnt++;
+	}
 	return cnt;
 }
 
@@ -3918,13 +4064,14 @@ StartAutovacuumWorker(void)
 	Backend	   *bn;
 
 	/*
-	 * do nothing if not in condition to run a process.  This should not
-	 * actually happen, since the signal is only supposed to be sent by
-	 * autovacuum launcher when it's OK to do it, but test for it just in case.
+	 * If not in condition to run a process, don't try, but handle it like a
+	 * fork failure.  This does not normally happen, since the signal is only
+	 * supposed to be sent by autovacuum launcher when it's OK to do it, but
+	 * we have to check to avoid race-condition problems during DB state
+	 * changes.
 	 */
-	if (StartupPID != 0 || FatalError || Shutdown != NoShutdown)
-		return;
-
+	if (canAcceptConnections() == CAC_OK)
+	{
 		/*
 		 * Compute the cancel key that will be assigned to this session.
 		 * We probably don't need cancel keys for autovac workers, but we'd
@@ -3941,6 +4088,7 @@ StartAutovacuumWorker(void)
 			{
 				bn->cancel_key = MyCancelKey;
 				bn->is_autovacuum = true;
+				bn->dead_end = false;
 				DLAddHead(BackendList, DLNewElem(bn));
 #ifdef EXEC_BACKEND
 				ShmemBackendArrayAdd(bn);
@@ -3959,11 +4107,18 @@ StartAutovacuumWorker(void)
 			ereport(LOG,
 					(errcode(ERRCODE_OUT_OF_MEMORY),
 					 errmsg("out of memory")));
+	}
 
-	/* report the failure to the launcher */
-	AutoVacWorkerFailed();
+	/*
+	 * Report the failure to the launcher, if it's running.  (If it's not,
+	 * we might not even be connected to shared memory, so don't try to
+	 * call AutoVacWorkerFailed.)
+	 */
 	if (AutoVacPID != 0)
+	{
+		AutoVacWorkerFailed();
 		kill(AutoVacPID, SIGUSR1);
+	}
 }
 
 /*