Install a "dead man switch" to allow the postmaster to detect cases where

a backend has done exit(0) or exit(1) without having disengaged itself
from shared memory.  We are at risk for this whenever third-party code is
loaded into a backend, since such code might not know it's supposed to go
through proc_exit() instead.  Also, it is reported that under Windows
there are ways to externally kill a process that cause the status code
returned to the postmaster to be indistinguishable from a voluntary exit
(thank you, Microsoft).  If this does happen then the system is probably
hosed --- for instance, the dead session might still be holding locks.
So the best recovery method is to treat this like a backend crash.

The dead man switch is armed for a particular child process when it
acquires a regular PGPROC, and disarmed when the PGPROC is released;
these should be the first and last touches of shared memory resources
in a backend, or close enough anyway.  This choice means there is no
coverage for auxiliary processes, but I doubt we need that, since they
shouldn't be executing any user-provided code anyway.

This patch also improves the management of the EXEC_BACKEND
ShmemBackendArray array a bit, by reducing search costs.

Although this problem is of long standing, the lack of field complaints
seems to mean it's not critical enough to risk back-patching; at least
not till we get some more testing of this mechanism.

src/backend/storage/ipc/ipci.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/ipc/ipci.c,v 1.99 2009/01/03 17:08:39 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/ipc/ipci.c,v 1.100 2009/05/05 19:59:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -111,6 +111,7 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 		size = add_size(size, ProcArrayShmemSize());
 		size = add_size(size, BackendStatusShmemSize());
 		size = add_size(size, SInvalShmemSize());
+		size = add_size(size, PMSignalShmemSize());
 		size = add_size(size, BgWriterShmemSize());
 		size = add_size(size, AutoVacuumShmemSize());
 		size = add_size(size, BTreeShmemSize());
@@ -206,7 +207,7 @@ CreateSharedMemoryAndSemaphores(bool makePrivate, int port)
 	/*
 	 * Set up interprocess signaling mechanisms
 	 */
-	PMSignalInit();
+	PMSignalShmemInit();
 	BgWriterShmemInit();
 	AutoVacuumShmemInit();
 

src/backend/storage/ipc/pmsignal.c

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/ipc/pmsignal.c,v 1.26 2009/01/01 17:23:47 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/ipc/pmsignal.c,v 1.27 2009/05/05 19:59:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -34,26 +34,69 @@
  * The flags are actually declared as "volatile sig_atomic_t" for maximum
  * portability.  This should ensure that loads and stores of the flag
  * values are atomic, allowing us to dispense with any explicit locking.
+ *
+ * In addition to the per-reason flags, we store a set of per-child-process
+ * flags that are currently used only for detecting whether a backend has
+ * exited without performing proper shutdown.  The per-child-process flags
+ * have three possible states: UNUSED, ASSIGNED, ACTIVE.  An UNUSED slot is
+ * available for assignment.  An ASSIGNED slot is associated with a postmaster
+ * child process, but either the process has not touched shared memory yet,
+ * or it has successfully cleaned up after itself.  A ACTIVE slot means the
+ * process is actively using shared memory.  The slots are assigned to
+ * child processes at random, and postmaster.c is responsible for tracking
+ * which one goes with which PID.
+ */
+
+#define PM_CHILD_UNUSED		0	/* these values must fit in sig_atomic_t */
+#define PM_CHILD_ASSIGNED	1
+#define PM_CHILD_ACTIVE		2
+
+/* "typedef struct PMSignalData PMSignalData" appears in pmsignal.h */
+struct PMSignalData
+{
+	/* per-reason flags */
+	sig_atomic_t PMSignalFlags[NUM_PMSIGNALS];
+	/* per-child-process flags */
+	int			num_child_flags;		/* # of entries in PMChildFlags[] */
+	int			next_child_flag;		/* next slot to try to assign */
+	sig_atomic_t PMChildFlags[1];		/* VARIABLE LENGTH ARRAY */
+};
+
+NON_EXEC_STATIC volatile PMSignalData *PMSignalState = NULL;
+
+
+/*
+ * PMSignalShmemSize
+ *		Compute space needed for pmsignal.c's shared memory
  */
+Size
+PMSignalShmemSize(void)
+{
+	Size		size;
 
-static volatile sig_atomic_t *PMSignalFlags;
+	size = offsetof(PMSignalData, PMChildFlags);
+	size = add_size(size, mul_size(MaxLivePostmasterChildren(),
+								   sizeof(sig_atomic_t)));
 
+	return size;
+}
 
 /*
- * PMSignalInit - initialize during shared-memory creation
+ * PMSignalShmemInit - initialize during shared-memory creation
  */
 void
-PMSignalInit(void)
+PMSignalShmemInit(void)
 {
 	bool		found;
 
-	PMSignalFlags = (sig_atomic_t *)
-		ShmemInitStruct("PMSignalFlags",
-						NUM_PMSIGNALS * sizeof(sig_atomic_t),
-						&found);
+	PMSignalState = (PMSignalData *)
+		ShmemInitStruct("PMSignalState", PMSignalShmemSize(), &found);
 
 	if (!found)
-		MemSet(PMSignalFlags, 0, NUM_PMSIGNALS * sizeof(sig_atomic_t));
+	{
+		MemSet(PMSignalState, 0, PMSignalShmemSize());
+		PMSignalState->num_child_flags = MaxLivePostmasterChildren();
+	}
 }
 
 /*
@@ -66,7 +109,7 @@ SendPostmasterSignal(PMSignalReason reason)
 	if (!IsUnderPostmaster)
 		return;
 	/* Atomically set the proper flag */
-	PMSignalFlags[reason] = true;
+	PMSignalState->PMSignalFlags[reason] = true;
 	/* Send signal to postmaster */
 	kill(PostmasterPid, SIGUSR1);
 }
@@ -80,14 +123,105 @@ bool
 CheckPostmasterSignal(PMSignalReason reason)
 {
 	/* Careful here --- don't clear flag if we haven't seen it set */
-	if (PMSignalFlags[reason])
+	if (PMSignalState->PMSignalFlags[reason])
 	{
-		PMSignalFlags[reason] = false;
+		PMSignalState->PMSignalFlags[reason] = false;
 		return true;
 	}
 	return false;
 }
 
+
+/*
+ * AssignPostmasterChildSlot - select an unused slot for a new postmaster
+ * child process, and set its state to ASSIGNED.  Returns a slot number
+ * (one to N).
+ *
+ * Only the postmaster is allowed to execute this routine, so we need no
+ * special locking.
+ */
+int
+AssignPostmasterChildSlot(void)
+{
+	int			slot = PMSignalState->next_child_flag;
+	int			n;
+
+	/*
+	 * Scan for a free slot.  We track the last slot assigned so as not to
+	 * waste time repeatedly rescanning low-numbered slots.
+	 */
+	for (n = PMSignalState->num_child_flags; n > 0; n--)
+	{
+		if (--slot < 0)
+			slot = PMSignalState->num_child_flags - 1;
+		if (PMSignalState->PMChildFlags[slot] == PM_CHILD_UNUSED)
+		{
+			PMSignalState->PMChildFlags[slot] = PM_CHILD_ASSIGNED;
+			PMSignalState->next_child_flag = slot;
+			return slot + 1;
+		}
+	}
+
+	/* Out of slots ... should never happen, else postmaster.c messed up */
+	elog(FATAL, "no free slots in PMChildFlags array");
+	return 0;					/* keep compiler quiet */
+}
+
+/*
+ * ReleasePostmasterChildSlot - release a slot after death of a postmaster
+ * child process.  This must be called in the postmaster process.
+ *
+ * Returns true if the slot had been in ASSIGNED state (the expected case),
+ * false otherwise (implying that the child failed to clean itself up).
+ */
+bool
+ReleasePostmasterChildSlot(int slot)
+{
+	bool		result;
+
+	Assert(slot > 0 && slot <= PMSignalState->num_child_flags);
+	slot--;
+	/*
+	 * Note: the slot state might already be unused, because the logic in
+	 * postmaster.c is such that this might get called twice when a child
+	 * crashes.  So we don't try to Assert anything about the state.
+	 */
+	result = (PMSignalState->PMChildFlags[slot] == PM_CHILD_ASSIGNED);
+	PMSignalState->PMChildFlags[slot] = PM_CHILD_UNUSED;
+	return result;
+}
+
+/*
+ * MarkPostmasterChildActive - mark a postmaster child as about to begin
+ * actively using shared memory.  This is called in the child process.
+ */
+void
+MarkPostmasterChildActive(void)
+{
+	int			slot = MyPMChildSlot;
+
+	Assert(slot > 0 && slot <= PMSignalState->num_child_flags);
+	slot--;
+	Assert(PMSignalState->PMChildFlags[slot] == PM_CHILD_ASSIGNED);
+	PMSignalState->PMChildFlags[slot] = PM_CHILD_ACTIVE;
+}
+
+/*
+ * MarkPostmasterChildInactive - mark a postmaster child as done using
+ * shared memory.  This is called in the child process.
+ */
+void
+MarkPostmasterChildInactive(void)
+{
+	int			slot = MyPMChildSlot;
+
+	Assert(slot > 0 && slot <= PMSignalState->num_child_flags);
+	slot--;
+	Assert(PMSignalState->PMChildFlags[slot] == PM_CHILD_ACTIVE);
+	PMSignalState->PMChildFlags[slot] = PM_CHILD_ASSIGNED;
+}
+
+
 /*
  * PostmasterIsAlive - check whether postmaster process is still alive
  *

src/backend/storage/lmgr/proc.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/lmgr/proc.c,v 1.205 2009/01/01 17:23:48 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/lmgr/proc.c,v 1.206 2009/05/05 19:59:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -41,6 +41,7 @@
 #include "postmaster/autovacuum.h"
 #include "storage/ipc.h"
 #include "storage/lmgr.h"
+#include "storage/pmsignal.h"
 #include "storage/proc.h"
 #include "storage/procarray.h"
 #include "storage/spin.h"
@@ -274,6 +275,14 @@ InitProcess(void)
 				 errmsg("sorry, too many clients already")));
 	}
 
+	/*
+	 * Now that we have a PGPROC, mark ourselves as an active postmaster
+	 * child; this is so that the postmaster can detect it if we exit
+	 * without cleaning up.
+	 */
+	if (IsUnderPostmaster)
+		MarkPostmasterChildActive();
+
 	/*
 	 * Initialize all fields of MyProc, except for the semaphore which was
 	 * prepared for us by InitProcGlobal.
@@ -614,6 +623,13 @@ ProcKill(int code, Datum arg)
 
 	SpinLockRelease(ProcStructLock);
 
+	/*
+	 * This process is no longer present in shared memory in any meaningful
+	 * way, so tell the postmaster we've cleaned up acceptably well.
+	 */
+	if (IsUnderPostmaster)
+		MarkPostmasterChildInactive();
+
 	/* wake autovac launcher if needed -- see comments in FreeWorkerInfo */
 	if (AutovacuumLauncherPid != 0)
 		kill(AutovacuumLauncherPid, SIGUSR1);

src/backend/utils/init/globals.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/utils/init/globals.c,v 1.107 2009/01/01 17:23:51 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/utils/init/globals.c,v 1.108 2009/05/05 19:59:00 tgl Exp $
  *
  * NOTES
  *	  Globals used all over the place should be declared here and not
@@ -36,6 +36,7 @@ int			MyProcPid;
 pg_time_t	MyStartTime;
 struct Port *MyProcPort;
 long		MyCancelKey;
+int			MyPMChildSlot;
 
 /*
  * DataDir is the absolute path to the top level of the PGDATA directory tree.

src/include/miscadmin.h

@@ -13,7 +13,7 @@
  * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/miscadmin.h,v 1.209 2009/01/05 02:27:45 alvherre Exp $
+ * $PostgreSQL: pgsql/src/include/miscadmin.h,v 1.210 2009/05/05 19:59:00 tgl Exp $
  *
  * NOTES
  *	  some of the information in this file should be moved to other files.
@@ -137,6 +137,7 @@ extern PGDLLIMPORT int MyProcPid;
 extern PGDLLIMPORT pg_time_t MyStartTime;
 extern PGDLLIMPORT struct Port *MyProcPort;
 extern long MyCancelKey;
+extern int	MyPMChildSlot;
 
 extern char OutputFileName[];
 extern PGDLLIMPORT char my_exec_path[];

src/include/postmaster/postmaster.h

@@ -6,7 +6,7 @@
  * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/postmaster/postmaster.h,v 1.19 2009/01/01 17:24:01 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/postmaster/postmaster.h,v 1.20 2009/05/05 19:59:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -38,6 +38,8 @@ extern const char *progname;
 extern int	PostmasterMain(int argc, char *argv[]);
 extern void ClosePostmasterPorts(bool am_syslogger);
 
+extern int	MaxLivePostmasterChildren(void);
+
 #ifdef EXEC_BACKEND
 extern pid_t postmaster_forkexec(int argc, char *argv[]);
 extern int	SubPostmasterMain(int argc, char *argv[]);

src/include/storage/pmsignal.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/storage/pmsignal.h,v 1.23 2009/02/23 09:28:50 heikki Exp $
+ * $PostgreSQL: pgsql/src/include/storage/pmsignal.h,v 1.24 2009/05/05 19:59:00 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -33,12 +33,20 @@ typedef enum
 	NUM_PMSIGNALS				/* Must be last value of enum! */
 } PMSignalReason;
 
+/* PMSignalData is an opaque struct, details known only within pmsignal.c */
+typedef struct PMSignalData PMSignalData;
+
 /*
  * prototypes for functions in pmsignal.c
  */
-extern void PMSignalInit(void);
+extern Size PMSignalShmemSize(void);
+extern void PMSignalShmemInit(void);
 extern void SendPostmasterSignal(PMSignalReason reason);
 extern bool CheckPostmasterSignal(PMSignalReason reason);
+extern int	AssignPostmasterChildSlot(void);
+extern bool ReleasePostmasterChildSlot(int slot);
+extern void MarkPostmasterChildActive(void);
+extern void MarkPostmasterChildInactive(void);
 extern bool PostmasterIsAlive(bool amDirectChild);
 
 #endif   /* PMSIGNAL_H */