Fix an old bug in multixact and two-phase commit. Prepared transactions can

be part of multixacts, so allocate a slot for each prepared transaction in
the "oldest member" array in multixact.c. On PREPARE TRANSACTION, transfer
the oldest member value from the current backends slot to the prepared xact
slot. Also save and recover the value from the 2pc state file.

The symptom of the bug was that after a transaction prepared, a shared lock
still held by the prepared transaction was sometimes ignored by other
transactions.

Fix back to 8.1, where both 2PC and multixact were introduced.

src/backend/access/transam/multixact.c

@@ -42,7 +42,7 @@
  * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/backend/access/transam/multixact.c,v 1.31 2009/06/26 20:29:04 tgl Exp $
+ * $PostgreSQL: pgsql/src/backend/access/transam/multixact.c,v 1.32 2009/11/23 09:58:36 heikki Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -51,6 +51,8 @@
 #include "access/multixact.h"
 #include "access/slru.h"
 #include "access/transam.h"
+#include "access/twophase.h"
+#include "access/twophase_rmgr.h"
 #include "access/xact.h"
 #include "miscadmin.h"
 #include "pg_trace.h"
@@ -118,8 +120,11 @@ typedef struct MultiXactStateData
 	/*
 	 * Per-backend data starts here.  We have two arrays stored in the area
 	 * immediately following the MultiXactStateData struct. Each is indexed by
-	 * BackendId.  (Note: valid BackendIds run from 1 to MaxBackends; element
-	 * zero of each array is never used.)
+	 * BackendId.
+	 *
+	 * In both arrays, there's a slot for all normal backends (1..MaxBackends)
+	 * followed by a slot for max_prepared_xacts prepared transactions. Valid
+	 * BackendIds start from 1; element zero of each array is never used.
 	 *
 	 * OldestMemberMXactId[k] is the oldest MultiXactId each backend's current
 	 * transaction(s) could possibly be a member of, or InvalidMultiXactId
@@ -152,6 +157,12 @@ typedef struct MultiXactStateData
 	MultiXactId perBackendXactIds[1];	/* VARIABLE LENGTH ARRAY */
 } MultiXactStateData;
 
+/*
+ * Last element of OldestMemberMXactID and OldestVisibleMXactId arrays.
+ * Valid elements are (1..MaxOldestSlot); element 0 is never used.
+ */
+#define MaxOldestSlot	(MaxBackends + max_prepared_xacts)
+
 /* Pointers to the state data in shared memory */
 static MultiXactStateData *MultiXactState;
 static MultiXactId *OldestMemberMXactId;
@@ -539,7 +550,7 @@ MultiXactIdSetOldestVisible(void)
 		if (oldestMXact < FirstMultiXactId)
 			oldestMXact = FirstMultiXactId;
 
-		for (i = 1; i <= MaxBackends; i++)
+		for (i = 1; i <= MaxOldestSlot; i++)
 		{
 			MultiXactId thisoldest = OldestMemberMXactId[i];
 
@@ -1275,6 +1286,119 @@ AtEOXact_MultiXact(void)
 	MXactCache = NULL;
 }
 
+/*
+ * AtPrepare_MultiXact
+ *		Save multixact state at 2PC tranasction prepare
+ *
+ * In this phase, we only store our OldestMemberMXactId value in the two-phase
+ * state file.
+ */
+void
+AtPrepare_MultiXact(void)
+{
+	MultiXactId myOldestMember = OldestMemberMXactId[MyBackendId];
+
+	if (MultiXactIdIsValid(myOldestMember))
+		RegisterTwoPhaseRecord(TWOPHASE_RM_MULTIXACT_ID, 0,
+							   &myOldestMember, sizeof(MultiXactId));
+}
+
+/*
+ * PostPrepare_MultiXact
+ *		Clean up after successful PREPARE TRANSACTION
+ */
+void
+PostPrepare_MultiXact(TransactionId xid)
+{
+	MultiXactId myOldestMember;
+
+	/*
+	 * Transfer our OldestMemberMXactId value to the slot reserved for the
+	 * prepared transaction.
+	 */
+	myOldestMember = OldestMemberMXactId[MyBackendId];
+	if (MultiXactIdIsValid(myOldestMember))
+	{
+		BackendId dummyBackendId = TwoPhaseGetDummyBackendId(xid);
+
+		/*
+		 * Even though storing MultiXactId is atomic, acquire lock to make sure
+		 * others see both changes, not just the reset of the slot of the
+		 * current backend. Using a volatile pointer might suffice, but this
+		 * isn't a hot spot.
+		 */
+		LWLockAcquire(MultiXactGenLock, LW_EXCLUSIVE);
+
+		OldestMemberMXactId[dummyBackendId] = myOldestMember;
+		OldestMemberMXactId[MyBackendId] = InvalidMultiXactId;
+
+		LWLockRelease(MultiXactGenLock);
+	}
+
+	/*
+	 * We don't need to transfer OldestVisibleMXactId value, because the
+	 * transaction is not going to be looking at any more multixacts once
+	 * it's prepared.
+	 *
+	 * We assume that storing a MultiXactId is atomic and so we need not take
+	 * MultiXactGenLock to do this.
+	 */
+	OldestVisibleMXactId[MyBackendId] = InvalidMultiXactId;
+
+	/*
+	 * Discard the local MultiXactId cache like in AtEOX_MultiXact
+	 */
+	MXactContext = NULL;
+	MXactCache = NULL;
+}
+
+/*
+ * multixact_twophase_recover
+ *		Recover the state of a prepared transaction at startup
+ */
+void
+multixact_twophase_recover(TransactionId xid, uint16 info,
+						   void *recdata, uint32 len)
+{
+	BackendId	dummyBackendId = TwoPhaseGetDummyBackendId(xid);
+	MultiXactId	oldestMember;
+
+	/*
+	 * Get the oldest member XID from the state file record, and set it in
+	 * the OldestMemberMXactId slot reserved for this prepared transaction.
+	 */
+	Assert(len == sizeof(MultiXactId));
+	oldestMember = *((MultiXactId *)recdata);
+
+	OldestMemberMXactId[dummyBackendId] = oldestMember;
+}
+
+/*
+ * multixact_twophase_postcommit
+ *		Similar to AtEOX_MultiXact but for COMMIT PREPARED
+ */
+void
+multixact_twophase_postcommit(TransactionId xid, uint16 info,
+							  void *recdata, uint32 len)
+{
+	BackendId	dummyBackendId = TwoPhaseGetDummyBackendId(xid);
+
+	Assert(len == sizeof(MultiXactId));
+
+	OldestMemberMXactId[dummyBackendId] = InvalidMultiXactId;
+}
+
+/*
+ * multixact_twophase_postabort
+ *		This is actually just the same as the COMMIT case.
+ */
+void
+multixact_twophase_postabort(TransactionId xid, uint16 info,
+						void *recdata, uint32 len)
+{
+	multixact_twophase_postcommit(xid, info, recdata, len);
+}
+
 /*
  * Initialization of shared memory for MultiXact.  We use two SLRU areas,
  * thus double memory.	Also, reserve space for the shared MultiXactState
@@ -1287,7 +1411,7 @@ MultiXactShmemSize(void)
 
 #define SHARED_MULTIXACT_STATE_SIZE \
 	add_size(sizeof(MultiXactStateData), \
-			 mul_size(sizeof(MultiXactId) * 2, MaxBackends))
+			 mul_size(sizeof(MultiXactId) * 2, MaxOldestSlot))
 
 	size = SHARED_MULTIXACT_STATE_SIZE;
 	size = add_size(size, SimpleLruShmemSize(NUM_MXACTOFFSET_BUFFERS, 0));
@@ -1329,10 +1453,10 @@ MultiXactShmemInit(void)
 
 	/*
 	 * Set up array pointers.  Note that perBackendXactIds[0] is wasted space
-	 * since we only use indexes 1..MaxBackends in each array.
+	 * since we only use indexes 1..MaxOldestSlot in each array.
 	 */
 	OldestMemberMXactId = MultiXactState->perBackendXactIds;
-	OldestVisibleMXactId = OldestMemberMXactId + MaxBackends;
+	OldestVisibleMXactId = OldestMemberMXactId + MaxOldestSlot;
 }
 
 /*
@@ -1702,7 +1826,7 @@ TruncateMultiXact(void)
 		nextMXact = FirstMultiXactId;
 
 	oldestMXact = nextMXact;
-	for (i = 1; i <= MaxBackends; i++)
+	for (i = 1; i <= MaxOldestSlot; i++)
 	{
 		MultiXactId thisoldest;
 

src/backend/access/transam/twophase.c

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *		$PostgreSQL: pgsql/src/backend/access/transam/twophase.c,v 1.55 2009/09/01 04:15:45 tgl Exp $
+ *		$PostgreSQL: pgsql/src/backend/access/transam/twophase.c,v 1.56 2009/11/23 09:58:36 heikki Exp $
  *
  * NOTES
  *		Each global transaction is associated with a global transaction
@@ -109,6 +109,7 @@ int			max_prepared_xacts = 0;
 typedef struct GlobalTransactionData
 {
 	PGPROC		proc;			/* dummy proc */
+	BackendId	dummyBackendId;	/* similar to backend id for backends */
 	TimestampTz prepared_at;	/* time of preparation */
 	XLogRecPtr	prepare_lsn;	/* XLOG offset of prepare record */
 	Oid			owner;			/* ID of user that executed the xact */
@@ -200,6 +201,20 @@ TwoPhaseShmemInit(void)
 		{
 			gxacts[i].proc.links.next = (SHM_QUEUE *) TwoPhaseState->freeGXacts;
 			TwoPhaseState->freeGXacts = &gxacts[i];
+
+			/*
+			 * Assign a unique ID for each dummy proc, so that the range of
+			 * dummy backend IDs immediately follows the range of normal
+			 * backend IDs. We don't dare to assign a real backend ID to
+			 * dummy procs, because prepared transactions don't take part in
+			 * cache invalidation like a real backend ID would imply, but
+			 * having a unique ID for them is nevertheless handy. This
+			 * arrangement allows you to allocate an array of size
+			 * (MaxBackends + max_prepared_xacts + 1), and have a slot for
+			 * every backend and prepared transaction. Currently multixact.c
+			 * uses that technique.
+			 */
+			gxacts[i].dummyBackendId = MaxBackends + 1 + i;
 		}
 	}
 	else
@@ -647,6 +662,22 @@ pg_prepared_xact(PG_FUNCTION_ARGS)
 	SRF_RETURN_DONE(funcctx);
 }
 
+/*
+ * TwoPhaseGetDummyProc
+ *		Get the dummy backend ID for prepared transaction specified by XID
+ *
+ * Dummy backend IDs are similar to real backend IDs of real backends.
+ * They start at MaxBackends + 1, and are unique across all currently active
+ * real backends and prepared transactions.
+ */
+BackendId
+TwoPhaseGetDummyBackendId(TransactionId xid)
+{
+	PGPROC *proc = TwoPhaseGetDummyProc(xid);
+
+	return ((GlobalTransaction) proc)->dummyBackendId;
+}
+
 /*
  * TwoPhaseGetDummyProc
  *		Get the PGPROC that represents a prepared transaction specified by XID

src/backend/access/transam/twophase_rmgr.c

@@ -8,12 +8,13 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/transam/twophase_rmgr.c,v 1.9 2009/09/01 02:54:51 alvherre Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/transam/twophase_rmgr.c,v 1.10 2009/11/23 09:58:36 heikki Exp $
  *
  *-------------------------------------------------------------------------
  */
 #include "postgres.h"
 
+#include "access/multixact.h"
 #include "access/twophase_rmgr.h"
 #include "commands/async.h"
 #include "pgstat.h"
@@ -27,7 +28,8 @@ const TwoPhaseCallback twophase_recover_callbacks[TWOPHASE_RM_MAX_ID + 1] =
 	lock_twophase_recover,		/* Lock */
 	NULL,						/* Inval */
 	NULL,						/* notify/listen */
-	NULL						/* pgstat */
+	NULL,						/* pgstat */
+	multixact_twophase_recover	/* MultiXact */
 };
 
 const TwoPhaseCallback twophase_postcommit_callbacks[TWOPHASE_RM_MAX_ID + 1] =
@@ -36,7 +38,8 @@ const TwoPhaseCallback twophase_postcommit_callbacks[TWOPHASE_RM_MAX_ID + 1] =
 	lock_twophase_postcommit,	/* Lock */
 	inval_twophase_postcommit,	/* Inval */
 	notify_twophase_postcommit, /* notify/listen */
-	pgstat_twophase_postcommit	/* pgstat */
+	pgstat_twophase_postcommit,	/* pgstat */
+	multixact_twophase_postcommit /* MultiXact */
 };
 
 const TwoPhaseCallback twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] =
@@ -45,5 +48,6 @@ const TwoPhaseCallback twophase_postabort_callbacks[TWOPHASE_RM_MAX_ID + 1] =
 	lock_twophase_postabort,	/* Lock */
 	NULL,						/* Inval */
 	NULL,						/* notify/listen */
-	pgstat_twophase_postabort	/* pgstat */
+	pgstat_twophase_postabort,	/* pgstat */
+	multixact_twophase_postabort /* MultiXact */
 };

src/backend/access/transam/xact.c

@@ -10,7 +10,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/transam/xact.c,v 1.275 2009/09/01 02:54:51 alvherre Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/transam/xact.c,v 1.276 2009/11/23 09:58:36 heikki Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -1856,6 +1856,7 @@ PrepareTransaction(void)
 	AtPrepare_Inval();
 	AtPrepare_Locks();
 	AtPrepare_PgStat();
+	AtPrepare_MultiXact();
 
 	/*
 	 * Here is where we really truly prepare.
@@ -1909,7 +1910,7 @@ PrepareTransaction(void)
 
 	PostPrepare_smgr();
 
-	AtEOXact_MultiXact();
+	PostPrepare_MultiXact(xid);
 
 	PostPrepare_Locks(xid);
 

src/include/access/multixact.h

@@ -6,7 +6,7 @@
  * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/multixact.h,v 1.14 2009/01/01 17:23:56 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/access/multixact.h,v 1.15 2009/11/23 09:58:36 heikki Exp $
  */
 #ifndef MULTIXACT_H
 #define MULTIXACT_H
@@ -52,6 +52,8 @@ extern void MultiXactIdSetOldestMember(void);
 extern int	GetMultiXactIdMembers(MultiXactId multi, TransactionId **xids);
 
 extern void AtEOXact_MultiXact(void);
+extern void AtPrepare_MultiXact(void);
+extern void PostPrepare_MultiXact(TransactionId xid);
 
 extern Size MultiXactShmemSize(void);
 extern void MultiXactShmemInit(void);
@@ -67,6 +69,13 @@ extern void MultiXactSetNextMXact(MultiXactId nextMulti,
 extern void MultiXactAdvanceNextMXact(MultiXactId minMulti,
 						  MultiXactOffset minMultiOffset);
 
+extern void multixact_twophase_recover(TransactionId xid, uint16 info,
+						   void *recdata, uint32 len);
+extern void multixact_twophase_postcommit(TransactionId xid, uint16 info,
+							  void *recdata, uint32 len);
+extern void multixact_twophase_postabort(TransactionId xid, uint16 info,
+							 void *recdata, uint32 len);
+
 extern void multixact_redo(XLogRecPtr lsn, XLogRecord *record);
 extern void multixact_desc(StringInfo buf, uint8 xl_info, char *rec);
 

src/include/access/twophase.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/twophase.h,v 1.11 2009/01/01 17:23:56 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/access/twophase.h,v 1.12 2009/11/23 09:58:36 heikki Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -15,6 +15,7 @@
 #define TWOPHASE_H
 
 #include "access/xlogdefs.h"
+#include "storage/backendid.h"
 #include "storage/proc.h"
 #include "utils/timestamp.h"
 
@@ -31,6 +32,7 @@ extern Size TwoPhaseShmemSize(void);
 extern void TwoPhaseShmemInit(void);
 
 extern PGPROC *TwoPhaseGetDummyProc(TransactionId xid);
+extern BackendId TwoPhaseGetDummyBackendId(TransactionId xid);
 
 extern GlobalTransaction MarkAsPreparing(TransactionId xid, const char *gid,
 				TimestampTz prepared_at,

src/include/access/twophase_rmgr.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2009, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/twophase_rmgr.h,v 1.8 2009/09/01 02:54:52 alvherre Exp $
+ * $PostgreSQL: pgsql/src/include/access/twophase_rmgr.h,v 1.9 2009/11/23 09:58:36 heikki Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -26,7 +26,8 @@ typedef uint8 TwoPhaseRmgrId;
 #define TWOPHASE_RM_INVAL_ID		2
 #define TWOPHASE_RM_NOTIFY_ID		3
 #define TWOPHASE_RM_PGSTAT_ID		4
-#define TWOPHASE_RM_MAX_ID			TWOPHASE_RM_PGSTAT_ID
+#define TWOPHASE_RM_MULTIXACT_ID	5
+#define TWOPHASE_RM_MAX_ID			TWOPHASE_RM_MULTIXACT_ID
 
 extern const TwoPhaseCallback twophase_recover_callbacks[];
 extern const TwoPhaseCallback twophase_postcommit_callbacks[];