Clean up some longstanding problems in shared-cache invalidation.

SI messages now include the relevant database OID, so that operations
in one database do not cause useless cache flushes in backends attached
to other databases.  Declare SI messages properly using a union, to
eliminate the former assumption that Oid is the same size as int or Index.
Rewrite the nearly-unreadable code in inval.c, and document it better.
Arrange for catcache flushes at end of command/transaction to happen before
relcache flushes do --- this avoids loading a new tuple into the catcache
while setting up new relcache entry, only to have it be flushed again
immediately.

src/backend/access/transam/xact.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/access/transam/xact.c,v 1.102 2001/05/04 18:39:16 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/access/transam/xact.c,v 1.103 2001/06/19 19:42:15 tgl Exp $
  *
  * NOTES
  *		Transaction aborts can now occur two ways:
@@ -544,7 +544,6 @@ CommandCounterIncrement(void)
 	 */
 	AtCommit_LocalCache();
 	AtStart_Cache();
-
 }
 
 void
@@ -577,7 +576,7 @@ InitializeTransactionSystem(void)
 static void
 AtStart_Cache(void)
 {
-	DiscardInvalid();
+	AcceptInvalidationMessages();
 }
 
 /* --------------------------------
@@ -725,11 +724,10 @@ RecordTransactionCommit()
 static void
 AtCommit_Cache(void)
 {
-
 	/*
-	 * Make catalog changes visible to all backend.
+	 * Make catalog changes visible to all backends.
 	 */
-	RegisterInvalid(true);
+	AtEOXactInvalidationMessages(true);
 }
 
 /* --------------------------------
@@ -739,11 +737,10 @@ AtCommit_Cache(void)
 static void
 AtCommit_LocalCache(void)
 {
-
 	/*
 	 * Make catalog changes visible to me for the next command.
 	 */
-	ImmediateLocalInvalidation(true);
+	CommandEndInvalidationMessages(true);
 }
 
 /* --------------------------------
@@ -753,7 +750,6 @@ AtCommit_LocalCache(void)
 static void
 AtCommit_Locks(void)
 {
-
 	/*
 	 * XXX What if ProcReleaseLocks fails?	(race condition?)
 	 *
@@ -769,7 +765,6 @@ AtCommit_Locks(void)
 static void
 AtCommit_Memory(void)
 {
-
 	/*
 	 * Now that we're "out" of a transaction, have the system allocate
 	 * things in the top memory context instead of per-transaction
@@ -844,7 +839,7 @@ static void
 AtAbort_Cache(void)
 {
 	RelationCacheAbort();
-	RegisterInvalid(false);
+	AtEOXactInvalidationMessages(false);
 }
 
 /* --------------------------------

src/backend/storage/ipc/sinval.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/storage/ipc/sinval.c,v 1.33 2001/06/16 22:58:13 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/storage/ipc/sinval.c,v 1.34 2001/06/19 19:42:15 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -55,56 +55,31 @@ InitBackendSharedInvalidationState(void)
 }
 
 /*
- * RegisterSharedInvalid
+ * SendSharedInvalidMessage
  *	Add a shared-cache-invalidation message to the global SI message queue.
- *
- * Note:
- *	Assumes hash index is valid.
- *	Assumes item pointer is valid.
  */
 void
-RegisterSharedInvalid(int cacheId,		/* XXX */
-					  Index hashIndex,
-					  ItemPointer pointer)
+SendSharedInvalidMessage(SharedInvalidationMessage *msg)
 {
-	SharedInvalidData newInvalid;
 	bool		insertOK;
 
-	/*
-	 * This code has been hacked to accept two types of messages.  This
-	 * might be treated more generally in the future.
-	 *
-	 * (1) cacheId= system cache id hashIndex= system cache hash index for a
-	 * (possibly) cached tuple pointer= pointer of (possibly) cached tuple
-	 *
-	 * (2) cacheId= special non-syscache id hashIndex= object id contained in
-	 * (possibly) cached relation descriptor pointer= null
-	 */
-
-	newInvalid.cacheId = cacheId;
-	newInvalid.hashIndex = hashIndex;
-
-	if (ItemPointerIsValid(pointer))
-		ItemPointerCopy(pointer, &newInvalid.pointerData);
-	else
-		ItemPointerSetInvalid(&newInvalid.pointerData);
-
 	SpinAcquire(SInvalLock);
-	insertOK = SIInsertDataEntry(shmInvalBuffer, &newInvalid);
+	insertOK = SIInsertDataEntry(shmInvalBuffer, msg);
 	SpinRelease(SInvalLock);
 	if (!insertOK)
-		elog(DEBUG, "RegisterSharedInvalid: SI buffer overflow");
+		elog(DEBUG, "SendSharedInvalidMessage: SI buffer overflow");
 }
 
 /*
- * InvalidateSharedInvalid
+ * ReceiveSharedInvalidMessages
  *		Process shared-cache-invalidation messages waiting for this backend
  */
 void
-			InvalidateSharedInvalid(void (*invalFunction) (),
-									void (*resetFunction) ())
+ReceiveSharedInvalidMessages(
+	void (*invalFunction) (SharedInvalidationMessage *msg),
+	void (*resetFunction) (void))
 {
-	SharedInvalidData data;
+	SharedInvalidationMessage data;
 	int			getResult;
 	bool		gotMessage = false;
 
@@ -118,15 +93,13 @@ void
 		if (getResult < 0)
 		{
 			/* got a reset message */
-			elog(DEBUG, "InvalidateSharedInvalid: cache state reset");
+			elog(DEBUG, "ReceiveSharedInvalidMessages: cache state reset");
 			resetFunction();
 		}
 		else
 		{
 			/* got a normal data message */
-			invalFunction(data.cacheId,
-						  data.hashIndex,
-						  &data.pointerData);
+			invalFunction(&data);
 		}
 		gotMessage = true;
 	}

src/backend/storage/ipc/sinvaladt.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/storage/ipc/sinvaladt.c,v 1.39 2001/06/16 22:58:15 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/storage/ipc/sinvaladt.c,v 1.40 2001/06/19 19:42:15 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -20,7 +20,6 @@
 #include "miscadmin.h"
 #include "storage/backendid.h"
 #include "storage/proc.h"
-#include "storage/sinval.h"
 #include "storage/sinvaladt.h"
 
 SISeg	   *shmInvalBuffer;
@@ -35,7 +34,6 @@ static void SISetProcStateInvalid(SISeg *segP);
 int
 SInvalShmemSize(int maxBackends)
 {
-
 	/*
 	 * Figure space needed. Note sizeof(SISeg) includes the first
 	 * ProcState entry.
@@ -183,14 +181,13 @@ CleanupInvalidationState(int status, Datum arg)
  * Returns true for normal successful insertion, false if had to reset.
  */
 bool
-SIInsertDataEntry(SISeg *segP, SharedInvalidData *data)
+SIInsertDataEntry(SISeg *segP, SharedInvalidationMessage *data)
 {
 	int			numMsgs = segP->maxMsgNum - segP->minMsgNum;
 
 	/* Is the buffer full? */
 	if (numMsgs >= MAXNUMMESSAGES)
 	{
-
 		/*
 		 * Don't panic just yet: slowest backend might have consumed some
 		 * messages but not yet have done SIDelExpiredDataEntries() to
@@ -273,13 +270,12 @@ SISetProcStateInvalid(SISeg *segP)
  */
 int
 SIGetDataEntry(SISeg *segP, int backendId,
-			   SharedInvalidData *data)
+			   SharedInvalidationMessage *data)
 {
 	ProcState  *stateP = &segP->procState[backendId - 1];
 
 	if (stateP->resetState)
 	{
-
 		/*
 		 * Force reset.  We can say we have dealt with any messages added
 		 * since the reset, as well...

src/backend/storage/lmgr/lmgr.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/storage/lmgr/lmgr.c,v 1.46 2001/06/12 05:55:49 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/storage/lmgr/lmgr.c,v 1.47 2001/06/19 19:42:16 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -144,7 +144,7 @@ LockRelation(Relation relation, LOCKMODE lockmode)
 	 * rebuild it and not just delete it.
 	 */
 	RelationIncrementReferenceCount(relation);
-	DiscardInvalid();
+	AcceptInvalidationMessages();
 	RelationDecrementReferenceCount(relation);
 }
 

src/backend/utils/cache/catcache.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/cache/catcache.c,v 1.79 2001/06/18 03:35:07 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/cache/catcache.c,v 1.80 2001/06/19 19:42:16 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -31,8 +31,18 @@
 
 /* #define CACHEDEBUG */		/* turns DEBUG elogs on */
 
-/* voodoo constants */
-#define NCCBUCKETS 257			/* Hash buckets per CatCache (prime!) */
+/*
+ * Constants related to size of the catcache.
+ *
+ * NCCBUCKETS should be prime and must be less than 64K (because
+ * SharedInvalCatcacheMsg crams hash indexes into a uint16 field).  In
+ * practice it should be a lot less, anyway, to avoid chewing up too much
+ * space on hash bucket headers.
+ *
+ * MAXCCTUPLES could be as small as a few hundred, if per-backend memory
+ * consumption is at a premium.
+ */
+#define NCCBUCKETS 257			/* Hash buckets per CatCache */
 #define MAXCCTUPLES 5000		/* Maximum # of tuples in all caches */
 
 
@@ -218,6 +228,11 @@ CatalogCacheInitializeCache(CatCache *cache)
 	 */
 	tupdesc = CreateTupleDescCopyConstr(RelationGetDescr(relation));
 
+	/*
+	 * get the relation's relisshared flag, too
+	 */
+	cache->cc_relisshared = RelationGetForm(relation)->relisshared;
+
 	/*
 	 * return to the caller's memory context and close the rel
 	 */
@@ -737,6 +752,7 @@ InitCatCache(int id,
 	cp->cc_relname = relname;
 	cp->cc_indname = indname;
 	cp->cc_reloidattr = reloidattr;
+	cp->cc_relisshared = false;	/* temporary */
 	cp->cc_tupdesc = (TupleDesc) NULL;
 	cp->cc_ntup = 0;
 	cp->cc_size = NCCBUCKETS;
@@ -1116,7 +1132,8 @@ ReleaseCatCache(HeapTuple tuple)
  *
  *	Note that it is irrelevant whether the given tuple is actually loaded
  *	into the catcache at the moment.  Even if it's not there now, it might
- *	be by the end of the command, so we have to be prepared to flush it.
+ *	be by the end of the command --- or might be in other backends' caches
+ * --- so we have to be prepared to flush it.
  *
  *	Also note that it's not an error if there are no catcaches for the
  *	specified relation.  inval.c doesn't know exactly which rels have
@@ -1126,7 +1143,7 @@ ReleaseCatCache(HeapTuple tuple)
 void
 PrepareToInvalidateCacheTuple(Relation relation,
 							  HeapTuple tuple,
-							  void (*function) (int, Index, ItemPointer))
+							  void (*function) (int, Index, ItemPointer, Oid))
 {
 	CatCache   *ccp;
 
@@ -1159,6 +1176,7 @@ PrepareToInvalidateCacheTuple(Relation relation,
 
 		(*function) (ccp->id,
 					 CatalogCacheComputeTupleHashIndex(ccp, tuple),
-					 &tuple->t_self);
+					 &tuple->t_self,
+					 ccp->cc_relisshared ? (Oid) 0 : MyDatabaseId);
 	}
 }

src/include/storage/sinval.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $Id: sinval.h,v 1.18 2001/02/26 00:50:08 tgl Exp $
+ * $Id: sinval.h,v 1.19 2001/06/19 19:42:16 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -17,15 +17,61 @@
 #include "storage/itemptr.h"
 #include "storage/spin.h"
 
+
+/*
+ * We currently support two types of shared-invalidation messages: one that
+ * invalidates an entry in a catcache, and one that invalidates a relcache
+ * entry.  More types could be added if needed.  The message type is
+ * identified by the first "int16" field of the message struct.  Zero or
+ * positive means a catcache inval message (and also serves as the catcache
+ * ID field).  -1 means a relcache inval message.  Other negative values
+ * are available to identify other inval message types.
+ *
+ * Shared-inval events are initially driven by detecting tuple inserts,
+ * updates and deletions in system catalogs (see RelationInvalidateHeapTuple
+ * and RelationMark4RollbackHeapTuple).  Note that some system catalogs have
+ * multiple caches on them (with different indexes).  On detecting a tuple
+ * invalidation in such a catalog, a separate catcache inval message must be
+ * generated for each of its caches.  The catcache inval message carries the
+ * hash index for the target tuple, so that the catcache only needs to search
+ * one hash chain not all its chains.  Of course this assumes that all the
+ * backends are using identical hashing code, but that should be OK.
+ */
+
+typedef struct
+{
+	int16		id;				/* cache ID --- must be first */
+	uint16		hashIndex;		/* hashchain index within this catcache */
+	Oid			dbId;			/* database ID, or 0 if a shared relation */
+	ItemPointerData tuplePtr;	/* tuple identifier in cached relation */
+} SharedInvalCatcacheMsg;
+
+#define SHAREDINVALRELCACHE_ID  (-1)
+
+typedef struct
+{
+	int16		id;				/* type field --- must be first */
+	Oid			dbId;			/* database ID, or 0 if a shared relation */
+	Oid			relId;			/* relation ID */
+} SharedInvalRelcacheMsg;
+
+typedef union
+{
+	int16		id;				/* type field --- must be first */
+	SharedInvalCatcacheMsg cc;
+	SharedInvalRelcacheMsg rc;
+} SharedInvalidationMessage;
+
+
 extern SPINLOCK SInvalLock;
 
 extern int	SInvalShmemSize(int maxBackends);
 extern void CreateSharedInvalidationState(int maxBackends);
 extern void InitBackendSharedInvalidationState(void);
-extern void RegisterSharedInvalid(int cacheId, Index hashIndex,
-					  ItemPointer pointer);
-extern void InvalidateSharedInvalid(void (*invalFunction) (),
-												void (*resetFunction) ());
+extern void SendSharedInvalidMessage(SharedInvalidationMessage *msg);
+extern void ReceiveSharedInvalidMessages(
+	void (*invalFunction) (SharedInvalidationMessage *msg),
+	void (*resetFunction) (void));
 
 extern bool DatabaseHasActiveBackends(Oid databaseId, bool ignoreMyself);
 extern bool TransactionIdIsInProgress(TransactionId xid);

src/include/storage/sinvaladt.h

@@ -7,15 +7,15 @@
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $Id: sinvaladt.h,v 1.26 2001/03/22 04:01:09 momjian Exp $
+ * $Id: sinvaladt.h,v 1.27 2001/06/19 19:42:16 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
 #ifndef SINVALADT_H
 #define SINVALADT_H
 
-#include "storage/itemptr.h"
 #include "storage/shmem.h"
+#include "storage/sinval.h"
 
 /*
  * The shared cache invalidation manager is responsible for transmitting
@@ -45,6 +45,9 @@
  * large so that we don't need to do this often.  It must be a multiple of
  * MAXNUMMESSAGES so that the existing circular-buffer entries don't need
  * to be moved when we do it.
+ *
+ * The struct type SharedInvalidationMessage, defining the contents of
+ * a single message, is defined in sinval.h.
  */
 
 
@@ -61,15 +64,6 @@
 #define MAXNUMMESSAGES 4096
 #define MSGNUMWRAPAROUND (MAXNUMMESSAGES * 4096)
 
-/* The content of one shared-invalidation message */
-typedef struct SharedInvalidData
-{
-	int			cacheId;		/* XXX */
-	Index		hashIndex;
-	ItemPointerData pointerData;
-} SharedInvalidData;
-
-typedef SharedInvalidData *SharedInvalid;
 
 /* Per-backend state in shared invalidation structure */
 typedef struct ProcState
@@ -83,7 +77,6 @@ typedef struct ProcState
 /* Shared cache invalidation memory segment */
 typedef struct SISeg
 {
-
 	/*
 	 * General state information
 	 */
@@ -96,7 +89,7 @@ typedef struct SISeg
 	/*
 	 * Circular buffer holding shared-inval messages
 	 */
-	SharedInvalidData buffer[MAXNUMMESSAGES];
+	SharedInvalidationMessage buffer[MAXNUMMESSAGES];
 
 	/*
 	 * Per-backend state info.
@@ -117,9 +110,9 @@ extern SISeg *shmInvalBuffer;	/* pointer to the shared inval buffer */
 extern void SIBufferInit(int maxBackends);
 extern int	SIBackendInit(SISeg *segP);
 
-extern bool SIInsertDataEntry(SISeg *segP, SharedInvalidData *data);
+extern bool SIInsertDataEntry(SISeg *segP, SharedInvalidationMessage *data);
 extern int SIGetDataEntry(SISeg *segP, int backendId,
-			   SharedInvalidData *data);
+						  SharedInvalidationMessage *data);
 extern void SIDelExpiredDataEntries(SISeg *segP);
 
 #endif	 /* SINVALADT_H */

src/include/utils/catcache.h

@@ -13,7 +13,7 @@
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $Id: catcache.h,v 1.33 2001/06/18 03:35:07 tgl Exp $
+ * $Id: catcache.h,v 1.34 2001/06/19 19:42:16 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -36,6 +36,7 @@ typedef struct catcache
 	char	   *cc_relname;		/* name of relation the tuples come from */
 	char	   *cc_indname;		/* name of index matching cache keys */
 	int			cc_reloidattr;	/* AttrNumber of relation OID, or 0 */
+	bool		cc_relisshared;	/* is relation shared? */
 	TupleDesc	cc_tupdesc;		/* tuple descriptor (copied from reldesc) */
 	int			cc_ntup;		/* # of tuples currently in this cache */
 	int			cc_size;		/* # of hash buckets in this cache */
@@ -98,7 +99,7 @@ extern void CatalogCacheFlushRelation(Oid relId);
 extern void CatalogCacheIdInvalidate(int cacheId, Index hashIndex,
 						 ItemPointer pointer);
 extern void PrepareToInvalidateCacheTuple(Relation relation,
-							  HeapTuple tuple,
-							  void (*function) (int, Index, ItemPointer));
+						 HeapTuple tuple,
+						 void (*function) (int, Index, ItemPointer, Oid));
 
 #endif	 /* CATCACHE_H */

src/include/utils/inval.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2001, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $Id: inval.h,v 1.19 2001/01/24 19:43:28 momjian Exp $
+ * $Id: inval.h,v 1.20 2001/06/19 19:42:16 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -16,11 +16,12 @@
 
 #include "access/htup.h"
 
-extern void DiscardInvalid(void);
 
-extern void RegisterInvalid(bool send);
+extern void AcceptInvalidationMessages(void);
 
-extern void ImmediateLocalInvalidation(bool send);
+extern void AtEOXactInvalidationMessages(bool isCommit);
+
+extern void CommandEndInvalidationMessages(bool isCommit);
 
 extern void RelationInvalidateHeapTuple(Relation relation, HeapTuple tuple);