Add pg_visibility contrib module.

This lets you examine the visibility map as well as page-level
visibility information.  I initially wrote it as a debugging aid,
but was encouraged to polish it for commit.

Patch by me, reviewed by Masahiko Sawada.

Discussion: 56D77803.6080503@BlueTreble.com

contrib/Makefile

@@ -37,6 +37,7 @@ SUBDIRS = \
 		pgcrypto	\
 		pgrowlocks	\
 		pgstattuple	\
+		pg_visibility	\
 		postgres_fdw	\
 		seg		\
 		spi		\

contrib/pg_visibility/Makefile

+# contrib/pg_visibility/Makefile
+
+MODULE_big = pg_visibility
+OBJS = pg_visibility.o $(WIN32RES)
+
+EXTENSION = pg_visibility
+DATA = pg_visibility--1.0.sql
+PGFILEDESC = "pg_visibility - page visibility information"
+
+ifdef USE_PGXS
+PG_CONFIG = pg_config
+PGXS := $(shell $(PG_CONFIG) --pgxs)
+include $(PGXS)
+else
+subdir = contrib/pg_visibility
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+include $(top_srcdir)/contrib/contrib-global.mk
+endif

contrib/pg_visibility/pg_visibility--1.0.sql

+/* contrib/pg_visibility/pg_visibility--1.0.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION pg_visibility" to load this file. \quit
+
+-- Show visibility map information.
+CREATE FUNCTION pg_visibility_map(regclass, blkno bigint,
+								  all_visible OUT boolean,
+								  all_frozen OUT boolean)
+RETURNS record
+AS 'MODULE_PATHNAME', 'pg_visibility_map'
+LANGUAGE C STRICT;
+
+-- Show visibility map and page-level visibility information.
+CREATE FUNCTION pg_visibility(regclass, blkno bigint,
+							  all_visible OUT boolean,
+							  all_frozen OUT boolean,
+							  pd_all_visible OUT boolean)
+RETURNS record
+AS 'MODULE_PATHNAME', 'pg_visibility'
+LANGUAGE C STRICT;
+
+-- Show visibility map information for each block in a relation.
+CREATE FUNCTION pg_visibility_map(regclass, blkno OUT bigint,
+								  all_visible OUT boolean,
+								  all_frozen OUT boolean)
+RETURNS SETOF record
+AS 'MODULE_PATHNAME', 'pg_visibility_map_rel'
+LANGUAGE C STRICT;
+
+-- Show visibility map and page-level visibility information for each block.
+CREATE FUNCTION pg_visibility(regclass, blkno OUT bigint,
+							  all_visible OUT boolean,
+							  all_frozen OUT boolean,
+							  pd_all_visible OUT boolean)
+RETURNS SETOF record
+AS 'MODULE_PATHNAME', 'pg_visibility_rel'
+LANGUAGE C STRICT;
+
+-- Show summary of visibility map bits for a relation.
+CREATE FUNCTION pg_visibility_map_summary(regclass,
+    OUT all_visible bigint, OUT all_frozen bigint)
+RETURNS record
+AS 'MODULE_PATHNAME', 'pg_visibility_map_summary'
+LANGUAGE C STRICT;
+
+-- Don't want these to be available to public.
+REVOKE ALL ON FUNCTION pg_visibility_map(regclass, bigint) FROM PUBLIC;
+REVOKE ALL ON FUNCTION pg_visibility(regclass, bigint) FROM PUBLIC;
+REVOKE ALL ON FUNCTION pg_visibility_map(regclass) FROM PUBLIC;
+REVOKE ALL ON FUNCTION pg_visibility(regclass) FROM PUBLIC;
+REVOKE ALL ON FUNCTION pg_visibility_map_summary(regclass) FROM PUBLIC;

contrib/pg_visibility/pg_visibility.c

+/*-------------------------------------------------------------------------
+ *
+ * pg_visibility.c
+ *	  display visibility map information and page-level visibility bits
+ *
+ *	  contrib/pg_visibility/pg_visibility.c
+ *-------------------------------------------------------------------------
+ */
+#include "postgres.h"
+
+#include "access/htup_details.h"
+#include "access/visibilitymap.h"
+#include "catalog/pg_type.h"
+#include "funcapi.h"
+#include "miscadmin.h"
+#include "storage/bufmgr.h"
+#include "utils/rel.h"
+
+PG_MODULE_MAGIC;
+
+typedef struct vbits
+{
+	BlockNumber	next;
+	BlockNumber	count;
+	uint8		bits[FLEXIBLE_ARRAY_MEMBER];
+} vbits;
+
+PG_FUNCTION_INFO_V1(pg_visibility_map);
+PG_FUNCTION_INFO_V1(pg_visibility_map_rel);
+PG_FUNCTION_INFO_V1(pg_visibility);
+PG_FUNCTION_INFO_V1(pg_visibility_rel);
+PG_FUNCTION_INFO_V1(pg_visibility_map_summary);
+
+static TupleDesc pg_visibility_tupdesc(bool include_blkno, bool include_pd);
+static vbits *collect_visibility_data(Oid relid, bool include_pd);
+
+/*
+ * Visibility map information for a single block of a relation.
+ */
+Datum
+pg_visibility_map(PG_FUNCTION_ARGS)
+{
+	Oid			relid = PG_GETARG_OID(0);
+	int64		blkno = PG_GETARG_INT64(1);
+	int32		mapbits;
+	Relation	rel;
+	Buffer		vmbuffer = InvalidBuffer;
+	TupleDesc	tupdesc;
+	Datum		values[2];
+	bool		nulls[2];
+
+	rel = relation_open(relid, AccessShareLock);
+
+	if (blkno < 0 || blkno > MaxBlockNumber)
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid block number")));
+
+	tupdesc = pg_visibility_tupdesc(false, false);
+	MemSet(nulls, 0, sizeof(nulls));
+
+	mapbits = (int32) visibilitymap_get_status(rel, blkno, &vmbuffer);
+	if (vmbuffer != InvalidBuffer)
+		ReleaseBuffer(vmbuffer);
+	values[0] = BoolGetDatum((mapbits & VISIBILITYMAP_ALL_VISIBLE) != 0);
+	values[1] = BoolGetDatum((mapbits & VISIBILITYMAP_ALL_FROZEN) != 0);
+
+	relation_close(rel, AccessShareLock);
+
+	PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls)));
+}
+
+/*
+ * Visibility map information for a single block of a relation, plus the
+ * page-level information for the same block.
+ */
+Datum
+pg_visibility(PG_FUNCTION_ARGS)
+{
+	Oid			relid = PG_GETARG_OID(0);
+	int64		blkno = PG_GETARG_INT64(1);
+	int32		mapbits;
+	Relation	rel;
+	Buffer		vmbuffer = InvalidBuffer;
+	Buffer		buffer;
+	Page		page;
+	TupleDesc	tupdesc;
+	Datum		values[3];
+	bool		nulls[3];
+
+	rel = relation_open(relid, AccessShareLock);
+
+	if (blkno < 0 || blkno > MaxBlockNumber)
+		ereport(ERROR,
+				(errcode(ERRCODE_INVALID_PARAMETER_VALUE),
+				 errmsg("invalid block number")));
+
+	tupdesc = pg_visibility_tupdesc(false, true);
+	MemSet(nulls, 0, sizeof(nulls));
+
+	mapbits = (int32) visibilitymap_get_status(rel, blkno, &vmbuffer);
+	if (vmbuffer != InvalidBuffer)
+		ReleaseBuffer(vmbuffer);
+	values[0] = BoolGetDatum((mapbits & VISIBILITYMAP_ALL_VISIBLE) != 0);
+	values[1] = BoolGetDatum((mapbits & VISIBILITYMAP_ALL_FROZEN) != 0);
+
+	buffer = ReadBuffer(rel, blkno);
+	LockBuffer(buffer, BUFFER_LOCK_SHARE);
+
+	page = BufferGetPage(buffer);
+	values[2] = BoolGetDatum(PageIsAllVisible(page));
+
+	UnlockReleaseBuffer(buffer);
+
+	relation_close(rel, AccessShareLock);
+
+	PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls)));
+}
+
+/*
+ * Visibility map information for every block in a relation.
+ */
+Datum
+pg_visibility_map_rel(PG_FUNCTION_ARGS)
+{
+	FuncCallContext *funcctx;
+	vbits	   *info;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		Oid			relid = PG_GETARG_OID(0);
+		MemoryContext	oldcontext;
+
+		funcctx = SRF_FIRSTCALL_INIT();
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+		funcctx->tuple_desc = pg_visibility_tupdesc(true, false);
+		funcctx->user_fctx = collect_visibility_data(relid, false);
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	info = (vbits *) funcctx->user_fctx;
+
+	if (info->next < info->count)
+	{
+		Datum		values[3];
+		bool		nulls[3];
+		HeapTuple	tuple;
+
+		MemSet(nulls, 0, sizeof(nulls));
+		values[0] = Int64GetDatum(info->next);
+		values[1] = BoolGetDatum((info->bits[info->next] & (1 << 0)) != 0);
+		values[2] = BoolGetDatum((info->bits[info->next] & (1 << 1)) != 0);
+		info->next++;
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple));
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Visibility map information for every block in a relation, plus the page
+ * level information for each block.
+ */
+Datum
+pg_visibility_rel(PG_FUNCTION_ARGS)
+{
+	FuncCallContext *funcctx;
+	vbits	   *info;
+
+	if (SRF_IS_FIRSTCALL())
+	{
+		Oid			relid = PG_GETARG_OID(0);
+		MemoryContext	oldcontext;
+
+		funcctx = SRF_FIRSTCALL_INIT();
+		oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);
+		funcctx->tuple_desc = pg_visibility_tupdesc(true, true);
+		funcctx->user_fctx = collect_visibility_data(relid, true);
+		MemoryContextSwitchTo(oldcontext);
+	}
+
+	funcctx = SRF_PERCALL_SETUP();
+	info = (vbits *) funcctx->user_fctx;
+
+	if (info->next < info->count)
+	{
+		Datum		values[4];
+		bool		nulls[4];
+		HeapTuple	tuple;
+
+		MemSet(nulls, 0, sizeof(nulls));
+		values[0] = Int64GetDatum(info->next);
+		values[1] = BoolGetDatum((info->bits[info->next] & (1 << 0)) != 0);
+		values[2] = BoolGetDatum((info->bits[info->next] & (1 << 1)) != 0);
+		values[3] = BoolGetDatum((info->bits[info->next] & (1 << 2)) != 0);
+		info->next++;
+
+		tuple = heap_form_tuple(funcctx->tuple_desc, values, nulls);
+		SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tuple));
+	}
+
+	SRF_RETURN_DONE(funcctx);
+}
+
+/*
+ * Count the number of all-visible and all-frozen pages in the visibility
+ * map for a particular relation.
+ */
+Datum
+pg_visibility_map_summary(PG_FUNCTION_ARGS)
+{
+	Oid			relid = PG_GETARG_OID(0);
+	Relation	rel;
+	BlockNumber	nblocks;
+	BlockNumber	blkno;
+	Buffer		vmbuffer = InvalidBuffer;
+	int64		all_visible = 0;
+	int64		all_frozen = 0;
+	TupleDesc	tupdesc;
+	Datum		values[2];
+	bool		nulls[2];
+
+	rel = relation_open(relid, AccessShareLock);
+	nblocks = RelationGetNumberOfBlocks(rel);
+
+	for (blkno = 0; blkno < nblocks; ++blkno)
+	{
+		int32		mapbits;
+
+		/* Make sure we are interruptible. */
+		CHECK_FOR_INTERRUPTS();
+
+		/* Get map info. */
+		mapbits = (int32) visibilitymap_get_status(rel, blkno, &vmbuffer);
+		if ((mapbits & VISIBILITYMAP_ALL_VISIBLE) != 0)
+			++all_visible;
+		if ((mapbits & VISIBILITYMAP_ALL_FROZEN) != 0)
+			++all_frozen;
+	}
+
+	/* Clean up. */
+	if (vmbuffer != InvalidBuffer)
+		ReleaseBuffer(vmbuffer);
+	relation_close(rel, AccessShareLock);
+
+	tupdesc = CreateTemplateTupleDesc(2, false);
+	TupleDescInitEntry(tupdesc, (AttrNumber) 1, "all_visible", INT8OID, -1, 0);
+	TupleDescInitEntry(tupdesc, (AttrNumber) 2, "all_frozen", INT8OID, -1, 0);
+	tupdesc = BlessTupleDesc(tupdesc);
+
+	MemSet(nulls, 0, sizeof(nulls));
+	values[0] = Int64GetDatum(all_visible);
+	values[1] = Int64GetDatum(all_frozen);
+
+	PG_RETURN_DATUM(HeapTupleGetDatum(heap_form_tuple(tupdesc, values, nulls)));
+}
+
+/*
+ * Helper function to construct whichever TupleDesc we need for a particular
+ * call.
+ */
+static TupleDesc
+pg_visibility_tupdesc(bool include_blkno, bool include_pd)
+{
+	TupleDesc	tupdesc;
+	AttrNumber	maxattr = 2;
+	AttrNumber	a = 0;
+
+	if (include_blkno)
+		++maxattr;
+	if (include_pd)
+		++maxattr;
+	tupdesc = CreateTemplateTupleDesc(maxattr, false);
+	if (include_blkno)
+		TupleDescInitEntry(tupdesc, ++a, "blkno", INT8OID, -1, 0);
+	TupleDescInitEntry(tupdesc, ++a, "all_visible", BOOLOID, -1, 0);
+	TupleDescInitEntry(tupdesc, ++a, "all_frozen", BOOLOID, -1, 0);
+	if (include_pd)
+		TupleDescInitEntry(tupdesc, ++a, "pd_all_visible", BOOLOID, -1, 0);
+	Assert(a == maxattr);
+
+	return BlessTupleDesc(tupdesc);
+}
+
+/*
+ * Collect visibility data about a relation.
+ */
+static vbits *
+collect_visibility_data(Oid relid, bool include_pd)
+{
+	Relation	rel;
+	BlockNumber	nblocks;
+	vbits	   *info;
+	BlockNumber	blkno;
+	Buffer		vmbuffer = InvalidBuffer;
+	BufferAccessStrategy	bstrategy = GetAccessStrategy(BAS_BULKREAD);
+
+	rel = relation_open(relid, AccessShareLock);
+
+	nblocks = RelationGetNumberOfBlocks(rel);
+	info = palloc0(offsetof(vbits, bits) + nblocks);
+	info->next = 0;
+	info->count = nblocks;
+
+	for (blkno = 0; blkno < nblocks; ++blkno)
+	{
+		int32		mapbits;
+
+		/* Make sure we are interruptible. */
+		CHECK_FOR_INTERRUPTS();
+
+		/* Get map info. */
+		mapbits = (int32) visibilitymap_get_status(rel, blkno, &vmbuffer);
+		if ((mapbits & VISIBILITYMAP_ALL_VISIBLE) != 0)
+			info->bits[blkno] |= (1 << 0);
+		if ((mapbits & VISIBILITYMAP_ALL_FROZEN) != 0)
+			info->bits[blkno] |= (1 << 1);
+
+		/*
+		 * Page-level data requires reading every block, so only get it if
+		 * the caller needs it.  Use a buffer access strategy, too, to prevent
+		 * cache-trashing.
+		 */
+		if (include_pd)
+		{
+			Buffer		buffer;
+			Page		page;
+
+			buffer = ReadBufferExtended(rel, MAIN_FORKNUM, blkno, RBM_NORMAL,
+										bstrategy);
+			LockBuffer(buffer, BUFFER_LOCK_SHARE);
+
+			page = BufferGetPage(buffer);
+			if (PageIsAllVisible(page))
+				info->bits[blkno] |= (1 << 2);
+
+			UnlockReleaseBuffer(buffer);
+		}
+	}
+
+	/* Clean up. */
+	if (vmbuffer != InvalidBuffer)
+		ReleaseBuffer(vmbuffer);
+	relation_close(rel, AccessShareLock);
+
+	return info;
+}

contrib/pg_visibility/pg_visibility.control

+# pg_visibility extension
+comment = 'examine the visibility map (VM) and page-level visibility info'
+default_version = '1.0'
+module_pathname = '$libdir/pg_visibility'
+relocatable = true

doc/src/sgml/contrib.sgml

@@ -132,6 +132,7 @@ CREATE EXTENSION <replaceable>module_name</> FROM unpackaged;
  &pgstatstatements;
  &pgstattuple;
  &pgtrgm;
+ &pgvisibility;
  &postgres-fdw;
  &seg;
  &sepgsql;

doc/src/sgml/filelist.sgml

@@ -136,6 +136,7 @@
 <!ENTITY pgstatstatements SYSTEM "pgstatstatements.sgml">
 <!ENTITY pgstattuple     SYSTEM "pgstattuple.sgml">
 <!ENTITY pgtrgm          SYSTEM "pgtrgm.sgml">
+<!ENTITY pgvisibility    SYSTEM "pgvisibility.sgml">
 <!ENTITY postgres-fdw    SYSTEM "postgres-fdw.sgml">
 <!ENTITY seg             SYSTEM "seg.sgml">
 <!ENTITY contrib-spi     SYSTEM "contrib-spi.sgml">

doc/src/sgml/pgvisibility.sgml

+<!-- doc/src/sgml/pgvisibility.sgml -->
+
+<sect1 id="pgvisibility" xreflabel="pg_visibility">
+ <title>pg_visibility</title>
+
+ <indexterm zone="pgvisibility">
+  <primary>pg_visibility</primary>
+ </indexterm>
+
+ <para>
+  The <filename>pg_visibility</> module provides a means for examining the
+  visibility map (VM) and page-level visibility information.
+ </para>
+
+ <para>
+  These routines return information about three different bits.  The
+  all-visible bit in the visibility map indicates that every tuple on
+  a given page of a relation is visible to every current transaction.  The
+  all-frozen bit in the visibility map indicates that every tuple on the
+  page is frozen; that is, no future vacuum will need to modify the page
+  until such time as a tuple is inserted, updated, deleted, or locked on
+  that page.  The page-level <literal>PD_ALL_VISIBLE</literal> bit has the
+  same meaning as the all-visible bit in the visibility map, but is stored
+  within the data page itself rather than a separate data tructure.  These
+  will normally agree, but the page-level bit can sometimes be set while the
+  visibility map bit is clear after a crash recovery; or they can disagree
+  because of a change which occurs after <literal>pg_visibility</> examines
+  the visibility map and before it examines the data page.
+ </para>
+
+ <para>
+  Functions which display information about <literal>PG_ALL_VISIBLE</>
+  are much more costly than those which only consult the visibility map,
+  because they must read the relation's data blocks rather than only the
+  (much smaller) visibility map.
+ </para>
+
+ <sect2>
+  <title>Functions</title>
+
+  <variablelist>
+   <varlistentry>
+    <term><function>pg_visibility_map(regclass, blkno bigint, all_visible OUT boolean, all_frozen OUT boolean) returns record</function></term>
+    <listitem>
+     <para>
+      Returns the all-visible and all-frozen bits in the visibility map for
+      the given block of the given relation.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry>
+    <term><function>pg_visibility(regclass, blkno bigint, all_visible OUT boolean, all_frozen OUT boolean, pd_all_visible OUT boolean) returns record</function></term>
+    <listitem>
+     <para>
+      Returns the all-visible and all-frozen bits in the visibility map for
+      the given block of the given relation, plus the
+      <literal>PD_ALL_VISIBILE</> bit for that block.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry>
+    <term><function>pg_visibility_map(regclass, blkno OUT bigint, all_visible OUT boolean, all_frozen OUT boolean) returns record</function></term>
+    <listitem>
+     <para>
+      Returns the all-visible and all-frozen bits in the visibility map for
+      each block the given relation.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry>
+    <term><function>pg_visibility(regclass, blkno OUT bigint, all_visible OUT boolean, all_frozen OUT boolean, pd_all_visible OUT boolean) returns record</function></term>
+
+    <listitem>
+     <para>
+      Returns the all-visible and all-frozen bits in the visibility map for
+      each block the given relation, plus the <literal>PD_ALL_VISIBLE</>
+      bit for each block.
+     </para>
+    </listitem>
+   </varlistentry>
+
+   <varlistentry>
+    <term><function>pg_visibility_map_summary(regclass, all_visible OUT bigint, all_frozen OUT bigint) returns record</function></term>
+
+    <listitem>
+     <para>
+      Returns the number of all-visible pages and the number of all-frozen
+      pages in the relation according to the visibility map.
+     </para>
+    </listitem>
+   </varlistentry>
+  </variablelist>
+
+  <para>
+   By default, these functions are not publicly executable.
+  </para>
+ </sect2>
+
+ <sect2>
+  <title>Author</title>
+
+  <para>
+   Robert Haas <email>rhaas@postgresql.org</email>
+  </para>
+ </sect2>
+
+</sect1>

doc/src/sgml/storage.sgml

@@ -648,6 +648,11 @@ might not be true. Visibility map bits are only set by vacuum, but are
 cleared by any data-modifying operations on a page.
 </para>
 
+<para>
+The <xref linkend="pgvisibility"> module can be used to examine the
+information stored in the visibility map.
+</para>
+
 </sect1>
 
 <sect1 id="storage-init">