Split backend status and progress related functionality out of pgstat.c.

Backend status (supporting pg_stat_activity) and command
progress (supporting pg_stat_progress*) related code is largely
independent from the rest of pgstat.[ch] (supporting views like
pg_stat_all_tables that accumulate data over time). See also
a333476b.

This commit doesn't rename the function names to make the distinction
from the rest of pgstat_ clearer - that'd be more invasive and not
clearly beneficial. If we were to decide to do such a rename at some
point, it's better done separately from moving the code as well.

Robert's review was of an earlier version.
Reviewed-By: Robert Haas <robertmhaas@gmail.com>
Discussion: https://postgr.es/m/20210316195440.twxmlov24rr2nxrg@alap3.anarazel.de

src/backend/bootstrap/bootstrap.c

@@ -407,8 +407,11 @@ AuxiliaryProcessMain(int argc, char *argv[])
 		 */
 		CreateAuxProcessResourceOwner();
 
-		/* Initialize backend status information */
+		/* Initialize statistics reporting */
 		pgstat_initialize();
+
+		/* Initialize backend status information */
+		pgstat_beinit();
 		pgstat_bestart();
 
 		/* register a before-shutdown callback for LWLock cleanup */

src/backend/utils/activity/Makefile

@@ -14,6 +14,8 @@ top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
 OBJS = \
+	backend_progress.o \
+	backend_status.o \
 	wait_event.o
 
 include $(top_srcdir)/src/backend/common.mk

src/backend/utils/activity/backend_progress.c

+/* ----------
+ * progress.c
+ *
+ *	Command progress reporting infrastructure.
+ *
+ *	Copyright (c) 2001-2021, PostgreSQL Global Development Group
+ *
+ *	src/backend/postmaster/progress.c
+ * ----------
+ */
+#include "postgres.h"
+
+#include "port/atomics.h" /* for memory barriers */
+#include "utils/backend_progress.h"
+#include "utils/backend_status.h"
+
+
+/*-----------
+ * pgstat_progress_start_command() -
+ *
+ * Set st_progress_command (and st_progress_command_target) in own backend
+ * entry.  Also, zero-initialize st_progress_param array.
+ *-----------
+ */
+void
+pgstat_progress_start_command(ProgressCommandType cmdtype, Oid relid)
+{
+	volatile PgBackendStatus *beentry = MyBEEntry;
+
+	if (!beentry || !pgstat_track_activities)
+		return;
+
+	PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+	beentry->st_progress_command = cmdtype;
+	beentry->st_progress_command_target = relid;
+	MemSet(&beentry->st_progress_param, 0, sizeof(beentry->st_progress_param));
+	PGSTAT_END_WRITE_ACTIVITY(beentry);
+}
+
+/*-----------
+ * pgstat_progress_update_param() -
+ *
+ * Update index'th member in st_progress_param[] of own backend entry.
+ *-----------
+ */
+void
+pgstat_progress_update_param(int index, int64 val)
+{
+	volatile PgBackendStatus *beentry = MyBEEntry;
+
+	Assert(index >= 0 && index < PGSTAT_NUM_PROGRESS_PARAM);
+
+	if (!beentry || !pgstat_track_activities)
+		return;
+
+	PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+	beentry->st_progress_param[index] = val;
+	PGSTAT_END_WRITE_ACTIVITY(beentry);
+}
+
+/*-----------
+ * pgstat_progress_update_multi_param() -
+ *
+ * Update multiple members in st_progress_param[] of own backend entry.
+ * This is atomic; readers won't see intermediate states.
+ *-----------
+ */
+void
+pgstat_progress_update_multi_param(int nparam, const int *index,
+								   const int64 *val)
+{
+	volatile PgBackendStatus *beentry = MyBEEntry;
+	int			i;
+
+	if (!beentry || !pgstat_track_activities || nparam == 0)
+		return;
+
+	PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+
+	for (i = 0; i < nparam; ++i)
+	{
+		Assert(index[i] >= 0 && index[i] < PGSTAT_NUM_PROGRESS_PARAM);
+
+		beentry->st_progress_param[index[i]] = val[i];
+	}
+
+	PGSTAT_END_WRITE_ACTIVITY(beentry);
+}
+
+/*-----------
+ * pgstat_progress_end_command() -
+ *
+ * Reset st_progress_command (and st_progress_command_target) in own backend
+ * entry.  This signals the end of the command.
+ *-----------
+ */
+void
+pgstat_progress_end_command(void)
+{
+	volatile PgBackendStatus *beentry = MyBEEntry;
+
+	if (!beentry || !pgstat_track_activities)
+		return;
+
+	if (beentry->st_progress_command == PROGRESS_COMMAND_INVALID)
+		return;
+
+	PGSTAT_BEGIN_WRITE_ACTIVITY(beentry);
+	beentry->st_progress_command = PROGRESS_COMMAND_INVALID;
+	beentry->st_progress_command_target = InvalidOid;
+	PGSTAT_END_WRITE_ACTIVITY(beentry);
+}

src/backend/utils/init/postinit.c

@@ -681,6 +681,10 @@ InitPostgres(const char *in_dbname, Oid dboid, const char *username,
 	if (!bootstrap)
 		pgstat_initialize();
 
+	/* Initialize status reporting */
+	if (!bootstrap)
+		pgstat_beinit();
+
 	/*
 	 * Load relcache entries for the shared system catalogs.  This must create
 	 * at least entries for pg_database and catalogs used for authentication.

src/backend/utils/misc/guc.c

@@ -90,6 +90,7 @@
 #include "tcop/tcopprot.h"
 #include "tsearch/ts_cache.h"
 #include "utils/acl.h"
+#include "utils/backend_status.h"
 #include "utils/builtins.h"
 #include "utils/bytea.h"
 #include "utils/float.h"

src/include/commands/progress.h

@@ -2,7 +2,7 @@
  *
  * progress.h
  *	  Constants used with the progress reporting facilities defined in
- *	  pgstat.h.  These are possibly interesting to extensions, so we
+ *	  backend_status.h.  These are possibly interesting to extensions, so we
  *	  expose them via this header file.  Note that if you update these
  *	  constants, you probably also need to update the views based on them
  *	  in system_views.sql.

src/include/utils/backend_progress.h

+/* ----------
+ * backend_progress.h
+ *	  Command progress reporting definition.
+ *
+ * Note that this file provides the infrastructure for storing a single
+ * backend's command progress counters, without ascribing meaning to the
+ * individual fields. See commands/progress.h and system_views.sql for that.
+ *
+ * Copyright (c) 2001-2021, PostgreSQL Global Development Group
+ *
+ * src/include/utils/backend_progress.h
+ * ----------
+ */
+#ifndef BACKEND_PROGRESS_H
+#define BACKEND_PROGRESS_H
+
+
+/* ----------
+ * Command type for progress reporting purposes
+ * ----------
+ */
+typedef enum ProgressCommandType
+{
+	PROGRESS_COMMAND_INVALID,
+	PROGRESS_COMMAND_VACUUM,
+	PROGRESS_COMMAND_ANALYZE,
+	PROGRESS_COMMAND_CLUSTER,
+	PROGRESS_COMMAND_CREATE_INDEX,
+	PROGRESS_COMMAND_BASEBACKUP,
+	PROGRESS_COMMAND_COPY
+} ProgressCommandType;
+
+#define PGSTAT_NUM_PROGRESS_PARAM	20
+
+
+extern void pgstat_progress_start_command(ProgressCommandType cmdtype,
+										  Oid relid);
+extern void pgstat_progress_update_param(int index, int64 val);
+extern void pgstat_progress_update_multi_param(int nparam, const int *index,
+											   const int64 *val);
+extern void pgstat_progress_end_command(void);
+
+
+#endif /* BACKEND_PROGRESS_H */

src/include/utils/backend_status.h

+/* ----------
+ * backend_status.h
+ *	  Definitions related to backend status reporting
+ *
+ * Copyright (c) 2001-2021, PostgreSQL Global Development Group
+ *
+ * src/include/utils/backend_status.h
+ * ----------
+ */
+#ifndef BACKEND_STATUS_H
+#define BACKEND_STATUS_H
+
+#include "datatype/timestamp.h"
+#include "libpq/pqcomm.h"
+#include "miscadmin.h" /* for BackendType */
+#include "utils/backend_progress.h"
+
+
+/* ----------
+ * Backend states
+ * ----------
+ */
+typedef enum BackendState
+{
+	STATE_UNDEFINED,
+	STATE_IDLE,
+	STATE_RUNNING,
+	STATE_IDLEINTRANSACTION,
+	STATE_FASTPATH,
+	STATE_IDLEINTRANSACTION_ABORTED,
+	STATE_DISABLED
+} BackendState;
+
+
+/* ----------
+ * Shared-memory data structures
+ * ----------
+ */
+
+/*
+ * PgBackendSSLStatus
+ *
+ * For each backend, we keep the SSL status in a separate struct, that
+ * is only filled in if SSL is enabled.
+ *
+ * All char arrays must be null-terminated.
+ */
+typedef struct PgBackendSSLStatus
+{
+	/* Information about SSL connection */
+	int			ssl_bits;
+	char		ssl_version[NAMEDATALEN];
+	char		ssl_cipher[NAMEDATALEN];
+	char		ssl_client_dn[NAMEDATALEN];
+
+	/*
+	 * serial number is max "20 octets" per RFC 5280, so this size should be
+	 * fine
+	 */
+	char		ssl_client_serial[NAMEDATALEN];
+
+	char		ssl_issuer_dn[NAMEDATALEN];
+} PgBackendSSLStatus;
+
+/*
+ * PgBackendGSSStatus
+ *
+ * For each backend, we keep the GSS status in a separate struct, that
+ * is only filled in if GSS is enabled.
+ *
+ * All char arrays must be null-terminated.
+ */
+typedef struct PgBackendGSSStatus
+{
+	/* Information about GSSAPI connection */
+	char		gss_princ[NAMEDATALEN]; /* GSSAPI Principal used to auth */
+	bool		gss_auth;		/* If GSSAPI authentication was used */
+	bool		gss_enc;		/* If encryption is being used */
+
+} PgBackendGSSStatus;
+
+
+/* ----------
+ * PgBackendStatus
+ *
+ * Each live backend maintains a PgBackendStatus struct in shared memory
+ * showing its current activity.  (The structs are allocated according to
+ * BackendId, but that is not critical.)  Note that the collector process
+ * has no involvement in, or even access to, these structs.
+ *
+ * Each auxiliary process also maintains a PgBackendStatus struct in shared
+ * memory.
+ * ----------
+ */
+typedef struct PgBackendStatus
+{
+	/*
+	 * To avoid locking overhead, we use the following protocol: a backend
+	 * increments st_changecount before modifying its entry, and again after
+	 * finishing a modification.  A would-be reader should note the value of
+	 * st_changecount, copy the entry into private memory, then check
+	 * st_changecount again.  If the value hasn't changed, and if it's even,
+	 * the copy is valid; otherwise start over.  This makes updates cheap
+	 * while reads are potentially expensive, but that's the tradeoff we want.
+	 *
+	 * The above protocol needs memory barriers to ensure that the apparent
+	 * order of execution is as it desires.  Otherwise, for example, the CPU
+	 * might rearrange the code so that st_changecount is incremented twice
+	 * before the modification on a machine with weak memory ordering.  Hence,
+	 * use the macros defined below for manipulating st_changecount, rather
+	 * than touching it directly.
+	 */
+	int			st_changecount;
+
+	/* The entry is valid iff st_procpid > 0, unused if st_procpid == 0 */
+	int			st_procpid;
+
+	/* Type of backends */
+	BackendType st_backendType;
+
+	/* Times when current backend, transaction, and activity started */
+	TimestampTz st_proc_start_timestamp;
+	TimestampTz st_xact_start_timestamp;
+	TimestampTz st_activity_start_timestamp;
+	TimestampTz st_state_start_timestamp;
+
+	/* Database OID, owning user's OID, connection client address */
+	Oid			st_databaseid;
+	Oid			st_userid;
+	SockAddr	st_clientaddr;
+	char	   *st_clienthostname;	/* MUST be null-terminated */
+
+	/* Information about SSL connection */
+	bool		st_ssl;
+	PgBackendSSLStatus *st_sslstatus;
+
+	/* Information about GSSAPI connection */
+	bool		st_gss;
+	PgBackendGSSStatus *st_gssstatus;
+
+	/* current state */
+	BackendState st_state;
+
+	/* application name; MUST be null-terminated */
+	char	   *st_appname;
+
+	/*
+	 * Current command string; MUST be null-terminated. Note that this string
+	 * possibly is truncated in the middle of a multi-byte character. As
+	 * activity strings are stored more frequently than read, that allows to
+	 * move the cost of correct truncation to the display side. Use
+	 * pgstat_clip_activity() to truncate correctly.
+	 */
+	char	   *st_activity_raw;
+
+	/*
+	 * Command progress reporting.  Any command which wishes can advertise
+	 * that it is running by setting st_progress_command,
+	 * st_progress_command_target, and st_progress_param[].
+	 * st_progress_command_target should be the OID of the relation which the
+	 * command targets (we assume there's just one, as this is meant for
+	 * utility commands), but the meaning of each element in the
+	 * st_progress_param array is command-specific.
+	 */
+	ProgressCommandType st_progress_command;
+	Oid			st_progress_command_target;
+	int64		st_progress_param[PGSTAT_NUM_PROGRESS_PARAM];
+} PgBackendStatus;
+
+
+/*
+ * Macros to load and store st_changecount with appropriate memory barriers.
+ *
+ * Use PGSTAT_BEGIN_WRITE_ACTIVITY() before, and PGSTAT_END_WRITE_ACTIVITY()
+ * after, modifying the current process's PgBackendStatus data.  Note that,
+ * since there is no mechanism for cleaning up st_changecount after an error,
+ * THESE MACROS FORM A CRITICAL SECTION.  Any error between them will be
+ * promoted to PANIC, causing a database restart to clean up shared memory!
+ * Hence, keep the critical section as short and straight-line as possible.
+ * Aside from being safer, that minimizes the window in which readers will
+ * have to loop.
+ *
+ * Reader logic should follow this sketch:
+ *
+ *		for (;;)
+ *		{
+ *			int before_ct, after_ct;
+ *
+ *			pgstat_begin_read_activity(beentry, before_ct);
+ *			... copy beentry data to local memory ...
+ *			pgstat_end_read_activity(beentry, after_ct);
+ *			if (pgstat_read_activity_complete(before_ct, after_ct))
+ *				break;
+ *			CHECK_FOR_INTERRUPTS();
+ *		}
+ *
+ * For extra safety, we generally use volatile beentry pointers, although
+ * the memory barriers should theoretically be sufficient.
+ */
+#define PGSTAT_BEGIN_WRITE_ACTIVITY(beentry) \
+	do { \
+		START_CRIT_SECTION(); \
+		(beentry)->st_changecount++; \
+		pg_write_barrier(); \
+	} while (0)
+
+#define PGSTAT_END_WRITE_ACTIVITY(beentry) \
+	do { \
+		pg_write_barrier(); \
+		(beentry)->st_changecount++; \
+		Assert(((beentry)->st_changecount & 1) == 0); \
+		END_CRIT_SECTION(); \
+	} while (0)
+
+#define pgstat_begin_read_activity(beentry, before_changecount) \
+	do { \
+		(before_changecount) = (beentry)->st_changecount; \
+		pg_read_barrier(); \
+	} while (0)
+
+#define pgstat_end_read_activity(beentry, after_changecount) \
+	do { \
+		pg_read_barrier(); \
+		(after_changecount) = (beentry)->st_changecount; \
+	} while (0)
+
+#define pgstat_read_activity_complete(before_changecount, after_changecount) \
+	((before_changecount) == (after_changecount) && \
+	 ((before_changecount) & 1) == 0)
+
+
+/* ----------
+ * LocalPgBackendStatus
+ *
+ * When we build the backend status array, we use LocalPgBackendStatus to be
+ * able to add new values to the struct when needed without adding new fields
+ * to the shared memory. It contains the backend status as a first member.
+ * ----------
+ */
+typedef struct LocalPgBackendStatus
+{
+	/*
+	 * Local version of the backend status entry.
+	 */
+	PgBackendStatus backendStatus;
+
+	/*
+	 * The xid of the current transaction if available, InvalidTransactionId
+	 * if not.
+	 */
+	TransactionId backend_xid;
+
+	/*
+	 * The xmin of the current session if available, InvalidTransactionId if
+	 * not.
+	 */
+	TransactionId backend_xmin;
+} LocalPgBackendStatus;
+
+
+/* ----------
+ * GUC parameters
+ * ----------
+ */
+extern PGDLLIMPORT bool pgstat_track_activities;
+extern PGDLLIMPORT int pgstat_track_activity_query_size;
+
+
+/* ----------
+ * Other global variables
+ * ----------
+ */
+extern PGDLLIMPORT  PgBackendStatus *MyBEEntry;
+
+
+/* ----------
+ * Functions called from postmaster
+ * ----------
+ */
+extern Size BackendStatusShmemSize(void);
+extern void CreateSharedBackendStatus(void);
+
+
+/* ----------
+ * Functions called from backends
+ * ----------
+ */
+
+/* Initialization functions */
+extern void pgstat_beinit(void);
+extern void pgstat_bestart(void);
+
+extern void pgstat_clear_backend_activity_snapshot(void);
+
+/* Activity reporting functions */
+extern void pgstat_report_activity(BackendState state, const char *cmd_str);
+extern void pgstat_report_tempfile(size_t filesize);
+extern void pgstat_report_appname(const char *appname);
+extern void pgstat_report_xact_timestamp(TimestampTz tstamp);
+extern const char *pgstat_get_backend_current_activity(int pid, bool checkUser);
+extern const char *pgstat_get_crashed_backend_activity(int pid, char *buffer,
+													   int buflen);
+
+
+/* ----------
+ * Support functions for the SQL-callable functions to
+ * generate the pgstat* views.
+ * ----------
+ */
+extern int	pgstat_fetch_stat_numbackends(void);
+extern PgBackendStatus *pgstat_fetch_stat_beentry(int beid);
+extern LocalPgBackendStatus *pgstat_fetch_stat_local_beentry(int beid);
+extern char *pgstat_clip_activity(const char *raw_activity);
+
+
+#endif /* BACKEND_STATUS_H */