Modify pgstats code to reduce performance penalties from oversized stats data

files: avoid creating stats hashtable entries for tables that aren't being
touched except by vacuum/analyze, ensure that entries for dropped tables are
removed promptly, and tweak the data layout to avoid storing useless struct
padding.  Also improve the performance of pgstat_vacuum_tabstat(), and make
sure that autovacuum invokes it exactly once per autovac cycle rather than
multiple times or not at all.  This should cure recent complaints about 8.1
showing much higher stats I/O volume than was seen in 8.0.  It'd still be a
good idea to revisit the design with an eye to not re-writing the entire
stats dataset every half second ... but that would be too much to backpatch,
I fear.

src/backend/commands/vacuum.c

@@ -13,7 +13,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/commands/vacuum.c,v 1.320 2006/01/04 19:16:24 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/commands/vacuum.c,v 1.321 2006/01/18 20:35:05 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -35,6 +35,7 @@
 #include "commands/vacuum.h"
 #include "executor/executor.h"
 #include "miscadmin.h"
+#include "postmaster/autovacuum.h"
 #include "storage/freespace.h"
 #include "storage/procarray.h"
 #include "storage/smgr.h"
@@ -324,9 +325,10 @@ vacuum(VacuumStmt *vacstmt, List *relids)
 				 errhint("Use VACUUM FULL, then VACUUM FREEZE.")));
 
 	/*
-	 * Send info about dead objects to the statistics collector
+	 * Send info about dead objects to the statistics collector, unless
+	 * we are in autovacuum --- autovacuum.c does this for itself.
 	 */
-	if (vacstmt->vacuum)
+	if (vacstmt->vacuum && !IsAutoVacuumProcess())
 		pgstat_vacuum_tabstat();
 
 	/*

src/backend/postmaster/autovacuum.c

@@ -10,7 +10,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/postmaster/autovacuum.c,v 1.9 2006/01/04 21:06:31 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/postmaster/autovacuum.c,v 1.10 2006/01/18 20:35:05 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -508,6 +508,11 @@ process_whole_db(void)
 	 /* functions in indexes may want a snapshot set */
 	ActiveSnapshot = CopySnapshot(GetTransactionSnapshot());
 
+	/*
+	 * Clean up any dead statistics collector entries for this DB.
+	 */
+	pgstat_vacuum_tabstat();
+
 	dbRel = heap_open(DatabaseRelationId, AccessShareLock);
 
 	/* Must use a table scan, since there's no syscache for pg_database */
@@ -572,6 +577,13 @@ do_autovacuum(PgStat_StatDBEntry *dbentry)
 	 /* functions in indexes may want a snapshot set */
 	ActiveSnapshot = CopySnapshot(GetTransactionSnapshot());
 
+	/*
+	 * Clean up any dead statistics collector entries for this DB.
+	 * We always want to do this exactly once per DB-processing cycle,
+	 * even if we find nothing worth vacuuming in the database.
+	 */
+	pgstat_vacuum_tabstat();
+
 	/*
 	 * StartTransactionCommand and CommitTransactionCommand will automatically
 	 * switch to other contexts.  We need this one to keep the list of

src/backend/storage/smgr/smgr.c

@@ -11,7 +11,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/smgr/smgr.c,v 1.94 2005/11/22 18:17:21 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/smgr/smgr.c,v 1.95 2006/01/18 20:35:06 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -19,6 +19,7 @@
 
 #include "access/xact.h"
 #include "commands/tablespace.h"
+#include "pgstat.h"
 #include "storage/bufmgr.h"
 #include "storage/freespace.h"
 #include "storage/ipc.h"
@@ -469,6 +470,9 @@ smgr_internal_unlink(RelFileNode rnode, int which, bool isTemp, bool isRedo)
 	 */
 	FreeSpaceMapForgetRel(&rnode);
 
+	/* Tell the stats collector to forget it immediately, too. */
+	pgstat_drop_relation(rnode.relNode);
+
 	/*
 	 * And delete the physical files.
 	 *

src/include/pgstat.h

@@ -5,7 +5,7 @@
  *
  *	Copyright (c) 2001-2005, PostgreSQL Global Development Group
  *
- *	$PostgreSQL: pgsql/src/include/pgstat.h,v 1.40 2005/12/31 19:39:11 momjian Exp $
+ *	$PostgreSQL: pgsql/src/include/pgstat.h,v 1.41 2006/01/18 20:35:06 tgl Exp $
  * ----------
  */
 #ifndef PGSTAT_H
@@ -271,7 +271,7 @@ typedef union PgStat_Msg
  * ------------------------------------------------------------
  */
 
-#define PGSTAT_FILE_FORMAT_ID	0x01A5BC93
+#define PGSTAT_FILE_FORMAT_ID	0x01A5BC94
 
 /* ----------
  * PgStat_StatDBEntry			The collector's data per database
@@ -280,14 +280,19 @@ typedef union PgStat_Msg
 typedef struct PgStat_StatDBEntry
 {
 	Oid			databaseid;
-	HTAB	   *tables;
+	int			destroy;
 	int			n_backends;
 	PgStat_Counter n_xact_commit;
 	PgStat_Counter n_xact_rollback;
 	PgStat_Counter n_blocks_fetched;
 	PgStat_Counter n_blocks_hit;
-	int			destroy;
 	TimestampTz last_autovac_time;
+
+	/*
+	 * tables must be last in the struct, because we don't write the pointer
+	 * out to the stats file.
+	 */
+	HTAB	   *tables;
 } PgStat_StatDBEntry;
 
 
@@ -301,10 +306,9 @@ typedef struct PgStat_StatBeEntry
 	int			procpid;
 	TimestampTz start_timestamp;
 	TimestampTz activity_start_timestamp;
-	char		activity[PGSTAT_ACTIVITY_SIZE];
 
 	/*
-	 * The following fields are initialized by the BESTART message. If we have
+	 * These fields are initialized by the BESTART message. If we have
 	 * received messages from a backend before we have received its BESTART,
 	 * these fields will be uninitialized: userid and databaseid will be
 	 * InvalidOid, and clientaddr will be undefined.
@@ -312,6 +316,12 @@ typedef struct PgStat_StatBeEntry
 	Oid			userid;
 	Oid			databaseid;
 	SockAddr	clientaddr;
+
+	/*
+	 * activity[] must be last in the struct, because we only write as much
+	 * of it as needed to the stats file.
+	 */
+	char		activity[PGSTAT_ACTIVITY_SIZE];
 } PgStat_StatBeEntry;
 
 
@@ -321,6 +331,8 @@ typedef struct PgStat_StatBeEntry
  *								about backends that are known to be
  *								dead for some seconds. This info is held
  *								in a hash table of these structs.
+ *
+ * (This struct is not used in the stats file.)
  * ----------
  */
 typedef struct PgStat_StatBeDead
@@ -338,6 +350,7 @@ typedef struct PgStat_StatBeDead
 typedef struct PgStat_StatTabEntry
 {
 	Oid			tableid;
+	int			destroy;
 
 	PgStat_Counter numscans;
 
@@ -354,8 +367,6 @@ typedef struct PgStat_StatTabEntry
 
 	PgStat_Counter blocks_fetched;
 	PgStat_Counter blocks_hit;
-
-	int			destroy;
 } PgStat_StatTabEntry;
 
 
@@ -400,7 +411,8 @@ extern void pgstat_report_vacuum(Oid tableoid, bool shared,
 extern void pgstat_report_analyze(Oid tableoid, bool shared,
 					  PgStat_Counter livetuples,
 					  PgStat_Counter deadtuples);
-extern int	pgstat_vacuum_tabstat(void);
+extern void pgstat_vacuum_tabstat(void);
+extern void pgstat_drop_relation(Oid relid);
 
 extern void pgstat_reset_counters(void);