Change memory-space accounting mechanism in tuplesort.c and tuplestore.c

to make a reasonable attempt at accounting for palloc overhead, not just
the requested size of each memory chunk.  Since in many scenarios this
will make for a significant reduction in the amount of space acquired,
partially compensate by doubling the default value of SORT_MEM to 1Mb.
Per discussion in pgsql-general around 9-Jun-2002..

doc/src/sgml/runtime.sgml

 <!--
-$Header: /cvsroot/pgsql/doc/src/sgml/runtime.sgml,v 1.123 2002/08/09 22:52:04 petere Exp $
+$Header: /cvsroot/pgsql/doc/src/sgml/runtime.sgml,v 1.124 2002/08/12 00:36:11 tgl Exp $
 -->
 
 <Chapter Id="runtime">
@@ -1640,8 +1640,8 @@ dynamic_library_path = '/usr/local/lib/postgresql:/home/my_project/lib:$libdir'
        <para>
 	Specifies the amount of memory to be used by internal sorts and
 	hashes before switching to temporary disk files. The value is
-	specified in kilobytes, and defaults to 512 kilobytes. Note that
-	for a complex query, several sorts and/or hashes might be
+	specified in kilobytes, and defaults to 1024 kilobytes (1MB).
+	Note that for a complex query, several sorts and/or hashes might be
 	running in parallel, and each one will be allowed to use as much
 	memory as this value specifies before it starts to put data into
 	temporary files. Also, each running backend could be doing one

src/backend/utils/init/globals.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/init/globals.c,v 1.65 2002/06/20 20:29:40 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/init/globals.c,v 1.66 2002/08/12 00:36:11 tgl Exp $
  *
  * NOTES
  *	  Globals used all over the place should be declared here and not
@@ -71,6 +71,6 @@ char		FloatFormat[20] = "%f";
 
 bool		enableFsync = true;
 bool		allowSystemTableMods = false;
-int			SortMem = 512;
+int			SortMem = 1024;
 int			VacuumMem = 8192;
 int			NBuffers = DEF_NBUFFERS;

src/backend/utils/misc/guc.c

@@ -5,7 +5,7 @@
  * command, configuration file, and command line options.
  * See src/backend/utils/misc/README for more information.
  *
- * $Header: /cvsroot/pgsql/src/backend/utils/misc/guc.c,v 1.78 2002/08/07 17:26:24 tgl Exp $
+ * $Header: /cvsroot/pgsql/src/backend/utils/misc/guc.c,v 1.79 2002/08/12 00:36:11 tgl Exp $
  *
  * Copyright 2000 by PostgreSQL Global Development Group
  * Written by Peter Eisentraut <peter_e@gmx.net>.
@@ -556,7 +556,7 @@ static struct config_int
 
 	{
 		{ "sort_mem", PGC_USERSET }, &SortMem,
-		512, 4 * BLCKSZ / 1024, INT_MAX, NULL, NULL
+		1024, 8 * BLCKSZ / 1024, INT_MAX, NULL, NULL
 	},
 
 	{

src/backend/utils/misc/postgresql.conf.sample

@@ -57,7 +57,7 @@
 #
 #	Non-shared Memory Sizes
 #
-#sort_mem = 512			# min 32
+#sort_mem = 1024		# min 64
 #vacuum_mem = 8192		# min 1024
 
 

src/backend/utils/mmgr/aset.c

@@ -11,7 +11,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/mmgr/aset.c,v 1.46 2002/06/20 20:29:40 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/mmgr/aset.c,v 1.47 2002/08/12 00:36:12 tgl Exp $
  *
  * NOTE:
  *	This is a new (Feb. 05, 1999) implementation of the allocation set
@@ -204,11 +204,11 @@ static void *AllocSetRealloc(MemoryContext context, void *pointer, Size size);
 static void AllocSetInit(MemoryContext context);
 static void AllocSetReset(MemoryContext context);
 static void AllocSetDelete(MemoryContext context);
-
+static Size AllocSetGetChunkSpace(MemoryContext context, void *pointer);
+static void AllocSetStats(MemoryContext context);
 #ifdef MEMORY_CONTEXT_CHECKING
 static void AllocSetCheck(MemoryContext context);
 #endif
-static void AllocSetStats(MemoryContext context);
 
 /*
  * This is the virtual function table for AllocSet contexts.
@@ -220,10 +220,11 @@ static MemoryContextMethods AllocSetMethods = {
 	AllocSetInit,
 	AllocSetReset,
 	AllocSetDelete,
+	AllocSetGetChunkSpace,
+	AllocSetStats
 #ifdef MEMORY_CONTEXT_CHECKING
-	AllocSetCheck,
+	, AllocSetCheck
 #endif
-	AllocSetStats
 };
 
 
@@ -953,6 +954,19 @@ AllocSetRealloc(MemoryContext context, void *pointer, Size size)
 	}
 }
 
+/*
+ * AllocSetGetChunkSpace
+ *		Given a currently-allocated chunk, determine the total space
+ *		it occupies (including all memory-allocation overhead).
+ */
+static Size
+AllocSetGetChunkSpace(MemoryContext context, void *pointer)
+{
+	AllocChunk	chunk = AllocPointerGetChunk(pointer);
+
+	return chunk->size + ALLOC_CHUNKHDRSZ;
+}
+
 /*
  * AllocSetStats
  *		Displays stats about memory consumption of an allocset.

src/backend/utils/mmgr/mcxt.c

@@ -14,7 +14,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/mmgr/mcxt.c,v 1.31 2002/08/10 20:29:18 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/mmgr/mcxt.c,v 1.32 2002/08/12 00:36:12 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -224,6 +224,39 @@ MemoryContextResetAndDeleteChildren(MemoryContext context)
 	(*context->methods->reset) (context);
 }
 
+/*
+ * GetMemoryChunkSpace
+ *		Given a currently-allocated chunk, determine the total space
+ *		it occupies (including all memory-allocation overhead).
+ *
+ * This is useful for measuring the total space occupied by a set of
+ * allocated chunks.
+ */
+Size
+GetMemoryChunkSpace(void *pointer)
+{
+	StandardChunkHeader *header;
+
+	/*
+	 * Try to detect bogus pointers handed to us, poorly though we can.
+	 * Presumably, a pointer that isn't MAXALIGNED isn't pointing at an
+	 * allocated chunk.
+	 */
+	Assert(pointer != NULL);
+	Assert(pointer == (void *) MAXALIGN(pointer));
+
+	/*
+	 * OK, it's probably safe to look at the chunk header.
+	 */
+	header = (StandardChunkHeader *)
+		((char *) pointer - STANDARDCHUNKHEADERSIZE);
+
+	AssertArg(MemoryContextIsValid(header->context));
+
+	return (*header->context->methods->get_chunk_space) (header->context,
+														 pointer);
+}
+
 /*
  * MemoryContextStats
  *		Print statistics about the named context and all its descendants.

src/backend/utils/sort/tuplestore.c

@@ -26,7 +26,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/sort/tuplestore.c,v 1.6 2002/06/20 20:29:40 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/sort/tuplestore.c,v 1.7 2002/08/12 00:36:12 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -154,30 +154,15 @@ struct Tuplestorestate
  *
  * NOTES about memory consumption calculations:
  *
- * We count space requested for tuples against the maxKBytes limit.
- * Fixed-size space (primarily the BufFile I/O buffer) is not
- * counted, nor do we count the variable-size memtuples array.
- * (Even though that could grow pretty large, it should be small compared
- * to the tuples proper, so this is not unreasonable.)
+ * We count space allocated for tuples against the maxKBytes limit,
+ * plus the space used by the variable-size array memtuples.
+ * Fixed-size space (primarily the BufFile I/O buffer) is not counted.
  *
- * The major deficiency in this approach is that it ignores palloc overhead.
- * The memory space actually allocated for a palloc chunk is always more
- * than the request size, and could be considerably more (as much as 2X
- * larger, in the current aset.c implementation).  So the space used could
- * be considerably more than maxKBytes says.
- *
- * One way to fix this is to add a memory management function that, given
- * a pointer to a palloc'd chunk, returns the actual space consumed by the
- * chunk.  This would be very easy in the current aset.c module, but I'm
- * hesitant to do it because it might be unpleasant to support in future
- * implementations of memory management.  (For example, a direct
- * implementation of palloc as malloc could not support such a function
- * portably.)
- *
- * A cruder answer is just to apply a fudge factor, say by initializing
- * availMem to only three-quarters of what maxKBytes indicates.  This is
- * probably the right answer if anyone complains that maxKBytes is not being
- * obeyed very faithfully.
+ * Note that we count actual space used (as shown by GetMemoryChunkSpace)
+ * rather than the originally-requested size.  This is important since
+ * palloc can add substantial overhead.  It's not a complete answer since
+ * we won't count any wasted space in palloc allocation blocks, but it's
+ * a lot better than what we were doing before 7.3.
  *
  *--------------------
  */
@@ -228,6 +213,8 @@ tuplestore_begin_common(bool randomAccess, int maxKBytes)
 		state->memtupsize = 1;	/* won't really need any space */
 	state->memtuples = (void **) palloc(state->memtupsize * sizeof(void *));
 
+	USEMEM(state, GetMemoryChunkSpace(state->memtuples));
+
 	return state;
 }
 
@@ -286,10 +273,12 @@ tuplestore_puttuple(Tuplestorestate *state, void *tuple)
 			if (state->memtupcount >= state->memtupsize)
 			{
 				/* Grow the array as needed. */
+				FREEMEM(state, GetMemoryChunkSpace(state->memtuples));
 				state->memtupsize *= 2;
 				state->memtuples = (void **)
 					repalloc(state->memtuples,
 							 state->memtupsize * sizeof(void *));
+				USEMEM(state, GetMemoryChunkSpace(state->memtuples));
 			}
 			state->memtuples[state->memtupcount++] = tuple;
 
@@ -631,8 +620,9 @@ copytup_heap(Tuplestorestate *state, void *tup)
 {
 	HeapTuple	tuple = (HeapTuple) tup;
 
-	USEMEM(state, HEAPTUPLESIZE + tuple->t_len);
-	return (void *) heap_copytuple(tuple);
+	tuple = heap_copytuple(tuple);
+	USEMEM(state, GetMemoryChunkSpace(tuple));
+	return (void *) tuple;
 }
 
 /*
@@ -657,7 +647,7 @@ writetup_heap(Tuplestorestate *state, void *tup)
 						 sizeof(tuplen)) != sizeof(tuplen))
 			elog(ERROR, "tuplestore: write failed");
 
-	FREEMEM(state, HEAPTUPLESIZE + tuple->t_len);
+	FREEMEM(state, GetMemoryChunkSpace(tuple));
 	heap_freetuple(tuple);
 }
 
@@ -667,7 +657,7 @@ readtup_heap(Tuplestorestate *state, unsigned int len)
 	unsigned int tuplen = len - sizeof(unsigned int) + HEAPTUPLESIZE;
 	HeapTuple	tuple = (HeapTuple) palloc(tuplen);
 
-	USEMEM(state, tuplen);
+	USEMEM(state, GetMemoryChunkSpace(tuple));
 	/* reconstruct the HeapTupleData portion */
 	tuple->t_len = len - sizeof(unsigned int);
 	ItemPointerSetInvalid(&(tuple->t_self));

src/include/nodes/memnodes.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2002, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $Id: memnodes.h,v 1.24 2002/06/20 20:29:50 momjian Exp $
+ * $Id: memnodes.h,v 1.25 2002/08/12 00:36:12 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -42,10 +42,11 @@ typedef struct MemoryContextMethods
 	void		(*init) (MemoryContext context);
 	void		(*reset) (MemoryContext context);
 	void		(*delete) (MemoryContext context);
+	Size		(*get_chunk_space) (MemoryContext context, void *pointer);
+	void		(*stats) (MemoryContext context);
 #ifdef MEMORY_CONTEXT_CHECKING
 	void		(*check) (MemoryContext context);
 #endif
-	void		(*stats) (MemoryContext context);
 } MemoryContextMethods;
 
 

src/include/utils/memutils.h

@@ -10,7 +10,7 @@
  * Portions Copyright (c) 1996-2002, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $Id: memutils.h,v 1.46 2002/06/20 20:29:53 momjian Exp $
+ * $Id: memutils.h,v 1.47 2002/08/12 00:36:12 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -82,8 +82,11 @@ extern void MemoryContextDelete(MemoryContext context);
 extern void MemoryContextResetChildren(MemoryContext context);
 extern void MemoryContextDeleteChildren(MemoryContext context);
 extern void MemoryContextResetAndDeleteChildren(MemoryContext context);
+extern Size GetMemoryChunkSpace(void *pointer);
 extern void MemoryContextStats(MemoryContext context);
+#ifdef MEMORY_CONTEXT_CHECKING
 extern void MemoryContextCheck(MemoryContext context);
+#endif
 extern bool MemoryContextContains(MemoryContext context, void *pointer);
 
 /*