Final stage of psort reconstruction work: replace psort.c with

a generalized module 'tuplesort.c' that can sort either HeapTuples or
IndexTuples, and is not tied to execution of a Sort node.  Clean up
memory leakages in sorting, and replace nbtsort.c's private implementation
of mergesorting with calls to tuplesort.c.

src/backend/access/nbtree/nbtree.c

 /*-------------------------------------------------------------------------
  *
- * btree.c
+ * nbtree.c
  *	  Implementation of Lehman and Yao's btree management algorithm for
  *	  Postgres.
  *
- * Copyright (c) 1994, Regents of the University of California
+ * NOTES
+ *	  This file contains only the public interface routines.
  *
  *
- * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/access/nbtree/nbtree.c,v 1.46 1999/09/18 19:06:10 tgl Exp $
+ * Copyright (c) 1994, Regents of the University of California
  *
- * NOTES
- *	  This file contains only the public interface routines.
+ * IDENTIFICATION
+ *	  $Header: /cvsroot/pgsql/src/backend/access/nbtree/nbtree.c,v 1.47 1999/10/17 22:15:03 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -76,7 +76,7 @@ btbuild(Relation heap,
 #endif
 	Node	   *pred,
 			   *oldPred;
-	void	   *spool = (void *) NULL;
+	BTSpool	   *spool = NULL;
 	bool		isunique;
 	bool		usefast;
 
@@ -147,7 +147,7 @@ btbuild(Relation heap,
 
 	if (usefast)
 	{
-		spool = _bt_spoolinit(index, 7, isunique);
+		spool = _bt_spoolinit(index, isunique);
 		res = (InsertIndexResult) NULL;
 	}
 
@@ -249,11 +249,11 @@ btbuild(Relation heap,
 
 		/*
 		 * if we are doing bottom-up btree build, we insert the index into
-		 * a spool page for subsequent processing.	otherwise, we insert
+		 * a spool file for subsequent processing.	otherwise, we insert
 		 * into the btree.
 		 */
 		if (usefast)
-			_bt_spool(index, btitem, spool);
+			_bt_spool(btitem, spool);
 		else
 			res = _bt_doinsert(index, btitem, isunique, heap);
 
@@ -275,15 +275,13 @@ btbuild(Relation heap,
 	}
 
 	/*
-	 * if we are doing bottom-up btree build, we now have a bunch of
-	 * sorted runs in the spool pages.	finish the build by (1) merging
-	 * the runs, (2) inserting the sorted tuples into btree pages and (3)
-	 * building the upper levels.
+	 * if we are doing bottom-up btree build, finish the build by
+	 * (1) completing the sort of the spool file, (2) inserting the
+	 * sorted tuples into btree pages and (3) building the upper levels.
 	 */
 	if (usefast)
 	{
-		_bt_spool(index, (BTItem) NULL, spool); /* flush the spool */
-		_bt_leafbuild(index, spool);
+		_bt_leafbuild(spool);
 		_bt_spooldestroy(spool);
 	}
 

src/backend/executor/nodeSort.c

@@ -7,16 +7,17 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/executor/nodeSort.c,v 1.23 1999/07/17 20:16:58 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/executor/nodeSort.c,v 1.24 1999/10/17 22:15:02 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
 
 #include "postgres.h"
+
 #include "executor/executor.h"
 #include "executor/execdebug.h"
 #include "executor/nodeSort.h"
-#include "utils/psort.h"
+#include "utils/tuplesort.h"
 
 /* ----------------------------------------------------------------
  *		FormSortKeys(node)
@@ -83,11 +84,9 @@ FormSortKeys(Sort *sortnode)
 /* ----------------------------------------------------------------
  *		ExecSort
  *
- * old comments
- *		Sorts tuples from the outer subtree of the node in psort,
+ *		Sorts tuples from the outer subtree of the node using tuplesort,
  *		which saves the results in a temporary file or memory. After the
  *		initial call, returns a tuple from the file with each call.
- *		Assumes that heap access method is used.
  *
  *		Conditions:
  *		  -- none.
@@ -101,10 +100,8 @@ ExecSort(Sort *node)
 {
 	EState	   *estate;
 	SortState  *sortstate;
-	Plan	   *outerNode;
 	ScanDirection dir;
-	int			keycount;
-	ScanKey		sortkeys;
+	Tuplesortstate *tuplesortstate;
 	HeapTuple	heapTuple;
 	TupleTableSlot *slot;
 	bool		should_free;
@@ -119,43 +116,71 @@ ExecSort(Sort *node)
 	sortstate = node->sortstate;
 	estate = node->plan.state;
 	dir = estate->es_direction;
+	tuplesortstate = (Tuplesortstate *) sortstate->tuplesortstate;
 
 	/* ----------------
-	 *	the first time we call this, psort sorts this into a file.
-	 *	Subsequent calls return tuples from psort.
+	 *	If first time through, read all tuples from outer plan and
+	 *	pass them to tuplesort.c.
+	 *	Subsequent calls just fetch tuples from tuplesort.
 	 * ----------------
 	 */
 
-	if (sortstate->sort_Flag == false)
+	if (! sortstate->sort_Done)
 	{
+		Plan	   *outerNode;
+		TupleDesc	tupDesc;
+		int			keycount;
+		ScanKey		sortkeys;
+
 		SO1_printf("ExecSort: %s\n",
-				   "sortstate == false -> sorting subplan");
+				   "sorting subplan");
 		/* ----------------
-		 *	set all relations to be scanned in the forward direction
-		 *	while creating the temporary relation.
+		 *	Want to scan subplan in the forward direction while creating
+		 *	the sorted data.  (Does setting my direction actually affect
+		 *	the subplan?  I bet this is useless code...)
 		 * ----------------
 		 */
 		estate->es_direction = ForwardScanDirection;
 
 		/* ----------------
-		 *	 prepare information for psort_begin()
+		 *	 Initialize tuplesort module.
 		 * ----------------
 		 */
-		outerNode = outerPlan((Plan *) node);
+		SO1_printf("ExecSort: %s\n",
+				   "calling tuplesort_begin");
 
+		outerNode = outerPlan((Plan *) node);
+		tupDesc = ExecGetTupType(outerNode);
 		keycount = node->keycount;
 		sortkeys = (ScanKey) sortstate->sort_Keys;
-		SO1_printf("ExecSort: %s\n",
-				   "calling psort_begin");
 
-		if (!psort_begin(node,	/* this node */
-						 keycount,		/* number keys */
-						 sortkeys))		/* keys */
+		tuplesortstate = tuplesort_begin_heap(tupDesc, keycount, sortkeys,
+											  true /* randomAccess */);
+
+		sortstate->tuplesortstate = (void *) tuplesortstate;
+
+		/* ----------------
+		 *	 Scan the subplan and feed all the tuples to tuplesort.
+		 * ----------------
+		 */
+
+		for (;;)
 		{
-			/* Psort says, there are no tuples to be sorted */
-			return NULL;
+			slot = ExecProcNode(outerNode, (Plan *) node);
+
+			if (TupIsNull(slot))
+				break;
+
+			tuplesort_puttuple(tuplesortstate, (void *) slot->val);
+			ExecClearTuple(slot);
 		}
 
+		/* ----------------
+		 *	 Complete the sort.
+		 * ----------------
+		 */
+		tuplesort_performsort(tuplesortstate);
+
 		/* ----------------
 		 *	 restore to user specified direction
 		 * ----------------
@@ -167,25 +192,29 @@ ExecSort(Sort *node)
 		 * ----------------
 		 */
 		slot = (TupleTableSlot *) sortstate->csstate.cstate.cs_ResultTupleSlot;
-		slot->ttc_tupleDescriptor = ExecGetTupType(outerNode);
+		slot->ttc_tupleDescriptor = tupDesc;
+
 		/* ----------------
 		 *	finally set the sorted flag to true
 		 * ----------------
 		 */
-		sortstate->sort_Flag = true;
+		sortstate->sort_Done = true;
 		SO1_printf(stderr, "ExecSort: sorting done.\n");
 	}
 	else
 		slot = (TupleTableSlot *) sortstate->csstate.cstate.cs_ResultTupleSlot;
 
 	SO1_printf("ExecSort: %s\n",
-			   "retrieving tuple from sorted relation");
+			   "retrieving tuple from tuplesort");
 
 	/* ----------------
-	 *	at this point we grab a tuple from psort
+	 *	Get the first or next tuple from tuplesort.
+	 *	Returns NULL if no more tuples.
 	 * ----------------
 	 */
-	heapTuple = psort_grabtuple(node, &should_free);
+	heapTuple = tuplesort_getheaptuple(tuplesortstate,
+									   ScanDirectionIsForward(dir),
+									   &should_free);
 
 	return ExecStoreTuple(heapTuple, slot, InvalidBuffer, should_free);
 }
@@ -193,7 +222,6 @@ ExecSort(Sort *node)
 /* ----------------------------------------------------------------
  *		ExecInitSort
  *
- * old comments
  *		Creates the run-time state information for the sort node
  *		produced by the planner and initailizes its outer subtree.
  * ----------------------------------------------------------------
@@ -203,7 +231,6 @@ ExecInitSort(Sort *node, EState *estate, Plan *parent)
 {
 	SortState  *sortstate;
 	Plan	   *outerPlan;
-	ScanKey		sortkeys;
 
 	SO1_printf("ExecInitSort: %s\n",
 			   "initializing sort node");
@@ -219,14 +246,14 @@ ExecInitSort(Sort *node, EState *estate, Plan *parent)
 	 * ----------------
 	 */
 	sortstate = makeNode(SortState);
-	sortstate->sort_Flag = 0;
+	sortstate->sort_Done = false;
 	sortstate->sort_Keys = NULL;
-	node->cleaned = FALSE;
+	sortstate->tuplesortstate = NULL;
 
 	node->sortstate = sortstate;
 
 	/* ----------------
-	 *	Miscellanious initialization
+	 *	Miscellaneous initialization
 	 *
 	 *		 +	assign node's base_id
 	 *		 +	assign debugging hooks
@@ -259,9 +286,7 @@ ExecInitSort(Sort *node, EState *estate, Plan *parent)
 	 *	initialize sortstate information
 	 * ----------------
 	 */
-	sortkeys = FormSortKeys(node);
-	sortstate->sort_Keys = sortkeys;
-	sortstate->sort_Flag = false;
+	sortstate->sort_Keys = FormSortKeys(node);
 
 	/* ----------------
 	 *	initialize tuple type.	no need to initialize projection
@@ -275,11 +300,6 @@ ExecInitSort(Sort *node, EState *estate, Plan *parent)
 	SO1_printf("ExecInitSort: %s\n",
 			   "sort node initialized");
 
-	/* ----------------
-	 *	return relation oid of temporary sort relation in a list
-	 *	(someday -- for now we return LispTrue... cim 10/12/89)
-	 * ----------------
-	 */
 	return TRUE;
 }
 
@@ -293,8 +313,6 @@ ExecCountSlotsSort(Sort *node)
 
 /* ----------------------------------------------------------------
  *		ExecEndSort(node)
- *
- * old comments
  * ----------------------------------------------------------------
  */
 void
@@ -325,8 +343,13 @@ ExecEndSort(Sort *node)
 	 */
 	ExecClearTuple(sortstate->csstate.css_ScanTupleSlot);
 
-	/* Clean up after psort */
-	psort_end(node);
+	/* ----------------
+	 *	Release tuplesort resources
+	 * ----------------
+	 */
+	if (sortstate->tuplesortstate != NULL)
+		tuplesort_end((Tuplesortstate *) sortstate->tuplesortstate);
+	sortstate->tuplesortstate = NULL;
 
 	SO1_printf("ExecEndSort: %s\n",
 			   "sort node shutdown");
@@ -335,51 +358,47 @@ ExecEndSort(Sort *node)
 /* ----------------------------------------------------------------
  *		ExecSortMarkPos
  *
- *		Calls psort to save the current position in the sorted file.
+ *		Calls tuplesort to save the current position in the sorted file.
  * ----------------------------------------------------------------
  */
 void
 ExecSortMarkPos(Sort *node)
 {
-	SortState  *sortstate;
+	SortState  *sortstate = node->sortstate;
 
 	/* ----------------
 	 *	if we haven't sorted yet, just return
 	 * ----------------
 	 */
-	sortstate = node->sortstate;
-	if (sortstate->sort_Flag == false)
+	if (! sortstate->sort_Done)
 		return;
 
-	psort_markpos(node);
-
-	return;
+	tuplesort_markpos((Tuplesortstate *) sortstate->tuplesortstate);
 }
 
 /* ----------------------------------------------------------------
  *		ExecSortRestrPos
  *
- *		Calls psort to restore the last saved sort file position.
+ *		Calls tuplesort to restore the last saved sort file position.
  * ----------------------------------------------------------------
  */
 void
 ExecSortRestrPos(Sort *node)
 {
-	SortState  *sortstate;
+	SortState  *sortstate = node->sortstate;
 
 	/* ----------------
 	 *	if we haven't sorted yet, just return.
 	 * ----------------
 	 */
-	sortstate = node->sortstate;
-	if (sortstate->sort_Flag == false)
+	if (! sortstate->sort_Done)
 		return;
 
 	/* ----------------
 	 *	restore the scan to the previously marked position
 	 * ----------------
 	 */
-	psort_restorepos(node);
+	tuplesort_restorepos((Tuplesortstate *) sortstate->tuplesortstate);
 }
 
 void
@@ -392,17 +411,25 @@ ExecReScanSort(Sort *node, ExprContext *exprCtxt, Plan *parent)
 	 * not NULL then it will be re-scanned by ExecProcNode, else - no
 	 * reason to re-scan it at all.
 	 */
-	if (sortstate->sort_Flag == false)
+	if (! sortstate->sort_Done)
 		return;
 
 	ExecClearTuple(sortstate->csstate.cstate.cs_ResultTupleSlot);
 
-	psort_rescan(node);
-
 	/*
-	 * If subnode is to be rescanned then we aren't sorted
+	 * If subnode is to be rescanned then we forget previous sort
+	 * results; we have to re-read the subplan and re-sort.
+	 *
+	 * Otherwise we can just rewind and rescan the sorted output.
 	 */
 	if (((Plan *) node)->lefttree->chgParam != NULL)
-		sortstate->sort_Flag = false;
-
+	{
+		sortstate->sort_Done = false;
+		tuplesort_end((Tuplesortstate *) sortstate->tuplesortstate);
+		sortstate->tuplesortstate = NULL;
+	}
+	else
+	{
+		tuplesort_rescan((Tuplesortstate *) sortstate->tuplesortstate);
+	}
 }

src/backend/utils/sort/Makefile

@@ -4,7 +4,7 @@
 #    Makefile for utils/sort
 #
 # IDENTIFICATION
-#    $Header: /cvsroot/pgsql/src/backend/utils/sort/Makefile,v 1.6 1999/10/16 19:49:27 tgl Exp $
+#    $Header: /cvsroot/pgsql/src/backend/utils/sort/Makefile,v 1.7 1999/10/17 22:15:05 tgl Exp $
 #
 #-------------------------------------------------------------------------
 
@@ -13,7 +13,7 @@ include ../../../Makefile.global
 
 CFLAGS += -I../..
 
-OBJS = logtape.o lselect.o psort.o
+OBJS = logtape.o tuplesort.o
 
 all: SUBSYS.o
 

src/backend/utils/sort/logtape.c

@@ -4,8 +4,8 @@
  *	  Management of "logical tapes" within temporary files.
  *
  * This module exists to support sorting via multiple merge passes (see
- * psort.c).  Merging is an ideal algorithm for tape devices, but if we
- * implement it on disk by creating a separate file for each "tape",
+ * tuplesort.c).  Merging is an ideal algorithm for tape devices, but if
+ * we implement it on disk by creating a separate file for each "tape",
  * there is an annoying problem: the peak space usage is at least twice
  * the volume of actual data to be sorted.  (This must be so because each
  * datum will appear in both the input and output tapes of the final
@@ -23,7 +23,7 @@
  * Few OSes allow arbitrary parts of a file to be released back to the OS,
  * so we have to implement this space-recycling ourselves within a single
  * logical file.  logtape.c exists to perform this bookkeeping and provide
- * the illusion of N independent tape devices to psort.c.  Note that
+ * the illusion of N independent tape devices to tuplesort.c.  Note that
  * logtape.c itself depends on buffile.c to provide a "logical file" of
  * larger size than the underlying OS may support.
  *
@@ -63,7 +63,7 @@
  * Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/utils/sort/logtape.c,v 1.1 1999/10/16 19:49:27 tgl Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/utils/sort/logtape.c,v 1.2 1999/10/17 22:15:05 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */

src/include/access/nbtree.h

@@ -6,7 +6,7 @@
  *
  * Copyright (c) 1994, Regents of the University of California
  *
- * $Id: nbtree.h,v 1.31 1999/08/08 20:12:49 tgl Exp $
+ * $Id: nbtree.h,v 1.32 1999/10/17 22:15:03 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -288,9 +288,12 @@ extern BTItem _bt_formitem(IndexTuple itup);
 /*
  * prototypes for functions in nbtsort.c
  */
-extern void *_bt_spoolinit(Relation index, int ntapes, bool isunique);
-extern void _bt_spooldestroy(void *spool);
-extern void _bt_spool(Relation index, BTItem btitem, void *spool);
-extern void _bt_leafbuild(Relation index, void *spool);
+
+typedef struct BTSpool BTSpool;	/* opaque type known only within nbtsort.c */
+
+extern BTSpool *_bt_spoolinit(Relation index, bool isunique);
+extern void _bt_spooldestroy(BTSpool *btspool);
+extern void _bt_spool(BTItem btitem, BTSpool *btspool);
+extern void _bt_leafbuild(BTSpool *btspool);
 
 #endif	 /* NBTREE_H */

src/include/nodes/execnodes.h

@@ -6,7 +6,7 @@
  *
  * Copyright (c) 1994, Regents of the University of California
  *
- * $Id: execnodes.h,v 1.36 1999/09/26 21:21:04 tgl Exp $
+ * $Id: execnodes.h,v 1.37 1999/10/17 22:15:07 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -597,17 +597,9 @@ typedef struct GroupState
 /* ----------------
  *	 SortState information
  *
- *|		sort nodes are really just a kind of a scan since
- *|		we implement sorts by retrieving the entire subplan
- *|		into a temp relation, sorting the temp relation into
- *|		another sorted relation, and then preforming a simple
- *|		unqualified sequential scan on the sorted relation..
- *|		-cim 10/15/89
- *
- *		Flag			indicated whether relation has been sorted
- *		Keys			scan key structures used to keep info on sort keys
- *		TempRelation	temporary relation containing result of executing
- *						the subplan.
+ *		sort_Done		indicates whether sort has been performed yet
+ *		sort_Keys		scan key structures describing the sort keys
+ *		tuplesortstate	private state of tuplesort.c
  *
  *	 CommonScanState information
  *
@@ -628,9 +620,9 @@ typedef struct GroupState
 typedef struct SortState
 {
 	CommonScanState csstate;	/* its first field is NodeTag */
-	bool		sort_Flag;
+	bool		sort_Done;
 	ScanKey		sort_Keys;
-	bool		cleaned;
+	void	   *tuplesortstate;
 } SortState;
 
 /* ----------------

src/include/nodes/plannodes.h

@@ -6,7 +6,7 @@
  *
  * Copyright (c) 1994, Regents of the University of California
  *
- * $Id: plannodes.h,v 1.30 1999/08/21 03:49:09 tgl Exp $
+ * $Id: plannodes.h,v 1.31 1999/10/17 22:15:07 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -288,8 +288,6 @@ typedef struct Sort
 	Oid			nonameid;
 	int			keycount;
 	SortState  *sortstate;
-	void	   *psortstate;
-	bool		cleaned;
 } Sort;
 
 /* ----------------

src/include/utils/tuplesort.h

+/*-------------------------------------------------------------------------
+ *
+ * tuplesort.h
+ *	  Generalized tuple sorting routines.
+ *
+ * This module handles sorting of either heap tuples or index tuples
+ * (and could fairly easily support other kinds of sortable objects,
+ * if necessary).  It works efficiently for both small and large amounts
+ * of data.  Small amounts are sorted in-memory using qsort().  Large
+ * amounts are sorted using temporary files and a standard external sort
+ * algorithm.
+ *
+ * Copyright (c) 1994, Regents of the University of California
+ *
+ * $Id: tuplesort.h,v 1.1 1999/10/17 22:15:09 tgl Exp $
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef TUPLESORT_H
+#define TUPLESORT_H
+
+#include "access/htup.h"
+#include "access/itup.h"
+#include "access/skey.h"
+#include "access/tupdesc.h"
+#include "utils/rel.h"
+
+/* Tuplesortstate is an opaque type whose details are not known outside tuplesort.c. */
+
+typedef struct Tuplesortstate Tuplesortstate;
+
+/*
+ * We provide two different interfaces to what is essentially the same
+ * code: one for sorting HeapTuples and one for sorting IndexTuples.
+ * They differ primarily in the way that the sort key information is
+ * supplied.
+ */
+
+extern Tuplesortstate *tuplesort_begin_heap(TupleDesc tupDesc,
+											int nkeys, ScanKey keys,
+											bool randomAccess);
+extern Tuplesortstate *tuplesort_begin_index(Relation indexRel,
+											 bool enforceUnique,
+											 bool randomAccess);
+
+extern void tuplesort_puttuple(Tuplesortstate *state, void *tuple);
+
+extern void tuplesort_performsort(Tuplesortstate *state);
+
+extern void *tuplesort_gettuple(Tuplesortstate *state, bool forward,
+								bool *should_free);
+#define tuplesort_getheaptuple(state, forward, should_free) \
+	((HeapTuple) tuplesort_gettuple(state, forward, should_free))
+#define tuplesort_getindextuple(state, forward, should_free) \
+	((IndexTuple) tuplesort_gettuple(state, forward, should_free))
+
+extern void tuplesort_end(Tuplesortstate *state);
+
+/*
+ * These routines may only be called if randomAccess was specified 'true'.
+ * Backwards scan in gettuple is likewise only allowed if randomAccess.
+ */
+
+extern void tuplesort_rescan(Tuplesortstate *state);
+extern void tuplesort_markpos(Tuplesortstate *state);
+extern void tuplesort_restorepos(Tuplesortstate *state);
+
+#endif	 /* TUPLESORT_H */