2nd try for the index tuple toast hack. This time as suggested

by Tom.

Jan

src/backend/access/common/indextuple.c

@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/access/common/indextuple.c,v 1.43 2000/04/12 17:14:37 momjian Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/access/common/indextuple.c,v 1.44 2000/07/22 11:18:45 wieck Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -17,6 +17,7 @@
 #include "postgres.h"
 
 #include "access/heapam.h"
+#include "access/tuptoaster.h"
 #include "access/itup.h"
 #include "catalog/pg_type.h"
 
@@ -44,11 +45,40 @@ index_formtuple(TupleDesc tupleDescriptor,
 	bool		hasnull = false;
 	uint16		tupmask = 0;
 	int			numberOfAttributes = tupleDescriptor->natts;
+#ifdef TOAST_INDEX_HACK
+	Datum		untoasted_value[MaxHeapAttributeNumber];
+	bool		untoasted_free[MaxHeapAttributeNumber];
+#endif
 
 	if (numberOfAttributes > INDEX_MAX_KEYS)
 		elog(ERROR, "index_formtuple: numberOfAttributes %d > %d",
 			 numberOfAttributes, INDEX_MAX_KEYS);
 
+#ifdef TOAST_INDEX_HACK
+	for (i = 0; i < numberOfAttributes; i++)
+	{
+		if (null[i] != ' ' || tupleDescriptor->attrs[i]->attlen >= 0)
+		{
+			untoasted_value[i] = value[i];
+			untoasted_free[i] = false;
+		}
+		else
+		{
+			if (VARATT_IS_EXTERNAL(value[i]))
+			{
+				untoasted_value[i] = PointerGetDatum(
+						heap_tuple_fetch_attr(
+						(varattrib *)DatumGetPointer(value[i])));
+				untoasted_free[i] = true;
+			}
+			else
+			{
+				untoasted_value[i] = value[i];
+				untoasted_free[i] = false;
+			}
+		}
+	}
+#endif
 	for (i = 0; i < numberOfAttributes && !hasnull; i++)
 	{
 		if (null[i] != ' ')
@@ -59,7 +89,11 @@ index_formtuple(TupleDesc tupleDescriptor,
 		infomask |= INDEX_NULL_MASK;
 
 	hoff = IndexInfoFindDataOffset(infomask);
+#ifdef TOAST_INDEX_HACK
+	size = hoff + ComputeDataSize(tupleDescriptor, untoasted_value, null);
+#else
 	size = hoff + ComputeDataSize(tupleDescriptor, value, null);
+#endif
 	size = MAXALIGN(size);		/* be conservative */
 
 	tp = (char *) palloc(size);
@@ -68,11 +102,23 @@ index_formtuple(TupleDesc tupleDescriptor,
 
 	DataFill((char *) tp + hoff,
 			 tupleDescriptor,
+#ifdef TOAST_INDEX_HACK
+			 untoasted_value,
+#else
 			 value,
+#endif
 			 null,
 			 &tupmask,
 			 (hasnull ? (bits8 *) tp + sizeof(*tuple) : NULL));
 
+#ifdef TOAST_INDEX_HACK
+	for (i = 0; i < numberOfAttributes; i++)
+	{
+		if (untoasted_free[i])
+			pfree(DatumGetPointer(untoasted_value[i]));
+	}
+#endif
+
 	/*
 	 * We do this because DataFill wants to initialize a "tupmask" which
 	 * is used for HeapTuples, but we want an indextuple infomask.	The

src/backend/access/heap/heapam.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/access/heap/heapam.c,v 1.81 2000/07/21 11:18:51 wieck Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/access/heap/heapam.c,v 1.82 2000/07/22 11:18:46 wieck Exp $
  *
  *
  * INTERFACE ROUTINES
@@ -1274,10 +1274,6 @@ Oid
 heap_insert(Relation relation, HeapTuple tup)
 {
 	Buffer buffer;
-#ifndef TOAST_INDICES
-	HeapTupleHeader plaintdata = NULL;
-	int32			plaintlen = 0;
-#endif
 
 	/* increment access statistics */
 	tup->tableOid = relation->rd_id;
@@ -1313,11 +1309,7 @@ heap_insert(Relation relation, HeapTuple tup)
 	 */
     if (HeapTupleHasExtended(tup) ||
 				(MAXALIGN(tup->t_len) > (MaxTupleSize / 4)))
-#ifdef TOAST_INDICES
 		heap_tuple_toast_attrs(relation, tup, NULL);
-#else
-		heap_tuple_toast_attrs(relation, tup, NULL, &plaintdata, &plaintlen);
-#endif
 #endif
 
 	/* Find buffer for this tuple */
@@ -1355,20 +1347,6 @@ heap_insert(Relation relation, HeapTuple tup)
 	if (IsSystemRelationName(RelationGetRelationName(relation)))
 		RelationMark4RollbackHeapTuple(relation, tup);
 
-#ifndef TOAST_INDICES
-    if (plaintdata != NULL && tup->t_data != plaintdata)
-	{
-		if (tup->t_datamcxt != NULL && (char *) (tup->t_data) !=
-					((char *) tup + HEAPTUPLESIZE))
-		{
-			MemoryContext oldcxt = MemoryContextSwitchTo(tup->t_datamcxt);
-			pfree(tup->t_data);
-			MemoryContextSwitchTo(oldcxt);
-		}
-	    tup->t_data = plaintdata;
-		tup->t_len  = plaintlen;
-	}
-#endif
 	return tup->t_data->t_oid;
 }
 
@@ -1483,11 +1461,7 @@ l1:
 	 * ----------
 	 */
 	if (HeapTupleHasExtended(&tp))
-#ifdef TOAST_INDICES
 		heap_tuple_toast_attrs(relation, NULL, &(tp));
-#else
-		heap_tuple_toast_attrs(relation, NULL, &(tp), NULL, NULL);
-#endif
 #endif
 
 	LockBuffer(buffer, BUFFER_LOCK_UNLOCK);
@@ -1512,10 +1486,6 @@ heap_update(Relation relation, ItemPointer otid, HeapTuple newtup,
 	PageHeader	dp;
 	Buffer		buffer, newbuf;
 	int			result;
-#ifndef TOAST_INDICES
-	HeapTupleHeader plaintdata = NULL;
-	int32			plaintlen  = 0;
-#endif
 
 	newtup->tableOid = relation->rd_id;
 	/* increment access statistics */
@@ -1604,11 +1574,7 @@ l2:
 	if (HeapTupleHasExtended(&oldtup) || 
 			HeapTupleHasExtended(newtup) ||
 			(MAXALIGN(newtup->t_len) > (MaxTupleSize / 4)))
-#ifdef TOAST_INDICES
 		heap_tuple_toast_attrs(relation, newtup, &oldtup);
-#else
-		heap_tuple_toast_attrs(relation, newtup, &oldtup, &plaintdata, &plaintlen);
-#endif
 #endif
 
 	/* Find buffer for new tuple */
@@ -1671,21 +1637,6 @@ l2:
 	RelationInvalidateHeapTuple(relation, &oldtup);
 	RelationMark4RollbackHeapTuple(relation, newtup);
 
-#ifndef TOAST_INDICES
-    if (plaintdata != NULL && newtup->t_data != plaintdata)
-	{
-		if (newtup->t_datamcxt != NULL && (char *) (newtup->t_data) !=
-					((char *) newtup + HEAPTUPLESIZE))
-		{
-			MemoryContext oldcxt = MemoryContextSwitchTo(newtup->t_datamcxt);
-			pfree(newtup->t_data);
-			MemoryContextSwitchTo(oldcxt);
-		}
-	    newtup->t_data = plaintdata;
-		newtup->t_len  = plaintlen;
-	}
-#endif
-
 	return HeapTupleMayBeUpdated;
 }
 

src/backend/access/heap/tuptoaster.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $Header: /cvsroot/pgsql/src/backend/access/heap/tuptoaster.c,v 1.8 2000/07/21 10:31:30 wieck Exp $
+ *	  $Header: /cvsroot/pgsql/src/backend/access/heap/tuptoaster.c,v 1.9 2000/07/22 11:18:46 wieck Exp $
  *
  *
  * INTERFACE ROUTINES
@@ -43,14 +43,8 @@
 
 static void			toast_delete(Relation rel, HeapTuple oldtup);
 static void			toast_delete_datum(Relation rel, Datum value);
-#ifdef TOAST_INDICES
 static void			toast_insert_or_update(Relation rel, HeapTuple newtup,
 								HeapTuple oldtup);
-#else
-static void			toast_insert_or_update(Relation rel, HeapTuple newtup,
-								HeapTuple oldtup, HeapTupleHeader *plaintdata,
-								int32 *plaintlen);
-#endif
 static Datum		toast_compress_datum(Datum value);
 static Datum		toast_save_datum(Relation rel, Oid mainoid, int16 attno, Datum value);
 								
@@ -65,7 +59,6 @@ static varattrib   *toast_fetch_datum(varattrib *attr);
  *	Calls the appropriate event specific action.
  * ----------
  */
-#ifdef TOAST_INDICES
 void
 heap_tuple_toast_attrs(Relation rel, HeapTuple newtup, HeapTuple oldtup)
 {
@@ -74,17 +67,39 @@ heap_tuple_toast_attrs(Relation rel, HeapTuple newtup, HeapTuple oldtup)
 	else
 		toast_insert_or_update(rel, newtup, oldtup);
 }
-#else
-void
-heap_tuple_toast_attrs(Relation rel, HeapTuple newtup, 
-			HeapTuple oldtup, HeapTupleHeader *plaintdata, int32 *plaintlen)
+
+
+/* ----------
+ * heap_tuple_fetch_attr -
+ *
+ *	Public entry point to get back a toasted value 
+ *	external storage (possibly still in compressed format).
+ * ----------
+ */
+varattrib *
+heap_tuple_fetch_attr(varattrib *attr)
 {
-	if (newtup == NULL)
-		toast_delete(rel, oldtup);
+	varattrib	*result;
+
+	if (VARATT_IS_EXTERNAL(attr))
+	{
+		/* ----------
+		 * This is an external stored plain value
+		 * ----------
+		 */
+		result = toast_fetch_datum(attr);
+	}
 	else
-		toast_insert_or_update(rel, newtup, oldtup, plaintdata, plaintlen);
+	{
+		/* ----------
+		 * This is a plain value inside of the main tuple - why am I called?
+		 * ----------
+		 */
+		result = attr;
+    }
+
+	return result;
 }
-#endif
 
 
 /* ----------
@@ -199,12 +214,7 @@ toast_delete(Relation rel, HeapTuple oldtup)
  * ----------
  */
 static void
-#ifdef TOAST_INDICES
 toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup)
-#else
-toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
-				HeapTupleHeader *plaintdata, int32 *plaintlen)
-#endif
 {
 	TupleDesc			tupleDesc;
 	Form_pg_attribute  *att;
@@ -227,12 +237,6 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 	bool				toast_free[MaxHeapAttributeNumber];
 	bool				toast_delold[MaxHeapAttributeNumber];
 
-#ifndef TOAST_INDICES
-	bool				need_plain  = false;
-	Datum				toast_plains[MaxHeapAttributeNumber];
-	bool				toast_freeplain[MaxHeapAttributeNumber];
-#endif
-
 	/* ----------
 	 * Get the tuple descriptor, the number of and attribute
 	 * descriptors and the location of the tuple values.
@@ -249,7 +253,6 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 	memset(toast_action,    ' ', numAttrs * sizeof(char));
 	memset(toast_nulls,     ' ', numAttrs * sizeof(char));
 	memset(toast_free,      0,   numAttrs * sizeof(bool));
-	memset(toast_freeplain, 0,   numAttrs * sizeof(bool));
 	memset(toast_delold,    0,   numAttrs * sizeof(bool));
 	for (i = 0; i < numAttrs; i++)
 	{
@@ -300,25 +303,6 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 					 */
 					toast_action[i] = 'p';
 					toast_sizes[i] = VARATT_SIZE(toast_values[i]);
-
-#ifndef TOAST_INDICES
-					/* ----------
-					 * But the tuple returned by the heap-am
-					 * function must not contain external references.
-					 * So we have to construct another plain tuple
-					 * later.
-					 * ----------
-					 */
-					if (att[i]->attstorage == 'x' || att[i]->attstorage == 'm')
-						toast_plains[i] = PointerGetDatum(
-								toast_fetch_datum(new_value));
-					else
-						toast_plains[i] = PointerGetDatum(
-								heap_tuple_untoast_attr(new_value));
-					toast_freeplain[i] = true;
-					need_plain = true;
-#endif
-
 					continue;
 				}
 			}
@@ -369,17 +353,10 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 			{
 				toast_values[i] = PointerGetDatum(heap_tuple_untoast_attr(
 					(varattrib *)DatumGetPointer(toast_values[i])));
-#ifndef TOAST_INDICES
-				toast_plains[i] = toast_values[i];
-#endif
 				toast_free[i] = true;
 				need_change = true;
 				need_free = true;
 			}
-#ifndef TOAST_INDICES
-			else
-				toast_plains[i] = toast_values[i];
-#endif
 
 			/* ----------
 			 * Remember the size of this attribute
@@ -395,9 +372,6 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 			 */
 			toast_action[i] = 'p';
 			toast_sizes[i]  = att[i]->attlen;
-#ifndef TOAST_INDICES
-			toast_plains[i] = toast_values[i];
-#endif
 		}
 	}
 
@@ -456,13 +430,8 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 		old_value			= toast_values[i];
 
 		toast_values[i]		= toast_compress_datum(toast_values[i]);
-#ifndef TOAST_INDICES
-		toast_plains[i]		= toast_values[i];
-#endif
-									
 		if (toast_free[i])
 			pfree(DatumGetPointer(old_value));
-
 		toast_free[i]		= true;
 		toast_sizes[i]		= VARATT_SIZE(toast_values[i]);
 
@@ -516,14 +485,8 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 									newtup->t_data->t_oid,
 									i + 1,
 									toast_values[i]);
-#ifndef TOAST_INDICES
-		need_plain = true;
-		if (toast_free[i])
-			toast_freeplain[i] = true;
-#else
 		if (toast_free[i])
 			pfree(DatumGetPointer(old_value));
-#endif
 
 		toast_free[i]		= true;
 		toast_sizes[i]		= VARATT_SIZE(toast_values[i]);
@@ -574,9 +537,6 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 		old_value			= toast_values[i];
 
 		toast_values[i]		= toast_compress_datum(toast_values[i]);
-#ifndef TOAST_INDICES
-		toast_plains[i]		= toast_values[i];
-#endif
 									
 		if (toast_free[i])
 			pfree(DatumGetPointer(old_value));
@@ -633,14 +593,8 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 									newtup->t_data->t_oid,
 									i + 1,
 									toast_values[i]);
-#ifndef TOAST_INDICES
-		need_plain = true;
-		if (toast_free[i])
-			toast_freeplain[i] = true;
-#else
 		if (toast_free[i])
 			pfree(DatumGetPointer(old_value));
-#endif
 
 		toast_free[i]		= true;
 		toast_sizes[i]		= VARATT_SIZE(toast_values[i]);
@@ -713,78 +667,14 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
 		MemoryContextSwitchTo(oldcxt);
 	}
 
-
-#ifndef TOAST_INDICES
-	/* ----------
-	 * In the case we toasted any values, we need to build
-	 * a new heap tuple with the changed values.
-	 * ----------
-	 */
-	if (need_plain)
-	{
-		int32			new_len;
-		MemoryContext	oldcxt;
-
-		/* ----------
-		 * Calculate the new size of the tuple
-		 * ----------
-		 */
-		new_len = offsetof(HeapTupleHeaderData, t_bits);
-		if (has_nulls)
-			new_len += BITMAPLEN(numAttrs);
-		new_len = MAXALIGN(new_len);
-		new_len += ComputeDataSize(tupleDesc, toast_plains, toast_nulls);
-
-		/* ----------
-		 * Switch to the memory context of the HeapTuple structure
-		 * and allocate the new tuple.
-		 * ----------
-		 */
-		oldcxt = MemoryContextSwitchTo(newtup->t_datamcxt);
-		*plaintdata = palloc(new_len);
-		*plaintlen  = new_len;
-
-		/* ----------
-		 * Put the tuple header and the changed values into place
-		 * ----------
-		 */
-		memcpy(*plaintdata, newtup->t_data, newtup->t_data->t_hoff);
-
-		DataFill((char *)(MAXALIGN((long)(*plaintdata) +
-						offsetof(HeapTupleHeaderData, t_bits) + 
-						((has_nulls) ? BITMAPLEN(numAttrs) : 0))),
-				tupleDesc,
-				toast_plains,
-				toast_nulls,
-				&((*plaintdata)->t_infomask),
-				has_nulls ? (*plaintdata)->t_bits : NULL);
-
-		/* ----------
-		 * Switch back to the old memory context
-		 * ----------
-		 */
-		MemoryContextSwitchTo(oldcxt);
-	}
-#endif
-
-
 	/* ----------
 	 * Free allocated temp values
 	 * ----------
 	 */
 	if (need_free)
 		for (i = 0; i < numAttrs; i++)
-#ifndef TOAST_INDICES
-		{
-			if (toast_free[i])
-				pfree(DatumGetPointer(toast_values[i]));
-			if (toast_freeplain[i])
-				pfree(DatumGetPointer(toast_plains[i]));
-		}
-#else
 			if (toast_free[i])
 				pfree(DatumGetPointer(toast_values[i]));
-#endif
 
 	/* ----------
 	 * Delete external values from the old tuple

src/include/access/tuptoaster.h

@@ -6,7 +6,7 @@
  *
  * Copyright (c) 2000, PostgreSQL Development Team
  *
- * $Id: tuptoaster.h,v 1.6 2000/07/21 10:31:31 wieck Exp $
+ * $Id: tuptoaster.h,v 1.7 2000/07/22 11:18:47 wieck Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -20,13 +20,7 @@
 #include "access/tupmacs.h"
 #include "utils/rel.h"
 
-/*
- * DO NOT ENABLE THIS
- * until we have crash safe file versioning and you've
- * changed VACUUM to recreate indices that use possibly
- * toasted values. 2000/07/20 Jan
- */
-#undef TOAST_INDICES
+#define TOAST_INDEX_HACK
 
 
 #define	TOAST_MAX_CHUNK_SIZE	((MaxTupleSize -							\
@@ -37,15 +31,36 @@
 					MAXALIGN(VARHDRSZ))) / 4)
 
 
-#ifdef TOAST_INDICES
+/* ----------
+ * heap_tuple_toast_attrs() -
+ *
+ *		Called by heap_insert(), heap_update() and heap_delete().
+ *		Outdates not any longer needed toast entries referenced
+ *		by oldtup and creates new ones until newtup is smaller
+ *		that ~2K (or running out of toastable values).
+ *		Possibly modifies newtup by replacing the t_data part!
+ * ----------
+ */
 extern void heap_tuple_toast_attrs(Relation rel,
 				HeapTuple newtup, HeapTuple oldtup);
-#else
-extern void heap_tuple_toast_attrs(Relation rel,
-				HeapTuple newtup, HeapTuple oldtup, 
-				HeapTupleHeader *plaintdata, int32 *plaintlen);
-#endif
 
+/* ----------
+ * heap_tuple_fetch_attr() -
+ *
+ *		Fetches an external stored attribute from the toast
+ *		relation. Does NOT decompress it, if stored external
+ *		in compressed format.
+ * ----------
+ */
+extern varattrib *heap_tuple_fetch_attr(varattrib * attr);
+
+/* ----------
+ * heap_tuple_untoast_attr() -
+ *
+ *		Fully detoasts one attribute, fetching and/or decompressing
+ *		it as needed.
+ * ----------
+ */
 extern varattrib *heap_tuple_untoast_attr(varattrib * attr);
 
 #endif	 /* TUPLE_TOASTER_ACTIVE */