Create an API for inserting and deleting rows in TOAST tables.

This moves much of the non-heap-specific logic from toast_delete and
toast_insert_or_update into a helper functions accessible via a new
header, toast_helper.h.  Using the functions in this module, a table
AM can implement creation and deletion of TOAST table rows with
much less code duplication than was possible heretofore.  Some
table AMs won't want to use the TOAST logic at all, but for those
that do this will make that easier.

Patch by me, reviewed and tested by Prabhat Sabu, Thomas Munro,
Andres Freund, and Álvaro Herrera.

Discussion: http://postgr.es/m/CA+TgmoZv-=2iWM4jcw5ZhJeL18HF96+W1yJeYrnGMYdkFFnEpQ@mail.gmail.com

src/backend/access/table/Makefile

@@ -12,6 +12,6 @@ subdir = src/backend/access/table
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = table.o tableam.o tableamapi.o
+OBJS = table.o tableam.o tableamapi.o toast_helper.o
 
 include $(top_srcdir)/src/backend/common.mk

src/backend/access/table/toast_helper.c

+/*-------------------------------------------------------------------------
+ *
+ * toast_helper.c
+ *	  Helper functions for table AMs implementing compressed or
+ *    out-of-line storage of varlena attributes.
+ *
+ * Copyright (c) 2000-2019, PostgreSQL Global Development Group
+ *
+ * IDENTIFICATION
+ *	  src/backend/access/common/toast_helper.c
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#include "postgres.h"
+
+#include "access/detoast.h"
+#include "access/table.h"
+#include "access/toast_helper.h"
+#include "access/toast_internals.h"
+
+/*
+ * Prepare to TOAST a tuple.
+ *
+ * tupleDesc, toast_values, and toast_isnull are required parameters; they
+ * provide the necessary details about the tuple to be toasted.
+ *
+ * toast_oldvalues and toast_oldisnull should be NULL for a newly-inserted
+ * tuple; for an update, they should describe the existing tuple.
+ *
+ * All of these arrays should have a length equal to tupleDesc->natts.
+ *
+ * On return, toast_flags and toast_attr will have been initialized.
+ * toast_flags is just a single uint8, but toast_attr is an caller-provided
+ * array with a length equal to tupleDesc->natts.  The caller need not
+ * perform any initialization of the array before calling this function.
+ */
+void
+toast_tuple_init(ToastTupleContext *ttc)
+{
+	TupleDesc	tupleDesc = ttc->ttc_rel->rd_att;
+	int			numAttrs = tupleDesc->natts;
+	int			i;
+
+	ttc->ttc_flags = 0;
+
+	for (i = 0; i < numAttrs; i++)
+	{
+		Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
+		struct varlena *old_value;
+		struct varlena *new_value;
+
+		ttc->ttc_attr[i].tai_colflags = 0;
+		ttc->ttc_attr[i].tai_oldexternal = NULL;
+
+		if (ttc->ttc_oldvalues != NULL)
+		{
+			/*
+			 * For UPDATE get the old and new values of this attribute
+			 */
+			old_value =
+				(struct varlena *) DatumGetPointer(ttc->ttc_oldvalues[i]);
+			new_value =
+				(struct varlena *) DatumGetPointer(ttc->ttc_values[i]);
+
+			/*
+			 * If the old value is stored on disk, check if it has changed so
+			 * we have to delete it later.
+			 */
+			if (att->attlen == -1 && !ttc->ttc_oldisnull[i] &&
+				VARATT_IS_EXTERNAL_ONDISK(old_value))
+			{
+				if (ttc->ttc_isnull[i] ||
+					!VARATT_IS_EXTERNAL_ONDISK(new_value) ||
+					memcmp((char *) old_value, (char *) new_value,
+						   VARSIZE_EXTERNAL(old_value)) != 0)
+				{
+					/*
+					 * The old external stored value isn't needed any more
+					 * after the update
+					 */
+					ttc->ttc_attr[i].tai_colflags |= TOASTCOL_NEEDS_DELETE_OLD;
+					ttc->ttc_flags |= TOAST_NEEDS_DELETE_OLD;
+				}
+				else
+				{
+					/*
+					 * This attribute isn't changed by this update so we reuse
+					 * the original reference to the old value in the new
+					 * tuple.
+					 */
+					ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
+					continue;
+				}
+			}
+		}
+		else
+		{
+			/*
+			 * For INSERT simply get the new value
+			 */
+			new_value = (struct varlena *) DatumGetPointer(ttc->ttc_values[i]);
+		}
+
+		/*
+		 * Handle NULL attributes
+		 */
+		if (ttc->ttc_isnull[i])
+		{
+			ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
+			ttc->ttc_flags |= TOAST_HAS_NULLS;
+			continue;
+		}
+
+		/*
+		 * Now look at varlena attributes
+		 */
+		if (att->attlen == -1)
+		{
+			/*
+			 * If the table's attribute says PLAIN always, force it so.
+			 */
+			if (att->attstorage == 'p')
+				ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
+
+			/*
+			 * We took care of UPDATE above, so any external value we find
+			 * still in the tuple must be someone else's that we cannot reuse
+			 * (this includes the case of an out-of-line in-memory datum).
+			 * Fetch it back (without decompression, unless we are forcing
+			 * PLAIN storage).  If necessary, we'll push it out as a new
+			 * external value below.
+			 */
+			if (VARATT_IS_EXTERNAL(new_value))
+			{
+				ttc->ttc_attr[i].tai_oldexternal = new_value;
+				if (att->attstorage == 'p')
+					new_value = heap_tuple_untoast_attr(new_value);
+				else
+					new_value = heap_tuple_fetch_attr(new_value);
+				ttc->ttc_values[i] = PointerGetDatum(new_value);
+				ttc->ttc_attr[i].tai_colflags |= TOASTCOL_NEEDS_FREE;
+				ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
+			}
+
+			/*
+			 * Remember the size of this attribute
+			 */
+			ttc->ttc_attr[i].tai_size = VARSIZE_ANY(new_value);
+		}
+		else
+		{
+			/*
+			 * Not a varlena attribute, plain storage always
+			 */
+			ttc->ttc_attr[i].tai_colflags |= TOASTCOL_IGNORE;
+		}
+	}
+}
+
+/*
+ * Find the largest varlena attribute that satisfies certain criteria.
+ *
+ * The relevant column must not be marked TOASTCOL_IGNORE, and if the
+ * for_compression flag is passed as true, it must also not be marked
+ * TOASTCOL_INCOMPRESSIBLE.
+ *
+ * The column must have attstorage 'e' or 'x' if check_main is false, and
+ * must have attstorage 'm' if check_main is true.
+ *
+ * The column must have a minimum size of MAXALIGN(TOAST_POINTER_SIZE);
+ * if not, no benefit is to be expected by compressing it.
+ *
+ * The return value is the index of the biggest suitable column, or
+ * -1 if there is none.
+ */
+int
+toast_tuple_find_biggest_attribute(ToastTupleContext *ttc,
+								   bool for_compression, bool check_main)
+{
+	TupleDesc	tupleDesc = ttc->ttc_rel->rd_att;
+	int			numAttrs = tupleDesc->natts;
+	int			biggest_attno = -1;
+	int32		biggest_size = MAXALIGN(TOAST_POINTER_SIZE);
+	int32		skip_colflags = TOASTCOL_IGNORE;
+	int			i;
+
+	if (for_compression)
+		skip_colflags |= TOASTCOL_INCOMPRESSIBLE;
+
+	for (i = 0; i < numAttrs; i++)
+	{
+		Form_pg_attribute att = TupleDescAttr(tupleDesc, i);
+
+		if ((ttc->ttc_attr[i].tai_colflags & skip_colflags) != 0)
+			continue;
+		if (VARATT_IS_EXTERNAL(DatumGetPointer(ttc->ttc_values[i])))
+			continue;			/* can't happen, toast_action would be 'p' */
+		if (for_compression &&
+			VARATT_IS_COMPRESSED(DatumGetPointer(ttc->ttc_values[i])))
+			continue;
+		if (check_main && att->attstorage != 'm')
+			continue;
+		if (!check_main && att->attstorage != 'x' && att->attstorage != 'e')
+			continue;
+
+		if (ttc->ttc_attr[i].tai_size > biggest_size)
+		{
+			biggest_attno = i;
+			biggest_size = ttc->ttc_attr[i].tai_size;
+		}
+	}
+
+	return biggest_attno;
+}
+
+/*
+ * Try compression for an attribute.
+ *
+ * If we find that the attribute is not compressible, mark it so.
+ */
+void
+toast_tuple_try_compression(ToastTupleContext *ttc, int attribute)
+{
+	Datum	   *value = &ttc->ttc_values[attribute];
+	Datum		new_value = toast_compress_datum(*value);
+	ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
+
+	if (DatumGetPointer(new_value) != NULL)
+	{
+		/* successful compression */
+		if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
+			pfree(DatumGetPointer(*value));
+		*value = new_value;
+		attr->tai_colflags |= TOASTCOL_NEEDS_FREE;
+		attr->tai_size = VARSIZE(DatumGetPointer(*value));
+		ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
+	}
+	else
+	{
+		/* incompressible, ignore on subsequent compression passes */
+		attr->tai_colflags |= TOASTCOL_INCOMPRESSIBLE;
+	}
+}
+
+/*
+ * Move an attribute to external storage.
+ */
+void
+toast_tuple_externalize(ToastTupleContext *ttc, int attribute, int options)
+{
+	Datum	   *value = &ttc->ttc_values[attribute];
+	Datum		old_value = *value;
+	ToastAttrInfo *attr = &ttc->ttc_attr[attribute];
+
+	attr->tai_colflags |= TOASTCOL_IGNORE;
+	*value = toast_save_datum(ttc->ttc_rel, old_value, attr->tai_oldexternal,
+							  options);
+	if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
+		pfree(DatumGetPointer(old_value));
+	attr->tai_colflags |= TOASTCOL_NEEDS_FREE;
+	ttc->ttc_flags |= (TOAST_NEEDS_CHANGE | TOAST_NEEDS_FREE);
+}
+
+/*
+ * Perform appropriate cleanup after one tuple has been subjected to TOAST.
+ */
+void
+toast_tuple_cleanup(ToastTupleContext *ttc)
+{
+	TupleDesc	tupleDesc = ttc->ttc_rel->rd_att;
+	int			numAttrs = tupleDesc->natts;
+
+	/*
+	 * Free allocated temp values
+	 */
+	if ((ttc->ttc_flags & TOAST_NEEDS_FREE) != 0)
+	{
+		int			i;
+
+		for (i = 0; i < numAttrs; i++)
+		{
+			ToastAttrInfo *attr = &ttc->ttc_attr[i];
+
+			if ((attr->tai_colflags & TOASTCOL_NEEDS_FREE) != 0)
+				pfree(DatumGetPointer(ttc->ttc_values[i]));
+		}
+	}
+
+	/*
+	 * Delete external values from the old tuple
+	 */
+	if ((ttc->ttc_flags & TOAST_NEEDS_DELETE_OLD) != 0)
+	{
+		int			i;
+
+		for (i = 0; i < numAttrs; i++)
+		{
+			ToastAttrInfo *attr = &ttc->ttc_attr[i];
+
+			if ((attr->tai_colflags & TOASTCOL_NEEDS_DELETE_OLD) != 0)
+				toast_delete_datum(ttc->ttc_rel, ttc->ttc_oldvalues[i], false);
+		}
+	}
+}
+
+/*
+ * Check for external stored attributes and delete them from the secondary
+ * relation.
+ */
+void
+toast_delete_external(Relation rel, Datum *values, bool *isnull,
+					  bool is_speculative)
+{
+	TupleDesc	tupleDesc = rel->rd_att;
+	int			numAttrs = tupleDesc->natts;
+	int			i;
+
+	for (i = 0; i < numAttrs; i++)
+	{
+		if (TupleDescAttr(tupleDesc, i)->attlen == -1)
+		{
+			Datum		value = values[i];
+
+			if (isnull[i])
+				continue;
+			else if (VARATT_IS_EXTERNAL_ONDISK(PointerGetDatum(value)))
+				toast_delete_datum(rel, value, is_speculative);
+		}
+	}
+}

src/include/access/toast_helper.h

+/*-------------------------------------------------------------------------
+ *
+ * toast_helper.h
+ *	  Helper functions for table AMs implementing compressed or
+ *    out-of-line storage of varlena attributes.
+ *
+ * Copyright (c) 2000-2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/toast_helper.h
+ *
+ *-------------------------------------------------------------------------
+ */
+
+#ifndef TOAST_HELPER_H
+#define TOAST_HELPER_H
+
+#include "utils/rel.h"
+
+/*
+ * Information about one column of a tuple being toasted.
+ *
+ * NOTE: toast_action[i] can have these values:
+ *		' '		default handling
+ *		'p'		already processed --- don't touch it
+ *		'x'		incompressible, but OK to move off
+ *
+ * NOTE: toast_attr[i].tai_size is only made valid for varlena attributes with
+ * toast_action[i] different from 'p'.
+ */
+typedef struct
+{
+	struct varlena *tai_oldexternal;
+	int32		tai_size;
+	uint8		tai_colflags;
+} ToastAttrInfo;
+
+/*
+ * Information about one tuple being toasted.
+ */
+typedef struct
+{
+	/*
+	 * Before calling toast_tuple_init, the caller must initialize the
+	 * following fields.  Each array must have a length equal to
+	 * ttc_rel->rd_att->natts.  The tts_oldvalues and tts_oldisnull fields
+	 * should be NULL in the case of an insert.
+	 */
+	Relation	ttc_rel;		/* the relation that contains the tuple */
+	Datum	   *ttc_values;		/* values from the tuple columns */
+	bool	   *ttc_isnull;		/* null flags for the tuple columns */
+	Datum	   *ttc_oldvalues;	/* values from previous tuple */
+	bool	   *ttc_oldisnull;	/* null flags from previous tuple */
+
+	/*
+	 * Before calling toast_tuple_init, the caller should set tts_attr to
+	 * point to an array of ToastAttrInfo structures of a length equal to
+	 * tts_rel->rd_att->natts.  The contents of the array need not be
+	 * initialized.  ttc_flags also does not need to be initialized.
+	 */
+	uint8		ttc_flags;
+	ToastAttrInfo *ttc_attr;
+} ToastTupleContext;
+
+/*
+ * Flags indicating the overall state of a TOAST operation.
+ *
+ * TOAST_NEEDS_DELETE_OLD indicates that one or more old TOAST datums need
+ * to be deleted.
+ *
+ * TOAST_NEEDS_FREE indicates that one or more TOAST values need to be freed.
+ *
+ * TOAST_HAS_NULLS indicates that nulls were found in the tuple being toasted.
+ *
+ * TOAST_NEEDS_CHANGE indicates that a new tuple needs to built; in other
+ * words, the toaster did something.
+ */
+#define TOAST_NEEDS_DELETE_OLD				0x0001
+#define TOAST_NEEDS_FREE					0x0002
+#define TOAST_HAS_NULLS						0x0004
+#define TOAST_NEEDS_CHANGE					0x0008
+
+/*
+ * Flags indicating the status of a TOAST operation with respect to a
+ * particular column.
+ *
+ * TOASTCOL_NEEDS_DELETE_OLD indicates that the old TOAST datums for this
+ * column need to be deleted.
+ *
+ * TOASTCOL_NEEDS_FREE indicates that the value for this column needs to
+ * be freed.
+ *
+ * TOASTCOL_IGNORE indicates that the toaster should not further process
+ * this column.
+ *
+ * TOASTCOL_INCOMPRESSIBLE indicates that this column has been found to
+ * be incompressible, but could be moved out-of-line.
+ */
+#define TOASTCOL_NEEDS_DELETE_OLD			TOAST_NEEDS_DELETE_OLD
+#define TOASTCOL_NEEDS_FREE					TOAST_NEEDS_FREE
+#define TOASTCOL_IGNORE						0x0010
+#define TOASTCOL_INCOMPRESSIBLE				0x0020
+
+extern void toast_tuple_init(ToastTupleContext *ttc);
+extern int	toast_tuple_find_biggest_attribute(ToastTupleContext *ttc,
+											   bool for_compression,
+											   bool check_main);
+extern void toast_tuple_try_compression(ToastTupleContext *ttc, int attribute);
+extern void toast_tuple_externalize(ToastTupleContext *ttc, int attribute,
+									int options);
+extern void toast_tuple_cleanup(ToastTupleContext *ttc);
+
+extern void toast_delete_external(Relation rel, Datum *values, bool *isnull,
+								  bool is_speculative);
+
+#endif

src/tools/pgindent/typedefs.list

@@ -2349,6 +2349,8 @@ TBlockState
 TIDBitmap
 TM_FailureData
 TM_Result
+ToastAttrInfo
+ToastTupleContext
 TOKEN_DEFAULT_DACL
 TOKEN_INFORMATION_CLASS
 TOKEN_PRIVILEGES