Split tuptoaster.c into three separate files.

detoast.c/h contain functions required to detoast a datum, partially
or completely, plus a few other utility functions for examining the
size of toasted datums.

toast_internals.c/h contain functions that are used internally to the
TOAST subsystem but which (mostly) do not need to be accessed from
outside.

heaptoast.c/h contains code that is intrinsically specific to the
heap AM, either because it operates on HeapTuples or is based on the
layout of a heap page.

detoast.c and toast_internals.c are placed in
src/backend/access/common rather than src/backend/access/heap.  At
present, both files still have dependencies on the heap, but that will
be improved in a future commit.

Patch by me, reviewed and tested by Prabhat Sabu, Thomas Munro,
Andres Freund, and Álvaro Herrera.

Discussion: http://postgr.es/m/CA+TgmoZv-=2iWM4jcw5ZhJeL18HF96+W1yJeYrnGMYdkFFnEpQ@mail.gmail.com

doc/src/sgml/storage.sgml

@@ -382,7 +382,7 @@ The oldest and most common type is a pointer to out-of-line data stored in
 a <firstterm><acronym>TOAST</acronym> table</firstterm> that is separate from, but
 associated with, the table containing the <acronym>TOAST</acronym> pointer datum
 itself.  These <firstterm>on-disk</firstterm> pointer datums are created by the
-<acronym>TOAST</acronym> management code (in <filename>access/heap/tuptoaster.c</filename>)
+<acronym>TOAST</acronym> management code (in <filename>access/common/toast_internals.c</filename>)
 when a tuple to be stored on disk is too large to be stored as-is.
 Further details appear in <xref linkend="storage-toast-ondisk"/>.
 Alternatively, a <acronym>TOAST</acronym> pointer datum can contain a pointer to

src/backend/access/common/Makefile

@@ -12,7 +12,8 @@ subdir = src/backend/access/common
 top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
-OBJS = bufmask.o heaptuple.o indextuple.o printsimple.o printtup.o \
-	relation.o reloptions.o scankey.o session.o tupconvert.o tupdesc.o
+OBJS = bufmask.o detoast.o heaptuple.o indextuple.o printsimple.o \
+	printtup.o relation.o reloptions.o scankey.o session.o toast_internals.o \
+	tupconvert.o tupdesc.o
 
 include $(top_srcdir)/src/backend/common.mk

src/backend/access/common/heaptuple.c

@@ -18,7 +18,7 @@
  * (In performance-critical code paths we can use pg_detoast_datum_packed
  * and the appropriate access macros to avoid that overhead.)  Note that this
  * conversion is performed directly in heap_form_tuple, without invoking
- * tuptoaster.c.
+ * heaptoast.c.
  *
  * This change will break any code that assumes it needn't detoast values
  * that have been put into a tuple but never sent to disk.  Hopefully there
@@ -57,9 +57,9 @@
 
 #include "postgres.h"
 
+#include "access/heaptoast.h"
 #include "access/sysattr.h"
 #include "access/tupdesc_details.h"
-#include "access/tuptoaster.h"
 #include "executor/tuptable.h"
 #include "utils/expandeddatum.h"
 

src/backend/access/common/indextuple.c

@@ -16,10 +16,17 @@
 
 #include "postgres.h"
 
+#include "access/detoast.h"
+#include "access/heaptoast.h"
 #include "access/htup_details.h"
 #include "access/itup.h"
-#include "access/tuptoaster.h"
+#include "access/toast_internals.h"
 
+/*
+ * This enables de-toasting of index entries.  Needed until VACUUM is
+ * smart enough to rebuild indexes from scratch.
+ */
+#define TOAST_INDEX_HACK
 
 /* ----------------------------------------------------------------
  *				  index_ tuple interface routines

src/backend/access/common/reloptions.c

@@ -19,11 +19,11 @@
 
 #include "access/gist_private.h"
 #include "access/hash.h"
+#include "access/heaptoast.h"
 #include "access/htup_details.h"
 #include "access/nbtree.h"
 #include "access/reloptions.h"
 #include "access/spgist.h"
-#include "access/tuptoaster.h"
 #include "catalog/pg_type.h"
 #include "commands/defrem.h"
 #include "commands/tablespace.h"

src/backend/access/heap/Makefile

@@ -13,6 +13,6 @@ top_builddir = ../../../..
 include $(top_builddir)/src/Makefile.global
 
 OBJS = heapam.o heapam_handler.o heapam_visibility.o hio.o pruneheap.o rewriteheap.o \
-	syncscan.o tuptoaster.o vacuumlazy.o visibilitymap.o
+	syncscan.o heaptoast.o vacuumlazy.o visibilitymap.o
 
 include $(top_srcdir)/src/backend/common.mk

src/backend/access/heap/heapam.c

@@ -36,6 +36,7 @@
 #include "access/genam.h"
 #include "access/heapam.h"
 #include "access/heapam_xlog.h"
+#include "access/heaptoast.h"
 #include "access/hio.h"
 #include "access/multixact.h"
 #include "access/parallel.h"
@@ -43,7 +44,6 @@
 #include "access/sysattr.h"
 #include "access/tableam.h"
 #include "access/transam.h"
-#include "access/tuptoaster.h"
 #include "access/valid.h"
 #include "access/visibilitymap.h"
 #include "access/xact.h"

src/backend/access/heap/heapam_handler.c

@@ -23,11 +23,11 @@
 
 #include "access/genam.h"
 #include "access/heapam.h"
+#include "access/heaptoast.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
 #include "access/tableam.h"
 #include "access/tsmapi.h"
-#include "access/tuptoaster.h"
 #include "access/xact.h"
 #include "catalog/catalog.h"
 #include "catalog/index.h"

src/backend/access/heap/rewriteheap.c

@@ -109,9 +109,9 @@
 
 #include "access/heapam.h"
 #include "access/heapam_xlog.h"
+#include "access/heaptoast.h"
 #include "access/rewriteheap.h"
 #include "access/transam.h"
-#include "access/tuptoaster.h"
 #include "access/xact.h"
 #include "access/xloginsert.h"
 

src/backend/access/transam/xlog.c

@@ -24,12 +24,12 @@
 
 #include "access/clog.h"
 #include "access/commit_ts.h"
+#include "access/heaptoast.h"
 #include "access/multixact.h"
 #include "access/rewriteheap.h"
 #include "access/subtrans.h"
 #include "access/timeline.h"
 #include "access/transam.h"
-#include "access/tuptoaster.h"
 #include "access/twophase.h"
 #include "access/xact.h"
 #include "access/xlog_internal.h"

src/backend/commands/analyze.c

@@ -16,6 +16,7 @@
 
 #include <math.h>
 
+#include "access/detoast.h"
 #include "access/genam.h"
 #include "access/multixact.h"
 #include "access/relation.h"
@@ -24,7 +25,6 @@
 #include "access/tableam.h"
 #include "access/transam.h"
 #include "access/tupconvert.h"
-#include "access/tuptoaster.h"
 #include "access/visibilitymap.h"
 #include "access/xact.h"
 #include "catalog/catalog.h"

src/backend/commands/cluster.c

@@ -23,7 +23,7 @@
 #include "access/relscan.h"
 #include "access/tableam.h"
 #include "access/transam.h"
-#include "access/tuptoaster.h"
+#include "access/toast_internals.h"
 #include "access/xact.h"
 #include "access/xlog.h"
 #include "catalog/pg_am.h"

src/backend/executor/execExprInterp.c

@@ -56,7 +56,7 @@
  */
 #include "postgres.h"
 
-#include "access/tuptoaster.h"
+#include "access/heaptoast.h"
 #include "catalog/pg_type.h"
 #include "commands/sequence.h"
 #include "executor/execExpr.h"

src/backend/executor/execTuples.c

@@ -57,9 +57,9 @@
  */
 #include "postgres.h"
 
+#include "access/heaptoast.h"
 #include "access/htup_details.h"
 #include "access/tupdesc_details.h"
-#include "access/tuptoaster.h"
 #include "funcapi.h"
 #include "catalog/pg_type.h"
 #include "nodes/nodeFuncs.h"

src/backend/executor/tstoreReceiver.c

@@ -20,7 +20,7 @@
 
 #include "postgres.h"
 
-#include "access/tuptoaster.h"
+#include "access/detoast.h"
 #include "executor/tstoreReceiver.h"
 
 

src/backend/replication/logical/reorderbuffer.c

@@ -56,10 +56,10 @@
 #include <unistd.h>
 #include <sys/stat.h>
 
+#include "access/detoast.h"
 #include "access/heapam.h"
 #include "access/rewriteheap.h"
 #include "access/transam.h"
-#include "access/tuptoaster.h"
 #include "access/xact.h"
 #include "access/xlog_internal.h"
 #include "catalog/catalog.h"

src/backend/statistics/extended_stats.c

@@ -16,10 +16,10 @@
  */
 #include "postgres.h"
 
+#include "access/detoast.h"
 #include "access/genam.h"
 #include "access/htup_details.h"
 #include "access/table.h"
-#include "access/tuptoaster.h"
 #include "catalog/indexing.h"
 #include "catalog/pg_collation.h"
 #include "catalog/pg_statistic_ext.h"

src/backend/storage/large_object/inv_api.c

@@ -32,10 +32,11 @@
 
 #include <limits.h>
 
+#include "access/detoast.h"
 #include "access/genam.h"
+#include "access/htup_details.h"
 #include "access/sysattr.h"
 #include "access/table.h"
-#include "access/tuptoaster.h"
 #include "access/xact.h"
 #include "catalog/dependency.h"
 #include "catalog/indexing.h"

src/backend/utils/adt/array_typanalyze.c

@@ -14,7 +14,7 @@
  */
 #include "postgres.h"
 
-#include "access/tuptoaster.h"
+#include "access/detoast.h"
 #include "commands/vacuum.h"
 #include "utils/array.h"
 #include "utils/builtins.h"

src/backend/utils/adt/datum.c

@@ -42,7 +42,7 @@
 
 #include "postgres.h"
 
-#include "access/tuptoaster.h"
+#include "access/detoast.h"
 #include "fmgr.h"
 #include "utils/datum.h"
 #include "utils/expandeddatum.h"

src/backend/utils/adt/expandedrecord.c

@@ -18,8 +18,9 @@
  */
 #include "postgres.h"
 
+#include "access/detoast.h"
+#include "access/heaptoast.h"
 #include "access/htup_details.h"
-#include "access/tuptoaster.h"
 #include "catalog/heap.h"
 #include "catalog/pg_type.h"
 #include "utils/builtins.h"

src/backend/utils/adt/rowtypes.c

@@ -16,8 +16,8 @@
 
 #include <ctype.h>
 
+#include "access/detoast.h"
 #include "access/htup_details.h"
-#include "access/tuptoaster.h"
 #include "catalog/pg_type.h"
 #include "funcapi.h"
 #include "libpq/pqformat.h"

src/backend/utils/adt/tsgistidx.c

@@ -15,7 +15,7 @@
 #include "postgres.h"
 
 #include "access/gist.h"
-#include "access/tuptoaster.h"
+#include "access/heaptoast.h"
 #include "port/pg_bitutils.h"
 #include "tsearch/ts_utils.h"
 #include "utils/builtins.h"

src/backend/utils/adt/varchar.c

@@ -14,7 +14,7 @@
  */
 #include "postgres.h"
 
-#include "access/tuptoaster.h"
+#include "access/detoast.h"
 #include "catalog/pg_collation.h"
 #include "catalog/pg_type.h"
 #include "libpq/pqformat.h"

src/backend/utils/adt/varlena.c

@@ -17,7 +17,7 @@
 #include <ctype.h>
 #include <limits.h>
 
-#include "access/tuptoaster.h"
+#include "access/detoast.h"
 #include "catalog/pg_collation.h"
 #include "catalog/pg_type.h"
 #include "common/int.h"

src/backend/utils/cache/catcache.c

@@ -15,10 +15,10 @@
 #include "postgres.h"
 
 #include "access/genam.h"
+#include "access/heaptoast.h"
 #include "access/relscan.h"
 #include "access/sysattr.h"
 #include "access/table.h"
-#include "access/tuptoaster.h"
 #include "access/valid.h"
 #include "access/xact.h"
 #include "catalog/pg_collation.h"

src/backend/utils/fmgr/fmgr.c

@@ -15,7 +15,7 @@
 
 #include "postgres.h"
 
-#include "access/tuptoaster.h"
+#include "access/detoast.h"
 #include "catalog/pg_language.h"
 #include "catalog/pg_proc.h"
 #include "executor/functions.h"

src/bin/pg_resetwal/pg_resetwal.c

@@ -45,7 +45,7 @@
 #include <unistd.h>
 
 #include "access/transam.h"
-#include "access/tuptoaster.h"
+#include "access/heaptoast.h"
 #include "access/multixact.h"
 #include "access/xlog.h"
 #include "access/xlog_internal.h"

src/include/access/detoast.h

+/*-------------------------------------------------------------------------
+ *
+ * detoast.h
+ *	  Access to compressed and external varlena values.
+ *
+ * Copyright (c) 2000-2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/detoast.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef DETOAST_H
+#define DETOAST_H
+
+/*
+ * Testing whether an externally-stored value is compressed now requires
+ * comparing extsize (the actual length of the external data) to rawsize
+ * (the original uncompressed datum's size).  The latter includes VARHDRSZ
+ * overhead, the former doesn't.  We never use compression unless it actually
+ * saves space, so we expect either equality or less-than.
+ */
+#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
+	((toast_pointer).va_extsize < (toast_pointer).va_rawsize - VARHDRSZ)
+
+/*
+ * Macro to fetch the possibly-unaligned contents of an EXTERNAL datum
+ * into a local "struct varatt_external" toast pointer.  This should be
+ * just a memcpy, but some versions of gcc seem to produce broken code
+ * that assumes the datum contents are aligned.  Introducing an explicit
+ * intermediate "varattrib_1b_e *" variable seems to fix it.
+ */
+#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr) \
+do { \
+	varattrib_1b_e *attre = (varattrib_1b_e *) (attr); \
+	Assert(VARATT_IS_EXTERNAL(attre)); \
+	Assert(VARSIZE_EXTERNAL(attre) == sizeof(toast_pointer) + VARHDRSZ_EXTERNAL); \
+	memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \
+} while (0)
+
+/* Size of an EXTERNAL datum that contains a standard TOAST pointer */
+#define TOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external))
+
+/* Size of an EXTERNAL datum that contains an indirection pointer */
+#define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))
+
+/* ----------
+ * heap_tuple_fetch_attr() -
+ *
+ *		Fetches an external stored attribute from the toast
+ *		relation. Does NOT decompress it, if stored external
+ *		in compressed format.
+ * ----------
+ */
+extern struct varlena *heap_tuple_fetch_attr(struct varlena *attr);
+
+/* ----------
+ * heap_tuple_untoast_attr() -
+ *
+ *		Fully detoasts one attribute, fetching and/or decompressing
+ *		it as needed.
+ * ----------
+ */
+extern struct varlena *heap_tuple_untoast_attr(struct varlena *attr);
+
+/* ----------
+ * heap_tuple_untoast_attr_slice() -
+ *
+ *		Fetches only the specified portion of an attribute.
+ *		(Handles all cases for attribute storage)
+ * ----------
+ */
+extern struct varlena *heap_tuple_untoast_attr_slice(struct varlena *attr,
+							  int32 sliceoffset,
+							  int32 slicelength);
+
+/* ----------
+ * toast_raw_datum_size -
+ *
+ *	Return the raw (detoasted) size of a varlena datum
+ * ----------
+ */
+extern Size toast_raw_datum_size(Datum value);
+
+/* ----------
+ * toast_datum_size -
+ *
+ *	Return the storage size of a varlena datum
+ * ----------
+ */
+extern Size toast_datum_size(Datum value);
+
+#endif							/* DETOAST_H */

src/include/access/tuptoaster.h → src/include/access/heaptoast.h

 /*-------------------------------------------------------------------------
  *
- * tuptoaster.h
- *	  POSTGRES definitions for external and compressed storage
+ * heaptoast.h
+ *	  Heap-specific definitions for external and compressed storage
  *	  of variable size attributes.
  *
  * Copyright (c) 2000-2019, PostgreSQL Global Development Group
  *
- * src/include/access/tuptoaster.h
+ * src/include/access/heaptoast.h
  *
  *-------------------------------------------------------------------------
  */
-#ifndef TUPTOASTER_H
-#define TUPTOASTER_H
+#ifndef HEAPTOAST_H
+#define HEAPTOAST_H
 
 #include "access/htup_details.h"
 #include "storage/lockdefs.h"
 #include "utils/relcache.h"
 
-/*
- * This enables de-toasting of index entries.  Needed until VACUUM is
- * smart enough to rebuild indexes from scratch.
- */
-#define TOAST_INDEX_HACK
-
-
 /*
  * Find the maximum size of a tuple if there are to be N tuples per page.
  */
@@ -95,37 +88,6 @@
 	 sizeof(int32) -									\
 	 VARHDRSZ)
 
-/* Size of an EXTERNAL datum that contains a standard TOAST pointer */
-#define TOAST_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_external))
-
-/* Size of an EXTERNAL datum that contains an indirection pointer */
-#define INDIRECT_POINTER_SIZE (VARHDRSZ_EXTERNAL + sizeof(varatt_indirect))
-
-/*
- * Testing whether an externally-stored value is compressed now requires
- * comparing extsize (the actual length of the external data) to rawsize
- * (the original uncompressed datum's size).  The latter includes VARHDRSZ
- * overhead, the former doesn't.  We never use compression unless it actually
- * saves space, so we expect either equality or less-than.
- */
-#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
-	((toast_pointer).va_extsize < (toast_pointer).va_rawsize - VARHDRSZ)
-
-/*
- * Macro to fetch the possibly-unaligned contents of an EXTERNAL datum
- * into a local "struct varatt_external" toast pointer.  This should be
- * just a memcpy, but some versions of gcc seem to produce broken code
- * that assumes the datum contents are aligned.  Introducing an explicit
- * intermediate "varattrib_1b_e *" variable seems to fix it.
- */
-#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr) \
-do { \
-	varattrib_1b_e *attre = (varattrib_1b_e *) (attr); \
-	Assert(VARATT_IS_EXTERNAL(attre)); \
-	Assert(VARSIZE_EXTERNAL(attre) == sizeof(toast_pointer) + VARHDRSZ_EXTERNAL); \
-	memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \
-} while (0)
-
 /* ----------
  * toast_insert_or_update -
  *
@@ -144,36 +106,6 @@ extern HeapTuple toast_insert_or_update(Relation rel,
  */
 extern void toast_delete(Relation rel, HeapTuple oldtup, bool is_speculative);
 
-/* ----------
- * heap_tuple_fetch_attr() -
- *
- *		Fetches an external stored attribute from the toast
- *		relation. Does NOT decompress it, if stored external
- *		in compressed format.
- * ----------
- */
-extern struct varlena *heap_tuple_fetch_attr(struct varlena *attr);
-
-/* ----------
- * heap_tuple_untoast_attr() -
- *
- *		Fully detoasts one attribute, fetching and/or decompressing
- *		it as needed.
- * ----------
- */
-extern struct varlena *heap_tuple_untoast_attr(struct varlena *attr);
-
-/* ----------
- * heap_tuple_untoast_attr_slice() -
- *
- *		Fetches only the specified portion of an attribute.
- *		(Handles all cases for attribute storage)
- * ----------
- */
-extern struct varlena *heap_tuple_untoast_attr_slice(struct varlena *attr,
-													 int32 sliceoffset,
-													 int32 slicelength);
-
 /* ----------
  * toast_flatten_tuple -
  *
@@ -204,36 +136,4 @@ extern HeapTuple toast_build_flattened_tuple(TupleDesc tupleDesc,
 											 Datum *values,
 											 bool *isnull);
 
-/* ----------
- * toast_compress_datum -
- *
- *	Create a compressed version of a varlena datum, if possible
- * ----------
- */
-extern Datum toast_compress_datum(Datum value);
-
-/* ----------
- * toast_raw_datum_size -
- *
- *	Return the raw (detoasted) size of a varlena datum
- * ----------
- */
-extern Size toast_raw_datum_size(Datum value);
-
-/* ----------
- * toast_datum_size -
- *
- *	Return the storage size of a varlena datum
- * ----------
- */
-extern Size toast_datum_size(Datum value);
-
-/* ----------
- * toast_get_valid_index -
- *
- *	Return OID of valid index associated to a toast relation
- * ----------
- */
-extern Oid	toast_get_valid_index(Oid toastoid, LOCKMODE lock);
-
-#endif							/* TUPTOASTER_H */
+#endif							/* HEAPTOAST_H */

src/include/access/toast_internals.h

+/*-------------------------------------------------------------------------
+ *
+ * toast_internals.h
+ *	  Internal definitions for the TOAST system.
+ *
+ * Copyright (c) 2000-2019, PostgreSQL Global Development Group
+ *
+ * src/include/access/toast_internals.h
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifndef TOAST_INTERNALS_H
+#define TOAST_INTERNALS_H
+
+#include "storage/lockdefs.h"
+#include "utils/relcache.h"
+#include "utils/snapshot.h"
+
+/*
+ *	The information at the start of the compressed toast data.
+ */
+typedef struct toast_compress_header
+{
+	int32		vl_len_;		/* varlena header (do not touch directly!) */
+	int32		rawsize;
+} toast_compress_header;
+
+/*
+ * Utilities for manipulation of header information for compressed
+ * toast entries.
+ */
+#define TOAST_COMPRESS_HDRSZ		((int32) sizeof(toast_compress_header))
+#define TOAST_COMPRESS_RAWSIZE(ptr) (((toast_compress_header *) (ptr))->rawsize)
+#define TOAST_COMPRESS_RAWDATA(ptr) \
+	(((char *) (ptr)) + TOAST_COMPRESS_HDRSZ)
+#define TOAST_COMPRESS_SET_RAWSIZE(ptr, len) \
+	(((toast_compress_header *) (ptr))->rawsize = (len))
+
+extern Datum toast_compress_datum(Datum value);
+extern Oid	toast_get_valid_index(Oid toastoid, LOCKMODE lock);
+
+extern void toast_delete_datum(Relation rel, Datum value, bool is_speculative);
+extern Datum toast_save_datum(Relation rel, Datum value,
+							  struct varlena *oldexternal, int options);
+
+extern int	toast_open_indexes(Relation toastrel,
+							   LOCKMODE lock,
+							   Relation **toastidxs,
+							   int *num_indexes);
+extern void toast_close_indexes(Relation *toastidxs, int num_indexes,
+								LOCKMODE lock);
+extern void init_toast_snapshot(Snapshot toast_snapshot);
+
+#endif							/* TOAST_INTERNALS_H */

src/pl/plpgsql/src/pl_exec.c

@@ -17,10 +17,10 @@
 
 #include <ctype.h>
 
+#include "access/detoast.h"
 #include "access/htup_details.h"
 #include "access/transam.h"
 #include "access/tupconvert.h"
-#include "access/tuptoaster.h"
 #include "catalog/pg_proc.h"
 #include "catalog/pg_type.h"
 #include "commands/defrem.h"

src/test/regress/regress.c

@@ -19,9 +19,9 @@
 #include <math.h>
 #include <signal.h>
 
+#include "access/detoast.h"
 #include "access/htup_details.h"
 #include "access/transam.h"
-#include "access/tuptoaster.h"
 #include "access/xact.h"
 #include "catalog/pg_operator.h"
 #include "catalog/pg_type.h"