Prevent stack overflow in container-type functions.

A range type can name another range type as its subtype, and a record
type can bear a column of another record type.  Consequently, functions
like range_cmp() and record_recv() are recursive.  Functions at risk
include operator family members and referents of pg_type regproc
columns.  Treat as recursive any such function that looks up and calls
the same-purpose function for a record column type or the range subtype.
Back-patch to 9.0 (all supported versions).

An array type's element type is never itself an array type, so array
functions are unaffected.  Recursion depth proportional to array
dimensionality, found in array_dim_to_jsonb(), is fine thanks to MAXDIM.

src/backend/utils/adt/rangetypes.c

@@ -33,6 +33,7 @@
 #include "access/hash.h"
 #include "lib/stringinfo.h"
 #include "libpq/pqformat.h"
+#include "miscadmin.h"
 #include "utils/builtins.h"
 #include "utils/date.h"
 #include "utils/int8.h"
@@ -89,6 +90,8 @@ range_in(PG_FUNCTION_ARGS)
 	RangeBound	lower;
 	RangeBound	upper;
 
+	check_stack_depth();		/* recurses when subtype is a range type */
+
 	cache = get_range_io_data(fcinfo, rngtypoid, IOFunc_input);
 
 	/* parse */
@@ -128,6 +131,8 @@ range_out(PG_FUNCTION_ARGS)
 	RangeBound	upper;
 	bool		empty;
 
+	check_stack_depth();		/* recurses when subtype is a range type */
+
 	cache = get_range_io_data(fcinfo, RangeTypeGetOid(range), IOFunc_output);
 
 	/* deserialize */
@@ -165,6 +170,8 @@ range_recv(PG_FUNCTION_ARGS)
 	RangeBound	lower;
 	RangeBound	upper;
 
+	check_stack_depth();		/* recurses when subtype is a range type */
+
 	cache = get_range_io_data(fcinfo, rngtypoid, IOFunc_receive);
 
 	/* receive the flags... */
@@ -245,6 +252,8 @@ range_send(PG_FUNCTION_ARGS)
 	RangeBound	upper;
 	bool		empty;
 
+	check_stack_depth();		/* recurses when subtype is a range type */
+
 	cache = get_range_io_data(fcinfo, RangeTypeGetOid(range), IOFunc_send);
 
 	/* deserialize */
@@ -1142,6 +1151,8 @@ range_cmp(PG_FUNCTION_ARGS)
 				empty2;
 	int			cmp;
 
+	check_stack_depth();		/* recurses when subtype is a range type */
+
 	/* Different types should be prevented by ANYRANGE matching rules */
 	if (RangeTypeGetOid(r1) != RangeTypeGetOid(r2))
 		elog(ERROR, "range types do not match");
@@ -1221,6 +1232,8 @@ hash_range(PG_FUNCTION_ARGS)
 	uint32		lower_hash;
 	uint32		upper_hash;
 
+	check_stack_depth();		/* recurses when subtype is a range type */
+
 	typcache = range_get_typcache(fcinfo, RangeTypeGetOid(r));
 
 	/* deserialize */

src/backend/utils/adt/rowtypes.c

@@ -21,6 +21,7 @@
 #include "catalog/pg_type.h"
 #include "funcapi.h"
 #include "libpq/pqformat.h"
+#include "miscadmin.h"
 #include "utils/builtins.h"
 #include "utils/lsyscache.h"
 #include "utils/typcache.h"
@@ -86,6 +87,8 @@ record_in(PG_FUNCTION_ARGS)
 	bool	   *nulls;
 	StringInfoData buf;
 
+	check_stack_depth();		/* recurses for record-type columns */
+
 	/*
 	 * Give a friendly error message if we did not get enough info to identify
 	 * the target record type.  (lookup_rowtype_tupdesc would fail anyway, but
@@ -309,6 +312,8 @@ record_out(PG_FUNCTION_ARGS)
 	bool	   *nulls;
 	StringInfoData buf;
 
+	check_stack_depth();		/* recurses for record-type columns */
+
 	/* Extract type info from the tuple itself */
 	tupType = HeapTupleHeaderGetTypeId(rec);
 	tupTypmod = HeapTupleHeaderGetTypMod(rec);
@@ -458,6 +463,8 @@ record_recv(PG_FUNCTION_ARGS)
 	Datum	   *values;
 	bool	   *nulls;
 
+	check_stack_depth();		/* recurses for record-type columns */
+
 	/*
 	 * Give a friendly error message if we did not get enough info to identify
 	 * the target record type.  (lookup_rowtype_tupdesc would fail anyway, but
@@ -650,6 +657,8 @@ record_send(PG_FUNCTION_ARGS)
 	bool	   *nulls;
 	StringInfoData buf;
 
+	check_stack_depth();		/* recurses for record-type columns */
+
 	/* Extract type info from the tuple itself */
 	tupType = HeapTupleHeaderGetTypeId(rec);
 	tupTypmod = HeapTupleHeaderGetTypMod(rec);
@@ -793,6 +802,8 @@ record_cmp(FunctionCallInfo fcinfo)
 	int			i2;
 	int			j;
 
+	check_stack_depth();		/* recurses for record-type columns */
+
 	/* Extract type info from the tuples */
 	tupType1 = HeapTupleHeaderGetTypeId(record1);
 	tupTypmod1 = HeapTupleHeaderGetTypMod(record1);
@@ -1029,6 +1040,8 @@ record_eq(PG_FUNCTION_ARGS)
 	int			i2;
 	int			j;
 
+	check_stack_depth();		/* recurses for record-type columns */
+
 	/* Extract type info from the tuples */
 	tupType1 = HeapTupleHeaderGetTypeId(record1);
 	tupTypmod1 = HeapTupleHeaderGetTypMod(record1);