Support varlena fields with single-byte headers and unaligned storage.

This commit breaks any code that assumes that the mere act of forming a tuple
(without writing it to disk) does not "toast" any fields.  While all available
regression tests pass, I'm not totally sure that we've fixed every nook and
cranny, especially in contrib.

Greg Stark with some help from Tom Lane

configure

@@ -11143,7 +11143,237 @@ fi
 ## Types, structures, compiler characteristics
 ##
 
- echo "$as_me:$LINENO: checking for an ANSI C-conforming const" >&5
+ echo "$as_me:$LINENO: checking whether byte ordering is bigendian" >&5
+echo $ECHO_N "checking whether byte ordering is bigendian... $ECHO_C" >&6
+if test "${ac_cv_c_bigendian+set}" = set; then
+  echo $ECHO_N "(cached) $ECHO_C" >&6
+else
+  # See if sys/param.h defines the BYTE_ORDER macro.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if !BYTE_ORDER || !BIG_ENDIAN || !LITTLE_ENDIAN
+ bogus endian macros
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  # It does; now see whether it defined to BIG_ENDIAN or not.
+cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+#include <sys/types.h>
+#include <sys/param.h>
+
+int
+main ()
+{
+#if BYTE_ORDER != BIG_ENDIAN
+ not big endian
+#endif
+
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=yes
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+ac_cv_c_bigendian=no
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+# It does not; compile a test program.
+if test "$cross_compiling" = yes; then
+  # try to guess the endianness by grepping values into an object file
+  ac_cv_c_bigendian=unknown
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+short ascii_mm[] = { 0x4249, 0x4765, 0x6E44, 0x6961, 0x6E53, 0x7953, 0 };
+short ascii_ii[] = { 0x694C, 0x5454, 0x656C, 0x6E45, 0x6944, 0x6E61, 0 };
+void _ascii () { char *s = (char *) ascii_mm; s = (char *) ascii_ii; }
+short ebcdic_ii[] = { 0x89D3, 0xE3E3, 0x8593, 0x95C5, 0x89C4, 0x9581, 0 };
+short ebcdic_mm[] = { 0xC2C9, 0xC785, 0x95C4, 0x8981, 0x95E2, 0xA8E2, 0 };
+void _ebcdic () { char *s = (char *) ebcdic_mm; s = (char *) ebcdic_ii; }
+int
+main ()
+{
+ _ascii (); _ebcdic ();
+  ;
+  return 0;
+}
+_ACEOF
+rm -f conftest.$ac_objext
+if { (eval echo "$as_me:$LINENO: \"$ac_compile\"") >&5
+  (eval $ac_compile) 2>conftest.er1
+  ac_status=$?
+  grep -v '^ *+' conftest.er1 >conftest.err
+  rm -f conftest.er1
+  cat conftest.err >&5
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } &&
+	 { ac_try='test -z "$ac_c_werror_flag"
+			 || test ! -s conftest.err'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; } &&
+	 { ac_try='test -s conftest.$ac_objext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  if grep BIGenDianSyS conftest.$ac_objext >/dev/null ; then
+  ac_cv_c_bigendian=yes
+fi
+if grep LiTTleEnDian conftest.$ac_objext >/dev/null ; then
+  if test "$ac_cv_c_bigendian" = unknown; then
+    ac_cv_c_bigendian=no
+  else
+    # finding both strings is unlikely to happen, but who knows?
+    ac_cv_c_bigendian=unknown
+  fi
+fi
+else
+  echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+else
+  cat >conftest.$ac_ext <<_ACEOF
+/* confdefs.h.  */
+_ACEOF
+cat confdefs.h >>conftest.$ac_ext
+cat >>conftest.$ac_ext <<_ACEOF
+/* end confdefs.h.  */
+int
+main ()
+{
+  /* Are we little or big endian?  From Harbison&Steele.  */
+  union
+  {
+    long l;
+    char c[sizeof (long)];
+  } u;
+  u.l = 1;
+  exit (u.c[sizeof (long) - 1] == 1);
+}
+_ACEOF
+rm -f conftest$ac_exeext
+if { (eval echo "$as_me:$LINENO: \"$ac_link\"") >&5
+  (eval $ac_link) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); } && { ac_try='./conftest$ac_exeext'
+  { (eval echo "$as_me:$LINENO: \"$ac_try\"") >&5
+  (eval $ac_try) 2>&5
+  ac_status=$?
+  echo "$as_me:$LINENO: \$? = $ac_status" >&5
+  (exit $ac_status); }; }; then
+  ac_cv_c_bigendian=no
+else
+  echo "$as_me: program exited with status $ac_status" >&5
+echo "$as_me: failed program was:" >&5
+sed 's/^/| /' conftest.$ac_ext >&5
+
+( exit $ac_status )
+ac_cv_c_bigendian=yes
+fi
+rm -f core *.core gmon.out bb.out conftest$ac_exeext conftest.$ac_objext conftest.$ac_ext
+fi
+fi
+rm -f conftest.err conftest.$ac_objext conftest.$ac_ext
+fi
+echo "$as_me:$LINENO: result: $ac_cv_c_bigendian" >&5
+echo "${ECHO_T}$ac_cv_c_bigendian" >&6
+case $ac_cv_c_bigendian in
+  yes)
+
+cat >>confdefs.h <<\_ACEOF
+#define WORDS_BIGENDIAN 1
+_ACEOF
+ ;;
+  no)
+     ;;
+  *)
+    { { echo "$as_me:$LINENO: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&5
+echo "$as_me: error: unknown endianness
+presetting ac_cv_c_bigendian=no (or yes) will help" >&2;}
+   { (exit 1); exit 1; }; } ;;
+esac
+
+echo "$as_me:$LINENO: checking for an ANSI C-conforming const" >&5
 echo $ECHO_N "checking for an ANSI C-conforming const... $ECHO_C" >&6
 if test "${ac_cv_c_const+set}" = set; then
   echo $ECHO_N "(cached) $ECHO_C" >&6

configure.in

 dnl Process this file with autoconf to produce a configure script.
-dnl $PostgreSQL: pgsql/configure.in,v 1.507 2007/03/29 15:30:51 mha Exp $
+dnl $PostgreSQL: pgsql/configure.in,v 1.508 2007/04/06 04:21:41 tgl Exp $
 dnl
 dnl Developers, please strive to achieve this order:
 dnl
@@ -869,6 +869,7 @@ fi
 ##
 
 m4_defun([AC_PROG_CC_STDC], []) dnl We don't want that.
+AC_C_BIGENDIAN
 AC_C_CONST
 AC_C_INLINE
 AC_C_STRINGIZE

contrib/dblink/dblink.c

@@ -8,7 +8,7 @@
  * Darko Prenosil <Darko.Prenosil@finteh.hr>
  * Shridhar Daithankar <shridhar_daithankar@persistent.co.in>
  *
- * $PostgreSQL: pgsql/contrib/dblink/dblink.c,v 1.62 2007/02/07 00:52:35 petere Exp $
+ * $PostgreSQL: pgsql/contrib/dblink/dblink.c,v 1.63 2007/04/06 04:21:41 tgl Exp $
  * Copyright (c) 2001-2007, PostgreSQL Global Development Group
  * ALL RIGHTS RESERVED;
  *
@@ -1752,8 +1752,8 @@ get_text_array_contents(ArrayType *array, int *numitems)
 		{
 			values[i] = DatumGetCString(DirectFunctionCall1(textout,
 													  PointerGetDatum(ptr)));
-			ptr = att_addlength(ptr, typlen, PointerGetDatum(ptr));
-			ptr = (char *) att_align(ptr, typalign);
+			ptr = att_addlength_pointer(ptr, typlen, ptr);
+			ptr = (char *) att_align_nominal(ptr, typalign);
 		}
 
 		/* advance bitmap pointer if any */

contrib/hstore/hstore_gist.c

@@ -170,7 +170,25 @@ ghstore_compress(PG_FUNCTION_ARGS)
 Datum
 ghstore_decompress(PG_FUNCTION_ARGS)
 {
-	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+	GISTENTRY  *entry = (GISTENTRY *) PG_GETARG_POINTER(0);
+	GISTENTRY  *retval;
+	HStore *key;
+
+	key = (HStore *) PG_DETOAST_DATUM(entry->key);
+
+	if (key != (HStore *) DatumGetPointer(entry->key))
+	{
+		/* need to pass back the decompressed item */
+		retval = palloc(sizeof(GISTENTRY));
+		gistentryinit(*retval, PointerGetDatum(key),
+					  entry->rel, entry->page, entry->offset, entry->leafkey);
+		PG_RETURN_POINTER(retval);
+	}
+	else
+	{
+		/* we can return the entry as-is */
+		PG_RETURN_POINTER(entry);
+	}
 }
 
 Datum

contrib/intarray/_int_gist.c

@@ -232,7 +232,16 @@ g_int_decompress(PG_FUNCTION_ARGS)
 
 	CHECKARRVALID(in);
 	if (ARRISVOID(in))
+	{
+		if (in != (ArrayType *) DatumGetPointer(entry->key)) {
+			retval = palloc(sizeof(GISTENTRY));
+			gistentryinit(*retval, PointerGetDatum(in),
+				entry->rel, entry->page, entry->offset, FALSE);
+			PG_RETURN_POINTER(retval);
+		}
+
 		PG_RETURN_POINTER(entry);
+	}
 
 	lenin = ARRNELEMS(in);
 

contrib/pg_trgm/trgm_gist.c

@@ -97,7 +97,7 @@ gtrgm_compress(PG_FUNCTION_ARGS)
 	if (entry->leafkey)
 	{							/* trgm */
 		TRGM	   *res;
-		text	   *val = (text *) DatumGetPointer(PG_DETOAST_DATUM(entry->key));
+		text	   *val = DatumGetTextP(entry->key);
 
 		res = generate_trgm(VARDATA(val), VARSIZE(val) - VARHDRSZ);
 		retval = (GISTENTRY *) palloc(sizeof(GISTENTRY));
@@ -134,7 +134,25 @@ gtrgm_compress(PG_FUNCTION_ARGS)
 Datum
 gtrgm_decompress(PG_FUNCTION_ARGS)
 {
-	PG_RETURN_DATUM(PG_GETARG_DATUM(0));
+	GISTENTRY  *entry = (GISTENTRY *) PG_GETARG_POINTER(0);
+	GISTENTRY  *retval;
+	text *key;
+
+	key = DatumGetTextP(entry->key);
+
+	if (key != (text *) DatumGetPointer(entry->key))
+	{
+		/* need to pass back the decompressed item */
+		retval = palloc(sizeof(GISTENTRY));
+		gistentryinit(*retval, PointerGetDatum(key),
+					  entry->rel, entry->page, entry->offset, entry->leafkey);
+		PG_RETURN_POINTER(retval);
+	}
+	else
+	{
+		/* we can return the entry as-is */
+		PG_RETURN_POINTER(entry);
+	}
 }
 
 Datum

contrib/tsearch2/ts_cfg.c

@@ -62,9 +62,7 @@ init_cfg(Oid id, TSCfgInfo * cfg)
 		ts_error(ERROR, "SPI_execp return %d", stat);
 	if (SPI_processed > 0)
 	{
-		prsname = (text *) DatumGetPointer(
-										   SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull)
-			);
+		prsname = DatumGetTextP(SPI_getbinval(SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, &isnull));
 		oldcontext = MemoryContextSwitchTo(TopMemoryContext);
 		prsname = ptextdup(prsname);
 		MemoryContextSwitchTo(oldcontext);

doc/src/sgml/storage.sgml

-<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.16 2007/04/03 04:14:26 tgl Exp $ -->
+<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.17 2007/04/06 04:21:41 tgl Exp $ -->
 
 <chapter id="storage">
 
@@ -210,18 +210,27 @@ value, but in some cases more efficient approaches are possible.)
 </para>
 
 <para>
-<acronym>TOAST</> usurps the high-order two bits of the varlena length word,
+<acronym>TOAST</> usurps two bits of the varlena length word (the high-order
+bits on big-endian machines, the low-order bits on little-endian machines),
 thereby limiting the logical size of any value of a <acronym>TOAST</>-able
 data type to 1 GB (2<superscript>30</> - 1 bytes).  When both bits are zero,
-the value is an ordinary un-<acronym>TOAST</>ed value of the data type.  One
-of these bits, if set, indicates that the value has been compressed and must
-be decompressed before use.  The other bit, if set, indicates that the value
-has been stored out-of-line.  In this case the remainder of the value is
-actually just a pointer, and the correct data has to be found elsewhere.  When
-both bits are set, the out-of-line data has been compressed too.  In each case
-the length in the low-order bits of the varlena word indicates the actual size
-of the datum, not the size of the logical value that would be extracted by
-decompression or fetching of the out-of-line data.
+the value is an ordinary un-<acronym>TOAST</>ed value of the data type, and
+the remaining bits of the length word give the total datum size (including
+length word) in bytes.  When the highest-order or lowest-order bit is set,
+the value has only a single-byte header instead of the normal four-byte
+header, and the remaining bits give the total datum size (including length
+byte) in bytes.  As a special case, if the remaining bits are all zero
+(which would be impossible for a self-inclusive length), the value is a
+pointer to out-of-line data stored in a separate TOAST table.  (The size of
+a TOAST pointer is known a priori, so it doesn't need to be represented in
+the header.)  Values with single-byte headers aren't aligned on any particular
+boundary, either.  Lastly, when the highest-order or lowest-order bit is
+clear but the adjacent bit is set, the content of the datum has been
+compressed and must be decompressed before use.  In this case the remaining
+bits of the length word give the total size of the compressed datum, not the
+original data.  Note that compression is also possible for out-of-line data
+but the varlena header does not tell whether it has occurred &mdash;
+the content of the TOAST pointer tells that, instead.
 </para>
 
 <para>
@@ -254,8 +263,8 @@ retrieval of the values.  A pointer datum representing an out-of-line
 <acronym>TOAST</> table in which to look and the OID of the specific value
 (its <structfield>chunk_id</>).  For convenience, pointer datums also store the
 logical datum size (original uncompressed data length) and actual stored size
-(different if compression was applied).  Allowing for the varlena header word,
-the total size of a <acronym>TOAST</> pointer datum is therefore 20 bytes
+(different if compression was applied).  Allowing for the varlena header byte,
+the total size of a <acronym>TOAST</> pointer datum is therefore 17 bytes
 regardless of the actual size of the represented value.
 </para>
 
@@ -280,7 +289,9 @@ The <acronym>TOAST</> code recognizes four different strategies for storing
     <listitem>
      <para>
       <literal>PLAIN</literal> prevents either compression or
-      out-of-line storage.  This is the only possible strategy for
+      out-of-line storage; furthermore it disables use of single-byte headers
+      for varlena types.
+      This is the only possible strategy for
       columns of non-<acronym>TOAST</>-able data types.
      </para>
     </listitem>
@@ -562,7 +573,7 @@ data. Empty in ordinary tables.</entry>
  <para>
 
   All table rows are structured in the same way. There is a fixed-size
-  header (occupying 27 bytes on most machines), followed by an optional null
+  header (occupying 23 bytes on most machines), followed by an optional null
   bitmap, an optional object ID field, and the user data. The header is
   detailed
   in <xref linkend="heaptupleheaderdata-table">.  The actual user data
@@ -604,12 +615,6 @@ data. Empty in ordinary tables.</entry>
    <entry>4 bytes</entry>
    <entry>insert XID stamp</entry>
   </row>
-  <row>
-   <entry>t_cmin</entry>
-   <entry>CommandId</entry>
-   <entry>4 bytes</entry>
-   <entry>insert CID stamp</entry>
-  </row>
   <row>
    <entry>t_xmax</entry>
    <entry>TransactionId</entry>
@@ -617,10 +622,10 @@ data. Empty in ordinary tables.</entry>
    <entry>delete XID stamp</entry>
   </row>
   <row>
-   <entry>t_cmax</entry>
+   <entry>t_cid</entry>
    <entry>CommandId</entry>
    <entry>4 bytes</entry>
-   <entry>delete CID stamp (overlays with t_xvac)</entry>
+   <entry>insert and/or delete CID stamp (overlays with t_xvac)</entry>
   </row>
   <row>
    <entry>t_xvac</entry>
@@ -635,10 +640,10 @@ data. Empty in ordinary tables.</entry>
    <entry>current TID of this or newer row version</entry>
   </row>
   <row>
-   <entry>t_natts</entry>
+   <entry>t_infomask2</entry>
    <entry>int16</entry>
    <entry>2 bytes</entry>
-   <entry>number of attributes</entry>
+   <entry>number of attributes, plus various flag bits</entry>
   </row>
   <row>
    <entry>t_infomask</entry>
@@ -682,7 +687,7 @@ data. Empty in ordinary tables.</entry>
   fixed width field, then all the bytes are simply placed. If it's a
   variable length field (attlen = -1) then it's a bit more complicated.
   All variable-length datatypes share the common header structure
-  <type>varattrib</type>, which includes the total length of the stored
+  <type>struct varlena</type>, which includes the total length of the stored
   value and some flag bits.  Depending on the flags, the data can be either
   inline or in a <acronym>TOAST</> table;
   it might be compressed, too (see <xref linkend="storage-toast">).

src/backend/access/common/indextuple.c

@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/access/common/indextuple.c,v 1.81 2007/02/27 23:48:06 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/access/common/indextuple.c,v 1.82 2007/04/06 04:21:41 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -38,6 +38,7 @@ index_form_tuple(TupleDesc tupleDescriptor,
 	char	   *tp;				/* tuple pointer */
 	IndexTuple	tuple;			/* return tuple */
 	Size		size,
+				data_size,
 				hoff;
 	int			i;
 	unsigned short infomask = 0;
@@ -74,9 +75,9 @@ index_form_tuple(TupleDesc tupleDescriptor,
 		 */
 		if (VARATT_IS_EXTERNAL(values[i]))
 		{
-			untoasted_values[i] = PointerGetDatum(
-												  heap_tuple_fetch_attr(
-								  (varattrib *) DatumGetPointer(values[i])));
+			untoasted_values[i] =
+				PointerGetDatum(heap_tuple_fetch_attr((struct varlena *)
+													  DatumGetPointer(values[i])));
 			untoasted_free[i] = true;
 		}
 
@@ -84,8 +85,8 @@ index_form_tuple(TupleDesc tupleDescriptor,
 		 * If value is above size target, and is of a compressible datatype,
 		 * try to compress it in-line.
 		 */
-		if (VARSIZE(untoasted_values[i]) > TOAST_INDEX_TARGET &&
-			!VARATT_IS_EXTENDED(untoasted_values[i]) &&
+		if (!VARATT_IS_EXTENDED(untoasted_values[i]) &&
+			VARSIZE(untoasted_values[i]) > TOAST_INDEX_TARGET &&
 			(att->attstorage == 'x' || att->attstorage == 'm'))
 		{
 			Datum		cvalue = toast_compress_datum(untoasted_values[i]);
@@ -116,12 +117,13 @@ index_form_tuple(TupleDesc tupleDescriptor,
 
 	hoff = IndexInfoFindDataOffset(infomask);
 #ifdef TOAST_INDEX_HACK
-	size = hoff + heap_compute_data_size(tupleDescriptor,
-										 untoasted_values, isnull);
+	data_size = heap_compute_data_size(tupleDescriptor,
+									   untoasted_values, isnull);
 #else
-	size = hoff + heap_compute_data_size(tupleDescriptor,
-										 values, isnull);
+	data_size = heap_compute_data_size(tupleDescriptor,
+									   values, isnull);
 #endif
+	size = hoff + data_size;
 	size = MAXALIGN(size);		/* be conservative */
 
 	tp = (char *) palloc0(size);
@@ -135,6 +137,7 @@ index_form_tuple(TupleDesc tupleDescriptor,
 #endif
 					isnull,
 					(char *) tp + hoff,
+					data_size,
 					&tupmask,
 					(hasnull ? (bits8 *) tp + sizeof(IndexTupleData) : NULL));
 
@@ -201,17 +204,14 @@ nocache_index_getattr(IndexTuple tup,
 					  bool *isnull)
 {
 	Form_pg_attribute *att = tupleDesc->attrs;
-	char	   *tp;				/* ptr to att in tuple */
-	bits8	   *bp = NULL;		/* ptr to null bitmask in tuple */
-	bool		slow = false;	/* do we have to walk nulls? */
+	char	   *tp;				/* ptr to data part of tuple */
+	bits8	   *bp = NULL;		/* ptr to null bitmap in tuple */
+	bool		slow = false;	/* do we have to walk attrs? */
 	int			data_off;		/* tuple data offset */
+	int			off;			/* current offset within data */
 
 	(void) isnull;				/* not used */
 
-	/*
-	 * sanity checks
-	 */
-
 	/* ----------------
 	 *	 Three cases:
 	 *
@@ -237,7 +237,7 @@ nocache_index_getattr(IndexTuple tup,
 	{
 #ifdef IN_MACRO
 /* This is handled in the macro */
-		if (att[attnum]->attcacheoff != -1)
+		if (att[attnum]->attcacheoff >= 0)
 		{
 			return fetchatt(att[attnum],
 							(char *) tup + data_off +
@@ -295,21 +295,28 @@ nocache_index_getattr(IndexTuple tup,
 
 	tp = (char *) tup + data_off;
 
-	/*
-	 * now check for any non-fixed length attrs before our attribute
-	 */
 	if (!slow)
 	{
-		if (att[attnum]->attcacheoff != -1)
+		/*
+		 * If we get here, there are no nulls up to and including the target
+		 * attribute.  If we have a cached offset, we can use it.
+		 */
+		if (att[attnum]->attcacheoff >= 0)
 		{
 			return fetchatt(att[attnum],
 							tp + att[attnum]->attcacheoff);
 		}
-		else if (IndexTupleHasVarwidths(tup))
+
+		/*
+		 * Otherwise, check for non-fixed-length attrs up to and including
+		 * target.  If there aren't any, it's safe to cheaply initialize
+		 * the cached offsets for these attrs.
+		 */
+		if (IndexTupleHasVarwidths(tup))
 		{
 			int			j;
 
-			for (j = 0; j < attnum; j++)
+			for (j = 0; j <= attnum; j++)
 			{
 				if (att[j]->attlen <= 0)
 				{
@@ -320,80 +327,109 @@ nocache_index_getattr(IndexTuple tup,
 		}
 	}
 
-	/*
-	 * If slow is false, and we got here, we know that we have a tuple with no
-	 * nulls or var-widths before the target attribute. If possible, we also
-	 * want to initialize the remainder of the attribute cached offset values.
-	 */
 	if (!slow)
 	{
+		int			natts = tupleDesc->natts;
 		int			j = 1;
-		long		off;
 
 		/*
-		 * need to set cache for some atts
+		 * If we get here, we have a tuple with no nulls or var-widths up to
+		 * and including the target attribute, so we can use the cached offset
+		 * ... only we don't have it yet, or we'd not have got here.  Since
+		 * it's cheap to compute offsets for fixed-width columns, we take the
+		 * opportunity to initialize the cached offsets for *all* the leading
+		 * fixed-width columns, in hope of avoiding future visits to this
+		 * routine.
 		 */
-
 		att[0]->attcacheoff = 0;
 
-		while (j < attnum && att[j]->attcacheoff > 0)
+		/* we might have set some offsets in the slow path previously */
+		while (j < natts && att[j]->attcacheoff > 0)
 			j++;
 
 		off = att[j - 1]->attcacheoff + att[j - 1]->attlen;
 
-		for (; j <= attnum; j++)
+		for (; j < natts; j++)
 		{
-			off = att_align(off, att[j]->attalign);
+			if (att[j]->attlen <= 0)
+				break;
+
+			off = att_align_nominal(off, att[j]->attalign);
 
 			att[j]->attcacheoff = off;
 
 			off += att[j]->attlen;
 		}
 
-		return fetchatt(att[attnum], tp + att[attnum]->attcacheoff);
+		Assert(j > attnum);
+
+		off = att[attnum]->attcacheoff;
 	}
 	else
 	{
 		bool		usecache = true;
-		int			off = 0;
 		int			i;
 
 		/*
-		 * Now we know that we have to walk the tuple CAREFULLY.
+		 * Now we know that we have to walk the tuple CAREFULLY.  But we
+		 * still might be able to cache some offsets for next time.
+		 *
+		 * Note - This loop is a little tricky.  For each non-null attribute,
+		 * we have to first account for alignment padding before the attr,
+		 * then advance over the attr based on its length.	Nulls have no
+		 * storage and no alignment padding either.  We can use/set
+		 * attcacheoff until we reach either a null or a var-width attribute.
 		 */
-
-		for (i = 0; i < attnum; i++)
+		off = 0;
+		for (i = 0; ; i++)			/* loop exit is at "break" */
 		{
-			if (IndexTupleHasNulls(tup))
+			if (IndexTupleHasNulls(tup) && att_isnull(i, bp))
 			{
-				if (att_isnull(i, bp))
-				{
-					usecache = false;
-					continue;
-				}
+				usecache = false;
+				continue;			/* this cannot be the target att */
 			}
 
 			/* If we know the next offset, we can skip the rest */
-			if (usecache && att[i]->attcacheoff != -1)
+			if (usecache && att[i]->attcacheoff >= 0)
 				off = att[i]->attcacheoff;
+			else if (att[i]->attlen == -1)
+			{
+				/*
+				 * We can only cache the offset for a varlena attribute
+				 * if the offset is already suitably aligned, so that there
+				 * would be no pad bytes in any case: then the offset will
+				 * be valid for either an aligned or unaligned value.
+				 */
+				if (usecache &&
+					off == att_align_nominal(off, att[i]->attalign))
+					att[i]->attcacheoff = off;
+				else
+				{
+					off = att_align_pointer(off, att[i]->attalign, -1,
+											tp + off);
+					usecache = false;
+				}
+			}
 			else
 			{
-				off = att_align(off, att[i]->attalign);
+				/* not varlena, so safe to use att_align_nominal */
+				off = att_align_nominal(off, att[i]->attalign);
 
 				if (usecache)
 					att[i]->attcacheoff = off;
 			}
 
-			off = att_addlength(off, att[i]->attlen, tp + off);
+			if (i == attnum)
+				break;
+
+			off = att_addlength_pointer(off, att[i]->attlen, tp + off);
 
 			if (usecache && att[i]->attlen <= 0)
 				usecache = false;
 		}
-
-		off = att_align(off, att[attnum]->attalign);
-
-		return fetchatt(att[attnum], tp + off);
 	}
+
+	return fetchatt(att[attnum], tp + off);
 }
 
 /*

src/backend/catalog/toasting.c

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/catalog/toasting.c,v 1.5 2007/01/09 02:14:11 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/catalog/toasting.c,v 1.6 2007/04/06 04:21:42 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -314,7 +314,7 @@ needs_toast_table(Relation rel)
 	{
 		if (att[i]->attisdropped)
 			continue;
-		data_length = att_align(data_length, att[i]->attalign);
+		data_length = att_align_nominal(data_length, att[i]->attalign);
 		if (att[i]->attlen > 0)
 		{
 			/* Fixed-length types are never toastable */

src/backend/commands/analyze.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/commands/analyze.c,v 1.103 2007/01/09 02:14:11 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/commands/analyze.c,v 1.104 2007/04/06 04:21:42 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -1478,7 +1478,7 @@ compute_minimal_stats(VacAttrStatsP stats,
 		 */
 		if (is_varlena)
 		{
-			total_width += VARSIZE(DatumGetPointer(value));
+			total_width += VARSIZE_ANY(DatumGetPointer(value));
 
 			/*
 			 * If the value is toasted, we want to detoast it just once to
@@ -1792,7 +1792,7 @@ compute_scalar_stats(VacAttrStatsP stats,
 		 */
 		if (is_varlena)
 		{
-			total_width += VARSIZE(DatumGetPointer(value));
+			total_width += VARSIZE_ANY(DatumGetPointer(value));
 
 			/*
 			 * If the value is toasted, we want to detoast it just once to

src/backend/executor/execQual.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/executor/execQual.c,v 1.216 2007/03/27 23:21:08 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/executor/execQual.c,v 1.217 2007/04/06 04:21:42 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -1896,8 +1896,8 @@ ExecEvalScalarArrayOp(ScalarArrayOpExprState *sstate,
 		else
 		{
 			elt = fetch_att(s, typbyval, typlen);
-			s = att_addlength(s, typlen, PointerGetDatum(s));
-			s = (char *) att_align(s, typalign);
+			s = att_addlength_pointer(s, typlen, s);
+			s = (char *) att_align_nominal(s, typalign);
 			fcinfo.arg[1] = elt;
 			fcinfo.argnull[1] = false;
 		}

src/backend/storage/large_object/inv_api.c

@@ -17,7 +17,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/storage/large_object/inv_api.c,v 1.123 2007/03/03 19:52:46 momjian Exp $
+ *	  $PostgreSQL: pgsql/src/backend/storage/large_object/inv_api.c,v 1.124 2007/04/06 04:21:42 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -336,7 +336,7 @@ inv_getsize(LargeObjectDesc *obj_desc)
 		if (VARATT_IS_EXTENDED(datafield))
 		{
 			datafield = (bytea *)
-				heap_tuple_untoast_attr((varattrib *) datafield);
+				heap_tuple_untoast_attr((struct varlena *) datafield);
 			pfreeit = true;
 		}
 		lastbyte = data->pageno * LOBLKSIZE + getbytealen(datafield);
@@ -462,7 +462,7 @@ inv_read(LargeObjectDesc *obj_desc, char *buf, int nbytes)
 			if (VARATT_IS_EXTENDED(datafield))
 			{
 				datafield = (bytea *)
-					heap_tuple_untoast_attr((varattrib *) datafield);
+					heap_tuple_untoast_attr((struct varlena *) datafield);
 				pfreeit = true;
 			}
 			len = getbytealen(datafield);
@@ -580,7 +580,7 @@ inv_write(LargeObjectDesc *obj_desc, const char *buf, int nbytes)
 			if (VARATT_IS_EXTENDED(datafield))
 			{
 				datafield = (bytea *)
-					heap_tuple_untoast_attr((varattrib *) datafield);
+					heap_tuple_untoast_attr((struct varlena *) datafield);
 				pfreeit = true;
 			}
 			len = getbytealen(datafield);
@@ -756,7 +756,7 @@ inv_truncate(LargeObjectDesc *obj_desc, int len)
 		if (VARATT_IS_EXTENDED(datafield))
 		{
 			datafield = (bytea *)
-				heap_tuple_untoast_attr((varattrib *) datafield);
+				heap_tuple_untoast_attr((struct varlena *) datafield);
 			pfreeit = true;
 		}
 		pagelen = getbytealen(datafield);

src/backend/utils/adt/arrayfuncs.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/utils/adt/arrayfuncs.c,v 1.138 2007/03/27 23:21:10 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/utils/adt/arrayfuncs.c,v 1.139 2007/04/06 04:21:42 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -805,8 +805,8 @@ ReadArrayStr(char *arrayStr,
 			/* let's just make sure data is not toasted */
 			if (typlen == -1)
 				values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i]));
-			totbytes = att_addlength(totbytes, typlen, values[i]);
-			totbytes = att_align(totbytes, typalign);
+			totbytes = att_addlength_datum(totbytes, typlen, values[i]);
+			totbytes = att_align_nominal(totbytes, typalign);
 			/* check for overflow of total request */
 			if (!AllocSizeIsValid(totbytes))
 				ereport(ERROR,
@@ -1011,8 +1011,8 @@ array_out(PG_FUNCTION_ARGS)
 
 			itemvalue = fetch_att(p, typbyval, typlen);
 			values[i] = OutputFunctionCall(&my_extra->proc, itemvalue);
-			p = att_addlength(p, typlen, PointerGetDatum(p));
-			p = (char *) att_align(p, typalign);
+			p = att_addlength_pointer(p, typlen, p);
+			p = (char *) att_align_nominal(p, typalign);
 
 			/* count data plus backslashes; detect chars needing quotes */
 			if (values[i][0] == '\0')
@@ -1399,8 +1399,8 @@ ReadArrayBinary(StringInfo buf,
 			/* let's just make sure data is not toasted */
 			if (typlen == -1)
 				values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i]));
-			totbytes = att_addlength(totbytes, typlen, values[i]);
-			totbytes = att_align(totbytes, typalign);
+			totbytes = att_addlength_datum(totbytes, typlen, values[i]);
+			totbytes = att_align_nominal(totbytes, typalign);
 			/* check for overflow of total request */
 			if (!AllocSizeIsValid(totbytes))
 				ereport(ERROR,
@@ -1512,8 +1512,8 @@ array_send(PG_FUNCTION_ARGS)
 						 VARSIZE(outputbytes) - VARHDRSZ);
 			pfree(outputbytes);
 
-			p = att_addlength(p, typlen, PointerGetDatum(p));
-			p = (char *) att_align(p, typalign);
+			p = att_addlength_pointer(p, typlen, p);
+			p = (char *) att_align_nominal(p, typalign);
 		}
 
 		/* advance bitmap pointer if any */
@@ -2108,8 +2108,8 @@ array_set(ArrayType *array,
 			olditemlen = 0;
 		else
 		{
-			olditemlen = att_addlength(0, elmlen, PointerGetDatum(elt_ptr));
-			olditemlen = att_align(olditemlen, elmalign);
+			olditemlen = att_addlength_pointer(0, elmlen, elt_ptr);
+			olditemlen = att_align_nominal(olditemlen, elmalign);
 		}
 		lenafter = (int) (olddatasize - lenbefore - olditemlen);
 	}
@@ -2118,8 +2118,8 @@ array_set(ArrayType *array,
 		newitemlen = 0;
 	else
 	{
-		newitemlen = att_addlength(0, elmlen, dataValue);
-		newitemlen = att_align(newitemlen, elmalign);
+		newitemlen = att_addlength_datum(0, elmlen, dataValue);
+		newitemlen = att_align_nominal(newitemlen, elmalign);
 	}
 
 	newsize = overheadlen + lenbefore + newitemlen + lenafter;
@@ -2639,8 +2639,8 @@ array_map(FunctionCallInfo fcinfo, Oid inpType, Oid retType,
 		else
 		{
 			elt = fetch_att(s, inp_typbyval, inp_typlen);
-			s = att_addlength(s, inp_typlen, elt);
-			s = (char *) att_align(s, inp_typalign);
+			s = att_addlength_datum(s, inp_typlen, elt);
+			s = (char *) att_align_nominal(s, inp_typalign);
 			fcinfo->arg[0] = elt;
 			fcinfo->argnull[0] = false;
 		}
@@ -2679,8 +2679,8 @@ array_map(FunctionCallInfo fcinfo, Oid inpType, Oid retType,
 			if (typlen == -1)
 				values[i] = PointerGetDatum(PG_DETOAST_DATUM(values[i]));
 			/* Update total result size */
-			nbytes = att_addlength(nbytes, typlen, values[i]);
-			nbytes = att_align(nbytes, typalign);
+			nbytes = att_addlength_datum(nbytes, typlen, values[i]);
+			nbytes = att_align_nominal(nbytes, typalign);
 			/* check for overflow of total request */
 			if (!AllocSizeIsValid(nbytes))
 				ereport(ERROR,
@@ -2827,8 +2827,8 @@ construct_md_array(Datum *elems,
 		/* make sure data is not toasted */
 		if (elmlen == -1)
 			elems[i] = PointerGetDatum(PG_DETOAST_DATUM(elems[i]));
-		nbytes = att_addlength(nbytes, elmlen, elems[i]);
-		nbytes = att_align(nbytes, elmalign);
+		nbytes = att_addlength_datum(nbytes, elmlen, elems[i]);
+		nbytes = att_align_nominal(nbytes, elmalign);
 		/* check for overflow of total request */
 		if (!AllocSizeIsValid(nbytes))
 			ereport(ERROR,
@@ -2947,8 +2947,8 @@ deconstruct_array(ArrayType *array,
 			elems[i] = fetch_att(p, elmbyval, elmlen);
 			if (nulls)
 				nulls[i] = false;
-			p = att_addlength(p, elmlen, PointerGetDatum(p));
-			p = (char *) att_align(p, elmalign);
+			p = att_addlength_pointer(p, elmlen, p);
+			p = (char *) att_align_nominal(p, elmalign);
 		}
 
 		/* advance bitmap pointer if any */
@@ -3064,8 +3064,8 @@ array_eq(PG_FUNCTION_ARGS)
 			{
 				isnull1 = false;
 				elt1 = fetch_att(ptr1, typbyval, typlen);
-				ptr1 = att_addlength(ptr1, typlen, PointerGetDatum(ptr1));
-				ptr1 = (char *) att_align(ptr1, typalign);
+				ptr1 = att_addlength_pointer(ptr1, typlen, ptr1);
+				ptr1 = (char *) att_align_nominal(ptr1, typalign);
 			}
 
 			if (bitmap2 && (*bitmap2 & bitmask) == 0)
@@ -3077,8 +3077,8 @@ array_eq(PG_FUNCTION_ARGS)
 			{
 				isnull2 = false;
 				elt2 = fetch_att(ptr2, typbyval, typlen);
-				ptr2 = att_addlength(ptr2, typlen, PointerGetDatum(ptr2));
-				ptr2 = (char *) att_align(ptr2, typalign);
+				ptr2 = att_addlength_pointer(ptr2, typlen, ptr2);
+				ptr2 = (char *) att_align_nominal(ptr2, typalign);
 			}
 
 			/* advance bitmap pointers if any */
@@ -3265,8 +3265,8 @@ array_cmp(FunctionCallInfo fcinfo)
 		{
 			isnull1 = false;
 			elt1 = fetch_att(ptr1, typbyval, typlen);
-			ptr1 = att_addlength(ptr1, typlen, PointerGetDatum(ptr1));
-			ptr1 = (char *) att_align(ptr1, typalign);
+			ptr1 = att_addlength_pointer(ptr1, typlen, ptr1);
+			ptr1 = (char *) att_align_nominal(ptr1, typalign);
 		}
 
 		if (bitmap2 && (*bitmap2 & bitmask) == 0)
@@ -3278,8 +3278,8 @@ array_cmp(FunctionCallInfo fcinfo)
 		{
 			isnull2 = false;
 			elt2 = fetch_att(ptr2, typbyval, typlen);
-			ptr2 = att_addlength(ptr2, typlen, PointerGetDatum(ptr2));
-			ptr2 = (char *) att_align(ptr2, typalign);
+			ptr2 = att_addlength_pointer(ptr2, typlen, ptr2);
+			ptr2 = (char *) att_align_nominal(ptr2, typalign);
 		}
 
 		/* advance bitmap pointers if any */
@@ -3468,8 +3468,8 @@ array_contain_compare(ArrayType *array1, ArrayType *array2, bool matchall,
 		{
 			isnull1 = false;
 			elt1 = fetch_att(ptr1, typbyval, typlen);
-			ptr1 = att_addlength(ptr1, typlen, PointerGetDatum(ptr1));
-			ptr1 = (char *) att_align(ptr1, typalign);
+			ptr1 = att_addlength_pointer(ptr1, typlen, ptr1);
+			ptr1 = (char *) att_align_nominal(ptr1, typalign);
 		}
 
 		/* advance bitmap pointer if any */
@@ -3667,14 +3667,14 @@ ArrayCastAndSet(Datum src,
 			store_att_byval(dest, src, typlen);
 		else
 			memmove(dest, DatumGetPointer(src), typlen);
-		inc = att_align(typlen, typalign);
+		inc = att_align_nominal(typlen, typalign);
 	}
 	else
 	{
 		Assert(!typbyval);
-		inc = att_addlength(0, typlen, src);
+		inc = att_addlength_datum(0, typlen, src);
 		memmove(dest, DatumGetPointer(src), inc);
-		inc = att_align(inc, typalign);
+		inc = att_align_nominal(inc, typalign);
 	}
 
 	return inc;
@@ -3700,7 +3700,7 @@ array_seek(char *ptr, int offset, bits8 *nullbitmap, int nitems,
 
 	/* easy if fixed-size elements and no NULLs */
 	if (typlen > 0 && !nullbitmap)
-		return ptr + nitems * ((Size) att_align(typlen, typalign));
+		return ptr + nitems * ((Size) att_align_nominal(typlen, typalign));
 
 	/* seems worth having separate loops for NULL and no-NULLs cases */
 	if (nullbitmap)
@@ -3712,8 +3712,8 @@ array_seek(char *ptr, int offset, bits8 *nullbitmap, int nitems,
 		{
 			if (*nullbitmap & bitmask)
 			{
-				ptr = att_addlength(ptr, typlen, PointerGetDatum(ptr));
-				ptr = (char *) att_align(ptr, typalign);
+				ptr = att_addlength_pointer(ptr, typlen, ptr);
+				ptr = (char *) att_align_nominal(ptr, typalign);
 			}
 			bitmask <<= 1;
 			if (bitmask == 0x100)
@@ -3727,8 +3727,8 @@ array_seek(char *ptr, int offset, bits8 *nullbitmap, int nitems,
 	{
 		for (i = 0; i < nitems; i++)
 		{
-			ptr = att_addlength(ptr, typlen, PointerGetDatum(ptr));
-			ptr = (char *) att_align(ptr, typalign);
+			ptr = att_addlength_pointer(ptr, typlen, ptr);
+			ptr = (char *) att_align_nominal(ptr, typalign);
 		}
 	}
 	return ptr;
@@ -3883,7 +3883,7 @@ array_slice_size(char *arraydataptr, bits8 *arraynullsptr,
 
 	/* Pretty easy for fixed element length without nulls ... */
 	if (typlen > 0 && !arraynullsptr)
-		return ArrayGetNItems(ndim, span) * att_align(typlen, typalign);
+		return ArrayGetNItems(ndim, span) * att_align_nominal(typlen, typalign);
 
 	/* Else gotta do it the hard way */
 	src_offset = ArrayGetOffset(ndim, dim, lb, st);
@@ -3904,8 +3904,8 @@ array_slice_size(char *arraydataptr, bits8 *arraynullsptr,
 		}
 		if (!array_get_isnull(arraynullsptr, src_offset))
 		{
-			inc = att_addlength(0, typlen, PointerGetDatum(ptr));
-			inc = att_align(inc, typalign);
+			inc = att_addlength_pointer(0, typlen, ptr);
+			inc = att_align_nominal(inc, typalign);
 			ptr += inc;
 			count += inc;
 		}

src/backend/utils/adt/datum.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/utils/adt/datum.c,v 1.34 2007/02/27 23:48:07 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/utils/adt/datum.c,v 1.35 2007/04/06 04:21:42 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -47,7 +47,7 @@
  * Find the "real" size of a datum, given the datum value,
  * whether it is a "by value", and the declared type length.
  *
- * This is essentially an out-of-line version of the att_addlength()
+ * This is essentially an out-of-line version of the att_addlength_datum()
  * macro in access/tupmacs.h.  We do a tad more error checking though.
  *-------------------------------------------------------------------------
  */
@@ -79,7 +79,7 @@ datumGetSize(Datum value, bool typByVal, int typLen)
 						(errcode(ERRCODE_DATA_EXCEPTION),
 						 errmsg("invalid Datum pointer")));
 
-			size = (Size) VARSIZE(s);
+			size = (Size) VARSIZE_ANY(s);
 		}
 		else if (typLen == -2)
 		{

src/backend/utils/adt/network.c

 /*
  *	PostgreSQL type definitions for the INET and CIDR types.
  *
- *	$PostgreSQL: pgsql/src/backend/utils/adt/network.c,v 1.68 2007/02/27 23:48:08 tgl Exp $
+ *	$PostgreSQL: pgsql/src/backend/utils/adt/network.c,v 1.69 2007/04/06 04:21:43 tgl Exp $
  *
  *	Jon Postel RIP 16 Oct 1998
  */
@@ -30,23 +30,38 @@ static int	ip_addrsize(inet *inetptr);
 static inet *internal_inetpl(inet *ip, int64 addend);
 
 /*
- *	Access macros.
+ *	Access macros.  We use VARDATA_ANY so that we can process short-header
+ *	varlena values without detoasting them.  This requires a trick:
+ *	VARDATA_ANY assumes the varlena header is already filled in, which is
+ *	not the case when constructing a new value (until SET_INET_VARSIZE is
+ *	called, which we typically can't do till the end).  Therefore, we
+ *	always initialize the newly-allocated value to zeroes (using palloc0).
+ *	A zero length word will look like the not-1-byte case to VARDATA_ANY,
+ *	and so we correctly construct an uncompressed value.
+ *
+ *	Note that ip_maxbits() and SET_INET_VARSIZE() require
+ *	the family field to be set correctly.
  */
 
 #define ip_family(inetptr) \
-	(((inet_struct *)VARDATA(inetptr))->family)
+	(((inet_struct *) VARDATA_ANY(inetptr))->family)
 
 #define ip_bits(inetptr) \
-	(((inet_struct *)VARDATA(inetptr))->bits)
+	(((inet_struct *) VARDATA_ANY(inetptr))->bits)
 
 #define ip_addr(inetptr) \
-	(((inet_struct *)VARDATA(inetptr))->ipaddr)
+	(((inet_struct *) VARDATA_ANY(inetptr))->ipaddr)
 
 #define ip_maxbits(inetptr) \
 	(ip_family(inetptr) == PGSQL_AF_INET ? 32 : 128)
 
+#define SET_INET_VARSIZE(dst) \
+	SET_VARSIZE(dst, VARHDRSZ + offsetof(inet_struct, ipaddr) + \
+				ip_addrsize(dst))
+
+
 /*
- * Return the number of bytes of storage needed for this data type.
+ * Return the number of bytes of address storage needed for this data type.
  */
 static int
 ip_addrsize(inet *inetptr)
@@ -71,7 +86,7 @@ network_in(char *src, bool is_cidr)
 	int			bits;
 	inet	   *dst;
 
-	dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct));
+	dst = (inet *) palloc0(sizeof(inet));
 
 	/*
 	 * First, check to see if this is an IPv6 or IPv4 address.	IPv6 addresses
@@ -105,10 +120,8 @@ network_in(char *src, bool is_cidr)
 					 errdetail("Value has bits set to right of mask.")));
 	}
 
-	SET_VARSIZE(dst, VARHDRSZ +
-		((char *) ip_addr(dst) - (char *) VARDATA(dst)) +
-		ip_addrsize(dst));
 	ip_bits(dst) = bits;
+	SET_INET_VARSIZE(dst);
 
 	return dst;
 }
@@ -194,7 +207,7 @@ network_recv(StringInfo buf, bool is_cidr)
 				i;
 
 	/* make sure any unused bits in a CIDR value are zeroed */
-	addr = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct));
+	addr = (inet *) palloc0(sizeof(inet));
 
 	ip_family(addr) = pq_getmsgbyte(buf);
 	if (ip_family(addr) != PGSQL_AF_INET &&
@@ -220,9 +233,6 @@ network_recv(StringInfo buf, bool is_cidr)
 		/* translator: %s is inet or cidr */
 				 errmsg("invalid length in external \"%s\" value",
 						is_cidr ? "cidr" : "inet")));
-	SET_VARSIZE(addr, VARHDRSZ +
-		((char *) ip_addr(addr) - (char *) VARDATA(addr)) +
-		ip_addrsize(addr));
 
 	addrptr = (char *) ip_addr(addr);
 	for (i = 0; i < nb; i++)
@@ -240,6 +250,8 @@ network_recv(StringInfo buf, bool is_cidr)
 					 errdetail("Value has bits set to right of mask.")));
 	}
 
+	SET_INET_VARSIZE(addr);
+
 	return addr;
 }
 
@@ -348,8 +360,8 @@ inet_to_cidr(PG_FUNCTION_ARGS)
 		elog(ERROR, "invalid inet bit length: %d", bits);
 
 	/* clone the original data */
-	dst = (inet *) palloc(VARSIZE(src));
-	memcpy(dst, src, VARSIZE(src));
+	dst = (inet *) palloc(VARSIZE_ANY(src));
+	memcpy(dst, src, VARSIZE_ANY(src));
 
 	/* zero out any bits to the right of the netmask */
 	byte = bits / 8;
@@ -387,8 +399,8 @@ inet_set_masklen(PG_FUNCTION_ARGS)
 				 errmsg("invalid mask length: %d", bits)));
 
 	/* clone the original data */
-	dst = (inet *) palloc(VARSIZE(src));
-	memcpy(dst, src, VARSIZE(src));
+	dst = (inet *) palloc(VARSIZE_ANY(src));
+	memcpy(dst, src, VARSIZE_ANY(src));
 
 	ip_bits(dst) = bits;
 
@@ -414,8 +426,8 @@ cidr_set_masklen(PG_FUNCTION_ARGS)
 				 errmsg("invalid mask length: %d", bits)));
 
 	/* clone the original data */
-	dst = (inet *) palloc(VARSIZE(src));
-	memcpy(dst, src, VARSIZE(src));
+	dst = (inet *) palloc(VARSIZE_ANY(src));
+	memcpy(dst, src, VARSIZE_ANY(src));
 
 	ip_bits(dst) = bits;
 
@@ -546,7 +558,7 @@ hashinet(PG_FUNCTION_ARGS)
 	int			addrsize = ip_addrsize(addr);
 
 	/* XXX this assumes there are no pad bytes in the data structure */
-	return hash_any((unsigned char *) VARDATA(addr), addrsize + 2);
+	return hash_any((unsigned char *) VARDATA_ANY(addr), addrsize + 2);
 }
 
 /*
@@ -762,7 +774,7 @@ network_broadcast(PG_FUNCTION_ARGS)
 			   *b;
 
 	/* make sure any unused bits are zeroed */
-	dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct));
+	dst = (inet *) palloc0(sizeof(inet));
 
 	if (ip_family(ip) == PGSQL_AF_INET)
 		maxbytes = 4;
@@ -793,9 +805,7 @@ network_broadcast(PG_FUNCTION_ARGS)
 
 	ip_family(dst) = ip_family(ip);
 	ip_bits(dst) = ip_bits(ip);
-	SET_VARSIZE(dst, VARHDRSZ +
-		((char *) ip_addr(dst) - (char *) VARDATA(dst)) +
-		ip_addrsize(dst));
+	SET_INET_VARSIZE(dst);
 
 	PG_RETURN_INET_P(dst);
 }
@@ -812,7 +822,7 @@ network_network(PG_FUNCTION_ARGS)
 			   *b;
 
 	/* make sure any unused bits are zeroed */
-	dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct));
+	dst = (inet *) palloc0(sizeof(inet));
 
 	bits = ip_bits(ip);
 	a = ip_addr(ip);
@@ -838,9 +848,7 @@ network_network(PG_FUNCTION_ARGS)
 
 	ip_family(dst) = ip_family(ip);
 	ip_bits(dst) = ip_bits(ip);
-	SET_VARSIZE(dst, VARHDRSZ +
-		((char *) ip_addr(dst) - (char *) VARDATA(dst)) +
-		ip_addrsize(dst));
+	SET_INET_VARSIZE(dst);
 
 	PG_RETURN_INET_P(dst);
 }
@@ -856,7 +864,7 @@ network_netmask(PG_FUNCTION_ARGS)
 	unsigned char *b;
 
 	/* make sure any unused bits are zeroed */
-	dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct));
+	dst = (inet *) palloc0(sizeof(inet));
 
 	bits = ip_bits(ip);
 	b = ip_addr(dst);
@@ -881,9 +889,7 @@ network_netmask(PG_FUNCTION_ARGS)
 
 	ip_family(dst) = ip_family(ip);
 	ip_bits(dst) = ip_maxbits(ip);
-	SET_VARSIZE(dst, VARHDRSZ +
-		((char *) ip_addr(dst) - (char *) VARDATA(dst)) +
-		ip_addrsize(dst));
+	SET_INET_VARSIZE(dst);
 
 	PG_RETURN_INET_P(dst);
 }
@@ -900,7 +906,7 @@ network_hostmask(PG_FUNCTION_ARGS)
 	unsigned char *b;
 
 	/* make sure any unused bits are zeroed */
-	dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct));
+	dst = (inet *) palloc0(sizeof(inet));
 
 	if (ip_family(ip) == PGSQL_AF_INET)
 		maxbytes = 4;
@@ -930,9 +936,7 @@ network_hostmask(PG_FUNCTION_ARGS)
 
 	ip_family(dst) = ip_family(ip);
 	ip_bits(dst) = ip_maxbits(ip);
-	SET_VARSIZE(dst, VARHDRSZ +
-		((char *) ip_addr(dst) - (char *) VARDATA(dst)) +
-		ip_addrsize(dst));
+	SET_INET_VARSIZE(dst);
 
 	PG_RETURN_INET_P(dst);
 }
@@ -1259,7 +1263,7 @@ inetnot(PG_FUNCTION_ARGS)
 	inet	   *ip = PG_GETARG_INET_P(0);
 	inet	   *dst;
 
-	dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct));
+	dst = (inet *) palloc0(sizeof(inet));
 
 	{
 		int			nb = ip_addrsize(ip);
@@ -1272,9 +1276,7 @@ inetnot(PG_FUNCTION_ARGS)
 	ip_bits(dst) = ip_bits(ip);
 
 	ip_family(dst) = ip_family(ip);
-	SET_VARSIZE(dst, VARHDRSZ +
-		((char *) ip_addr(dst) - (char *) VARDATA(dst)) +
-		ip_addrsize(dst));
+	SET_INET_VARSIZE(dst);
 
 	PG_RETURN_INET_P(dst);
 }
@@ -1287,7 +1289,7 @@ inetand(PG_FUNCTION_ARGS)
 	inet	   *ip2 = PG_GETARG_INET_P(1);
 	inet	   *dst;
 
-	dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct));
+	dst = (inet *) palloc0(sizeof(inet));
 
 	if (ip_family(ip) != ip_family(ip2))
 		ereport(ERROR,
@@ -1306,9 +1308,7 @@ inetand(PG_FUNCTION_ARGS)
 	ip_bits(dst) = Max(ip_bits(ip), ip_bits(ip2));
 
 	ip_family(dst) = ip_family(ip);
-	SET_VARSIZE(dst, VARHDRSZ +
-		((char *) ip_addr(dst) - (char *) VARDATA(dst)) +
-		ip_addrsize(dst));
+	SET_INET_VARSIZE(dst);
 
 	PG_RETURN_INET_P(dst);
 }
@@ -1321,7 +1321,7 @@ inetor(PG_FUNCTION_ARGS)
 	inet	   *ip2 = PG_GETARG_INET_P(1);
 	inet	   *dst;
 
-	dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct));
+	dst = (inet *) palloc0(sizeof(inet));
 
 	if (ip_family(ip) != ip_family(ip2))
 		ereport(ERROR,
@@ -1340,9 +1340,7 @@ inetor(PG_FUNCTION_ARGS)
 	ip_bits(dst) = Max(ip_bits(ip), ip_bits(ip2));
 
 	ip_family(dst) = ip_family(ip);
-	SET_VARSIZE(dst, VARHDRSZ +
-		((char *) ip_addr(dst) - (char *) VARDATA(dst)) +
-		ip_addrsize(dst));
+	SET_INET_VARSIZE(dst);
 
 	PG_RETURN_INET_P(dst);
 }
@@ -1353,7 +1351,7 @@ internal_inetpl(inet *ip, int64 addend)
 {
 	inet	   *dst;
 
-	dst = (inet *) palloc0(VARHDRSZ + sizeof(inet_struct));
+	dst = (inet *) palloc0(sizeof(inet));
 
 	{
 		int			nb = ip_addrsize(ip);
@@ -1391,12 +1389,10 @@ internal_inetpl(inet *ip, int64 addend)
 					(errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE),
 					 errmsg("result is out of range")));
 	}
-	ip_bits(dst) = ip_bits(ip);
 
+	ip_bits(dst) = ip_bits(ip);
 	ip_family(dst) = ip_family(ip);
-	SET_VARSIZE(dst, VARHDRSZ +
-		((char *) ip_addr(dst) - (char *) VARDATA(dst)) +
-		ip_addrsize(dst));
+	SET_INET_VARSIZE(dst);
 
 	return dst;
 }

src/backend/utils/adt/pg_lzcompress.c

@@ -166,7 +166,7 @@
  *
  * Copyright (c) 1999-2007, PostgreSQL Global Development Group
  *
- * $PostgreSQL: pgsql/src/backend/utils/adt/pg_lzcompress.c,v 1.25 2007/02/27 23:48:08 tgl Exp $
+ * $PostgreSQL: pgsql/src/backend/utils/adt/pg_lzcompress.c,v 1.26 2007/04/06 04:21:43 tgl Exp $
  * ----------
  */
 #include "postgres.h"
@@ -618,7 +618,7 @@ pglz_compress(const char *source, int32 slen, PGLZ_Header *dest,
 	/*
 	 * Success - need only fill in the actual length of the compressed datum.
 	 */
-	SET_VARSIZE(dest, result_size + sizeof(PGLZ_Header));
+	SET_VARSIZE_COMPRESSED(dest, result_size + sizeof(PGLZ_Header));
 
 	return true;
 }

src/backend/utils/adt/xml.c

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/backend/utils/adt/xml.c,v 1.41 2007/04/05 13:53:23 momjian Exp $
+ * $PostgreSQL: pgsql/src/backend/utils/adt/xml.c,v 1.42 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -3071,8 +3071,8 @@ xmlpath(PG_FUNCTION_ARGS)
 				else
 					ns_uris[i - ns_count] = DatumGetCString(DirectFunctionCall1(textout,
 														  PointerGetDatum(ptr)));
-				ptr = att_addlength(ptr, typlen, PointerGetDatum(ptr));
-				ptr = (char *) att_align(ptr, typalign);
+				ptr = att_addlength_pointer(ptr, typlen, ptr);
+				ptr = (char *) att_align_nominal(ptr, typalign);
 			}
 	
 			/* advance bitmap pointer if any */

src/backend/utils/fmgr/fmgr.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/utils/fmgr/fmgr.c,v 1.105 2007/03/27 23:21:10 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/utils/fmgr/fmgr.c,v 1.106 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -1962,7 +1962,7 @@ struct varlena *
 pg_detoast_datum(struct varlena * datum)
 {
 	if (VARATT_IS_EXTENDED(datum))
-		return (struct varlena *) heap_tuple_untoast_attr((varattrib *) datum);
+		return heap_tuple_untoast_attr(datum);
 	else
 		return datum;
 }
@@ -1971,7 +1971,7 @@ struct varlena *
 pg_detoast_datum_copy(struct varlena * datum)
 {
 	if (VARATT_IS_EXTENDED(datum))
-		return (struct varlena *) heap_tuple_untoast_attr((varattrib *) datum);
+		return heap_tuple_untoast_attr(datum);
 	else
 	{
 		/* Make a modifiable copy of the varlena object */
@@ -1987,7 +1987,16 @@ struct varlena *
 pg_detoast_datum_slice(struct varlena * datum, int32 first, int32 count)
 {
 	/* Only get the specified portion from the toast rel */
-	return (struct varlena *) heap_tuple_untoast_attr_slice((varattrib *) datum, first, count);
+	return heap_tuple_untoast_attr_slice(datum, first, count);
+}
+
+struct varlena *
+pg_detoast_datum_packed(struct varlena * datum)
+{
+	if (VARATT_IS_COMPRESSED(datum) || VARATT_IS_EXTERNAL(datum))
+		return heap_tuple_untoast_attr(datum);
+	else
+		return datum;
 }
 
 /*-------------------------------------------------------------------------

src/backend/utils/init/flatfiles.c

@@ -23,7 +23,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/backend/utils/init/flatfiles.c,v 1.24 2007/01/05 22:19:43 momjian Exp $
+ * $PostgreSQL: pgsql/src/backend/utils/init/flatfiles.c,v 1.25 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -467,7 +467,7 @@ write_auth_file(Relation rel_authid, Relation rel_authmem)
 				auth_info[curr_role].rolpassword = DatumGetCString(DirectFunctionCall1(textout, datum));
 
 			/* assume passwd has attlen -1 */
-			off = att_addlength(off, -1, tp + off);
+			off = att_addlength_pointer(off, -1, tp + off);
 		}
 
 		if (HeapTupleHasNulls(tuple) &&
@@ -482,7 +482,7 @@ write_auth_file(Relation rel_authid, Relation rel_authmem)
 			 * rolvaliduntil is timestamptz, which we assume is double
 			 * alignment and pass-by-reference.
 			 */
-			off = att_align(off, 'd');
+			off = att_align_nominal(off, 'd');
 			datum = PointerGetDatum(tp + off);
 			auth_info[curr_role].rolvaliduntil = DatumGetCString(DirectFunctionCall1(timestamptz_out, datum));
 		}

src/include/access/heapam.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.121 2007/03/29 00:15:39 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/heapam.h,v 1.122 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -200,7 +200,8 @@ extern Size heap_compute_data_size(TupleDesc tupleDesc,
 					   Datum *values, bool *isnull);
 extern void heap_fill_tuple(TupleDesc tupleDesc,
 				Datum *values, bool *isnull,
-				char *data, uint16 *infomask, bits8 *bit);
+				char *data, Size data_size,
+				uint16 *infomask, bits8 *bit);
 extern bool heap_attisnull(HeapTuple tup, int attnum);
 extern Datum nocachegetattr(HeapTuple tup, int attnum,
 			   TupleDesc att, bool *isnull);

src/include/access/htup.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/htup.h,v 1.92 2007/02/27 23:48:09 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/htup.h,v 1.93 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -160,9 +160,8 @@ typedef HeapTupleHeaderData *HeapTupleHeader;
 #define HEAP_HASNULL			0x0001	/* has null attribute(s) */
 #define HEAP_HASVARWIDTH		0x0002	/* has variable-width attribute(s) */
 #define HEAP_HASEXTERNAL		0x0004	/* has external stored attribute(s) */
-#define HEAP_HASCOMPRESSED		0x0008	/* has compressed stored attribute(s) */
-#define HEAP_HASEXTENDED		0x000C	/* the two above combined */
-#define HEAP_HASOID				0x0010	/* has an object-id field */
+#define HEAP_HASOID				0x0008	/* has an object-id field */
+/* bit 0x0010 is available */
 #define HEAP_COMBOCID			0x0020	/* t_cid is a combo cid */
 #define HEAP_XMAX_EXCL_LOCK		0x0040	/* xmax is exclusive locker */
 #define HEAP_XMAX_SHARED_LOCK	0x0080	/* xmax is shared locker */
@@ -341,7 +340,7 @@ do { \
  * MaxAttrSize is a somewhat arbitrary upper limit on the declared size of
  * data fields of char(n) and similar types.  It need not have anything
  * directly to do with the *actual* upper limit of varlena values, which
- * is currently 1Gb (see struct varattrib in postgres.h).  I've set it
+ * is currently 1Gb (see TOAST structures in postgres.h).  I've set it
  * at 10Mb which seems like a reasonable number --- tgl 8/6/00.
  */
 #define MaxAttrSize		(10 * 1024 * 1024)
@@ -485,12 +484,6 @@ typedef HeapTupleData *HeapTuple;
 #define HeapTupleHasExternal(tuple) \
 		(((tuple)->t_data->t_infomask & HEAP_HASEXTERNAL) != 0)
 
-#define HeapTupleHasCompressed(tuple) \
-		(((tuple)->t_data->t_infomask & HEAP_HASCOMPRESSED) != 0)
-
-#define HeapTupleHasExtended(tuple) \
-		(((tuple)->t_data->t_infomask & HEAP_HASEXTENDED) != 0)
-
 #define HeapTupleGetOid(tuple) \
 		HeapTupleHeaderGetOid((tuple)->t_data)
 

src/include/access/tupmacs.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/access/tupmacs.h,v 1.32 2007/02/27 23:48:09 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/tupmacs.h,v 1.33 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -91,26 +91,83 @@
 #endif   /* SIZEOF_DATUM == 8 */
 
 /*
- * att_align aligns the given offset as needed for a datum of alignment
- * requirement attalign.  The cases are tested in what is hopefully something
- * like their frequency of occurrence.
+ * att_align_datum aligns the given offset as needed for a datum of alignment
+ * requirement attalign and typlen attlen.  attdatum is the Datum variable
+ * we intend to pack into a tuple (it's only accessed if we are dealing with
+ * a varlena type).  Note that this assumes the Datum will be stored as-is;
+ * callers that are intending to convert non-short varlena datums to short
+ * format have to account for that themselves.
  */
-#define att_align(cur_offset, attalign) \
+#define att_align_datum(cur_offset, attalign, attlen, attdatum) \
+( \
+	((attlen) == -1 && VARATT_IS_SHORT(DatumGetPointer(attdatum))) ? (long) (cur_offset) : \
+	att_align_nominal(cur_offset, attalign) \
+)
+
+/*
+ * att_align_pointer performs the same calculation as att_align_datum,
+ * but is used when walking a tuple.  attptr is the current actual data
+ * pointer; when accessing a varlena field we have to "peek" to see if we
+ * are looking at a pad byte or the first byte of a 1-byte-header datum.
+ * (A zero byte must be either a pad byte, or the first byte of a correctly
+ * aligned 4-byte length word; in either case we can align safely.  A non-zero
+ * byte must be either a 1-byte length word, or the first byte of a correctly
+ * aligned 4-byte length word; in either case we need not align.)
+ *
+ * Note: some callers pass a "char *" pointer for cur_offset.  This is
+ * a bit of a hack but works OK on all known platforms.  It ought to be
+ * cleaned up someday, though.
+ */
+#define att_align_pointer(cur_offset, attalign, attlen, attptr) \
+( \
+	((attlen) == -1 && VARATT_NOT_PAD_BYTE(attptr)) ? (long) (cur_offset) : \
+	att_align_nominal(cur_offset, attalign) \
+)
+
+/*
+ * att_align_nominal aligns the given offset as needed for a datum of alignment
+ * requirement attalign, ignoring any consideration of packed varlena datums.
+ * There are three main use cases for using this macro directly:
+ * 	* we know that the att in question is not varlena (attlen != -1);
+ *	  in this case it is cheaper than the above macros and just as good.
+ *	* we need to estimate alignment padding cost abstractly, ie without
+ *	  reference to a real tuple.  We must assume the worst case that
+ *	  all varlenas are aligned.
+ *	* within arrays, we unconditionally align varlenas (XXX this should be
+ *	  revisited, probably).
+ *
+ * The attalign cases are tested in what is hopefully something like their
+ * frequency of occurrence.
+ */
+#define att_align_nominal(cur_offset, attalign) \
 ( \
 	((attalign) == 'i') ? INTALIGN(cur_offset) : \
-	 (((attalign) == 'c') ? ((long)(cur_offset)) : \
+	 (((attalign) == 'c') ? (long) (cur_offset) : \
 	  (((attalign) == 'd') ? DOUBLEALIGN(cur_offset) : \
-		( \
+	   ( \
 			AssertMacro((attalign) == 's'), \
 			SHORTALIGN(cur_offset) \
-		))) \
+	   ))) \
 )
 
 /*
- * att_addlength increments the given offset by the length of the attribute.
- * attval is only accessed if we are dealing with a variable-length attribute.
+ * att_addlength_datum increments the given offset by the space needed for
+ * the given Datum variable.  attdatum is only accessed if we are dealing
+ * with a variable-length attribute.
+ */
+#define att_addlength_datum(cur_offset, attlen, attdatum) \
+	att_addlength_pointer(cur_offset, attlen, DatumGetPointer(attdatum))
+
+/*
+ * att_addlength_pointer performs the same calculation as att_addlength_datum,
+ * but is used when walking a tuple --- attptr is the pointer to the field
+ * within the tuple.
+ *
+ * Note: some callers pass a "char *" pointer for cur_offset.  This is
+ * actually perfectly OK, but probably should be cleaned up along with
+ * the same practice for att_align_pointer.
  */
-#define att_addlength(cur_offset, attlen, attval) \
+#define att_addlength_pointer(cur_offset, attlen, attptr) \
 ( \
 	((attlen) > 0) ? \
 	( \
@@ -118,12 +175,12 @@
 	) \
 	: (((attlen) == -1) ? \
 	( \
-		(cur_offset) + VARSIZE(DatumGetPointer(attval)) \
+		(cur_offset) + VARSIZE_ANY(attptr) \
 	) \
 	: \
 	( \
 		AssertMacro((attlen) == -2), \
-		(cur_offset) + (strlen(DatumGetCString(attval)) + 1) \
+		(cur_offset) + (strlen((char *) (attptr)) + 1) \
 	)) \
 )
 

src/include/access/tuptoaster.h

@@ -6,7 +6,7 @@
  *
  * Copyright (c) 2000-2007, PostgreSQL Global Development Group
  *
- * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.34 2007/04/03 04:14:26 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/access/tuptoaster.h,v 1.35 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -111,7 +111,7 @@ extern void toast_delete(Relation rel, HeapTuple oldtup);
  *		in compressed format.
  * ----------
  */
-extern varattrib *heap_tuple_fetch_attr(varattrib *attr);
+extern struct varlena *heap_tuple_fetch_attr(struct varlena *attr);
 
 /* ----------
  * heap_tuple_untoast_attr() -
@@ -120,7 +120,7 @@ extern varattrib *heap_tuple_fetch_attr(varattrib *attr);
  *		it as needed.
  * ----------
  */
-extern varattrib *heap_tuple_untoast_attr(varattrib *attr);
+extern struct varlena *heap_tuple_untoast_attr(struct varlena *attr);
 
 /* ----------
  * heap_tuple_untoast_attr_slice() -
@@ -129,7 +129,7 @@ extern varattrib *heap_tuple_untoast_attr(varattrib *attr);
  *		(Handles all cases for attribute storage)
  * ----------
  */
-extern varattrib *heap_tuple_untoast_attr_slice(varattrib *attr,
+extern struct varlena *heap_tuple_untoast_attr_slice(struct varlena *attr,
 							  int32 sliceoffset,
 							  int32 slicelength);
 

src/include/catalog/catversion.h

@@ -37,7 +37,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.399 2007/04/02 03:49:40 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.400 2007/04/06 04:21:43 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -53,6 +53,6 @@
  */
 
 /*							yyyymmddN */
-#define CATALOG_VERSION_NO	200704012
+#define CATALOG_VERSION_NO	200704051
 
 #endif

src/include/catalog/pg_type.h

@@ -8,7 +8,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/catalog/pg_type.h,v 1.181 2007/04/02 03:49:41 tgl Exp $
+ * $PostgreSQL: pgsql/src/include/catalog/pg_type.h,v 1.182 2007/04/06 04:21:43 tgl Exp $
  *
  * NOTES
  *	  the genbki.sh script reads this file and generates .bki
@@ -130,8 +130,10 @@ CATALOG(pg_type,1247) BKI_BOOTSTRAP
 	 * 'i' = INT alignment (4 bytes on most machines).
 	 * 'd' = DOUBLE alignment (8 bytes on many machines, but by no means all).
 	 *
-	 * See include/utils/memutils.h for the macros that compute these
-	 * alignment requirements.
+	 * See include/access/tupmacs.h for the macros that compute these
+	 * alignment requirements.  Note also that we allow the nominal alignment
+	 * to be violated when storing "packed" varlenas; the TOAST mechanism
+	 * takes care of hiding that from most code.
 	 *
 	 * NOTE: for types used in system tables, it is critical that the
 	 * size and alignment defined in pg_type agree with the way that the
@@ -398,10 +400,10 @@ DATA(insert OID = 791 (  _money    PGNSP PGUID	-1 f b t \054 0  790 array_in arr
 DATA(insert OID = 829 ( macaddr    PGNSP PGUID	6 f b t \054 0 0 macaddr_in macaddr_out macaddr_recv macaddr_send - - - i p f 0 -1 0 _null_ _null_ ));
 DESCR("XX:XX:XX:XX:XX:XX, MAC address");
 #define MACADDROID 829
-DATA(insert OID = 869 ( inet	   PGNSP PGUID	-1 f b t \054 0 0 inet_in inet_out inet_recv inet_send - - - i p f 0 -1 0 _null_ _null_ ));
+DATA(insert OID = 869 ( inet	   PGNSP PGUID	-1 f b t \054 0 0 inet_in inet_out inet_recv inet_send - - - i m f 0 -1 0 _null_ _null_ ));
 DESCR("IP address/netmask, host address, netmask optional");
 #define INETOID 869
-DATA(insert OID = 650 ( cidr	   PGNSP PGUID	-1 f b t \054 0 0 cidr_in cidr_out cidr_recv cidr_send - - - i p f 0 -1 0 _null_ _null_ ));
+DATA(insert OID = 650 ( cidr	   PGNSP PGUID	-1 f b t \054 0 0 cidr_in cidr_out cidr_recv cidr_send - - - i m f 0 -1 0 _null_ _null_ ));
 DESCR("network IP address/netmask, network address");
 #define CIDROID 650
 

src/include/fmgr.h

@@ -11,7 +11,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/fmgr.h,v 1.49 2007/01/05 22:19:50 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/fmgr.h,v 1.50 2007/04/06 04:21:44 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -153,6 +153,11 @@ extern void fmgr_info_copy(FmgrInfo *dstinfo, FmgrInfo *srcinfo,
  * if you need a modifiable copy of the input.	Caller is expected to have
  * checked for null inputs first, if necessary.
  *
+ * pg_detoast_datum_packed() will return packed (1-byte header) datums
+ * unmodified.  It will still expand an externally toasted or compressed datum.
+ * The resulting datum can be accessed using VARSIZE_ANY() and VARDATA_ANY()
+ * (beware of multiple evaluations in those macros!)
+ *
  * Note: it'd be nice if these could be macros, but I see no way to do that
  * without evaluating the arguments multiple times, which is NOT acceptable.
  */
@@ -160,6 +165,7 @@ extern struct varlena *pg_detoast_datum(struct varlena * datum);
 extern struct varlena *pg_detoast_datum_copy(struct varlena * datum);
 extern struct varlena *pg_detoast_datum_slice(struct varlena * datum,
 					   int32 first, int32 count);
+extern struct varlena *pg_detoast_datum_packed(struct varlena * datum);
 
 #define PG_DETOAST_DATUM(datum) \
 	pg_detoast_datum((struct varlena *) DatumGetPointer(datum))
@@ -168,6 +174,8 @@ extern struct varlena *pg_detoast_datum_slice(struct varlena * datum,
 #define PG_DETOAST_DATUM_SLICE(datum,f,c) \
 		pg_detoast_datum_slice((struct varlena *) DatumGetPointer(datum), \
 		(int32) f, (int32) c)
+#define PG_DETOAST_DATUM_PACKED(datum) \
+	pg_detoast_datum_packed((struct varlena *) DatumGetPointer(datum))
 
 /*
  * Support for cleaning up detoasted copies of inputs.	This must only
@@ -207,9 +215,13 @@ extern struct varlena *pg_detoast_datum_slice(struct varlena * datum,
 #define PG_GETARG_VARLENA_P(n) PG_DETOAST_DATUM(PG_GETARG_DATUM(n))
 /* DatumGetFoo macros for varlena types will typically look like this: */
 #define DatumGetByteaP(X)			((bytea *) PG_DETOAST_DATUM(X))
+#define DatumGetByteaPP(X)			((bytea *) PG_DETOAST_DATUM_PACKED(X))
 #define DatumGetTextP(X)			((text *) PG_DETOAST_DATUM(X))
+#define DatumGetTextPP(X)			((text *) PG_DETOAST_DATUM_PACKED(X))
 #define DatumGetBpCharP(X)			((BpChar *) PG_DETOAST_DATUM(X))
+#define DatumGetBpCharPP(X)			((BpChar *) PG_DETOAST_DATUM_PACKED(X))
 #define DatumGetVarCharP(X)			((VarChar *) PG_DETOAST_DATUM(X))
+#define DatumGetVarCharPP(X)		((VarChar *) PG_DETOAST_DATUM_PACKED(X))
 #define DatumGetHeapTupleHeader(X)	((HeapTupleHeader) PG_DETOAST_DATUM(X))
 /* And we also offer variants that return an OK-to-write copy */
 #define DatumGetByteaPCopy(X)		((bytea *) PG_DETOAST_DATUM_COPY(X))
@@ -224,9 +236,13 @@ extern struct varlena *pg_detoast_datum_slice(struct varlena * datum,
 #define DatumGetVarCharPSlice(X,m,n) ((VarChar *) PG_DETOAST_DATUM_SLICE(X,m,n))
 /* GETARG macros for varlena types will typically look like this: */
 #define PG_GETARG_BYTEA_P(n)		DatumGetByteaP(PG_GETARG_DATUM(n))
+#define PG_GETARG_BYTEA_PP(n)		DatumGetByteaPP(PG_GETARG_DATUM(n))
 #define PG_GETARG_TEXT_P(n)			DatumGetTextP(PG_GETARG_DATUM(n))
+#define PG_GETARG_TEXT_PP(n)		DatumGetTextPP(PG_GETARG_DATUM(n))
 #define PG_GETARG_BPCHAR_P(n)		DatumGetBpCharP(PG_GETARG_DATUM(n))
+#define PG_GETARG_BPCHAR_PP(n)		DatumGetBpCharPP(PG_GETARG_DATUM(n))
 #define PG_GETARG_VARCHAR_P(n)		DatumGetVarCharP(PG_GETARG_DATUM(n))
+#define PG_GETARG_VARCHAR_PP(n)		DatumGetVarCharPP(PG_GETARG_DATUM(n))
 #define PG_GETARG_HEAPTUPLEHEADER(n)	DatumGetHeapTupleHeader(PG_GETARG_DATUM(n))
 /* And we also offer variants that return an OK-to-write copy */
 #define PG_GETARG_BYTEA_P_COPY(n)	DatumGetByteaPCopy(PG_GETARG_DATUM(n))

src/include/pg_config.h.in

@@ -671,6 +671,10 @@
 /* Define to select Win32-style shared memory. */
 #undef USE_WIN32_SHARED_MEMORY
 
+/* Define to 1 if your processor stores words with the most significant byte
+   first (like Motorola and SPARC, unlike Intel and VAX). */
+#undef WORDS_BIGENDIAN
+
 /* Number of bits in a file offset, on hosts where this is settable. */
 #undef _FILE_OFFSET_BITS
 

src/include/utils/inet.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/utils/inet.h,v 1.25 2007/01/05 22:19:59 momjian Exp $
+ * $PostgreSQL: pgsql/src/include/utils/inet.h,v 1.26 2007/04/06 04:21:44 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -39,13 +39,19 @@ typedef struct
 
 /*
  * Both INET and CIDR addresses are represented within Postgres as varlena
- * objects, ie, there is a varlena header (basically a length word) in front
- * of the struct type depicted above.
- *
- * Although these types are variable-length, the maximum length
- * is pretty short, so we make no provision for TOASTing them.
+ * objects, ie, there is a varlena header in front of the struct type
+ * depicted above.  This struct depicts what we actually have in memory
+ * in "uncompressed" cases.  Note that since the maximum data size is only
+ * 18 bytes, INET/CIDR will invariably be stored into tuples using the
+ * 1-byte-header varlena format.  However, we have to be prepared to cope
+ * with the 4-byte-header format too, because various code may helpfully
+ * try to "decompress" 1-byte-header datums.
  */
-typedef struct varlena inet;
+typedef struct
+{
+	int32		vl_len_;		/* Do not touch this field directly! */
+	inet_struct	inet_data;
+} inet;
 
 
 /*
@@ -64,7 +70,7 @@ typedef struct macaddr
 /*
  * fmgr interface macros
  */
-#define DatumGetInetP(X)	((inet *) DatumGetPointer(X))
+#define DatumGetInetP(X)	((inet *) PG_DETOAST_DATUM_PACKED(X))
 #define InetPGetDatum(X)	PointerGetDatum(X)
 #define PG_GETARG_INET_P(n) DatumGetInetP(PG_GETARG_DATUM(n))
 #define PG_RETURN_INET_P(x) return InetPGetDatum(x)

src/test/regress/expected/rowtypes.out

@@ -248,3 +248,20 @@ order by thousand, tenthous;
       999 |     9999
 (25 rows)
 
+-- Check some corner cases involving empty rowtypes
+select ROW();
+ row 
+-----
+ ()
+(1 row)
+
+select ROW() IS NULL;
+ ?column? 
+----------
+ t
+(1 row)
+
+select ROW() = ROW();
+ERROR:  cannot compare rows of zero length
+LINE 1: select ROW() = ROW();
+                     ^

src/test/regress/sql/rowtypes.sql

@@ -109,3 +109,7 @@ select thousand, tenthous from tenk1
 where (thousand, tenthous) >= (997, 5000)
 order by thousand, tenthous;
 
+-- Check some corner cases involving empty rowtypes
+select ROW();
+select ROW() IS NULL;
+select ROW() = ROW();