Skip to content

P
Postgres FD Implementation
Commit 27b89222 authored by Tom Lane's avatar Tom Lane
Browse files
Options

Add an extra header byte to TOAST-pointer datums to represent their size 

explicitly.  This means a TOAST pointer takes 18 bytes instead of 17 --- still
smaller than in 8.2 --- which seems a good tradeoff to ensure we won't have
painted ourselves into a corner if we want to support multiple types of TOAST
pointer later on.  Per discussion with Greg Stark.
parent 3ff0018c
Hide whitespace changes
Inline Side-by-side
Showing with 58 additions and 37 deletions
doc/src/sgml/storage.sgml
View file @ 27b89222
<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.19 2007/09/21 21:25:42 tgl Exp $ -->
<!-- $PostgreSQL: pgsql/doc/src/sgml/storage.sgml,v 1.20 2007/09/30 19:54:57 tgl Exp $ -->
<chapter id="storage">
......@@ -233,8 +233,8 @@ header, and the remaining bits give the total datum size (including length
byte) in bytes. As a special case, if the remaining bits are all zero
(which would be impossible for a self-inclusive length), the value is a
pointer to out-of-line data stored in a separate TOAST table. (The size of
a TOAST pointer is known a priori, so it doesn't need to be represented in
the header.) Values with single-byte headers aren't aligned on any particular
a TOAST pointer is given in the second byte of the datum.)
Values with single-byte headers aren't aligned on any particular
boundary, either. Lastly, when the highest-order or lowest-order bit is
clear but the adjacent bit is set, the content of the datum has been
compressed and must be decompressed before use. In this case the remaining
......@@ -274,8 +274,8 @@ retrieval of the values. A pointer datum representing an out-of-line
<acronym>TOAST</> table in which to look and the OID of the specific value
(its <structfield>chunk_id</>). For convenience, pointer datums also store the
logical datum size (original uncompressed data length) and actual stored size
(different if compression was applied). Allowing for the varlena header byte,
the total size of a <acronym>TOAST</> pointer datum is therefore 17 bytes
(different if compression was applied). Allowing for the varlena header bytes,
the total size of a <acronym>TOAST</> pointer datum is therefore 18 bytes
regardless of the actual size of the represented value.
</para>
......
src/backend/access/heap/tuptoaster.c
View file @ 27b89222
......@@ -8,7 +8,7 @@
*
*
* IDENTIFICATION
* $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.75 2007/09/26 23:29:10 tgl Exp $
* $PostgreSQL: pgsql/src/backend/access/heap/tuptoaster.c,v 1.76 2007/09/30 19:54:58 tgl Exp $
*
*
* INTERFACE ROUTINES
......@@ -52,6 +52,21 @@
#define VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer) \
((toast_pointer).va_extsize < (toast_pointer).va_rawsize - VARHDRSZ)
/*
* Macro to fetch the possibly-unaligned contents of an EXTERNAL datum
* into a local "struct varatt_external" toast pointer. This should be
* just a memcpy, but some versions of gcc seem to produce broken code
* that assumes the datum contents are aligned. Introducing an explicit
* intermediate "varattrib_1b_e *" variable seems to fix it.
*/
#define VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr) \
do { \
varattrib_1b_e *attre = (varattrib_1b_e *) (attr); \
Assert(VARSIZE_ANY_EXHDR(attre) == sizeof(toast_pointer)); \
memcpy(&(toast_pointer), VARDATA_EXTERNAL(attre), sizeof(toast_pointer)); \
} while (0)
static void toast_delete_datum(Relation rel, Datum value);
static Datum toast_save_datum(Relation rel, Datum value,
bool use_wal, bool use_fsm);
......@@ -172,7 +187,7 @@ heap_tuple_untoast_attr_slice(struct varlena *attr,
{
struct varatt_external toast_pointer;
memcpy(&toast_pointer, VARDATA_SHORT(attr), sizeof(toast_pointer));
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
/* fast path for non-compressed external datums */
if (!VARATT_EXTERNAL_IS_COMPRESSED(toast_pointer))
......@@ -249,7 +264,7 @@ toast_raw_datum_size(Datum value)
/* va_rawsize is the size of the original datum -- including header */
struct varatt_external toast_pointer;
memcpy(&toast_pointer, VARDATA_SHORT(attr), sizeof(toast_pointer));
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
result = toast_pointer.va_rawsize;
}
else if (VARATT_IS_COMPRESSED(attr))
......@@ -294,7 +309,7 @@ toast_datum_size(Datum value)
*/
struct varatt_external toast_pointer;
memcpy(&toast_pointer, VARDATA_SHORT(attr), sizeof(toast_pointer));
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
result = toast_pointer.va_extsize;
}
else if (VARATT_IS_SHORT(attr))
......@@ -470,9 +485,8 @@ toast_insert_or_update(Relation rel, HeapTuple newtup, HeapTuple oldtup,
VARATT_IS_EXTERNAL(old_value))
{
if (toast_isnull[i] || !VARATT_IS_EXTERNAL(new_value) ||
memcmp(VARDATA_SHORT(old_value),
VARDATA_SHORT(new_value),
sizeof(struct varatt_external)) != 0)
memcmp((char *) old_value, (char *) new_value,
VARSIZE_EXTERNAL(old_value)) != 0)
{
/*
* The old external stored value isn't needed any more
......@@ -1071,7 +1085,7 @@ toast_save_datum(Relation rel, Datum value,
Datum t_values[3];
bool t_isnull[3];
CommandId mycid = GetCurrentCommandId();
struct varlena *result;
varattrib_pointer *result;
struct varatt_external toast_pointer;
struct
{
......@@ -1192,9 +1206,9 @@ toast_save_datum(Relation rel, Datum value,
/*
* Create the TOAST pointer value that we'll return
*/
result = (struct varlena *) palloc(sizeof(varattrib_pointer));
SET_VARSIZE_EXTERNAL(result);
memcpy(VARDATA_SHORT(result), &toast_pointer, sizeof(toast_pointer));
result = (varattrib_pointer *) palloc(sizeof(varattrib_pointer));
SET_VARSIZE_EXTERNAL(result, sizeof(varattrib_pointer));
memcpy(VARDATA_EXTERNAL(result), &toast_pointer, sizeof(toast_pointer));
return PointerGetDatum(result);
}
......@@ -1221,8 +1235,7 @@ toast_delete_datum(Relation rel, Datum value)
return;
/* Must copy to access aligned fields */
memcpy(&toast_pointer, VARDATA_SHORT(attr),
sizeof(struct varatt_external));
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
/*
* Open the toast relation and its index
......@@ -1289,8 +1302,7 @@ toast_fetch_datum(struct varlena *attr)
int32 chunksize;
/* Must copy to access aligned fields */
memcpy(&toast_pointer, VARDATA_SHORT(attr),
sizeof(struct varatt_external));
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
ressize = toast_pointer.va_extsize;
numchunks = ((ressize - 1) / TOAST_MAX_CHUNK_SIZE) + 1;
......@@ -1452,8 +1464,7 @@ toast_fetch_datum_slice(struct varlena *attr, int32 sliceoffset, int32 length)
Assert(VARATT_IS_EXTERNAL(attr));
/* Must copy to access aligned fields */
memcpy(&toast_pointer, VARDATA_SHORT(attr),
sizeof(struct varatt_external));
VARATT_EXTERNAL_GET_POINTER(toast_pointer, attr);
/*
* It's nonsense to fetch slices of a compressed datum -- this isn't lo_*
......
src/include/catalog/catversion.h
View file @ 27b89222
......@@ -37,7 +37,7 @@
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.429 2007/09/25 22:21:55 tgl Exp $
* $PostgreSQL: pgsql/src/include/catalog/catversion.h,v 1.430 2007/09/30 19:54:58 tgl Exp $
*
*-------------------------------------------------------------------------
*/
......@@ -53,6 +53,6 @@
*/
/* yyyymmddN */
#define CATALOG_VERSION_NO 200709251
#define CATALOG_VERSION_NO 200709301
#endif
src/include/postgres.h
View file @ 27b89222
......@@ -10,7 +10,7 @@
* Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
* Portions Copyright (c) 1995, Regents of the University of California
*
* $PostgreSQL: pgsql/src/include/postgres.h,v 1.83 2007/09/27 21:01:59 tgl Exp $
* $PostgreSQL: pgsql/src/include/postgres.h,v 1.84 2007/09/30 19:54:58 tgl Exp $
*
*-------------------------------------------------------------------------
*/
......@@ -100,12 +100,20 @@ typedef union
typedef struct
{
uint8 va_header;
char va_data[1]; /* Data or TOAST pointer */
char va_data[1]; /* Data begins here */
} varattrib_1b;
typedef struct
{
uint8 va_header;
uint8 va_header; /* Always 0x80 or 0x01 */
uint8 va_len_1be; /* Physical length of datum */
char va_data[1]; /* Data (for now always a TOAST pointer) */
} varattrib_1b_e;
typedef struct
{
uint8 va_header; /* Always 0x80 or 0x01 */
uint8 va_len_1be; /* Physical length of datum */
char va_data[sizeof(struct varatt_external)];
} varattrib_pointer;
......@@ -161,9 +169,8 @@ typedef struct
(((varattrib_4b *) (PTR))->va_4byte.va_header & 0x3FFFFFFF)
#define VARSIZE_1B(PTR) \
(((varattrib_1b *) (PTR))->va_header & 0x7F)
/* Currently there is only one size of toast pointer, but someday maybe not */
#define VARSIZE_1B_E(PTR) \
(sizeof(varattrib_pointer))
(((varattrib_1b_e *) (PTR))->va_len_1be)
#define SET_VARSIZE_4B(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = (len) & 0x3FFFFFFF)
......@@ -171,8 +178,9 @@ typedef struct
(((varattrib_4b *) (PTR))->va_4byte.va_header = ((len) & 0x3FFFFFFF) | 0x40000000)
#define SET_VARSIZE_1B(PTR,len) \
(((varattrib_1b *) (PTR))->va_header = (len) | 0x80)
#define SET_VARSIZE_1B_E(PTR) \
(((varattrib_1b *) (PTR))->va_header = 0x80)
#define SET_VARSIZE_1B_E(PTR,len) \
(((varattrib_1b_e *) (PTR))->va_header = 0x80, \
((varattrib_1b_e *) (PTR))->va_len_1be = (len))
#else /* !WORDS_BIGENDIAN */
......@@ -194,9 +202,8 @@ typedef struct
((((varattrib_4b *) (PTR))->va_4byte.va_header >> 2) & 0x3FFFFFFF)
#define VARSIZE_1B(PTR) \
((((varattrib_1b *) (PTR))->va_header >> 1) & 0x7F)
/* Currently there is only one size of toast pointer, but someday maybe not */
#define VARSIZE_1B_E(PTR) \
(sizeof(varattrib_pointer))
(((varattrib_1b_e *) (PTR))->va_len_1be)
#define SET_VARSIZE_4B(PTR,len) \
(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2))
......@@ -204,8 +211,9 @@ typedef struct
(((varattrib_4b *) (PTR))->va_4byte.va_header = (((uint32) (len)) << 2) | 0x02)
#define SET_VARSIZE_1B(PTR,len) \
(((varattrib_1b *) (PTR))->va_header = (((uint8) (len)) << 1) | 0x01)
#define SET_VARSIZE_1B_E(PTR) \
(((varattrib_1b *) (PTR))->va_header = 0x01)
#define SET_VARSIZE_1B_E(PTR,len) \
(((varattrib_1b_e *) (PTR))->va_header = 0x01, \
((varattrib_1b_e *) (PTR))->va_len_1be = (len))
#endif /* WORDS_BIGENDIAN */
......@@ -220,6 +228,7 @@ typedef struct
#define VARDATA_4B(PTR) (((varattrib_4b *) (PTR))->va_4byte.va_data)
#define VARDATA_4B_C(PTR) (((varattrib_4b *) (PTR))->va_compressed.va_data)
#define VARDATA_1B(PTR) (((varattrib_1b *) (PTR))->va_data)
#define VARDATA_1B_E(PTR) (((varattrib_1b_e *) (PTR))->va_data)
#define VARRAWSIZE_4B_C(PTR) \
(((varattrib_4b *) (PTR))->va_compressed.va_rawsize)
......@@ -249,6 +258,7 @@ typedef struct
#define VARDATA_SHORT(PTR) VARDATA_1B(PTR)
#define VARSIZE_EXTERNAL(PTR) VARSIZE_1B_E(PTR)
#define VARDATA_EXTERNAL(PTR) VARDATA_1B_E(PTR)
#define VARATT_IS_COMPRESSED(PTR) VARATT_IS_4B_C(PTR)
#define VARATT_IS_EXTERNAL(PTR) VARATT_IS_1B_E(PTR)
......@@ -258,7 +268,7 @@ typedef struct
#define SET_VARSIZE(PTR, len) SET_VARSIZE_4B(PTR, len)
#define SET_VARSIZE_SHORT(PTR, len) SET_VARSIZE_1B(PTR, len)
#define SET_VARSIZE_COMPRESSED(PTR, len) SET_VARSIZE_4B_C(PTR, len)
#define SET_VARSIZE_EXTERNAL(PTR) SET_VARSIZE_1B_E(PTR)
#define SET_VARSIZE_EXTERNAL(PTR, len) SET_VARSIZE_1B_E(PTR, len)
#define VARSIZE_ANY(PTR) \
(VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR) : \
......@@ -266,7 +276,7 @@ typedef struct
VARSIZE_4B(PTR)))
#define VARSIZE_ANY_EXHDR(PTR) \
(VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR)-1 : \
(VARATT_IS_1B_E(PTR) ? VARSIZE_1B_E(PTR)-2 : \
(VARATT_IS_1B(PTR) ? VARSIZE_1B(PTR)-1 : \
VARSIZE_4B(PTR)-4))
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment