The new LZ compression and an lztext data type based on it.

Jan

src/backend/utils/adt/Makefile

@@ -4,7 +4,7 @@
 #    Makefile for utils/adt
 #
 # IDENTIFICATION
-#    $Header: /cvsroot/pgsql/src/backend/utils/adt/Makefile,v 1.26 1999/09/30 14:54:22 wieck Exp $
+#    $Header: /cvsroot/pgsql/src/backend/utils/adt/Makefile,v 1.27 1999/11/17 21:21:50 wieck Exp $
 #
 #-------------------------------------------------------------------------
 
@@ -35,7 +35,7 @@ OBJS = acl.o arrayfuncs.o arrayutils.o bool.o cash.o char.o chunk.o \
 	regexp.o regproc.o ruleutils.o selfuncs.o sets.o \
 	tid.o timestamp.o varchar.o varlena.o version.o \
 	network.o mac.o inet_net_ntop.o inet_net_pton.o \
-	ri_triggers.o
+	ri_triggers.o pg_lzcompress.o lztext.o
 
 all: SUBSYS.o
 

src/backend/utils/adt/lztext.c

+/* ----------
+ * lztext.c -
+ *
+ * $Header: /cvsroot/pgsql/src/backend/utils/adt/Attic/lztext.c,v 1.1 1999/11/17 21:21:50 wieck Exp $
+ *
+ *	Text type with internal LZ compressed representation. Uses the
+ *	standard PostgreSQL compression method.
+ * ----------
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <errno.h>
+
+#include "postgres.h"
+#include "utils/builtins.h"
+#include "utils/palloc.h"
+#include "utils/pg_lzcompress.h"
+
+
+/* ----------
+ * lztextin -
+ *
+ *		Input function for datatype lztext
+ * ----------
+ */
+lztext *
+lztextin(char *str)
+{
+	lztext		   *result;
+	int32			rawsize;
+	lztext		   *tmp;
+	int				tmp_size;
+
+	/* ----------
+	 * Handle NULL
+	 * ----------
+	 */
+	if (str == NULL)
+		return NULL;
+
+	/* ----------
+	 * Determine input size and eventually tuple size
+	 * ----------
+	 */
+	rawsize = strlen(str);
+	tmp_size = PGLZ_MAX_OUTPUT(rawsize);
+
+	/* ----------
+	 * Allocate a temporary result and compress into it
+	 * ----------
+	 */
+	tmp = (lztext *) palloc(tmp_size);
+	pglz_compress(str, rawsize, tmp, NULL);
+
+	/* ----------
+	 * If we miss less than x% bytes at the end of the temp value,
+	 * so be it. Therefore we save a memcpy().
+	 * ----------
+	 */
+	if (tmp_size - tmp->varsize < 256 || 
+					tmp_size - tmp->varsize < tmp_size / 4)
+	{
+		result = tmp;
+	} else {
+		result = (lztext *) palloc(tmp->varsize);
+		memcpy(result, tmp, tmp->varsize);
+		pfree(tmp);
+	}
+
+	return result;
+}
+
+
+/* ----------
+ * lztextout -
+ *
+ *		Output function for data type lztext
+ * ----------
+ */
+char *
+lztextout(lztext *lz)
+{
+	char			*result;
+
+	/* ----------
+	 * Handle NULL
+	 * ----------
+	 */
+	if (lz == NULL)
+	{
+		result = (char *) palloc(2);
+		result[0] = '-';
+		result[1] = '\0';
+		return result;
+	}
+
+	/* ----------
+	 * Allocate the result string - the required size is remembered
+	 * in the lztext header so we don't need a temporary buffer or
+	 * have to diddle with realloc's.
+	 * ----------
+	 */
+	result = (char *) palloc(PGLZ_RAW_SIZE(lz) + 1);
+
+	/* ----------
+	 * Decompress and add terminating ZERO
+	 * ----------
+	 */
+	pglz_decompress(lz, result);
+	result[lz->rawsize] = '\0';
+
+	/* ----------
+	 * Return the result
+	 * ----------
+	 */
+	return result;
+}
+
+
+/* ----------
+ * lztextlen -
+ *
+ *	Logical length of lztext field (it's the uncompressed size
+ *	of the original data).
+ * ----------
+ */
+int32
+lztextlen(lztext *lz)
+{
+	/* ----------
+	 * Handle NULL
+	 * ----------
+	 */
+	if (lz == NULL)
+		return 0;
+
+	/* ----------
+	 * without multibyte support, it's the remembered rawsize
+	 * ----------
+	 */
+	return lz->rawsize;
+}
+
+
+/* ----------
+ * lztextoctetlen -
+ *
+ *	Physical length of lztext field (it's the compressed size
+ *	plus the rawsize field).
+ * ----------
+ */
+int32
+lztextoctetlen(lztext *lz)
+{
+	/* ----------
+	 * Handle NULL
+	 * ----------
+	 */
+	if (lz == NULL)
+		return 0;
+
+	/* ----------
+	 * Return the varsize minus the VARSIZE field itself.
+	 * ----------
+	 */
+	return lz->varsize - sizeof(int32);
+}
+
+
+/* ----------
+ * text_lztext -
+ *
+ *	Convert text to lztext
+ * ----------
+ */
+lztext *
+text_lztext(text *txt)
+{
+	lztext		   *result;
+	int32			rawsize;
+	lztext		   *tmp;
+	int				tmp_size;
+	char		   *str;
+
+	/* ----------
+	 * Handle NULL
+	 * ----------
+	 */
+	if (txt == NULL)
+		return NULL;
+
+	/* ----------
+	 * Determine input size and eventually tuple size
+	 * ----------
+	 */
+	rawsize  = VARSIZE(txt) - VARHDRSZ;
+	str      = VARDATA(txt);
+	tmp_size = PGLZ_MAX_OUTPUT(rawsize);
+
+	/* ----------
+	 * Allocate a temporary result and compress into it
+	 * ----------
+	 */
+	tmp = (lztext *) palloc(tmp_size);
+	pglz_compress(str, rawsize, tmp, NULL);
+
+	/* ----------
+	 * If we miss less than x% bytes at the end of the temp value,
+	 * so be it. Therefore we save a memcpy().
+	 * ----------
+	 */
+	if (tmp_size - tmp->varsize < 256 || 
+					tmp_size - tmp->varsize < tmp_size / 4)
+	{
+		result = tmp;
+	} else {
+		result = (lztext *) palloc(tmp->varsize);
+		memcpy(result, tmp, tmp->varsize);
+		pfree(tmp);
+	}
+
+	return result;
+
+	
+}
+
+
+/* ----------
+ * lztext_text -
+ *
+ *	Convert lztext to text
+ * ----------
+ */
+text *
+lztext_text(lztext *lz)
+{
+	text	   *result;
+
+	/* ----------
+	 * Handle NULL
+	 * ----------
+	 */
+	if (lz == NULL)
+		return NULL;
+
+	/* ----------
+	 * Allocate and initialize the text result
+	 * ----------
+	 */
+	result = (text *) palloc(lz->rawsize + VARHDRSZ + 1);
+	VARSIZE(result) = lz->rawsize + VARHDRSZ;
+
+	/* ----------
+	 * Decompress directly into the text data area.
+	 * ----------
+	 */
+	pglz_decompress(lz, VARDATA(result));
+	VARDATA(result)[lz->rawsize] = 0;
+
+	return result;
+}
+
+

src/include/catalog/pg_proc.h

@@ -6,7 +6,7 @@
  *
  * Copyright (c) 1994, Regents of the University of California
  *
- * $Id: pg_proc.h,v 1.105 1999/10/11 06:28:28 inoue Exp $
+ * $Id: pg_proc.h,v 1.106 1999/11/17 21:21:50 wieck Exp $
  *
  * NOTES
  *	  The script catalog/genbki.sh reads this file and generates .bki
@@ -2338,6 +2338,28 @@ DESCR("larger of two numbers");
 DATA(insert OID = 1769 ( numeric_cmp			PGUID 11 f t t 2 f 23 "1700 1700" 100 0 0 100  numeric_cmp - ));
 DESCR("compare two numbers");
 
+/* OID's 1625 - 1639 LZTEXT data type */
+DATA(insert OID = 1626 ( lztextin				PGUID 11 f t t 1 f 1625 "0" 100 0 0 100  lztextin - ));
+DESCR("(internal)");
+DATA(insert OID = 1627 ( lztextout				PGUID 11 f t t 1 f 23 "0" 100 0 0 100  lztextout - ));
+DESCR("(internal)");
+DATA(insert OID = 1628 ( lztext_text	   		PGUID 11 f t t 1 f 25 "1625" 100 0 0 100  lztext_text -));
+DESCR("convert lztext to text");
+DATA(insert OID = 1629 ( text			   		PGUID 11 f t t 1 f 25 "1625" 100 0 0 100  lztext_text -));
+DESCR("convert lztext to text");
+DATA(insert OID = 1630 ( text_lztext	   		PGUID 11 f t t 1 f 1625 "25" 100 0 0 100  text_lztext -));
+DESCR("convert text to lztext");
+DATA(insert OID = 1631 ( lztext			   		PGUID 11 f t t 1 f 1625 "25" 100 0 0 100  text_lztext -));
+DESCR("convert text to lztext");
+DATA(insert OID = 1632 ( lztextlen		   		PGUID 11 f t t 1 f 23 "1625" 100 0 1 0  lztextlen - ));
+DESCR("length");
+DATA(insert OID = 1633 ( length			   		PGUID 11 f t t 1 f 23 "1625" 100 0 1 0  lztextlen - ));
+DESCR("length");
+DATA(insert OID = 1634 ( lztextoctetlen	   		PGUID 11 f t t 1 f 23 "1625" 100 0 1 0  lztextoctetlen - ));
+DESCR("octet length");
+DATA(insert OID = 1635 ( octet_length	   		PGUID 11 f t t 1 f 23 "1625" 100 0 1 0  lztextoctetlen - ));
+DESCR("octet length");
+
 
 /*
  * prototypes for functions pg_proc.c

src/include/catalog/pg_type.h

@@ -7,7 +7,7 @@
  *
  * Copyright (c) 1994, Regents of the University of California
  *
- * $Id: pg_type.h,v 1.70 1999/10/18 14:14:04 momjian Exp $
+ * $Id: pg_type.h,v 1.71 1999/11/17 21:21:51 wieck Exp $
  *
  * NOTES
  *	  the genbki.sh script reads this file and generates .bki
@@ -382,6 +382,11 @@ DATA(insert OID = 1296 ( timestamp	 PGUID	4  19 t b t \054 0	0 timestamp_in time
 DESCR("date time timezone, limited-range ISO-formated date and time");
 #define TIMESTAMPOID	1296
 
+/* OIDS 1625 - 1639 */
+DATA(insert OID = 1625 ( lztext	   PGUID -1  -1 f b t \054 0  0 lztextin lztextout lztextin lztextout i _null_ ));
+DESCR("variable-length string, stored compressed");
+#define LZTEXTOID		1625
+
 /* OIDS 1700 - 1799 */
 DATA(insert OID = 1700 ( numeric	   PGUID -1  -1 f b t \054 0  0 numeric_in numeric_out numeric_in numeric_out i _null_ ));
 DESCR("numeric(precision, decimal), arbitrary precision number");

src/include/utils/builtins.h

@@ -6,7 +6,7 @@
  *
  * Copyright (c) 1994, Regents of the University of California
  *
- * $Id: builtins.h,v 1.89 1999/10/11 06:28:28 inoue Exp $
+ * $Id: builtins.h,v 1.90 1999/11/17 21:21:51 wieck Exp $
  *
  * NOTES
  *	  This should normally only be included by fmgr.h.
@@ -30,6 +30,7 @@
 #include "utils/int8.h"
 #include "utils/nabstime.h"
 #include "utils/numeric.h"
+#include "utils/lztext.h"
 #include "access/heapam.h"		/* for HeapTuple */
 
 /*
@@ -627,4 +628,12 @@ HeapTuple	RI_FKey_setnull_upd(FmgrInfo *proinfo);
 HeapTuple	RI_FKey_setdefault_del(FmgrInfo *proinfo);
 HeapTuple	RI_FKey_setdefault_upd(FmgrInfo *proinfo);
 
+/* lztext.c */
+lztext	   *lztextin(char *str);
+char	   *lztextout(lztext *lz);
+text	   *lztext_text(lztext *lz);
+lztext	   *text_lztext(text *txt);
+int32		lztextlen(lztext *lz);
+int32		lztextoctetlen(lztext *lz);
+
 #endif	 /* BUILTINS_H */

src/include/utils/lztext.h

+/* ----------
+ * lztext.h
+ *
+ * $Header: /cvsroot/pgsql/src/include/utils/Attic/lztext.h,v 1.1 1999/11/17 21:21:51 wieck Exp $
+ *
+ *	Definitions for the lztext compressed data type
+ * ----------
+ */
+
+#ifndef _LZTEXT_H_
+#define _LZTEXT_H_
+
+#include "utils/pg_lzcompress.h"
+
+
+/* ----------
+ * The internal storage format of an LZ compressed text field
+ * ----------
+ */
+typedef PGLZ_Header		lztext;
+
+#endif /* _LZTEXT_H_ */

src/include/utils/pg_lzcompress.h

+/* ----------
+ * pg_lzcompress.h -
+ *
+ * $Header: /cvsroot/pgsql/src/include/utils/pg_lzcompress.h,v 1.1 1999/11/17 21:21:51 wieck Exp $
+ *
+ *	Definitions for the builtin LZ compressor
+ * ----------
+ */
+
+#ifndef _PG_LZCOMPRESS_H_
+#define _PG_LZCOMPRESS_H_
+
+
+/* ----------
+ * PGLZ_Header -
+ *
+ *      The information at the top of the compressed data.
+ *		The varsize must be kept the same data type as the value
+ *		in front of all variable size data types in PostgreSQL.
+ * ----------
+ */
+typedef struct PGLZ_Header {
+    int32                       varsize;
+    int32                       rawsize;
+} PGLZ_Header;
+
+
+/* ----------
+ * PGLZ_MAX_OUTPUT -
+ *
+ *		Macro to compute the maximum buffer required for the
+ *		compression output. It is larger than the input, because
+ *		in the worst case, we cannot write out one single tag but
+ *		need one control byte per 8 literal data bytes plus the
+ *		EOF mark at the end.
+ * ----------
+ */
+#define PGLZ_MAX_OUTPUT(_dlen)			((_dlen) + (((_dlen) | 0x07) >> 3)	\
+													 + sizeof(PGLZ_Header))
+#define PGLZ_RAW_SIZE(_lzdata)			(_lzdata->rawsize)
+#define PGLZ_IS_COMPRESSED(_lzdata)		(_lzdata->varsize != 				\
+										 _lzdata->rawsize + sizeof(PGLZ_Header))
+
+/* ----------
+ * PGLZ_Strategy -
+ *
+ *		Some values that control the compression algorithm.
+ *
+ *		min_input_size		Minimum input data size to start compression.
+ *
+ *		force_input_size	Input data size at which compressed storage is
+ *							forced even if the compression rate drops below
+ *							min_comp_rate (but not below 0).
+ *
+ *		min_comp_rate		Minimum compression rate (0-99%), the output
+ *							must be smaller than the input. If that isn't
+ *							the case, the compressor will throw away it's
+ *							output and copy the original, uncompressed data
+ *							to the output buffer.
+ *
+ *		match_size_good		The initial GOOD match size when starting history
+ *							lookup. When looking up the history to find a
+ *							match that could be expressed as a tag, the
+ *							algorithm does not allways walk back entirely.
+ *							A good match fast is usually better than the 
+ *							best possible one very late. For each iteration
+ *							in the lookup, this value is lowered so the
+ *							longer the lookup takes, the smaller matches
+ *							are considered good.
+ *
+ *		match_size_drop		The percentage, match_size_good is lowered
+ *							at each history check. Allowed values are
+ *							0 (no change until end) to 100 (only check
+ *							latest history entry at all).
+ * ----------
+ */
+typedef struct PGLZ_Strategy {
+	int32		min_input_size;
+	int32		force_input_size;
+	int32		min_comp_rate;
+	int32		match_size_good;
+	int32		match_size_drop;
+} PGLZ_Strategy;
+
+
+/* ----------
+ * The standard strategies
+ *
+ *		PGLZ_strategy_default		Starts compression only if input is
+ *									at least 256 bytes large. Stores output
+ *									uncompressed if compression does not
+ *									gain at least 20% size reducture but
+ *									input does not exceed 6K. Stops history
+ *									lookup if at least a 128 byte long
+ *									match has been found.
+ *
+ *									This is the default strategy if none
+ *									is given to pglz_compress().
+ *
+ *		PGLZ_strategy_allways		Starts compression on any infinitely
+ *									small input and does fallback to
+ *									uncompressed storage only if output
+ *									would be larger than input.
+ *
+ *		PGLZ_strategy_never			Force pglz_compress to act as a custom
+ *									interface for memcpy(). Only useful
+ *									for generic interfacing.
+ * ----------
+ */
+extern PGLZ_Strategy	*PGLZ_strategy_default;
+extern PGLZ_Strategy	*PGLZ_strategy_allways;
+extern PGLZ_Strategy	*PGLZ_strategy_never;
+
+
+/* ----------
+ * Global function declarations
+ * ----------
+ */
+int	pglz_compress (char *source, int32 slen, PGLZ_Header *dest,
+									 PGLZ_Strategy *strategy);
+int pglz_decompress (PGLZ_Header *source, char *dest);
+
+
+#endif /* _PG_LZCOMPRESS_H_ */
+