trgm - Trigram matching for PostgreSQL

--------------------------------------

	The pg_trgm contrib module provides functions and index classes
	for determining the similarity of text based on trigram
	matching.

contrib/pg_trgm/Makefile

+subdir = contrib/pg_trgm
+top_builddir = ../..
+include $(top_builddir)/src/Makefile.global
+
+
+override CPPFLAGS := -I. $(CPPFLAGS)
+
+MODULE_big = pg_trgm
+OBJS = trgm_op.o trgm_gist.o 
+
+DATA_built = pg_trgm.sql
+DOCS = README.pg_trgm
+REGRESS = pg_trgm
+
+include $(top_srcdir)/contrib/contrib-global.mk
+# DO NOT DELETE
+

contrib/pg_trgm/README.pg_trgm

+trgm - Trigram matching for PostgreSQL
+--------------------------------------
+
+Introduction
+
+	This module is sponsored by Delta-Soft Ltd., Moscow, Russia.
+
+	The pg_trgm contrib module provides functions and index classes
+	for determining the similarity of text based on trigram
+	matching.
+
+Definitions
+
+	Trigram (or Trigraph)
+
+	A trigram is a set of three consecutive characters taken
+	from a string.  A string is considered to have two spaces
+	prefixed and one space suffixed when determining the set
+	of trigrams that comprise the string.
+
+	eg. The set of trigrams in the word "cat" is "  c", " ca", 
+	"at " and "cat".
+
+Public Functions
+
+	real similarity(text, text)
+
+	Returns a number that indicates how closely matches the two
+	arguments are.  A zero result indicates that the two words
+	are completely dissimilar, and a result of one indicates that
+	the two words are identical.
+
+	real show_limit()
+
+	Returns the current similarity threshold used by the '%'
+	operator.  This in effect sets the minimum similarity between
+	two words in order that they be considered similar enough to
+	be misspellings of each other, for example.
+
+	real set_limit(real)
+
+	Sets the current similarity threshold that is used by the '%'
+	operator, and is returned by the show_limit() function.
+
+	text[] show_trgm(text)
+
+	Returns an array of all the trigrams of the supplied text
+	parameter.
+
+Public Operators
+
+	text % text (returns boolean)
+
+	The '%' operator returns TRUE if its two arguments have a similarity
+	that is greater than the similarity threshold set by set_limit(). It
+	will return FALSE if the similarity is less than the current
+	threshold.
+
+Public Index Operator Classes
+
+	gist_trgm_ops
+
+	The pg_trgm module comes with an index operator class that allows a
+	developer to create an index over a text column for the purpose
+	of very fast similarity searches.
+
+	To use this index, the '%' operator must be used and an appropriate
+	similarity threshold for the application must be set.
+
+	eg.
+
+	CREATE TABLE test_trgm (t text);
+	CREATE INDEX trgm_idx ON test_trgm USING gist (t gist_trgm_ops);
+	
+	At this point, you will have an index on the t text column that you
+	can use for similarity searching.
+
+	eg.
+
+	SELECT
+		t,
+		similarity(t, 'word') AS sml
+	FROM
+		test_trgm
+	WHERE
+		t % 'word'
+	ORDER BY
+		sml DESC, t;
+
+	This will return all values in the text column that are sufficiently
+	similar to 'word', sorted from best match to worst.  The index will
+	be used to make this a fast operation over very large data sets.
+
+Tsearch2 Integration
+
+	Trigram matching is a very useful tool when used in conjunction
+	with a text index created by the Tsearch2 contrib module. (See
+	contrib/tsearch2)
+
+	The first step is to generate an auxiliary table containing all
+	the unique words in the Tsearch2 index:
+
+	CREATE TABLE words AS 
+		SELECT word FROM stat('SELECT vector FROM documents');
+
+	Where 'documents' is the table that contains the Tsearch2 index
+	column 'vector', of type 'tsvector'.
+
+	Next, create a trigram index on the word column:
+
+	CREATE INDEX words_idx ON words USING gist(word gist_trgm_ops);
+
+	Now, a SELECT query similar to the example above can be used to
+	suggest spellings for misspelled words in user search terms. A
+	useful extra clause is to ensure that the similar words are also
+	of similar length to the misspelled word.
+
+	Note: Since the 'words' table has been generated as a separate,
+	static table, it will need to be periodically regenerated so that
+	it remains up to date with the word list in the Tsearch2 index.
+
+Authors
+
+	Oleg Bartunov <oleg@sai.msu.su>, Moscow, Moscow University, Russia
+	Teodor Sigaev <teodor@sigaev.ru>, Moscow, Delta-Soft Ltd.,Russia
+       
+Contributors
+
+	Christopher Kings-Lynne wrote this README file
+
+References
+
+	Tsearch2 Development Site
+	http://www.sai.msu.su/~megera/postgres/gist/tsearch/V2/
+	
+	GiST Development Site
+	http://www.sai.msu.su/~megera/postgres/gist/
+

contrib/pg_trgm/pg_trgm.sql.in

+SET search_path = public;
+
+BEGIN;
+
+create function set_limit(float4)
+returns float4
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C' with (isstrict,iscachable);
+
+create function show_limit()
+returns float4
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C' with (isstrict,iscachable);
+
+create function show_trgm(text)
+returns _text
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C' with (isstrict,iscachable);
+
+create function similarity(text,text)
+returns float4
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C' with (isstrict,iscachable);
+
+create function similarity_op(text,text)
+returns bool
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C' with (isstrict,iscachable);
+
+CREATE OPERATOR % (
+        LEFTARG = text,
+        RIGHTARG = text,
+        PROCEDURE = similarity_op,
+        COMMUTATOR = '%',
+        RESTRICT = contsel,
+        JOIN = contjoinsel
+);
+
+--gist key
+CREATE FUNCTION gtrgm_in(cstring)
+RETURNS gtrgm
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C' with (isstrict);
+
+CREATE FUNCTION gtrgm_out(gtrgm)
+RETURNS cstring
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C' with (isstrict);
+
+CREATE TYPE gtrgm (
+        INTERNALLENGTH = -1,
+        INPUT = gtrgm_in,
+        OUTPUT = gtrgm_out
+);
+
+-- support functions
+CREATE FUNCTION gtrgm_consistent(gtrgm,internal,int4)
+RETURNS bool
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C';
+ 
+CREATE FUNCTION gtrgm_compress(internal)
+RETURNS internal
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C';
+
+CREATE FUNCTION gtrgm_decompress(internal)
+RETURNS internal
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C';
+
+CREATE FUNCTION gtrgm_penalty(internal,internal,internal)
+RETURNS internal
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C' with (isstrict);
+
+CREATE FUNCTION gtrgm_picksplit(internal, internal)
+RETURNS internal
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C';
+
+CREATE FUNCTION gtrgm_union(bytea, internal)
+RETURNS _int4
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C';
+
+CREATE FUNCTION gtrgm_same(gtrgm, gtrgm, internal)
+RETURNS internal
+AS 'MODULE_PATHNAME'
+LANGUAGE 'C';
+
+-- create the operator class
+CREATE OPERATOR CLASS gist_trgm_ops
+FOR TYPE text USING gist
+AS
+        OPERATOR        1       % (text, text),
+        FUNCTION        1       gtrgm_consistent (gtrgm, internal, int4),
+        FUNCTION        2       gtrgm_union (bytea, internal),
+        FUNCTION        3       gtrgm_compress (internal),
+        FUNCTION        4       gtrgm_decompress (internal),
+        FUNCTION        5       gtrgm_penalty (internal, internal, internal),
+        FUNCTION        6       gtrgm_picksplit (internal, internal),
+        FUNCTION        7       gtrgm_same (gtrgm, gtrgm, internal),
+        STORAGE         gtrgm;
+
+
+COMMIT;

contrib/pg_trgm/sql/pg_trgm.sql

+\set ECHO none
+\i pg_trgm.sql
+\set ECHO all
+
+select show_trgm('');
+select show_trgm('(*&^$@%@');
+select show_trgm('a b c');
+select show_trgm(' a b c ');
+select show_trgm('aA bB cC');
+select show_trgm(' aA bB cC ');
+select show_trgm('a b C0*%^');
+
+select similarity('wow','WOWa ');
+select similarity('wow',' WOW ');
+
+CREATE TABLE test_trgm(t text);
+
+\copy test_trgm from 'data/trgm.data
+
+select t,similarity(t,'qwertyu0988') as sml from test_trgm where t % 'qwertyu0988' order by sml desc, t;
+select t,similarity(t,'gwertyu0988') as sml from test_trgm where t % 'gwertyu0988' order by sml desc, t;
+select t,similarity(t,'gwertyu1988') as sml from test_trgm where t % 'gwertyu1988' order by sml desc, t;
+
+create index trgm_idx on test_trgm using gist (t gist_trgm_ops);
+set enable_seqscan=off;
+
+select t,similarity(t,'qwertyu0988') as sml from test_trgm where t % 'qwertyu0988' order by sml desc, t;
+select t,similarity(t,'gwertyu0988') as sml from test_trgm where t % 'gwertyu0988' order by sml desc, t;
+select t,similarity(t,'gwertyu1988') as sml from test_trgm where t % 'gwertyu1988' order by sml desc, t;
+

contrib/pg_trgm/trgm.h

+#ifndef __TRGM_H__
+#define __TRGM_H__
+
+#include "postgres.h"
+
+#include "access/gist.h"
+#include "access/itup.h"
+#include "utils/elog.h"
+#include "utils/palloc.h"
+#include "utils/builtins.h"
+#include "storage/bufpage.h"
+
+/* options */
+#define LPADDING        2
+#define RPADDING        1
+#define KEEPONLYALNUM
+#define IGNORECASE  
+#define DIVUNION
+
+
+typedef char trgm[3];
+
+#define CMPCHAR(a,b) ( ((a)==(b)) ? 0 : ( ((a)<(b)) ? -1 : 1 ) )
+#define CMPPCHAR(a,b,i)  CMPCHAR( *(((char*)(a))+i), *(((char*)(b))+i) ) 
+#define CMPTRGM(a,b) ( CMPPCHAR(a,b,0) ? CMPPCHAR(a,b,0) : ( CMPPCHAR(a,b,1) ? CMPPCHAR(a,b,1) : CMPPCHAR(a,b,2) ) )
+
+#define CPTRGM(a,b) do {		      	\
+	*(((char*)(a))+0) = *(((char*)(b))+0);	\
+	*(((char*)(a))+1) = *(((char*)(b))+1);	\
+	*(((char*)(a))+2) = *(((char*)(b))+2);	\
+} while(0);
+
+
+typedef struct {
+	int4	len;
+	uint8	flag;
+	char	data[1];
+} TRGM;
+
+#define TRGMHRDSIZE       (sizeof(int4)+sizeof(uint8))
+
+/* gist */
+#define BITBYTE 8
+#define SIGLENINT  3               /* >122 => key will toast, so very slow!!! */
+#define SIGLEN  ( sizeof(int)*SIGLENINT )
+
+#define SIGLENBIT (SIGLEN*BITBYTE - 1) /* see makesign */ 
+
+typedef char BITVEC[SIGLEN];
+typedef char *BITVECP;
+
+#define LOOPBYTE(a) \
+                for(i=0;i<SIGLEN;i++) {\
+                                a;\
+                }
+
+#define LOOPBIT(a) \
+                for(i=0;i<SIGLENBIT;i++) {\
+                                a;\
+                }
+
+#define GETBYTE(x,i) ( *( (BITVECP)(x) + (int)( (i) / BITBYTE ) ) )
+#define GETBITBYTE(x,i) ( ((char)(x)) >> i & 0x01 )
+#define CLRBIT(x,i)   GETBYTE(x,i) &= ~( 0x01 << ( (i) % BITBYTE ) )
+#define SETBIT(x,i)   GETBYTE(x,i) |=  ( 0x01 << ( (i) % BITBYTE ) )
+#define GETBIT(x,i) ( (GETBYTE(x,i) >> ( (i) % BITBYTE )) & 0x01 )
+
+#define HASHVAL(val) (((unsigned int)(val)) % SIGLENBIT)
+#define HASH(sign, val) SETBIT((sign), HASHVAL(val))
+
+#define ARRKEY          0x01
+#define SIGNKEY         0x02
+#define ALLISTRUE       0x04
+   
+#define ISARRKEY(x) ( ((TRGM*)x)->flag & ARRKEY )
+#define ISSIGNKEY(x)    ( ((TRGM*)x)->flag & SIGNKEY )
+#define ISALLTRUE(x)    ( ((TRGM*)x)->flag & ALLISTRUE )
+
+#define CALCGTSIZE(flag, len) ( TRGMHRDSIZE + ( ( (flag) & ARRKEY ) ? ((len)*sizeof(trgm)) : (((flag) & ALLISTRUE) ? 0 : SIGLEN) ) )
+#define GETSIGN(x)      ( (BITVECP)( (char*)x+TRGMHRDSIZE ) )
+#define GETARR(x)       ( (trgm*)( (char*)x+TRGMHRDSIZE ) )
+#define ARRNELEM(x) ( ( ((TRGM*)x)->len - TRGMHRDSIZE )/sizeof(trgm) )
+
+extern float4 trgm_limit;
+TRGM* generate_trgm(char *str, int slen);
+float4 cnt_sml(TRGM *trg1, TRGM *trg2);
+
+#endif

contrib/pg_trgm/trgm_op.c

+#include "trgm.h"
+#include <ctype.h>
+#include "utils/array.h"
+#include "catalog/pg_type.h"
+
+float4 trgm_limit = 0.3;
+
+PG_FUNCTION_INFO_V1(set_limit);
+Datum set_limit(PG_FUNCTION_ARGS);
+Datum
+set_limit(PG_FUNCTION_ARGS) {
+	float4 nlimit = PG_GETARG_FLOAT4(0);
+	if ( nlimit < 0 || nlimit > 1.0 ) 
+		elog(ERROR,"Wrong limit, should be between 0 and 1");
+	trgm_limit = nlimit;
+	PG_RETURN_FLOAT4(trgm_limit);
+}
+
+PG_FUNCTION_INFO_V1(show_limit);
+Datum show_limit(PG_FUNCTION_ARGS);
+Datum
+show_limit(PG_FUNCTION_ARGS) {
+	PG_RETURN_FLOAT4(trgm_limit);
+}
+
+#define WORDWAIT        0
+#define INWORD          1
+
+static int
+comp_trgm(const void *a, const void *b) {
+	return CMPTRGM(a,b);
+}
+
+static int
+unique_array (trgm *a, int len) {
+	trgm *curend, *tmp;
+
+	curend = tmp = a;
+	while (tmp - a < len)
+		if ( CMPTRGM(tmp, curend) ) {
+			curend++;
+			CPTRGM(curend,tmp);
+			tmp++;
+		} else
+			tmp++;
+ 
+	return curend + 1 - a;
+}
+
+
+TRGM*
+generate_trgm(char *str, int slen) {
+	TRGM*	trg;
+	char	*buf,*sptr,*bufptr;
+	trgm	*tptr;
+	int     state=WORDWAIT;
+	int wl,len;
+
+	trg = (TRGM*) palloc(TRGMHRDSIZE+sizeof(trgm) * (slen/2 + 1) * 3);
+	trg->flag = ARRKEY;
+	trg->len = TRGMHRDSIZE;
+
+	if ( slen+LPADDING+RPADDING<3 || slen == 0 )
+		return trg;
+
+	tptr = GETARR(trg);
+
+	buf = palloc(sizeof(char) * (slen+4));
+	sptr = str;
+
+	if ( LPADDING > 0 ) {
+		*buf = ' ';
+		if ( LPADDING > 1 )
+			*(buf+1) = ' ';
+	}
+
+	bufptr = buf+LPADDING;
+	while( sptr-str < slen ) {
+		if ( state == WORDWAIT ) {
+			if ( 
+#ifdef	KEEPONLYALNUM
+	 			isalnum((unsigned char)*sptr)
+#else
+				!isspace( (unsigned char)*sptr )
+#endif
+				) {
+				*bufptr = *sptr; /* start put word in buffer */
+				bufptr++;
+				state = INWORD;
+				if ( sptr-str == slen-1 /* last char */ )
+					goto gettrg;
+			}
+		} else {
+			if (
+#ifdef	KEEPONLYALNUM
+	 			!isalnum((unsigned char)*sptr)
+#else
+				isspace( (unsigned char)*sptr )			
+#endif
+				) {
+gettrg:
+				/* word in buffer, so count trigrams */
+				*bufptr = ' ';
+				*(bufptr+1) = ' ';
+				wl = bufptr - (buf+LPADDING) - 2 + LPADDING + RPADDING;
+				if ( wl<=0 ) {
+					bufptr = buf+LPADDING;
+					state = WORDWAIT;
+					sptr++;
+					continue;
+				}
+
+#ifdef IGNORECASE
+				do { /* lower word */
+					int wwl = bufptr-buf;
+					bufptr = buf+LPADDING;
+					while( bufptr-buf < wwl ) {
+						*bufptr = tolower( (unsigned char) *bufptr );
+						bufptr++;
+					}
+				} while(0);
+#endif
+				bufptr = buf;
+				/* set trigrams */
+				while( bufptr-buf < wl ) {
+					CPTRGM(tptr, bufptr);
+					bufptr++;
+					tptr++;
+				}
+				bufptr = buf+LPADDING;
+				state = WORDWAIT;
+			} else {
+				*bufptr = *sptr; /* put in buffer */
+				bufptr++;
+				if ( sptr-str == slen-1 )
+					goto gettrg;
+			}
+		}
+		sptr++;
+	}
+
+	pfree(buf);
+
+	if ( (len=tptr-GETARR(trg)) == 0 ) 
+		return trg;
+
+	if ( len>0 ) {
+		qsort( (void*)GETARR(trg), len, sizeof(trgm), comp_trgm );
+		len = unique_array( GETARR(trg), len );
+	} 
+
+	trg->len = CALCGTSIZE(ARRKEY, len);
+
+	return trg;
+}
+
+
+PG_FUNCTION_INFO_V1(show_trgm);
+Datum show_trgm(PG_FUNCTION_ARGS);
+Datum
+show_trgm(PG_FUNCTION_ARGS) {
+	text       *in = PG_GETARG_TEXT_P(0);
+	TRGM	*trg;
+	Datum *d;
+	ArrayType *a;
+	trgm *ptr;
+
+	trg = generate_trgm(VARDATA(in), VARSIZE(in) - VARHDRSZ);
+	d = (Datum*)palloc( sizeof(Datum)*(1+ARRNELEM(trg)) );
+
+	ptr = GETARR(trg);
+	while( ptr-GETARR(trg) < ARRNELEM(trg) ) {
+		text *item=(text*)palloc(VARHDRSZ + 3);
+		VARATT_SIZEP(item) = VARHDRSZ+3;
+		CPTRGM(VARDATA(item), ptr);
+		d[ ptr-GETARR(trg) ] = PointerGetDatum(item);
+		ptr++;
+	}
+
+	a = construct_array(
+		d,
+		ARRNELEM(trg),
+		TEXTOID,
+		-1,
+		false,
+		'i'
+	);
+
+	ptr = GETARR(trg);
+	while( ptr-GETARR(trg) < ARRNELEM(trg) ) {
+		pfree(DatumGetPointer(d[ ptr-GETARR(trg) ]));
+		ptr++;
+	}
+	
+	pfree(d);
+	pfree(trg);
+	PG_FREE_IF_COPY(in,0);
+
+	PG_RETURN_POINTER(a);
+}
+
+float4
+cnt_sml(TRGM *trg1, TRGM *trg2) {
+	trgm	*ptr1, *ptr2;
+	int count=0;
+	int len1, len2;
+	
+	ptr1 = GETARR(trg1);
+	ptr2 = GETARR(trg2);
+	
+	len1 = ARRNELEM(trg1);
+	len2 = ARRNELEM(trg2);
+
+	while( ptr1 - GETARR(trg1) < len1 && ptr2 - GETARR(trg2) < len2 ) {
+		int res = CMPTRGM(ptr1,ptr2);
+		if ( res < 0 ) {
+			ptr1++;
+		} else if ( res > 0 ) {
+			ptr2++;
+		} else {
+			ptr1++;
+			ptr2++;
+			count++;
+		}
+	}
+
+#ifdef DIVUNION
+	return ( ( ((float4)count) / ((float4)(len1+len2-count)) ) );
+#else
+	return ( ((float)count) / ((float)( (len1>len2) ? len1 : len2 )) );
+#endif
+
+}
+
+PG_FUNCTION_INFO_V1(similarity);
+Datum similarity(PG_FUNCTION_ARGS);
+Datum
+similarity(PG_FUNCTION_ARGS) {
+	text       *in1 = PG_GETARG_TEXT_P(0);
+	text       *in2 = PG_GETARG_TEXT_P(1);
+	TRGM    *trg1, *trg2;
+	float4 res;
+
+	trg1 = generate_trgm(VARDATA(in1), VARSIZE(in1) - VARHDRSZ);
+	trg2 = generate_trgm(VARDATA(in2), VARSIZE(in2) - VARHDRSZ);
+
+	res = cnt_sml(trg1,trg2);
+	
+	pfree(trg1);
+	pfree(trg2);
+	PG_FREE_IF_COPY(in1,0);
+	PG_FREE_IF_COPY(in2,1);
+	
+	PG_RETURN_FLOAT4(res);
+}
+
+PG_FUNCTION_INFO_V1(similarity_op);
+Datum similarity_op(PG_FUNCTION_ARGS);
+Datum
+similarity_op(PG_FUNCTION_ARGS) {
+	float4 res=DatumGetFloat4( DirectFunctionCall2(
+		similarity,
+		PG_GETARG_DATUM(0),
+		PG_GETARG_DATUM(1)
+	) );
+	PG_RETURN_BOOL( res >= trgm_limit );
+}
+
+