Move the built-in conversions into the initial catalog data.

Instead of running a SQL script to create the standard conversion
functions and pg_conversion entries, put those entries into the
initial data in postgres.bki.

This shaves a few percent off the runtime of initdb, and also allows
accurate comments to be attached to the conversion functions; the
previous script labeled them with machine-generated comments that
were not quite right for multi-purpose conversion functions.
Also, we can get rid of the duplicative Makefile and MSVC perl
implementations of the generation code for that SQL script.

A functional change is that these pg_proc and pg_conversion entries
are now "pinned" by initdb.  Leaving them unpinned was perhaps a
good thing back while the conversions feature was under development,
but there seems no valid reason for it now.

Also, the conversion functions are now marked as immutable, where
before they were volatile by virtue of lacking any explicit
specification.  That seems like it was just an oversight.

To avoid using magic constants in pg_conversion.dat, extend
genbki.pl to allow encoding names to be converted, much as it
does for language, access method, etc names.

John Naylor

Discussion: https://postgr.es/m/CAJVSVGWtUqxpfAaxS88vEGvi+jKzWZb2EStu5io-UPc4p9rSJg@mail.gmail.com

doc/src/sgml/bki.sgml

@@ -410,7 +410,7 @@
     catalogs, <filename>genbki.pl</filename> provides mechanisms to write
     symbolic references instead.  Currently this is possible for references
     to access methods, functions, languages,
-    operators, opclasses, opfamilies, and types.
+    operators, opclasses, opfamilies, types, and encodings.
     The rules are as follows:
    </para>
 
@@ -424,11 +424,16 @@
       is <literal>pg_am</literal>, <literal>pg_proc</literal>,
       <literal>pg_language</literal>,
       <literal>pg_operator</literal>, <literal>pg_opclass</literal>,
-      <literal>pg_opfamily</literal>, or <literal>pg_type</literal>.
+      <literal>pg_opfamily</literal>,
+      <literal>pg_type</literal>,
+      or <literal>encoding</literal>.
       <literal>BKI_LOOKUP</literal> can be attached to columns of
       type <type>Oid</type>, <type>regproc</type>, <type>oidvector</type>,
       or <type>Oid[]</type>; in the latter two cases it implies performing a
       lookup on each element of the array.
+      It's also permissible to attach <literal>BKI_LOOKUP</literal>
+      to integer columns; this should be done only for encodings,
+      which are not currently represented as catalog OIDs.
      </para>
     </listitem>
 

src/backend/catalog/Makefile

@@ -60,7 +60,7 @@ POSTGRES_BKI_SRCS := $(addprefix $(top_srcdir)/src/include/catalog/,\
 # The .dat files we need can just be listed alphabetically.
 POSTGRES_BKI_DATA = $(addprefix $(top_srcdir)/src/include/catalog/,\
 	pg_aggregate.dat pg_am.dat pg_amop.dat pg_amproc.dat pg_authid.dat \
-	pg_cast.dat pg_class.dat pg_collation.dat \
+	pg_cast.dat pg_class.dat pg_collation.dat pg_conversion.dat \
 	pg_database.dat pg_language.dat \
 	pg_namespace.dat pg_opclass.dat pg_operator.dat pg_opfamily.dat \
 	pg_pltemplate.dat pg_proc.dat pg_range.dat pg_tablespace.dat \

src/backend/catalog/genbki.pl

@@ -57,11 +57,15 @@ die "No input files.\n" if !@input_files;
 die "--set-version must be specified.\n" if !defined $major_version;
 die "-I, the header include path, must be specified.\n" if !$include_path;
 
-# Make sure output_path ends in a slash.
+# Make sure paths end with a slash.
 if ($output_path ne '' && substr($output_path, -1) ne '/')
 {
 	$output_path .= '/';
 }
+if (substr($include_path, -1) ne '/')
+{
+	$include_path .= '/';
+}
 
 # Read all the files into internal data structures.
 my @catnames;
@@ -175,8 +179,7 @@ my $PG_CATALOG_NAMESPACE =
 	'PG_CATALOG_NAMESPACE');
 
 
-# Build lookup tables for OID macro substitutions and for pg_attribute
-# copies of pg_type values.
+# Build lookup tables.
 
 # access method OID lookup
 my %amoids;
@@ -259,11 +262,44 @@ my %typeoids;
 my %types;
 foreach my $row (@{ $catalog_data{pg_type} })
 {
+	# for OID macro substitutions
 	$typeoids{ $row->{typname} } = $row->{oid};
+
+	# for pg_attribute copies of pg_type values
 	$types{ $row->{typname} }    = $row;
 }
 
-# Map catalog name to OID lookup.
+# Encoding identifier lookup.  This uses the same replacement machinery
+# as for OIDs, but we have to dig the values out of pg_wchar.h.
+my %encids;
+
+my $encfile = $include_path . 'mb/pg_wchar.h';
+open(my $ef, '<', $encfile) || die "$encfile: $!";
+
+# We're parsing an enum, so start with 0 and increment
+# every time we find an enum member.
+my $encid = 0;
+my $collect_encodings = 0;
+while (<$ef>)
+{
+	if (/typedef\s+enum\s+pg_enc/)
+	{
+		$collect_encodings = 1;
+		next;
+	}
+
+	last if /_PG_LAST_ENCODING_/;
+
+	if ($collect_encodings and /^\s+(PG_\w+)/)
+	{
+		$encids{$1} = $encid;
+		$encid++;
+	}
+}
+
+close $ef;
+
+# Map lookup name to the corresponding hash table.
 my %lookup_kind = (
 	pg_am       => \%amoids,
 	pg_language => \%langoids,
@@ -271,7 +307,8 @@ my %lookup_kind = (
 	pg_operator => \%operoids,
 	pg_opfamily => \%opfoids,
 	pg_proc     => \%procoids,
-	pg_type     => \%typeoids);
+	pg_type     => \%typeoids,
+	encoding    => \%encids);
 
 
 # Open temp files

src/backend/utils/mb/conversion_procs/.gitignore

-/conversion_create.sql

src/bin/initdb/initdb.c

@@ -155,7 +155,6 @@ static char *shdesc_file;
 static char *hba_file;
 static char *ident_file;
 static char *conf_file;
-static char *conversion_file;
 static char *dictionary_file;
 static char *info_schema_file;
 static char *features_file;
@@ -254,7 +253,6 @@ static void setup_depend(FILE *cmdfd);
 static void setup_sysviews(FILE *cmdfd);
 static void setup_description(FILE *cmdfd);
 static void setup_collation(FILE *cmdfd);
-static void setup_conversion(FILE *cmdfd);
 static void setup_dictionary(FILE *cmdfd);
 static void setup_privileges(FILE *cmdfd);
 static void set_info_version(void);
@@ -1774,26 +1772,6 @@ setup_collation(FILE *cmdfd)
 	PG_CMD_PUTS("SELECT pg_import_system_collations('pg_catalog');\n\n");
 }
 
-/*
- * load conversion functions
- */
-static void
-setup_conversion(FILE *cmdfd)
-{
-	char	  **line;
-	char	  **conv_lines;
-
-	conv_lines = readfile(conversion_file);
-	for (line = conv_lines; *line != NULL; line++)
-	{
-		if (strstr(*line, "DROP CONVERSION") != *line)
-			PG_CMD_PUTS(*line);
-		free(*line);
-	}
-
-	free(conv_lines);
-}
-
 /*
  * load extra dictionaries (Snowball stemmers)
  */
@@ -2679,7 +2657,6 @@ setup_data_file_paths(void)
 	set_input(&hba_file, "pg_hba.conf.sample");
 	set_input(&ident_file, "pg_ident.conf.sample");
 	set_input(&conf_file, "postgresql.conf.sample");
-	set_input(&conversion_file, "conversion_create.sql");
 	set_input(&dictionary_file, "snowball_create.sql");
 	set_input(&info_schema_file, "information_schema.sql");
 	set_input(&features_file, "sql_features.txt");
@@ -2710,7 +2687,6 @@ setup_data_file_paths(void)
 	check_input(hba_file);
 	check_input(ident_file);
 	check_input(conf_file);
-	check_input(conversion_file);
 	check_input(dictionary_file);
 	check_input(info_schema_file);
 	check_input(features_file);
@@ -3070,8 +3046,6 @@ initialize_data_directory(void)
 
 	setup_collation(cmdfd);
 
-	setup_conversion(cmdfd);
-
 	setup_dictionary(cmdfd);
 
 	setup_privileges(cmdfd);

src/include/catalog/catversion.h

@@ -53,6 +53,6 @@
  */
 
 /*							yyyymmddN */
-#define CATALOG_VERSION_NO	201812311
+#define CATALOG_VERSION_NO	201901031
 
 #endif

src/include/catalog/genbki.h

@@ -35,7 +35,10 @@
 #define BKI_DEFAULT(value)
 /* Specifies a default value for auto-generated array types */
 #define BKI_ARRAY_DEFAULT(value)
-/* Indicates how to perform name lookups for an OID or OID-array field */
+/*
+ * Indicates how to perform name lookups, typically for an OID or
+ * OID-array field
+ */
 #define BKI_LOOKUP(catalog)
 
 /* The following are never defined; they are here only for documentation. */

src/include/catalog/pg_conversion.h

@@ -3,7 +3,6 @@
  * pg_conversion.h
  *	  definition of the "conversion" system catalog (pg_conversion)
  *
- *
  * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
@@ -23,30 +22,36 @@
 
 #include "catalog/objectaddress.h"
 
-/* ----------------------------------------------------------------
- *		pg_conversion definition.
- *
- *		cpp turns this into typedef struct FormData_pg_namespace
- *
- *	conname				name of the conversion
- *	connamespace		name space which the conversion belongs to
- *	conowner			owner of the conversion
- *	conforencoding		FOR encoding id
- *	contoencoding		TO encoding id
- *	conproc				OID of the conversion proc
- *	condefault			true if this is a default conversion
- * ----------------------------------------------------------------
+/* ----------------
+ *		pg_conversion definition.  cpp turns this into
+ *		typedef struct FormData_pg_conversion
+ * ----------------
  */
 CATALOG(pg_conversion,2607,ConversionRelationId)
 {
-	Oid			oid;			/* oid */
+	/* oid */
+	Oid			oid;
+
+	/* name of the conversion */
 	NameData	conname;
-	Oid			connamespace;
-	Oid			conowner;
-	int32		conforencoding;
-	int32		contoencoding;
-	regproc		conproc;
-	bool		condefault;
+
+	/* namespace that the conversion belongs to */
+	Oid			connamespace BKI_DEFAULT(PGNSP);
+
+	/* owner of the conversion */
+	Oid			conowner BKI_DEFAULT(PGUID);
+
+	/* FOR encoding id */
+	int32		conforencoding BKI_LOOKUP(encoding);
+
+	/* TO encoding id */
+	int32		contoencoding BKI_LOOKUP(encoding);
+
+	/* OID of the conversion proc */
+	regproc		conproc BKI_LOOKUP(pg_proc);
+
+	/* true if this is a default conversion */
+	bool		condefault BKI_DEFAULT(t);
 } FormData_pg_conversion;
 
 /* ----------------

src/test/regress/expected/misc_sanity.out

@@ -74,7 +74,6 @@ loop
 end loop;
 end$$;
 NOTICE:  pg_constraint contains unpinned initdb-created object(s)
-NOTICE:  pg_conversion contains unpinned initdb-created object(s)
 NOTICE:  pg_database contains unpinned initdb-created object(s)
 NOTICE:  pg_extension contains unpinned initdb-created object(s)
 NOTICE:  pg_rewrite contains unpinned initdb-created object(s)

src/tools/msvc/Install.pm

@@ -139,7 +139,6 @@ sub Install
 		CopyFiles(
 			'Error code data',    $target . '/share/',
 			'src/backend/utils/', 'errcodes.txt');
-		GenerateConversionScript($target);
 		GenerateTimezoneFiles($target, $conf);
 		GenerateTsearchFiles($target);
 		CopySetOfFiles(
@@ -348,44 +347,6 @@ sub CopySolutionOutput
 	return;
 }
 
-sub GenerateConversionScript
-{
-	my $target = shift;
-	my $sql    = "";
-	my $F;
-
-	print "Generating conversion proc script...";
-	my $mf = read_file('src/backend/utils/mb/conversion_procs/Makefile');
-	$mf =~ s{\\\r?\n}{}g;
-	$mf =~ /^CONVERSIONS\s*=\s*(.*)$/m
-	  || die "Could not find CONVERSIONS line in conversions Makefile\n";
-	my @pieces = split /\s+/, $1;
-	while ($#pieces > 0)
-	{
-		my $name = shift @pieces;
-		my $se   = shift @pieces;
-		my $de   = shift @pieces;
-		my $func = shift @pieces;
-		my $obj  = shift @pieces;
-		$sql .= "-- $se --> $de\n";
-		$sql .=
-		  "CREATE OR REPLACE FUNCTION $func (INTEGER, INTEGER, CSTRING, INTERNAL, INTEGER) RETURNS VOID AS '\$libdir/$obj', '$func' LANGUAGE C STRICT;\n";
-		$sql .=
-		  "COMMENT ON FUNCTION $func(INTEGER, INTEGER, CSTRING, INTERNAL, INTEGER) IS 'internal conversion function for $se to $de';\n";
-		$sql .= "DROP CONVERSION pg_catalog.$name;\n";
-		$sql .=
-		  "CREATE DEFAULT CONVERSION pg_catalog.$name FOR '$se' TO '$de' FROM $func;\n";
-		$sql .=
-		  "COMMENT ON CONVERSION pg_catalog.$name IS 'conversion for $se to $de';\n\n";
-	}
-	open($F, '>', "$target/share/conversion_create.sql")
-	  || die "Could not write to conversion_create.sql\n";
-	print $F $sql;
-	close($F);
-	print "\n";
-	return;
-}
-
 sub GenerateTimezoneFiles
 {
 	my $target = shift;