Avoid doing encoding conversions by double-conversion via MULE_INTERNAL.

Previously, we did many conversions for Cyrillic and Central European
single-byte encodings by converting to a related MULE_INTERNAL coding
scheme before converting to the destination.  This seems unnecessarily
inefficient.  Moreover, if the conversion encounters an untranslatable
character, the error message will confusingly complain about failure
to convert to or from MULE_INTERNAL, rather than the user-visible
encodings.  Worse still, this approach results in some completely
unnecessary conversion failures; there are cases where the chosen
MULE subset lacks characters that exist in both of the user-visible
encodings, causing a conversion failure that need not occur.

This patch fixes the first two of those deficiencies by introducing
a new local2local() conversion support subroutine for direct conversion
between any two single-byte character sets, and adding new conversion
tables where needed.  However, I generated the new conversion tables by
testing PG 9.5's behavior, so that the actual conversion behavior is
bug-compatible with previous releases; the only user-visible behavior
change is that the error messages for conversion failures are saner.
Changes in the conversion behavior will probably ensue after discussion.

Interestingly, although this approach requires more tables, the .so files
actually end up smaller (at least on my x86_64 machine); the tables are
smaller than the management code needed for double conversion.

Per a complaint from Albe Laurenz.

src/backend/utils/mb/conv.c

@@ -14,6 +14,51 @@
 #include "mb/pg_wchar.h"
 
 
+/*
+ * local2local: a generic single byte charset encoding
+ * conversion between two ASCII-superset encodings.
+ *
+ * l points to the source string of length len
+ * p is the output area (must be large enough!)
+ * src_encoding is the PG identifier for the source encoding
+ * dest_encoding is the PG identifier for the target encoding
+ * tab holds conversion entries for the source charset
+ * starting from 128 (0x80). each entry in the table holds the corresponding
+ * code point for the target charset, or 0 if there is no equivalent code.
+ */
+void
+local2local(const unsigned char *l,
+			unsigned char *p,
+			int len,
+			int src_encoding,
+			int dest_encoding,
+			const unsigned char *tab)
+{
+	unsigned char c1,
+				c2;
+
+	while (len > 0)
+	{
+		c1 = *l;
+		if (c1 == 0)
+			report_invalid_encoding(src_encoding, (const char *) l, len);
+		if (!IS_HIGHBIT_SET(c1))
+			*p++ = c1;
+		else
+		{
+			c2 = tab[c1 - HIGHBIT];
+			if (c2)
+				*p++ = c2;
+			else
+				report_untranslatable_char(src_encoding, dest_encoding,
+										   (const char *) l, len);
+		}
+		l++;
+		len--;
+	}
+	*p = '\0';
+}
+
 /*
  * LATINn ---> MIC when the charset's local codes map directly to MIC
  *
@@ -141,8 +186,8 @@ pg_mic2ascii(const unsigned char *mic, unsigned char *p, int len)
  * lc is the mule character set id for the local encoding
  * encoding is the PG identifier for the local encoding
  * tab holds conversion entries for the local charset
- * starting from 128 (0x80). each entry in the table
- * holds the corresponding code point for the mule internal code.
+ * starting from 128 (0x80). each entry in the table holds the corresponding
+ * code point for the mule encoding, or 0 if there is no equivalent code.
  */
 void
 latin2mic_with_table(const unsigned char *l,
@@ -188,9 +233,9 @@ latin2mic_with_table(const unsigned char *l,
  * p is the output area (must be large enough!)
  * lc is the mule character set id for the local encoding
  * encoding is the PG identifier for the local encoding
- * tab holds conversion entries for the mule internal code's
- * second byte, starting from 128 (0x80). each entry in the table
- * holds the corresponding code point for the local charset.
+ * tab holds conversion entries for the mule internal code's second byte,
+ * starting from 128 (0x80). each entry in the table holds the corresponding
+ * code point for the local charset, or 0 if there is no equivalent code.
  */
 void
 mic2latin_with_table(const unsigned char *mic,

src/backend/utils/mb/conversion_procs/latin2_and_win1250/latin2_and_win1250.c

@@ -15,8 +15,6 @@
 #include "fmgr.h"
 #include "mb/pg_wchar.h"
 
-#define ENCODING_GROWTH_RATE 4
-
 PG_MODULE_MAGIC;
 
 PG_FUNCTION_INFO_V1(latin2_to_mic);
@@ -37,10 +35,46 @@ PG_FUNCTION_INFO_V1(win1250_to_latin2);
  * ----------
  */
 
-static void latin22mic(const unsigned char *l, unsigned char *p, int len);
-static void mic2latin2(const unsigned char *mic, unsigned char *p, int len);
-static void win12502mic(const unsigned char *l, unsigned char *p, int len);
-static void mic2win1250(const unsigned char *mic, unsigned char *p, int len);
+/* WIN1250 to ISO-8859-2 */
+static const unsigned char win1250_2_iso88592[] = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0xA9, 0x8B, 0xA6, 0xAB, 0xAE, 0xAC,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0xB9, 0x9B, 0xB6, 0xBB, 0xBE, 0xBC,
+	0xA0, 0xB7, 0xA2, 0xA3, 0xA4, 0xA1, 0x00, 0xA7,
+	0xA8, 0x00, 0xAA, 0x00, 0x00, 0xAD, 0x00, 0xAF,
+	0xB0, 0x00, 0xB2, 0xB3, 0xB4, 0x00, 0x00, 0x00,
+	0xB8, 0xB1, 0xBA, 0x00, 0xA5, 0xBD, 0xB5, 0xBF,
+	0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
+	0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
+	0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
+	0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
+	0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
+	0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
+	0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
+	0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF
+};
+
+/* ISO-8859-2 to WIN1250 */
+static const unsigned char iso88592_2_win1250[] = {
+	0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
+	0x88, 0x89, 0x00, 0x8B, 0x00, 0x00, 0x00, 0x00,
+	0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
+	0x98, 0x99, 0x00, 0x9B, 0x00, 0x00, 0x00, 0x00,
+	0xA0, 0xA5, 0xA2, 0xA3, 0xA4, 0xBC, 0x8C, 0xA7,
+	0xA8, 0x8A, 0xAA, 0x8D, 0x8F, 0xAD, 0x8E, 0xAF,
+	0xB0, 0xB9, 0xB2, 0xB3, 0xB4, 0xBE, 0x9C, 0xA1,
+	0xB8, 0x9A, 0xBA, 0x9D, 0x9F, 0xBD, 0x9E, 0xBF,
+	0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
+	0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
+	0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
+	0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
+	0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
+	0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
+	0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
+	0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF
+};
+
 
 Datum
 latin2_to_mic(PG_FUNCTION_ARGS)
@@ -51,7 +85,7 @@ latin2_to_mic(PG_FUNCTION_ARGS)
 
 	CHECK_ENCODING_CONVERSION_ARGS(PG_LATIN2, PG_MULE_INTERNAL);
 
-	latin22mic(src, dest, len);
+	latin2mic(src, dest, len, LC_ISO8859_2, PG_LATIN2);
 
 	PG_RETURN_VOID();
 }
@@ -65,7 +99,7 @@ mic_to_latin2(PG_FUNCTION_ARGS)
 
 	CHECK_ENCODING_CONVERSION_ARGS(PG_MULE_INTERNAL, PG_LATIN2);
 
-	mic2latin2(src, dest, len);
+	mic2latin(src, dest, len, LC_ISO8859_2, PG_LATIN2);
 
 	PG_RETURN_VOID();
 }
@@ -79,7 +113,8 @@ win1250_to_mic(PG_FUNCTION_ARGS)
 
 	CHECK_ENCODING_CONVERSION_ARGS(PG_WIN1250, PG_MULE_INTERNAL);
 
-	win12502mic(src, dest, len);
+	latin2mic_with_table(src, dest, len, LC_ISO8859_2, PG_WIN1250,
+						 win1250_2_iso88592);
 
 	PG_RETURN_VOID();
 }
@@ -93,7 +128,8 @@ mic_to_win1250(PG_FUNCTION_ARGS)
 
 	CHECK_ENCODING_CONVERSION_ARGS(PG_MULE_INTERNAL, PG_WIN1250);
 
-	mic2win1250(src, dest, len);
+	mic2latin_with_table(src, dest, len, LC_ISO8859_2, PG_WIN1250,
+						 iso88592_2_win1250);
 
 	PG_RETURN_VOID();
 }
@@ -104,14 +140,10 @@ latin2_to_win1250(PG_FUNCTION_ARGS)
 	unsigned char *src = (unsigned char *) PG_GETARG_CSTRING(2);
 	unsigned char *dest = (unsigned char *) PG_GETARG_CSTRING(3);
 	int			len = PG_GETARG_INT32(4);
-	unsigned char *buf;
 
 	CHECK_ENCODING_CONVERSION_ARGS(PG_LATIN2, PG_WIN1250);
 
-	buf = palloc(len * ENCODING_GROWTH_RATE + 1);
-	latin22mic(src, buf, len);
-	mic2win1250(buf, dest, strlen((char *) buf));
-	pfree(buf);
+	local2local(src, dest, len, PG_LATIN2, PG_WIN1250, iso88592_2_win1250);
 
 	PG_RETURN_VOID();
 }
@@ -122,82 +154,10 @@ win1250_to_latin2(PG_FUNCTION_ARGS)
 	unsigned char *src = (unsigned char *) PG_GETARG_CSTRING(2);
 	unsigned char *dest = (unsigned char *) PG_GETARG_CSTRING(3);
 	int			len = PG_GETARG_INT32(4);
-	unsigned char *buf;
 
 	CHECK_ENCODING_CONVERSION_ARGS(PG_WIN1250, PG_LATIN2);
 
-	buf = palloc(len * ENCODING_GROWTH_RATE + 1);
-	win12502mic(src, buf, len);
-	mic2latin2(buf, dest, strlen((char *) buf));
-	pfree(buf);
+	local2local(src, dest, len, PG_WIN1250, PG_LATIN2, win1250_2_iso88592);
 
 	PG_RETURN_VOID();
 }
-
-static void
-latin22mic(const unsigned char *l, unsigned char *p, int len)
-{
-	latin2mic(l, p, len, LC_ISO8859_2, PG_LATIN2);
-}
-
-static void
-mic2latin2(const unsigned char *mic, unsigned char *p, int len)
-{
-	mic2latin(mic, p, len, LC_ISO8859_2, PG_LATIN2);
-}
-
-/*-----------------------------------------------------------------
- * WIN1250
- * Microsoft's CP1250(windows-1250)
- *-----------------------------------------------------------------*/
-static void
-win12502mic(const unsigned char *l, unsigned char *p, int len)
-{
-	static const unsigned char win1250_2_iso88592[] = {
-		0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
-		0x88, 0x89, 0xA9, 0x8B, 0xA6, 0xAB, 0xAE, 0xAC,
-		0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
-		0x98, 0x99, 0xB9, 0x9B, 0xB6, 0xBB, 0xBE, 0xBC,
-		0xA0, 0xB7, 0xA2, 0xA3, 0xA4, 0xA1, 0x00, 0xA7,
-		0xA8, 0x00, 0xAA, 0x00, 0x00, 0xAD, 0x00, 0xAF,
-		0xB0, 0x00, 0xB2, 0xB3, 0xB4, 0x00, 0x00, 0x00,
-		0xB8, 0xB1, 0xBA, 0x00, 0xA5, 0xBD, 0xB5, 0xBF,
-		0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
-		0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
-		0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
-		0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
-		0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
-		0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
-		0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
-		0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF
-	};
-
-	latin2mic_with_table(l, p, len, LC_ISO8859_2, PG_WIN1250,
-						 win1250_2_iso88592);
-}
-
-static void
-mic2win1250(const unsigned char *mic, unsigned char *p, int len)
-{
-	static const unsigned char iso88592_2_win1250[] = {
-		0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87,
-		0x88, 0x89, 0x00, 0x8B, 0x00, 0x00, 0x00, 0x00,
-		0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97,
-		0x98, 0x99, 0x00, 0x9B, 0x00, 0x00, 0x00, 0x00,
-		0xA0, 0xA5, 0xA2, 0xA3, 0xA4, 0xBC, 0x8C, 0xA7,
-		0xA8, 0x8A, 0xAA, 0x8D, 0x8F, 0xAD, 0x8E, 0xAF,
-		0xB0, 0xB9, 0xB2, 0xB3, 0xB4, 0xBE, 0x9C, 0xA1,
-		0xB8, 0x9A, 0xBA, 0x9D, 0x9F, 0xBD, 0x9E, 0xBF,
-		0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7,
-		0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF,
-		0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
-		0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF,
-		0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7,
-		0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF,
-		0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7,
-		0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF
-	};
-
-	mic2latin_with_table(mic, p, len, LC_ISO8859_2, PG_WIN1250,
-						 iso88592_2_win1250);
-}

src/backend/utils/mb/conversion_procs/latin_and_mic/latin_and_mic.c

@@ -35,12 +35,6 @@ PG_FUNCTION_INFO_V1(mic_to_latin4);
  * ----------
  */
 
-static void latin12mic(const unsigned char *l, unsigned char *p, int len);
-static void mic2latin1(const unsigned char *mic, unsigned char *p, int len);
-static void latin32mic(const unsigned char *l, unsigned char *p, int len);
-static void mic2latin3(const unsigned char *mic, unsigned char *p, int len);
-static void latin42mic(const unsigned char *l, unsigned char *p, int len);
-static void mic2latin4(const unsigned char *mic, unsigned char *p, int len);
 
 Datum
 latin1_to_mic(PG_FUNCTION_ARGS)
@@ -51,7 +45,7 @@ latin1_to_mic(PG_FUNCTION_ARGS)
 
 	CHECK_ENCODING_CONVERSION_ARGS(PG_LATIN1, PG_MULE_INTERNAL);
 
-	latin12mic(src, dest, len);
+	latin2mic(src, dest, len, LC_ISO8859_1, PG_LATIN1);
 
 	PG_RETURN_VOID();
 }
@@ -65,7 +59,7 @@ mic_to_latin1(PG_FUNCTION_ARGS)
 
 	CHECK_ENCODING_CONVERSION_ARGS(PG_MULE_INTERNAL, PG_LATIN1);
 
-	mic2latin1(src, dest, len);
+	mic2latin(src, dest, len, LC_ISO8859_1, PG_LATIN1);
 
 	PG_RETURN_VOID();
 }
@@ -79,7 +73,7 @@ latin3_to_mic(PG_FUNCTION_ARGS)
 
 	CHECK_ENCODING_CONVERSION_ARGS(PG_LATIN3, PG_MULE_INTERNAL);
 
-	latin32mic(src, dest, len);
+	latin2mic(src, dest, len, LC_ISO8859_3, PG_LATIN3);
 
 	PG_RETURN_VOID();
 }
@@ -93,7 +87,7 @@ mic_to_latin3(PG_FUNCTION_ARGS)
 
 	CHECK_ENCODING_CONVERSION_ARGS(PG_MULE_INTERNAL, PG_LATIN3);
 
-	mic2latin3(src, dest, len);
+	mic2latin(src, dest, len, LC_ISO8859_3, PG_LATIN3);
 
 	PG_RETURN_VOID();
 }
@@ -107,7 +101,7 @@ latin4_to_mic(PG_FUNCTION_ARGS)
 
 	CHECK_ENCODING_CONVERSION_ARGS(PG_LATIN4, PG_MULE_INTERNAL);
 
-	latin42mic(src, dest, len);
+	latin2mic(src, dest, len, LC_ISO8859_4, PG_LATIN4);
 
 	PG_RETURN_VOID();
 }
@@ -121,43 +115,7 @@ mic_to_latin4(PG_FUNCTION_ARGS)
 
 	CHECK_ENCODING_CONVERSION_ARGS(PG_MULE_INTERNAL, PG_LATIN4);
 
-	mic2latin4(src, dest, len);
+	mic2latin(src, dest, len, LC_ISO8859_4, PG_LATIN4);
 
 	PG_RETURN_VOID();
 }
-
-static void
-latin12mic(const unsigned char *l, unsigned char *p, int len)
-{
-	latin2mic(l, p, len, LC_ISO8859_1, PG_LATIN1);
-}
-
-static void
-mic2latin1(const unsigned char *mic, unsigned char *p, int len)
-{
-	mic2latin(mic, p, len, LC_ISO8859_1, PG_LATIN1);
-}
-
-static void
-latin32mic(const unsigned char *l, unsigned char *p, int len)
-{
-	latin2mic(l, p, len, LC_ISO8859_3, PG_LATIN3);
-}
-
-static void
-mic2latin3(const unsigned char *mic, unsigned char *p, int len)
-{
-	mic2latin(mic, p, len, LC_ISO8859_3, PG_LATIN3);
-}
-
-static void
-latin42mic(const unsigned char *l, unsigned char *p, int len)
-{
-	latin2mic(l, p, len, LC_ISO8859_4, PG_LATIN4);
-}
-
-static void
-mic2latin4(const unsigned char *mic, unsigned char *p, int len)
-{
-	mic2latin(mic, p, len, LC_ISO8859_4, PG_LATIN4);
-}

src/include/mb/pg_wchar.h

@@ -537,6 +537,8 @@ extern void report_invalid_encoding(int encoding, const char *mbstr, int len) pg
 extern void report_untranslatable_char(int src_encoding, int dest_encoding,
 						 const char *mbstr, int len) pg_attribute_noreturn();
 
+extern void local2local(const unsigned char *l, unsigned char *p, int len,
+			int src_encoding, int dest_encoding, const unsigned char *tab);
 extern void pg_ascii2mic(const unsigned char *l, unsigned char *p, int len);
 extern void pg_mic2ascii(const unsigned char *mic, unsigned char *p, int len);
 extern void latin2mic(const unsigned char *l, unsigned char *p, int len,