Implement a solution to the 'Turkish locale downcases I incorrectly'

problem, per previous discussion.  Make some additional changes to
centralize the knowledge of just how identifier downcasing is done,
in hopes of simplifying any future tweaking in this area.

src/backend/commands/define.c

@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/commands/define.c,v 1.85 2003/11/29 19:51:47 pgsql Exp $
+ *	  $PostgreSQL: pgsql/src/backend/commands/define.c,v 1.86 2004/02/21 00:34:52 tgl Exp $
  *
  * DESCRIPTION
  *	  The "DefineFoo" routines take the parse tree and pick out the
@@ -38,24 +38,19 @@
 #include "catalog/namespace.h"
 #include "commands/defrem.h"
 #include "parser/parse_type.h"
+#include "parser/scansup.h"
 #include "utils/int8.h"
 
 
 /*
- * Translate the input language name to lower case.
+ * Translate the input language name to lower case, and truncate if needed.
  *
- * Output buffer must be NAMEDATALEN long.
+ * Returns a palloc'd string
  */
-void
-case_translate_language_name(const char *input, char *output)
+char *
+case_translate_language_name(const char *input)
 {
-	int			i;
-
-	MemSet(output, 0, NAMEDATALEN);		/* ensure result Name is
-										 * zero-filled */
-
-	for (i = 0; i < NAMEDATALEN - 1 && input[i]; ++i)
-		output[i] = tolower((unsigned char) input[i]);
+	return downcase_truncate_identifier(input, strlen(input), false);
 }
 
 

src/backend/commands/functioncmds.c

@@ -10,7 +10,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/commands/functioncmds.c,v 1.43 2004/01/06 23:55:18 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/commands/functioncmds.c,v 1.44 2004/02/21 00:34:52 tgl Exp $
  *
  * DESCRIPTION
  *	  These routines take the parse tree and pick out the
@@ -401,7 +401,7 @@ CreateFunction(CreateFunctionStmt *stmt)
 	Oid			prorettype;
 	bool		returnsSet;
 	char	   *language;
-	char		languageName[NAMEDATALEN];
+	char	   *languageName;
 	Oid			languageOid;
 	Oid			languageValidator;
 	char	   *funcname;
@@ -437,7 +437,7 @@ CreateFunction(CreateFunctionStmt *stmt)
 			   &as_clause, &language, &volatility, &isStrict, &security);
 
 	/* Convert language name to canonical case */
-	case_translate_language_name(language, languageName);
+	languageName = case_translate_language_name(language);
 
 	/* Look up the language and validate permissions */
 	languageTuple = SearchSysCache(LANGNAME,

src/backend/commands/proclang.c

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/commands/proclang.c,v 1.52 2003/11/29 19:51:47 pgsql Exp $
+ *	  $PostgreSQL: pgsql/src/backend/commands/proclang.c,v 1.53 2004/02/21 00:34:52 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -40,11 +40,12 @@
 void
 CreateProceduralLanguage(CreatePLangStmt *stmt)
 {
-	char		languageName[NAMEDATALEN];
+	char	   *languageName;
 	Oid			procOid,
 				valProcOid;
 	Oid			funcrettype;
 	Oid			typev[FUNC_MAX_ARGS];
+	NameData	langname;
 	char		nulls[Natts_pg_language];
 	Datum		values[Natts_pg_language];
 	Relation	rel;
@@ -66,7 +67,7 @@ CreateProceduralLanguage(CreatePLangStmt *stmt)
 	 * Translate the language name and check that this language doesn't
 	 * already exist
 	 */
-	case_translate_language_name(stmt->plname, languageName);
+	languageName = case_translate_language_name(stmt->plname);
 
 	if (SearchSysCacheExists(LANGNAME,
 							 PointerGetDatum(languageName),
@@ -124,12 +125,13 @@ CreateProceduralLanguage(CreatePLangStmt *stmt)
 	}
 
 	i = 0;
-	values[i++] = PointerGetDatum(languageName);
-	values[i++] = BoolGetDatum(true);	/* lanispl */
-	values[i++] = BoolGetDatum(stmt->pltrusted);
-	values[i++] = ObjectIdGetDatum(procOid);
-	values[i++] = ObjectIdGetDatum(valProcOid);
-	nulls[i] = 'n';				/* lanacl */
+	namestrcpy(&langname, languageName);
+	values[i++] = NameGetDatum(&langname);			/* lanname */
+	values[i++] = BoolGetDatum(true);				/* lanispl */
+	values[i++] = BoolGetDatum(stmt->pltrusted);	/* lanpltrusted */
+	values[i++] = ObjectIdGetDatum(procOid);		/* lanplcallfoid */
+	values[i++] = ObjectIdGetDatum(valProcOid);		/* lanvalidator */
+	nulls[i] = 'n';									/* lanacl */
 
 	rel = heap_openr(LanguageRelationName, RowExclusiveLock);
 
@@ -173,7 +175,7 @@ CreateProceduralLanguage(CreatePLangStmt *stmt)
 void
 DropProceduralLanguage(DropPLangStmt *stmt)
 {
-	char		languageName[NAMEDATALEN];
+	char	   *languageName;
 	HeapTuple	langTup;
 	ObjectAddress object;
 
@@ -189,7 +191,7 @@ DropProceduralLanguage(DropPLangStmt *stmt)
 	 * Translate the language name, check that this language exist and is
 	 * a PL
 	 */
-	case_translate_language_name(stmt->plname, languageName);
+	languageName = case_translate_language_name(stmt->plname);
 
 	langTup = SearchSysCache(LANGNAME,
 							 CStringGetDatum(languageName),

src/backend/parser/keywords.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/parser/keywords.c,v 1.144 2003/11/29 19:51:51 pgsql Exp $
+ *	  $PostgreSQL: pgsql/src/backend/parser/keywords.c,v 1.145 2004/02/21 00:34:52 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -369,17 +369,13 @@ ScanKeywordLookup(const char *text)
 
 	/*
 	 * Apply an ASCII-only downcasing.	We must not use tolower() since it
-	 * may produce the wrong translation in some locales (eg, Turkish),
-	 * and we don't trust isupper() very much either.  In an ASCII-based
-	 * encoding the tests against A and Z are sufficient, but we also
-	 * check isupper() so that we will work correctly under EBCDIC.  The
-	 * actual case conversion step should work for either ASCII or EBCDIC.
+	 * may produce the wrong translation in some locales (eg, Turkish).
 	 */
 	for (i = 0; i < len; i++)
 	{
 		char		ch = text[i];
 
-		if (ch >= 'A' && ch <= 'Z' && isupper((unsigned char) ch))
+		if (ch >= 'A' && ch <= 'Z')
 			ch += 'a' - 'A';
 		word[i] = ch;
 	}

src/backend/parser/scan.l

@@ -10,7 +10,7 @@
  * Portions Copyright (c) 1994, Regents of the University of California
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/parser/scan.l,v 1.113 2004/02/19 19:11:30 tgl Exp $
+ *	  $PostgreSQL: pgsql/src/backend/parser/scan.l,v 1.114 2004/02/21 00:34:52 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -27,6 +27,7 @@
 #include "parser/keywords.h"
 /* Not needed now that this file is compiled as part of gram.y */
 /* #include "parser/parse.h" */
+#include "parser/scansup.h"
 #include "utils/builtins.h"
 #include "mb/pg_wchar.h"
 
@@ -395,23 +396,15 @@ other			.
 					startlit();
 				}
 <xd>{xdstop}	{
+					char		   *ident;
+
 					BEGIN(INITIAL);
 					if (literallen == 0)
 						yyerror("zero-length delimited identifier");
+					ident = litbufdup();
 					if (literallen >= NAMEDATALEN)
-					{
-						int len;
-
-						len = pg_mbcliplen(literalbuf, literallen,
-										   NAMEDATALEN-1);
-						ereport(NOTICE,
-								(errcode(ERRCODE_NAME_TOO_LONG),
-								 errmsg("identifier \"%s\" will be truncated to \"%.*s\"",
-										literalbuf, len, literalbuf)));
-						literalbuf[len] = '\0';
-						literallen = len;
-					}
-					yylval.str = litbufdup();
+						truncate_identifier(ident, literallen, true);
+					yylval.str = ident;
 					return IDENT;
 				}
 <xd>{xddouble} {
@@ -537,7 +530,6 @@ other			.
 {identifier}	{
 					const ScanKeyword *keyword;
 					char		   *ident;
-					int				i;
 
 					/* Is it a keyword? */
 					keyword = ScanKeywordLookup(yytext);
@@ -550,28 +542,8 @@ other			.
 					/*
 					 * No.  Convert the identifier to lower case, and truncate
 					 * if necessary.
-					 *
-					 * Note: here we use a locale-dependent case conversion,
-					 * which seems appropriate under standard SQL rules, whereas
-					 * the keyword comparison was NOT locale-dependent.
 					 */
-					ident = pstrdup(yytext);
-					for (i = 0; ident[i]; i++)
-					{
-						if (isupper((unsigned char) ident[i]))
-							ident[i] = tolower((unsigned char) ident[i]);
-					}
-					if (i >= NAMEDATALEN)
-                    {
-						int len;
-
-						len = pg_mbcliplen(ident, i, NAMEDATALEN-1);
-						ereport(NOTICE,
-								(errcode(ERRCODE_NAME_TOO_LONG),
-								 errmsg("identifier \"%s\" will be truncated to \"%.*s\"",
-										ident, len, ident)));
-						ident[len] = '\0';
-                    }
+					ident = downcase_truncate_identifier(yytext, yyleng, true);
 					yylval.str = ident;
 					return IDENT;
 				}

src/backend/parser/scansup.c

@@ -9,7 +9,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/parser/scansup.c,v 1.25 2003/11/29 19:51:52 pgsql Exp $
+ *	  $PostgreSQL: pgsql/src/backend/parser/scansup.c,v 1.26 2004/02/21 00:34:53 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -19,6 +19,8 @@
 
 #include "miscadmin.h"
 #include "parser/scansup.h"
+#include "mb/pg_wchar.h"
+
 
 /* ----------------
  *		scanstr
@@ -32,7 +34,7 @@
  */
 
 char *
-scanstr(char *s)
+scanstr(const char *s)
 {
 	char	   *newStr;
 	int			len,
@@ -109,3 +111,75 @@ scanstr(char *s)
 	newStr[j] = '\0';
 	return newStr;
 }
+
+
+/*
+ * downcase_truncate_identifier() --- do appropriate downcasing and
+ * truncation of an unquoted identifier.  Optionally warn of truncation.
+ *
+ * Returns a palloc'd string containing the adjusted identifier.
+ *
+ * Note: in some usages the passed string is not null-terminated.
+ *
+ * Note: the API of this function is designed to allow for downcasing
+ * transformations that increase the string length, but we don't yet
+ * support that.  If you want to implement it, you'll need to fix
+ * SplitIdentifierString() in utils/adt/varlena.c.
+ */
+char *
+downcase_truncate_identifier(const char *ident, int len, bool warn)
+{
+	char	   *result;
+	int			i;
+
+	result = palloc(len + 1);
+	/*
+	 * SQL99 specifies Unicode-aware case normalization, which we don't yet
+	 * have the infrastructure for.  Instead we use tolower() to provide a
+	 * locale-aware translation.  However, there are some locales where this
+	 * is not right either (eg, Turkish may do strange things with 'i' and
+	 * 'I').  Our current compromise is to use tolower() for characters with
+	 * the high bit set, and use an ASCII-only downcasing for 7-bit
+	 * characters.
+	 */
+	for (i = 0; i < len; i++)
+	{
+		unsigned char	ch = (unsigned char) ident[i];
+
+		if (ch >= 'A' && ch <= 'Z')
+			ch += 'a' - 'A';
+		else if (ch >= 0x80 && isupper(ch))
+			ch = tolower(ch);
+		result[i] = (char) ch;
+	}
+	result[i] = '\0';
+
+	if (i >= NAMEDATALEN)
+		truncate_identifier(result, i, warn);
+
+	return result;
+}
+
+/*
+ * truncate_identifier() --- truncate an identifier to NAMEDATALEN-1 bytes.
+ *
+ * The given string is modified in-place, if necessary.  A warning is
+ * issued if requested.
+ *
+ * We require the caller to pass in the string length since this saves a
+ * strlen() call in some common usages.
+ */
+void
+truncate_identifier(char *ident, int len, bool warn)
+{
+	if (len >= NAMEDATALEN)
+	{
+		len = pg_mbcliplen(ident, len, NAMEDATALEN-1);
+		if (warn)
+			ereport(NOTICE,
+					(errcode(ERRCODE_NAME_TOO_LONG),
+					 errmsg("identifier \"%s\" will be truncated to \"%.*s\"",
+							ident, len, ident)));
+		ident[len] = '\0';
+	}
+}

src/backend/utils/adt/varlena.c

@@ -8,7 +8,7 @@
  *
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/backend/utils/adt/varlena.c,v 1.111 2004/01/31 05:09:40 neilc Exp $
+ *	  $PostgreSQL: pgsql/src/backend/utils/adt/varlena.c,v 1.112 2004/02/21 00:34:53 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -16,17 +16,18 @@
 
 #include <ctype.h>
 
-#include "mb/pg_wchar.h"
-#include "miscadmin.h"
 #include "access/tuptoaster.h"
 #include "catalog/pg_type.h"
 #include "lib/stringinfo.h"
 #include "libpq/crypt.h"
 #include "libpq/pqformat.h"
+#include "mb/pg_wchar.h"
+#include "miscadmin.h"
+#include "parser/scansup.h"
 #include "utils/array.h"
 #include "utils/builtins.h"
-#include "utils/pg_locale.h"
 #include "utils/lsyscache.h"
+#include "utils/pg_locale.h"
 
 
 typedef struct varlena unknown;
@@ -1695,7 +1696,6 @@ SplitIdentifierString(char *rawstring, char separator,
 	{
 		char	   *curname;
 		char	   *endp;
-		int			curlen;
 
 		if (*nextp == '\"')
 		{
@@ -1718,21 +1718,30 @@ SplitIdentifierString(char *rawstring, char separator,
 		else
 		{
 			/* Unquoted name --- extends to separator or whitespace */
+			char	   *downname;
+			int			len;
+
 			curname = nextp;
 			while (*nextp && *nextp != separator &&
 				   !isspace((unsigned char) *nextp))
-			{
-				/*
-				 * It's important that this match the identifier
-				 * downcasing code used by backend/parser/scan.l.
-				 */
-				if (isupper((unsigned char) *nextp))
-					*nextp = tolower((unsigned char) *nextp);
 				nextp++;
-			}
 			endp = nextp;
 			if (curname == nextp)
 				return false;	/* empty unquoted name not allowed */
+			/*
+			 * Downcase the identifier, using same code as main lexer does.
+			 *
+			 * XXX because we want to overwrite the input in-place, we cannot
+			 * support a downcasing transformation that increases the
+			 * string length.  This is not a problem given the current
+			 * implementation of downcase_truncate_identifier, but we'll
+			 * probably have to do something about this someday.
+			 */
+			len = endp - curname;
+			downname = downcase_truncate_identifier(curname, len, false);
+			Assert(strlen(downname) <= len);
+			strncpy(curname, downname, len);
+			pfree(downname);
 		}
 
 		while (isspace((unsigned char) *nextp))
@@ -1753,13 +1762,8 @@ SplitIdentifierString(char *rawstring, char separator,
 		/* Now safe to overwrite separator with a null */
 		*endp = '\0';
 
-		/* Truncate name if it's overlength; again, should match scan.l */
-		curlen = strlen(curname);
-		if (curlen >= NAMEDATALEN)
-		{
-			curlen = pg_mbcliplen(curname, curlen, NAMEDATALEN - 1);
-			curname[curlen] = '\0';
-		}
+		/* Truncate name if it's overlength */
+		truncate_identifier(curname, strlen(curname), false);
 
 		/*
 		 * Finished isolating current name --- add it to list

src/include/commands/defrem.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/commands/defrem.h,v 1.53 2003/11/29 22:40:59 pgsql Exp $
+ * $PostgreSQL: pgsql/src/include/commands/defrem.h,v 1.54 2004/02/21 00:34:53 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -61,7 +61,7 @@ extern void RenameOpClass(List *name, const char *access_method, const char *new
 
 /* support routines in commands/define.c */
 
-extern void case_translate_language_name(const char *input, char *output);
+extern char *case_translate_language_name(const char *input);
 
 extern char *defGetString(DefElem *def);
 extern double defGetNumeric(DefElem *def);

src/include/parser/scansup.h

@@ -7,7 +7,7 @@
  * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
  * Portions Copyright (c) 1994, Regents of the University of California
  *
- * $PostgreSQL: pgsql/src/include/parser/scansup.h,v 1.14 2003/11/29 22:41:09 pgsql Exp $
+ * $PostgreSQL: pgsql/src/include/parser/scansup.h,v 1.15 2004/02/21 00:34:53 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -15,6 +15,11 @@
 #ifndef SCANSUP_H
 #define SCANSUP_H
 
-extern char *scanstr(char *s);
+extern char *scanstr(const char *s);
+
+extern char *downcase_truncate_identifier(const char *ident, int len,
+										  bool warn);
+
+extern void truncate_identifier(char *ident, int len, bool warn);
 
 #endif   /* SCANSUP_H */

src/pl/plpgsql/src/pl_funcs.c

@@ -3,7 +3,7 @@
  *			  procedural language
  *
  * IDENTIFICATION
- *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/pl_funcs.c,v 1.31 2003/11/29 19:52:12 pgsql Exp $
+ *	  $PostgreSQL: pgsql/src/pl/plpgsql/src/pl_funcs.c,v 1.32 2004/02/21 00:34:53 tgl Exp $
  *
  *	  This software is copyrighted by Jan Wieck - Hamburg.
  *
@@ -40,7 +40,7 @@
 
 #include <ctype.h>
 
-#include "mb/pg_wchar.h"
+#include "parser/scansup.h"
 
 
 /* ----------
@@ -348,15 +348,15 @@ plpgsql_convert_ident(const char *s, char **output, int numidents)
 	{
 		char	   *curident;
 		char	   *cp;
-		int			i;
 
 		/* Process current identifier */
-		curident = palloc(strlen(s) + 1);		/* surely enough room */
-		cp = curident;
 
 		if (*s == '"')
 		{
 			/* Quoted identifier: copy, collapsing out doubled quotes */
+
+			curident = palloc(strlen(s) + 1); /* surely enough room */
+			cp = curident;
 			s++;
 			while (*s)
 			{
@@ -373,35 +373,20 @@ plpgsql_convert_ident(const char *s, char **output, int numidents)
 						(errcode(ERRCODE_SYNTAX_ERROR),
 						 errmsg("unterminated \" in name: %s", sstart)));
 			s++;
+			*cp = '\0';
+			/* Truncate to NAMEDATALEN */
+			truncate_identifier(curident, cp-curident, false);
 		}
 		else
 		{
-			/*
-			 * Normal identifier: downcase, stop at dot or whitespace.
-			 *
-			 * Note that downcasing is locale-sensitive, following SQL99
-			 * rules for identifiers.  We have already decided that the
-			 * item is not a PLPGSQL keyword.
-			 */
-			while (*s && *s != '.' && !isspace((unsigned char) *s))
-			{
-				if (isupper((unsigned char) *s))
-					*cp++ = tolower((unsigned char) *s++);
-				else
-					*cp++ = *s++;
-			}
-		}
-
-		/* Truncate to NAMEDATALEN */
-		*cp = '\0';
-		i = cp - curident;
-
-		if (i >= NAMEDATALEN)
-		{
-			int			len;
+			/* Normal identifier: extends till dot or whitespace */
+			const char *thisstart = s;
 
-			len = pg_mbcliplen(curident, i, NAMEDATALEN - 1);
-			curident[len] = '\0';
+			while (*s && *s != '.' && !isspace((unsigned char) *s))
+				s++;
+			/* Downcase and truncate to NAMEDATALEN */
+			curident = downcase_truncate_identifier(thisstart, s-thisstart,
+													false);
 		}
 
 		/* Pass ident to caller */