ts_locale.c 6.77 KB
Newer Older
1 2 3
/*-------------------------------------------------------------------------
 *
 * ts_locale.c
4
 *		locale compatibility layer for tsearch
5 6 7 8 9
 *
 * Portions Copyright (c) 1996-2007, PostgreSQL Global Development Group
 *
 *
 * IDENTIFICATION
10
 *	  $PostgreSQL: pgsql/src/backend/tsearch/ts_locale.c,v 1.3 2007/11/09 22:37:35 tgl Exp $
11 12 13 14 15 16 17 18 19
 *
 *-------------------------------------------------------------------------
 */
#include "postgres.h"

#include "tsearch/ts_locale.h"
#include "tsearch/ts_public.h"


20
#ifdef TS_USE_WIDE
21

22 23 24 25 26 27 28
/*
 * wchar2char --- convert wide characters to multibyte format
 *
 * This has the same API as the standard wcstombs() function; in particular,
 * tolen is the maximum number of bytes to store at *to, and *from should be
 * zero-terminated.  The output will be zero-terminated iff there is room.
 */
29
size_t
30
wchar2char(char *to, const wchar_t *from, size_t tolen)
31
{
32
	if (tolen == 0)
33 34
		return 0;

35
#ifdef WIN32
36 37 38 39
	if (GetDatabaseEncoding() == PG_UTF8)
	{
		int			r;

40
		r = WideCharToMultiByte(CP_UTF8, 0, from, -1, to, tolen,
41 42
								NULL, NULL);

43 44 45 46
		if (r <= 0)
			return (size_t) -1;

		Assert(r <= tolen);
47

48 49
		/* Microsoft counts the zero terminator in the result */
		return r-1;
50
	}
51
#endif   /* WIN32 */
52

53
	return wcstombs(to, from, tolen);
54 55
}

56 57 58 59 60 61 62 63 64
/*
 * char2wchar --- convert multibyte characters to wide characters
 *
 * This has almost the API of mbstowcs(), except that *from need not be
 * null-terminated; instead, the number of input bytes is specified as
 * fromlen.  Also, we ereport() rather than returning -1 for invalid
 * input encoding.  tolen is the maximum number of wchar_t's to store at *to.
 * The output will be zero-terminated iff there is room.
 */
65
size_t
66
char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen)
67
{
68
	if (tolen == 0)
69 70 71 72 73 74 75
		return 0;

#ifdef WIN32
	if (GetDatabaseEncoding() == PG_UTF8)
	{
		int			r;

76
		r = MultiByteToWideChar(CP_UTF8, 0, from, fromlen, to, tolen);
77

78
		if (r <= 0)
79
		{
80
			pg_verifymbstr(from, fromlen, false);
81 82 83 84 85 86
			ereport(ERROR,
					(errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
					 errmsg("invalid multibyte character for locale"),
					 errhint("The server's LC_CTYPE locale is probably incompatible with the database encoding.")));
		}

87
		Assert(r <= tolen);
88

89 90
		/* Microsoft counts the zero terminator in the result */
		return r-1;
91 92
	}
#endif   /* WIN32 */
93

94 95 96 97 98 99
	if (lc_ctype_is_c())
	{
		/*
		 * pg_mb2wchar_with_len always adds trailing '\0', so 'to' should be
		 * allocated with sufficient space
		 */
100
		return pg_mb2wchar_with_len(from, (pg_wchar *) to, fromlen);
101 102 103 104
	}
	else
	{
		/*
105
		 * mbstowcs requires ending '\0'
106
		 */
107 108 109 110
		char	   *str = pnstrdup(from, fromlen);
		size_t		result;

		result = mbstowcs(to, str, tolen);
111 112 113

		pfree(str);

114 115 116 117 118 119 120 121 122 123 124 125 126
		if (result == (size_t) -1)
		{
			pg_verifymbstr(from, fromlen, false);
			ereport(ERROR,
					(errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
					 errmsg("invalid multibyte character for locale"),
					 errhint("The server's LC_CTYPE locale is probably incompatible with the database encoding.")));
		}

		if (result < tolen)
			to[result] = 0;

		return result;
127 128 129
	}
}

130

131
int
132
t_isdigit(const char *ptr)
133
{
134
	int			clen = pg_mblen(ptr);
135 136
	wchar_t		character[2];

137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
	if (clen == 1 || lc_ctype_is_c())
		return isdigit(TOUCHAR(ptr));

	char2wchar(character, 2, ptr, clen);

	return iswdigit((wint_t) character[0]);
}

int
t_isspace(const char *ptr)
{
	int			clen = pg_mblen(ptr);
	wchar_t		character[2];

	if (clen == 1 || lc_ctype_is_c())
		return isspace(TOUCHAR(ptr));

	char2wchar(character, 2, ptr, clen);

	return iswspace((wint_t) character[0]);
}

int
t_isalpha(const char *ptr)
{
	int			clen = pg_mblen(ptr);
	wchar_t		character[2];

	if (clen == 1 || lc_ctype_is_c())
166 167
		return isalpha(TOUCHAR(ptr));

168
	char2wchar(character, 2, ptr, clen);
169

170
	return iswalpha((wint_t) character[0]);
171 172 173
}

int
174
t_isprint(const char *ptr)
175
{
176
	int			clen = pg_mblen(ptr);
177 178
	wchar_t		character[2];

179
	if (clen == 1 || lc_ctype_is_c())
180 181
		return isprint(TOUCHAR(ptr));

182
	char2wchar(character, 2, ptr, clen);
183

184
	return iswprint((wint_t) character[0]);
185
}
186

187 188
#endif   /* TS_USE_WIDE */

189

190
/*
191 192 193
 * Read the next line from a tsearch data file (expected to be in UTF-8), and
 * convert it to database encoding if needed. The returned string is palloc'd.
 * NULL return means EOF.
194 195
 */
char *
196
t_readline(FILE *fp)
197
{
198 199 200 201 202 203
	int len;
	char *recoded;
	char buf[4096];		/* lines must not be longer than this */
	
	if (fgets(buf, sizeof(buf), fp) == NULL)
		return NULL;
204

205
	len = strlen(buf);
206

207 208
	/* Make sure the input is valid UTF-8 */
	(void) pg_verify_mbstr(PG_UTF8, buf, len, false);
209

210 211 212 213 214
	/* And convert */
	recoded = (char *) pg_do_encoding_conversion((unsigned char *) buf,
												 len,
												 PG_UTF8,
												 GetDatabaseEncoding());
215

216 217 218 219 220 221 222 223 224 225 226 227
	if (recoded == NULL)		/* should not happen */
		elog(ERROR, "encoding conversion failed");

	if (recoded == buf)
	{
		/*
		 * conversion didn't pstrdup, so we must.
		 * We can use the length of the original string, because
		 * no conversion was done.
		 */
		recoded = pnstrdup(recoded, len);
	}
228

229
	return recoded;
230 231
}

232 233 234 235 236
/*
 * lowerstr --- fold null-terminated string to lower case
 *
 * Returned string is palloc'd
 */
237
char *
238
lowerstr(const char *str)
239 240 241 242
{
	return lowerstr_with_len(str, strlen(str));
}

243
/*
244 245 246 247
 * lowerstr_with_len --- fold string to lower case
 *
 * Input string need not be null-terminated.
 *
248 249
 * Returned string is palloc'd
 */
250
char *
251
lowerstr_with_len(const char *str, int len)
252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273
{
	char	   *out;

	if (len == 0)
		return pstrdup("");

#ifdef TS_USE_WIDE

	/*
	 * Use wide char code only when max encoding length > 1 and ctype != C.
	 * Some operating systems fail with multi-byte encodings and a C locale.
	 * Also, for a C locale there is no need to process as multibyte. From
	 * backend/utils/adt/oracle_compat.c Teodor
	 */
	if (pg_database_encoding_max_length() > 1 && !lc_ctype_is_c())
	{
		wchar_t    *wstr,
				   *wptr;
		int			wlen;

		/*
		 * alloc number of wchar_t for worst case, len contains number of
274 275
		 * bytes >= number of characters and alloc 1 wchar_t for 0, because
		 * wchar2char wants zero-terminated string
276 277 278
		 */
		wptr = wstr = (wchar_t *) palloc(sizeof(wchar_t) * (len + 1));

279
		wlen = char2wchar(wstr, len+1, str, len);
280 281 282 283 284 285 286 287 288 289 290
		Assert(wlen <= len);

		while (*wptr)
		{
			*wptr = towlower((wint_t) *wptr);
			wptr++;
		}

		/*
		 * Alloc result string for worst case + '\0'
		 */
291
		len = pg_database_encoding_max_length() * wlen + 1;
292 293 294
		out = (char *) palloc(len);

		wlen = wchar2char(out, wstr, len);
295

296 297 298 299 300
		pfree(wstr);

		if (wlen < 0)
			ereport(ERROR,
					(errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
301 302
					 errmsg("translation from wchar_t to server encoding failed: %m")));
		Assert(wlen < len);
303 304
	}
	else
305
#endif   /* TS_USE_WIDE */
306
	{
307
		const char *ptr = str;
308 309 310
		char	   *outptr;

		outptr = out = (char *) palloc(sizeof(char) * (len + 1));
311
		while ((ptr - str) < len && *ptr)
312
		{
313
			*outptr++ = tolower(TOUCHAR(ptr));
314 315 316 317 318 319 320
			ptr++;
		}
		*outptr = '\0';
	}

	return out;
}