database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-15 19:21:59 +03:00
Code Activity

Further hacking on ICU collation creation and usage.

Browse Source 
pg_import_system_collations() refused to create any ICU collations if
the current database's encoding didn't support ICU.  This is wrongheaded:
initdb must initialize pg_collation in an encoding-independent way
since it might be used in other databases with different encodings.
The reason for the restriction seems to be that get_icu_locale_comment()
used icu_from_uchar() to convert the UChar-format display name, and that
unsurprisingly doesn't know what to do in unsupported encodings.
But by the same token that the initial catalog contents must be
encoding-independent, we can't allow non-ASCII characters in the comment
strings.  So we don't really need icu_from_uchar() here: just check for
Unicode codes outside the ASCII range, and if there are none, the format
conversion is trivial.  If there are some, we can simply not install the
comment.  (In my testing, this affects only Norwegian Bokmål, which has
given us trouble before.)

For paranoia's sake, also check for non-ASCII characters in ICU locale
names, and skip such locales, as we do for libc locales.  I don't
currently have a reason to believe that this will ever reject anything,
but then again the libc maintainers should have known better too.

With just the import changes, ICU collations can be found in pg_collation
in databases with unsupported encodings.  This resulted in more or less
clean failures at runtime, but that's not how things act for unsupported
encodings with libc collations.  Make it work the same as our traditional
behavior for libc collations by having collation lookup take into account
whether is_encoding_supported_by_icu().

Adjust documentation to match.  Also, expand Table 23.1 to show which
encodings are supported by ICU.

catversion bump because of likely change in pg_collation/pg_description
initial contents in ICU-enabled builds.

Discussion: https://postgr.es/m/20c74bc3-d6ca-243d-1bbc-12f17fa4fe9a@gmail.com
This commit is contained in:
Tom Lane
2017-06-24 13:54:15 -04:00
parent a15b47df35
commit ddb5fdc068
4 changed files with 194 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
src/backend/commands/collationcmds.c
View File

@ -353,6 +353,21 @@ pg_collation_actual_version(PG_FUNCTION_ARGS)
}
/*
* Check a string to see if it is pure ASCII
*/
static bool
is_all_ascii(const char *str)
{
while (*str)
{
if (IS_HIGHBIT_SET(*str))
return false;
str++;
}
return true;
}
/* will we use "locale -a" in pg_import_system_collations? */
#if defined(HAVE_LOCALE_T) && !defined(WIN32)
#define READ_LOCALE_A_OUTPUT
@ -431,7 +446,9 @@ get_icu_language_tag(const char *localename)
/*
* Get a comment (specifically, the display name) for an ICU locale.
* The result is a palloc'd string.
* The result is a palloc'd string, or NULL if we can't get a comment
* or find that it's not all ASCII. (We can *not* accept non-ASCII
* comments, because the contents of template0 must be encoding-agnostic.)
*/
static char *
get_icu_locale_comment(const char *localename)
@ -439,6 +456,7 @@ get_icu_locale_comment(const char *localename)
UErrorCode status;
UChar displayname[128];
int32 len_uchar;
int32 i;
char *result;
status = U_ZERO_ERROR;
@ -446,11 +464,20 @@ get_icu_locale_comment(const char *localename)
displayname, lengthof(displayname),
&status);
if (U_FAILURE(status))
ereport(ERROR,
(errmsg("could not get display name for locale \"%s\": %s",
localename, u_errorName(status))));
return NULL; /* no good reason to raise an error */
icu_from_uchar(&result, displayname, len_uchar);
/* Check for non-ASCII comment (can't use is_all_ascii for this) */
for (i = 0; i < len_uchar; i++)
{
if (displayname[i] > 127)
return NULL;
}
/* OK, transcribe */
result = palloc(len_uchar + 1);
for (i = 0; i < len_uchar; i++)
result[i] = displayname[i];
result[len_uchar] = '\0';
return result;
}
@ -502,7 +529,6 @@ pg_import_system_collations(PG_FUNCTION_ARGS)
{
size_t len;
int enc;
bool skip;
char alias[NAMEDATALEN];
len = strlen(localebuf);
@ -521,16 +547,7 @@ pg_import_system_collations(PG_FUNCTION_ARGS)
* interpret the non-ASCII characters. We can't do much with
* those, so we filter them out.
*/
skip = false;
for (i = 0; i < len; i++)
{
if (IS_HIGHBIT_SET(localebuf[i]))
{
skip = true;
break;
}
}
if (skip)
if (!is_all_ascii(localebuf))
{
elog(DEBUG1, "locale name has non-ASCII characters, skipped: \"%s\"", localebuf);
continue;
@ -642,14 +659,6 @@ pg_import_system_collations(PG_FUNCTION_ARGS)
/* Load collations known to ICU */
#ifdef USE_ICU
if (!is_encoding_supported_by_icu(GetDatabaseEncoding()))
{
ereport(NOTICE,
(errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
errmsg("encoding \"%s\" not supported by ICU",
pg_encoding_to_char(GetDatabaseEncoding()))));
}
else
{
int i;
@ -661,6 +670,7 @@ pg_import_system_collations(PG_FUNCTION_ARGS)
{
const char *name;
char *langtag;
char *icucomment;
const char *collcollate;
UEnumeration *en;
UErrorCode status;
@ -674,6 +684,14 @@ pg_import_system_collations(PG_FUNCTION_ARGS)
langtag = get_icu_language_tag(name);
collcollate = U_ICU_VERSION_MAJOR_NUM >= 54 ? langtag : name;
/*
* Be paranoid about not allowing any non-ASCII strings into
* pg_collation
*/
if (!is_all_ascii(langtag) || !is_all_ascii(collcollate))
continue;
collid = CollationCreate(psprintf("%s-x-icu", langtag),
nspid, GetUserId(),
COLLPROVIDER_ICU, -1,
@ -686,8 +704,10 @@ pg_import_system_collations(PG_FUNCTION_ARGS)
CommandCounterIncrement();
CreateComments(collid, CollationRelationId, 0,
get_icu_locale_comment(name));
icucomment = get_icu_locale_comment(name);
if (icucomment)
CreateComments(collid, CollationRelationId, 0,
icucomment);
}
/*
@ -708,6 +728,14 @@ pg_import_system_collations(PG_FUNCTION_ARGS)
langtag = get_icu_language_tag(localeid);
collcollate = U_ICU_VERSION_MAJOR_NUM >= 54 ? langtag : localeid;
/*
* Be paranoid about not allowing any non-ASCII strings into
* pg_collation
*/
if (!is_all_ascii(langtag) || !is_all_ascii(collcollate))
continue;
collid = CollationCreate(psprintf("%s-x-icu", langtag),
nspid, GetUserId(),
COLLPROVIDER_ICU, -1,
@ -720,8 +748,10 @@ pg_import_system_collations(PG_FUNCTION_ARGS)
CommandCounterIncrement();
CreateComments(collid, CollationRelationId, 0,
get_icu_locale_comment(localeid));
icucomment = get_icu_locale_comment(name);
if (icucomment)
CreateComments(collid, CollationRelationId, 0,
icucomment);
}
}
if (U_FAILURE(status))