database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-15 19:21:59 +03:00
Code Activity

Allow Unicode escapes in any server encoding, not only UTF-8.

Browse Source 
SQL includes provisions for numeric Unicode escapes in string
literals and identifiers.  Previously we only accepted those
if they represented ASCII characters or the server encoding
was UTF-8, making the conversion to internal form trivial.
This patch adjusts things so that we'll call the appropriate
encoding conversion function in less-trivial cases, allowing
the escape sequence to be accepted so long as it corresponds
to some character available in the server encoding.

This also applies to processing of Unicode escapes in JSONB.
However, the old restriction still applies to client-side
JSON processing, since that hasn't got access to the server's
encoding conversion infrastructure.

This patch includes some lexer infrastructure that simplifies
throwing errors with error cursors pointing into the middle of
a string (or other complex token).  For the moment I only used
it for errors relating to Unicode escapes, but we might later
expand the usage to some other cases.

Patch by me, reviewed by John Naylor.

Discussion: https://postgr.es/m/2393.1578958316@sss.pgh.pa.us
This commit is contained in:
Tom Lane
2020-03-06 14:17:43 -05:00
parent fe30e7ebfa
commit a6525588b7
20 changed files with 613 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
src/backend/utils/adt/jsonpath_scan.l
View File

@ -486,13 +486,6 @@ hexval(char c)
static void
addUnicodeChar(int ch)
{
/*
* For UTF8, replace the escape sequence by the actual
* utf8 character in lex->strval. Do this also for other
* encodings if the escape designates an ASCII character,
* otherwise raise an error.
*/
if (ch == 0)
{
/* We can't allow this, since our TEXT type doesn't */
@ -501,40 +494,20 @@ addUnicodeChar(int ch)
errmsg("unsupported Unicode escape sequence"),
errdetail("\\u0000 cannot be converted to text.")));
}
else if (GetDatabaseEncoding() == PG_UTF8)
{
char utf8str[5];
int utf8len;
unicode_to_utf8(ch, (unsigned char *) utf8str);
utf8len = pg_utf_mblen((unsigned char *) utf8str);
addstring(false, utf8str, utf8len);
}
else if (ch <= 0x007f)
{
/*
* This is the only way to designate things like a
* form feed character in JSON, so it's useful in all
* encodings.
*/
addchar(false, (char) ch);
}
else
{
ereport(ERROR,
(errcode(ERRCODE_INVALID_TEXT_REPRESENTATION),
errmsg("invalid input syntax for type %s", "jsonpath"),
errdetail("Unicode escape values cannot be used for code "
"point values above 007F when the server encoding "
"is not UTF8.")));
char cbuf[MAX_UNICODE_EQUIVALENT_STRING + 1];
pg_unicode_to_server(ch, (unsigned char *) cbuf);
addstring(false, cbuf, strlen(cbuf));
}
}
/* Add unicode character and process its hi surrogate */
/* Add unicode character, processing any surrogate pairs */
static void
addUnicode(int ch, int *hi_surrogate)
{
if (ch >= 0xd800 && ch <= 0xdbff)
if (is_utf16_surrogate_first(ch))
{
if (*hi_surrogate != -1)
ereport(ERROR,
@ -542,10 +515,10 @@ addUnicode(int ch, int *hi_surrogate)
errmsg("invalid input syntax for type %s", "jsonpath"),
errdetail("Unicode high surrogate must not follow "
"a high surrogate.")));
*hi_surrogate = (ch & 0x3ff) << 10;
*hi_surrogate = ch;
return;
}
else if (ch >= 0xdc00 && ch <= 0xdfff)
else if (is_utf16_surrogate_second(ch))
{
if (*hi_surrogate == -1)
ereport(ERROR,
@ -553,7 +526,7 @@ addUnicode(int ch, int *hi_surrogate)
errmsg("invalid input syntax for type %s", "jsonpath"),
errdetail("Unicode low surrogate must follow a high "
"surrogate.")));
ch = 0x10000 + *hi_surrogate + (ch & 0x3ff);
ch = surrogate_pair_to_codepoint(*hi_surrogate, ch);
*hi_surrogate = -1;
}
else if (*hi_surrogate != -1)