database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-10-25 13:17:41 +03:00
Code Activity

Allow Unicode escapes in any server encoding, not only UTF-8.

Browse Source 
SQL includes provisions for numeric Unicode escapes in string
literals and identifiers.  Previously we only accepted those
if they represented ASCII characters or the server encoding
was UTF-8, making the conversion to internal form trivial.
This patch adjusts things so that we'll call the appropriate
encoding conversion function in less-trivial cases, allowing
the escape sequence to be accepted so long as it corresponds
to some character available in the server encoding.

This also applies to processing of Unicode escapes in JSONB.
However, the old restriction still applies to client-side
JSON processing, since that hasn't got access to the server's
encoding conversion infrastructure.

This patch includes some lexer infrastructure that simplifies
throwing errors with error cursors pointing into the middle of
a string (or other complex token).  For the moment I only used
it for errors relating to Unicode escapes, but we might later
expand the usage to some other cases.

Patch by me, reviewed by John Naylor.

Discussion: https://postgr.es/m/2393.1578958316@sss.pgh.pa.us
This commit is contained in:
Tom Lane
2020-03-06 14:17:43 -05:00
parent fe30e7ebfa
commit a6525588b7
20 changed files with 613 additions and 227 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
src/include/parser/scanner.h
View File

@@ -99,9 +99,13 @@ typedef struct core_yy_extra_type
int literallen; /* actual current string length */
int literalalloc; /* current allocated buffer size */
/*
* Random assorted scanner state.
*/
int state_before_str_stop; /* start cond. before end quote */
int xcdepth; /* depth of nesting in slash-star comments */
char *dolqstart; /* current $foo$ quote start string */
YYLTYPE save_yylloc; /* one-element stack for PUSH_YYLLOC() */
/* first part of UTF16 surrogate pair for Unicode escapes */
int32 utf16_first_part;
@@ -116,6 +120,14 @@ typedef struct core_yy_extra_type
*/
typedef void *core_yyscan_t;
/* Support for scanner_errposition_callback function */
typedef struct ScannerCallbackState
{
core_yyscan_t yyscanner;
int location;
ErrorContextCallback errcallback;
} ScannerCallbackState;
/* Constant data exported from parser/scan.l */
extern PGDLLIMPORT const uint16 ScanKeywordTokens[];
@@ -129,6 +141,10 @@ extern void scanner_finish(core_yyscan_t yyscanner);
extern int core_yylex(core_YYSTYPE *lvalp, YYLTYPE *llocp,
core_yyscan_t yyscanner);
extern int scanner_errposition(int location, core_yyscan_t yyscanner);
extern void setup_scanner_errposition_callback(ScannerCallbackState *scbstate,
core_yyscan_t yyscanner,
int location);
extern void cancel_scanner_errposition_callback(ScannerCallbackState *scbstate);
extern void scanner_yyerror(const char *message, core_yyscan_t yyscanner) pg_attribute_noreturn();
#endif /* SCANNER_H */