database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-12-01 12:18:01 +03:00
Code Activity
Files
399a36a75d2d06bfdb91402713d408271b4833ba
postgres/src/backend/utils/mb/wchar.c
Tom Lane c61a2f5841 Change the backend to reject strings containing invalidly-encoded multibyte 
characters in all cases.  Formerly we mostly just threw warnings for invalid
input, and failed to detect it at all if no encoding conversion was required.
The tighter check is needed to defend against SQL-injection attacks as per
CVE-2006-2313 (further details will be published after release).  Embedded
zero (null) bytes will be rejected as well.  The checks are applied during
input to the backend (receipt from client or COPY IN), so it no longer seems
necessary to check in textin() and related routines; any string arriving at
those functions will already have been validated.  Conversion failure
reporting (for characters with no equivalent in the destination encoding)
has been cleaned up and made consistent while at it.

Also, fix a few longstanding errors in little-used encoding conversion
routines: win1251_to_iso, win866_to_iso, euc_tw_to_big5, euc_tw_to_mic,
mic_to_euc_tw were all broken to varying extents.

Patches by Tatsuo Ishii and Tom Lane.  Thanks to Akio Ishida and Yasuo Ohgaki
for identifying the security issues.
2006-05-21 20:05:21 +00:00

1523 lines
32 KiB
C
Raw Blame History

/*
* conversion functions between pg_wchar and multibyte streams.
* Tatsuo Ishii
* $PostgreSQL: pgsql/src/backend/utils/mb/wchar.c,v 1.55 2006/05/21 20:05:19 tgl Exp $
*
* WIN1250 client encoding updated by Pavel Behal
*
*/
/* can be used in either frontend or backend */
#ifdef FRONTEND
#include "postgres_fe.h"
#define Assert(condition)
#else
#include "postgres.h"
#endif
#include "mb/pg_wchar.h"
/*
* conversion to pg_wchar is done by "table driven."
* to add an encoding support, define mb2wchar_with_len(), mblen(), dsplen()
* for the particular encoding. Note that if the encoding is only
* supported in the client, you don't need to define
* mb2wchar_with_len() function (SJIS is the case).
*
* These functions generally assume that their input is validly formed.
* The "verifier" functions, further down in the file, have to be more
* paranoid. We expect that mblen() does not need to examine more than
* the first byte of the character to discover the correct length.
*
* Note: for the display output of psql to work properly, the return values
* of the dsplen functions must conform to the Unicode standard. In particular
* the NUL character is zero width and control characters are generally
* width -1. It is recommended that non-ASCII encodings refer their ASCII
* subset to the ASCII routines to ensure consistency.
*/
/*
* SQL/ASCII
*/
static int
pg_ascii2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
int cnt = 0;
while (len > 0 && *from)
{
*to++ = *from++;
len--;
cnt++;
}
*to = 0;
return cnt;
}
static int
pg_ascii_mblen(const unsigned char *s)
{
return 1;
}
static int
pg_ascii_dsplen(const unsigned char *s)
{
if (*s == '\0')
return 0;
if (*s < 0x20 || *s == 0x7f)
return -1;
return 1;
}
/*
* EUC
*/
static int pg_euc2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
int cnt = 0;
while (len > 0 && *from)
{
if (*from == SS2 && len >= 2) /* JIS X 0201 (so called "1 byte KANA") */
{
from++;
*to = (SS2 << 8) | *from++;
len -= 2;
}
else if (*from == SS3 && len >= 3) /* JIS X 0212 KANJI */
{
from++;
*to = (SS3 << 16) | (*from++ << 8);
*to |= *from++;
len -= 3;
}
else if (IS_HIGHBIT_SET(*from) && len >= 2) /* JIS X 0208 KANJI */
{
*to = *from++ << 8;
*to |= *from++;
len -= 2;
}
else /* must be ASCII */
{
*to = *from++;
len--;
}
to++;
cnt++;
}
*to = 0;
return cnt;
}
static inline int
pg_euc_mblen(const unsigned char *s)
{
int len;
if (*s == SS2)
len = 2;
else if (*s == SS3)
len = 3;
else if (IS_HIGHBIT_SET(*s))
len = 2;
else
len = 1;
return len;
}
static inline int
pg_euc_dsplen(const unsigned char *s)
{
int len;
if (*s == SS2)
len = 2;
else if (*s == SS3)
len = 2;
else if (IS_HIGHBIT_SET(*s))
len = 2;
else
len = pg_ascii_dsplen(s);
return len;
}
/*
* EUC_JP
*/
static int pg_eucjp2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
return pg_euc2wchar_with_len(from, to, len);
}
static int
pg_eucjp_mblen(const unsigned char *s)
{
return pg_euc_mblen(s);
}
static int
pg_eucjp_dsplen(const unsigned char *s)
{
int len;
if (*s == SS2)
len = 1;
else if (*s == SS3)
len = 2;
else if (IS_HIGHBIT_SET(*s))
len = 2;
else
len = pg_ascii_dsplen(s);
return len;
}
/*
* EUC_KR
*/
static int pg_euckr2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
return pg_euc2wchar_with_len(from, to, len);
}
static int
pg_euckr_mblen(const unsigned char *s)
{
return pg_euc_mblen(s);
}
static int
pg_euckr_dsplen(const unsigned char *s)
{
return pg_euc_dsplen(s);
}
/*
* EUC_CN
*
*/
static int pg_euccn2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
int cnt = 0;
while (len > 0 && *from)
{
if (*from == SS2 && len >= 3) /* code set 2 (unused?) */
{
from++;
*to = (SS2 << 16) | (*from++ << 8);
*to |= *from++;
len -= 3;
}
else if (*from == SS3 && len >= 3) /* code set 3 (unsed ?) */
{
from++;
*to = (SS3 << 16) | (*from++ << 8);
*to |= *from++;
len -= 3;
}
else if (IS_HIGHBIT_SET(*from) && len >= 2) /* code set 1 */
{
*to = *from++ << 8;
*to |= *from++;
len -= 2;
}
else
{
*to = *from++;
len--;
}
to++;
cnt++;
}
*to = 0;
return cnt;
}
static int
pg_euccn_mblen(const unsigned char *s)
{
int len;
if (IS_HIGHBIT_SET(*s))
len = 2;
else
len = 1;
return len;
}
static int
pg_euccn_dsplen(const unsigned char *s)
{
int len;
if (IS_HIGHBIT_SET(*s))
len = 2;
else
len = pg_ascii_dsplen(s);
return len;
}
/*
* EUC_TW
*
*/
static int pg_euctw2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
int cnt = 0;
while (len > 0 && *from)
{
if (*from == SS2 && len >= 4) /* code set 2 */
{
from++;
*to = (SS2 << 24) | (*from++ << 16) ;
*to |= *from++ << 8;
*to |= *from++;
len -= 4;
}
else if (*from == SS3 && len >= 3) /* code set 3 (unused?) */
{
from++;
*to = (SS3 << 16) | (*from++ << 8);
*to |= *from++;
len -= 3;
}
else if (IS_HIGHBIT_SET(*from) && len >= 2) /* code set 2 */
{
*to = *from++ << 8;
*to |= *from++;
len -= 2;
}
else
{
*to = *from++;
len--;
}
to++;
cnt++;
}
*to = 0;
return cnt;
}
static int
pg_euctw_mblen(const unsigned char *s)
{
int len;
if (*s == SS2)
len = 4;
else if (*s == SS3)
len = 3;
else if (IS_HIGHBIT_SET(*s))
len = 2;
else
len = 1;
return len;
}
static int
pg_euctw_dsplen(const unsigned char *s)
{
int len;
if (*s == SS2)
len = 2;
else if (*s == SS3)
len = 2;
else if (IS_HIGHBIT_SET(*s))
len = 2;
else
len = pg_ascii_dsplen(s);
return len;
}
/*
* JOHAB
*/
static int
pg_johab2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
return pg_euc2wchar_with_len(from, to, len);
}
static int
pg_johab_mblen(const unsigned char *s)
{
return pg_euc_mblen(s);
}
static int
pg_johab_dsplen(const unsigned char *s)
{
return pg_euc_dsplen(s);
}
/*
* convert UTF8 string to pg_wchar (UCS-2)
* caller should allocate enough space for "to"
* len: length of from.
* "from" not necessarily null terminated.
*/
static int
pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
unsigned char c1,
c2,
c3;
int cnt = 0;
while (len > 0 && *from)
{
if (!IS_HIGHBIT_SET(*from))
{
*to = *from++;
len--;
}
else if ((*from & 0xe0) == 0xc0 && len >= 2)
{
c1 = *from++ & 0x1f;
c2 = *from++ & 0x3f;
*to = c1 << 6;
*to |= c2;
len -= 2;
}
else if ((*from & 0xe0) == 0xe0 && len >= 3)
{
c1 = *from++ & 0x0f;
c2 = *from++ & 0x3f;
c3 = *from++ & 0x3f;
*to = c1 << 12;
*to |= c2 << 6;
*to |= c3;
len -= 3;
}
else
{
*to = *from++;
len--;
}
to++;
cnt++;
}
*to = 0;
return cnt;
}
/*
* returns the byte length of a UTF8 character pointed to by s
*/
int
pg_utf_mblen(const unsigned char *s)
{
int len = 1;
if ((*s & 0x80) == 0)
len = 1;
else if ((*s & 0xe0) == 0xc0)
len = 2;
else if ((*s & 0xf0) == 0xe0)
len = 3;
else if ((*s & 0xf8) == 0xf0)
len = 4;
else if ((*s & 0xfc) == 0xf8)
len = 5;
else if ((*s & 0xfe) == 0xfc)
len = 6;
return len;
}
/*
* This is an implementation of wcwidth() and wcswidth() as defined in
* "The Single UNIX Specification, Version 2, The Open Group, 1997"
* <http://www.UNIX-systems.org/online.html>
*
* Markus Kuhn -- 2001-09-08 -- public domain
*
* customised for PostgreSQL
*
* original available at : http://www.cl.cam.ac.uk/~mgk25/ucs/wcwidth.c
*/
struct mbinterval
{
unsigned short first;
unsigned short last;
};
/* auxiliary function for binary search in interval table */
static int
mbbisearch(pg_wchar ucs, const struct mbinterval *table, int max)
{
int min = 0;
int mid;
if (ucs < table[0].first || ucs > table[max].last)
return 0;
while (max >= min)
{
mid = (min + max) / 2;
if (ucs > table[mid].last)
min = mid + 1;
else if (ucs < table[mid].first)
max = mid - 1;
else
return 1;
}
return 0;
}
/* The following functions define the column width of an ISO 10646
* character as follows:
*
* - The null character (U+0000) has a column width of 0.
*
* - Other C0/C1 control characters and DEL will lead to a return
* value of -1.
*
* - Non-spacing and enclosing combining characters (general
* category code Mn or Me in the Unicode database) have a
* column width of 0.
*
* - Other format characters (general category code Cf in the Unicode
* database) and ZERO WIDTH SPACE (U+200B) have a column width of 0.
*
* - Hangul Jamo medial vowels and final consonants (U+1160-U+11FF)
* have a column width of 0.
*
* - Spacing characters in the East Asian Wide (W) or East Asian
* FullWidth (F) category as defined in Unicode Technical
* Report #11 have a column width of 2.
*
* - All remaining characters (including all printable
* ISO 8859-1 and WGL4 characters, Unicode control characters,
* etc.) have a column width of 1.
*
* This implementation assumes that wchar_t characters are encoded
* in ISO 10646.
*/
static int
ucs_wcwidth(pg_wchar ucs)
{
/* sorted list of non-overlapping intervals of non-spacing characters */
static const struct mbinterval combining[] = {
{0x0300, 0x034E}, {0x0360, 0x0362}, {0x0483, 0x0486},
{0x0488, 0x0489}, {0x0591, 0x05A1}, {0x05A3, 0x05B9},
{0x05BB, 0x05BD}, {0x05BF, 0x05BF}, {0x05C1, 0x05C2},
{0x05C4, 0x05C4}, {0x064B, 0x0655}, {0x0670, 0x0670},
{0x06D6, 0x06E4}, {0x06E7, 0x06E8}, {0x06EA, 0x06ED},
{0x070F, 0x070F}, {0x0711, 0x0711}, {0x0730, 0x074A},
{0x07A6, 0x07B0}, {0x0901, 0x0902}, {0x093C, 0x093C},
{0x0941, 0x0948}, {0x094D, 0x094D}, {0x0951, 0x0954},
{0x0962, 0x0963}, {0x0981, 0x0981}, {0x09BC, 0x09BC},
{0x09C1, 0x09C4}, {0x09CD, 0x09CD}, {0x09E2, 0x09E3},
{0x0A02, 0x0A02}, {0x0A3C, 0x0A3C}, {0x0A41, 0x0A42},
{0x0A47, 0x0A48}, {0x0A4B, 0x0A4D}, {0x0A70, 0x0A71},
{0x0A81, 0x0A82}, {0x0ABC, 0x0ABC}, {0x0AC1, 0x0AC5},
{0x0AC7, 0x0AC8}, {0x0ACD, 0x0ACD}, {0x0B01, 0x0B01},
{0x0B3C, 0x0B3C}, {0x0B3F, 0x0B3F}, {0x0B41, 0x0B43},
{0x0B4D, 0x0B4D}, {0x0B56, 0x0B56}, {0x0B82, 0x0B82},
{0x0BC0, 0x0BC0}, {0x0BCD, 0x0BCD}, {0x0C3E, 0x0C40},
{0x0C46, 0x0C48}, {0x0C4A, 0x0C4D}, {0x0C55, 0x0C56},
{0x0CBF, 0x0CBF}, {0x0CC6, 0x0CC6}, {0x0CCC, 0x0CCD},
{0x0D41, 0x0D43}, {0x0D4D, 0x0D4D}, {0x0DCA, 0x0DCA},
{0x0DD2, 0x0DD4}, {0x0DD6, 0x0DD6}, {0x0E31, 0x0E31},
{0x0E34, 0x0E3A}, {0x0E47, 0x0E4E}, {0x0EB1, 0x0EB1},
{0x0EB4, 0x0EB9}, {0x0EBB, 0x0EBC}, {0x0EC8, 0x0ECD},
{0x0F18, 0x0F19}, {0x0F35, 0x0F35}, {0x0F37, 0x0F37},
{0x0F39, 0x0F39}, {0x0F71, 0x0F7E}, {0x0F80, 0x0F84},
{0x0F86, 0x0F87}, {0x0F90, 0x0F97}, {0x0F99, 0x0FBC},
{0x0FC6, 0x0FC6}, {0x102D, 0x1030}, {0x1032, 0x1032},
{0x1036, 0x1037}, {0x1039, 0x1039}, {0x1058, 0x1059},
{0x1160, 0x11FF}, {0x17B7, 0x17BD}, {0x17C6, 0x17C6},
{0x17C9, 0x17D3}, {0x180B, 0x180E}, {0x18A9, 0x18A9},
{0x200B, 0x200F}, {0x202A, 0x202E}, {0x206A, 0x206F},
{0x20D0, 0x20E3}, {0x302A, 0x302F}, {0x3099, 0x309A},
{0xFB1E, 0xFB1E}, {0xFE20, 0xFE23}, {0xFEFF, 0xFEFF},
{0xFFF9, 0xFFFB}
};
/* test for 8-bit control characters */
if (ucs == 0)
return 0;
if (ucs < 0x20 || (ucs >= 0x7f && ucs < 0xa0) || ucs > 0x0010ffff)
return -1;
/* binary search in table of non-spacing characters */
if (mbbisearch(ucs, combining,
sizeof(combining) / sizeof(struct mbinterval) - 1))
return 0;
/*
* if we arrive here, ucs is not a combining or C0/C1 control character
*/
return 1 +
(ucs >= 0x1100 &&
(ucs <= 0x115f || /* Hangul Jamo init. consonants */
(ucs >= 0x2e80 && ucs <= 0xa4cf && (ucs & ~0x0011) != 0x300a &&
ucs != 0x303f) || /* CJK ... Yi */
(ucs >= 0xac00 && ucs <= 0xd7a3) || /* Hangul Syllables */
(ucs >= 0xf900 && ucs <= 0xfaff) || /* CJK Compatibility
* Ideographs */
(ucs >= 0xfe30 && ucs <= 0xfe6f) || /* CJK Compatibility Forms */
(ucs >= 0xff00 && ucs <= 0xff5f) || /* Fullwidth Forms */
(ucs >= 0xffe0 && ucs <= 0xffe6) ||
(ucs >= 0x20000 && ucs <= 0x2ffff)));
}
static pg_wchar
utf2ucs(const unsigned char *c)
{
/*
* one char version of pg_utf2wchar_with_len. no control here, c must
* point to a large enough string
*/
if ((*c & 0x80) == 0)
return (pg_wchar) c[0];
else if ((*c & 0xe0) == 0xc0)
return (pg_wchar) (((c[0] & 0x1f) << 6) |
(c[1] & 0x3f));
else if ((*c & 0xf0) == 0xe0)
return (pg_wchar) (((c[0] & 0x0f) << 12) |
((c[1] & 0x3f) << 6) |
(c[2] & 0x3f));
else if ((*c & 0xf0) == 0xf0)
return (pg_wchar) (((c[0] & 0x07) << 18) |
((c[1] & 0x3f) << 12) |
((c[2] & 0x3f) << 6) |
(c[3] & 0x3f));
else
/* that is an invalid code on purpose */
return 0xffffffff;
}
static int
pg_utf_dsplen(const unsigned char *s)
{
return ucs_wcwidth(utf2ucs(s));
}
/*
* convert mule internal code to pg_wchar
* caller should allocate enough space for "to"
* len: length of from.
* "from" not necessarily null terminated.
*/
static int
pg_mule2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
int cnt = 0;
while (len > 0 && *from)
{
if (IS_LC1(*from) && len >= 2)
{
*to = *from++ << 16;
*to |= *from++;
len -= 2;
}
else if (IS_LCPRV1(*from) && len >= 3)
{
from++;
*to = *from++ << 16;
*to |= *from++;
len -= 3;
}
else if (IS_LC2(*from) && len >= 3)
{
*to = *from++ << 16;
*to |= *from++ << 8;
*to |= *from++;
len -= 3;
}
else if (IS_LCPRV2(*from) && len >= 4)
{
from++;
*to = *from++ << 16;
*to |= *from++ << 8;
*to |= *from++;
len -= 4;
}
else
{ /* assume ASCII */
*to = (unsigned char) *from++;
len--;
}
to++;
cnt++;
}
*to = 0;
return cnt;
}
int
pg_mule_mblen(const unsigned char *s)
{
int len;
if (IS_LC1(*s))
len = 2;
else if (IS_LCPRV1(*s))
len = 3;
else if (IS_LC2(*s))
len = 3;
else if (IS_LCPRV2(*s))
len = 4;
else
len = 1; /* assume ASCII */
return len;
}
static int
pg_mule_dsplen(const unsigned char *s)
{
int len;
if (IS_LC1(*s))
len = 1;
else if (IS_LCPRV1(*s))
len = 1;
else if (IS_LC2(*s))
len = 2;
else if (IS_LCPRV2(*s))
len = 2;
else
len = 1; /* assume ASCII */
return len;
}
/*
* ISO8859-1
*/
static int
pg_latin12wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
int cnt = 0;
while (len > 0 && *from)
{
*to++ = *from++;
len--;
cnt++;
}
*to = 0;
return cnt;
}
static int
pg_latin1_mblen(const unsigned char *s)
{
return 1;
}
static int
pg_latin1_dsplen(const unsigned char *s)
{
return pg_ascii_dsplen(s);
}
/*
* SJIS
*/
static int
pg_sjis_mblen(const unsigned char *s)
{
int len;
if (*s >= 0xa1 && *s <= 0xdf)
len = 1; /* 1 byte kana? */
else if (IS_HIGHBIT_SET(*s))
len = 2; /* kanji? */
else
len = 1; /* should be ASCII */
return len;
}
static int
pg_sjis_dsplen(const unsigned char *s)
{
int len;
if (*s >= 0xa1 && *s <= 0xdf)
len = 1; /* 1 byte kana? */
else if (IS_HIGHBIT_SET(*s))
len = 2; /* kanji? */
else
len = pg_ascii_dsplen(s); /* should be ASCII */
return len;
}
/*
* Big5
*/
static int
pg_big5_mblen(const unsigned char *s)
{
int len;
if (IS_HIGHBIT_SET(*s))
len = 2; /* kanji? */
else
len = 1; /* should be ASCII */
return len;
}
static int
pg_big5_dsplen(const unsigned char *s)
{
int len;
if (IS_HIGHBIT_SET(*s))
len = 2; /* kanji? */
else
len = pg_ascii_dsplen(s); /* should be ASCII */
return len;
}
/*
* GBK
*/
static int
pg_gbk_mblen(const unsigned char *s)
{
int len;
if (IS_HIGHBIT_SET(*s))
len = 2; /* kanji? */
else
len = 1; /* should be ASCII */
return len;
}
static int
pg_gbk_dsplen(const unsigned char *s)
{
int len;
if (IS_HIGHBIT_SET(*s))
len = 2; /* kanji? */
else
len = pg_ascii_dsplen(s); /* should be ASCII */
return len;
}
/*
* UHC
*/
static int
pg_uhc_mblen(const unsigned char *s)
{
int len;
if (IS_HIGHBIT_SET(*s))
len = 2; /* 2byte? */
else
len = 1; /* should be ASCII */
return len;
}
static int
pg_uhc_dsplen(const unsigned char *s)
{
int len;
if (IS_HIGHBIT_SET(*s))
len = 2; /* 2byte? */
else
len = pg_ascii_dsplen(s); /* should be ASCII */
return len;
}
/*
* * GB18030
* * Added by Bill Huang <bhuang@redhat.com>,<bill_huanghb@ybb.ne.jp>
* */
static int
pg_gb18030_mblen(const unsigned char *s)
{
int len;
if (!IS_HIGHBIT_SET(*s))
len = 1; /* ASCII */
else
{
if ((*(s + 1) >= 0x40 && *(s + 1) <= 0x7e) || (*(s + 1) >= 0x80 && *(s + 1) <= 0xfe))
len = 2;
else if (*(s + 1) >= 0x30 && *(s + 1) <= 0x39)
len = 4;
else
len = 2;
}
return len;
}
static int
pg_gb18030_dsplen(const unsigned char *s)
{
int len;
if (IS_HIGHBIT_SET(*s))
len = 2;
else
len = pg_ascii_dsplen(s); /* ASCII */
return len;
}
/*
*-------------------------------------------------------------------
* multibyte sequence validators
*
* These functions accept "s", a pointer to the first byte of a string,
* and "len", the remaining length of the string. If there is a validly
* encoded character beginning at *s, return its length in bytes; else
* return -1.
*
* The functions can assume that len > 0 and that *s != '\0', but they must
* test for and reject zeroes in any additional bytes of a multibyte character.
*
* Note that this definition allows the function for a single-byte
* encoding to be just "return 1".
*-------------------------------------------------------------------
*/
static int
pg_ascii_verifier(const unsigned char *s, int len)
{
return 1;
}
#define IS_EUC_RANGE_VALID(c) ((c) >= 0xa1 && (c) <= 0xfe)
static int
pg_eucjp_verifier(const unsigned char *s, int len)
{
int l;
unsigned char c1, c2;
c1 = *s++;
switch (c1)
{
case SS2: /* JIS X 0201 */
l = 2;
if (l > len)
return -1;
c2 = *s++;
if (c2 < 0xa1 || c2 > 0xdf)
return -1;
break;
case SS3: /* JIS X 0212 */
l = 3;
if (l > len)
return -1;
c2 = *s++;
if (!IS_EUC_RANGE_VALID(c2))
return -1;
c2 = *s++;
if (!IS_EUC_RANGE_VALID(c2))
return -1;
break;
default:
if (IS_HIGHBIT_SET(c1)) /* JIS X 0208? */
{
l = 2;
if (l > len)
return -1;
if (!IS_EUC_RANGE_VALID(c1))
return -1;
c2 = *s++;
if (!IS_EUC_RANGE_VALID(c2))
return -1;
}
else /* must be ASCII */
{
l = 1;
}
break;
}
return l;
}
static int
pg_euckr_verifier(const unsigned char *s, int len)
{
int l;
unsigned char c1, c2;
c1 = *s++;
if (IS_HIGHBIT_SET(c1))
{
l = 2;
if (l > len)
return -1;
if (!IS_EUC_RANGE_VALID(c1))
return -1;
c2 = *s++;
if (!IS_EUC_RANGE_VALID(c2))
return -1;
}
else /* must be ASCII */
{
l = 1;
}
return l;
}
/* EUC-CN byte sequences are exactly same as EUC-KR */
#define pg_euccn_verifier pg_euckr_verifier
static int
pg_euctw_verifier(const unsigned char *s, int len)
{
int l;
unsigned char c1, c2;
c1 = *s++;
switch (c1)
{
case SS2: /* CNS 11643 Plane 1-7 */
l = 4;
if (l > len)
return -1;
c2 = *s++;
if (c2 < 0xa1 || c2 > 0xa7)
return -1;
c2 = *s++;
if (!IS_EUC_RANGE_VALID(c2))
return -1;
c2 = *s++;
if (!IS_EUC_RANGE_VALID(c2))
return -1;
break;
case SS3: /* unused */
return -1;
default:
if (IS_HIGHBIT_SET(c1)) /* CNS 11643 Plane 1 */
{
l = 2;
if (l > len)
return -1;
/* no further range check on c1? */
c2 = *s++;
if (!IS_EUC_RANGE_VALID(c2))
return -1;
}
else /* must be ASCII */
{
l = 1;
}
break;
}
return l;
}
static int
pg_johab_verifier(const unsigned char *s, int len)
{
int l, mbl;
unsigned char c;
l = mbl = pg_johab_mblen(s);
if (len < l)
return -1;
if (!IS_HIGHBIT_SET(*s))
return mbl;
while (--l > 0)
{
c = *++s;
if (!IS_EUC_RANGE_VALID(c))
return -1;
}
return mbl;
}
static int
pg_mule_verifier(const unsigned char *s, int len)
{
int l, mbl;
unsigned char c;
l = mbl = pg_mule_mblen(s);
if (len < l)
return -1;
while (--l > 0)
{
c = *++s;
if (!IS_HIGHBIT_SET(c))
return -1;
}
return mbl;
}
static int
pg_latin1_verifier(const unsigned char *s, int len)
{
return 1;
}
static int
pg_sjis_verifier(const unsigned char *s, int len)
{
int l, mbl;
unsigned char c1, c2;
l = mbl = pg_sjis_mblen(s);
if (len < l)
return -1;
if (l == 1) /* pg_sjis_mblen already verified it */
return mbl;
c1 = *s++;
c2 = *s;
if (!ISSJISHEAD(c1) || !ISSJISTAIL(c2))
return -1;
return mbl;
}
static int
pg_big5_verifier(const unsigned char *s, int len)
{
int l, mbl;
l = mbl = pg_big5_mblen(s);
if (len < l)
return -1;
while (--l > 0)
{
if (*++s == '\0')
return -1;
}
return mbl;
}
static int
pg_gbk_verifier(const unsigned char *s, int len)
{
int l, mbl;
l = mbl = pg_gbk_mblen(s);
if (len < l)
return -1;
while (--l > 0)
{
if (*++s == '\0')
return -1;
}
return mbl;
}
static int
pg_uhc_verifier(const unsigned char *s, int len)
{
int l, mbl;
l = mbl = pg_uhc_mblen(s);
if (len < l)
return -1;
while (--l > 0)
{
if (*++s == '\0')
return -1;
}
return mbl;
}
static int
pg_gb18030_verifier(const unsigned char *s, int len)
{
int l, mbl;
l = mbl = pg_gb18030_mblen(s);
if (len < l)
return -1;
while (--l > 0)
{
if (*++s == '\0')
return -1;
}
return mbl;
}
static int
pg_utf8_verifier(const unsigned char *s, int len)
{
int l = pg_utf_mblen(s);
if (len < l)
return -1;
if (!pg_utf8_islegal(s, l))
return -1;
return l;
}
/*
* Check for validity of a single UTF-8 encoded character
*
* This directly implements the rules in RFC3629. The bizarre-looking
* restrictions on the second byte are meant to ensure that there isn't
* more than one encoding of a given Unicode character point; that is,
* you may not use a longer-than-necessary byte sequence with high order
* zero bits to represent a character that would fit in fewer bytes.
* To do otherwise is to create security hazards (eg, create an apparent
* non-ASCII character that decodes to plain ASCII).
*
* length is assumed to have been obtained by pg_utf_mblen(), and the
* caller must have checked that that many bytes are present in the buffer.
*/
bool
pg_utf8_islegal(const unsigned char *source, int length)
{
unsigned char a;
switch (length)
{
default:
/* reject lengths 5 and 6 for now */
return false;
case 4:
a = source[3];
if (a < 0x80 || a > 0xBF)
return false;
/* FALL THRU */
case 3:
a = source[2];
if (a < 0x80 || a > 0xBF)
return false;
/* FALL THRU */
case 2:
a = source[1];
switch (*source)
{
case 0xE0:
if (a < 0xA0 || a > 0xBF)
return false;
break;
case 0xED:
if (a < 0x80 || a > 0x9F)
return false;
break;
case 0xF0:
if (a < 0x90 || a > 0xBF)
return false;
break;
case 0xF4:
if (a < 0x80 || a > 0x8F)
return false;
break;
default:
if (a < 0x80 || a > 0xBF)
return false;
break;
}
/* FALL THRU */
case 1:
a = *source;
if (a >= 0x80 && a < 0xC2)
return false;
if (a > 0xF4)
return false;
break;
}
return true;
}
/*
*-------------------------------------------------------------------
* encoding info table
*-------------------------------------------------------------------
*/
pg_wchar_tbl pg_wchar_table[] = {
{pg_ascii2wchar_with_len, pg_ascii_mblen, pg_ascii_dsplen, pg_ascii_verifier, 1}, /* 0; PG_SQL_ASCII */
{pg_eucjp2wchar_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifier, 3}, /* 1; PG_EUC_JP */
{pg_euccn2wchar_with_len, pg_euccn_mblen, pg_euccn_dsplen, pg_euccn_verifier, 3}, /* 2; PG_EUC_CN */
{pg_euckr2wchar_with_len, pg_euckr_mblen, pg_euckr_dsplen, pg_euckr_verifier, 3}, /* 3; PG_EUC_KR */
{pg_euctw2wchar_with_len, pg_euctw_mblen, pg_euctw_dsplen, pg_euctw_verifier, 3}, /* 4; PG_EUC_TW */
{pg_johab2wchar_with_len, pg_johab_mblen, pg_johab_dsplen, pg_johab_verifier, 3}, /* 5; PG_JOHAB */
{pg_utf2wchar_with_len, pg_utf_mblen, pg_utf_dsplen, pg_utf8_verifier, 4}, /* 6; PG_UTF8 */
{pg_mule2wchar_with_len, pg_mule_mblen, pg_mule_dsplen, pg_mule_verifier, 3}, /* 7; PG_MULE_INTERNAL */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 8; PG_LATIN1 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 9; PG_LATIN2 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 10; PG_LATIN3 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 11; PG_LATIN4 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 12; PG_LATIN5 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 13; PG_LATIN6 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 14; PG_LATIN7 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 15; PG_LATIN8 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 16; PG_LATIN9 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 17; PG_LATIN10 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 18; PG_WIN1256 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 19; PG_WIN1258 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 20; PG_WIN874 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 21; PG_KOI8 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 22; PG_WIN1251 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 22; PG_WIN1252 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 23; PG_WIN866 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 24; ISO-8859-5 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 25; ISO-8859-6 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 26; ISO-8859-7 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 27; ISO-8859-8 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 28; PG_WIN1250 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 29; PG_WIN1253 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 30; PG_WIN1254 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 31; PG_WIN1255 */
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* 32; PG_WIN1257 */
{0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifier, 2}, /* 33; PG_SJIS */
{0, pg_big5_mblen, pg_big5_dsplen, pg_big5_verifier, 2}, /* 34; PG_BIG5 */
{0, pg_gbk_mblen, pg_gbk_dsplen, pg_gbk_verifier, 2}, /* 35; PG_GBK */
{0, pg_uhc_mblen, pg_uhc_dsplen, pg_uhc_verifier, 2}, /* 36; PG_UHC */
{0, pg_gb18030_mblen, pg_gb18030_dsplen, pg_gb18030_verifier, 2} /* 37; PG_GB18030 */
};
/* returns the byte length of a word for mule internal code */
int
pg_mic_mblen(const unsigned char *mbstr)
{
return pg_mule_mblen(mbstr);
}
/*
* Returns the byte length of a multibyte character.
*/
int
pg_encoding_mblen(int encoding, const char *mbstr)
{
Assert(PG_VALID_ENCODING(encoding));
return ((encoding >= 0 &&
encoding < sizeof(pg_wchar_table) / sizeof(pg_wchar_tbl)) ?
((*pg_wchar_table[encoding].mblen) ((const unsigned char *) mbstr)) :
((*pg_wchar_table[PG_SQL_ASCII].mblen) ((const unsigned char *) mbstr)));
}
/*
* Returns the display length of a multibyte character.
*/
int
pg_encoding_dsplen(int encoding, const char *mbstr)
{
Assert(PG_VALID_ENCODING(encoding));
return ((encoding >= 0 &&
encoding < sizeof(pg_wchar_table) / sizeof(pg_wchar_tbl)) ?
((*pg_wchar_table[encoding].dsplen) ((const unsigned char *) mbstr)) :
((*pg_wchar_table[PG_SQL_ASCII].dsplen) ((const unsigned char *) mbstr)));
}
/*
* Verify the first multibyte character of the given string.
* Return its byte length if good, -1 if bad. (See comments above for
* full details of the mbverify API.)
*/
int
pg_encoding_verifymb(int encoding, const char *mbstr, int len)
{
Assert(PG_VALID_ENCODING(encoding));
return ((encoding >= 0 &&
encoding < sizeof(pg_wchar_table) / sizeof(pg_wchar_tbl)) ?
((*pg_wchar_table[encoding].mbverify) ((const unsigned char *) mbstr, len)) :
((*pg_wchar_table[PG_SQL_ASCII].mbverify) ((const unsigned char *) mbstr, len)));
}
/*
* fetch maximum length of a given encoding
*/
int
pg_encoding_max_length(int encoding)
{
Assert(PG_VALID_ENCODING(encoding));
return pg_wchar_table[encoding].maxmblen;
}
#ifndef FRONTEND
/*
* fetch maximum length of the encoding for the current database
*/
int
pg_database_encoding_max_length(void)
{
return pg_wchar_table[GetDatabaseEncoding()].maxmblen;
}
/*
* Verify mbstr to make sure that it is validly encoded in the current
* database encoding. Otherwise same as pg_verify_mbstr().
*/
bool
pg_verifymbstr(const char *mbstr, int len, bool noError)
{
return pg_verify_mbstr(GetDatabaseEncoding(), mbstr, len, noError);
}
/*
* Verify mbstr to make sure that it is validly encoded in the specified
* encoding.
*
* mbstr is not necessarily zero terminated; length of mbstr is
* specified by len.
*
* If OK, return TRUE. If a problem is found, return FALSE when noError is
* true; when noError is false, ereport() a descriptive message.
*/
bool
pg_verify_mbstr(int encoding, const char *mbstr, int len, bool noError)
{
mbverifier mbverify;
Assert(PG_VALID_ENCODING(encoding));
/*
* In single-byte encodings, we need only reject nulls (\0).
*/
if (pg_encoding_max_length(encoding) <= 1)
{
const char *nullpos = memchr(mbstr, 0, len);
if (nullpos == NULL)
return true;
if (noError)
return false;
report_invalid_encoding(encoding, nullpos, 1);
}
/* fetch function pointer just once */
mbverify = pg_wchar_table[encoding].mbverify;
while (len > 0)
{
int l;
/* fast path for ASCII-subset characters */
if (!IS_HIGHBIT_SET(*mbstr))
{
if (*mbstr != '\0')
{
mbstr++;
len--;
continue;
}
if (noError)
return false;
report_invalid_encoding(encoding, mbstr, len);
}
l = (*mbverify) ((const unsigned char *) mbstr, len);
if (l < 0)
{
if (noError)
return false;
report_invalid_encoding(encoding, mbstr, len);
}
mbstr += l;
len -= l;
}
return true;
}
/*
* report_invalid_encoding: complain about invalid multibyte character
*
* note: len is remaining length of string, not length of character;
* len must be greater than zero, as we always examine the first byte.
*/
void
report_invalid_encoding(int encoding, const char *mbstr, int len)
{
int l = pg_encoding_mblen(encoding, mbstr);
char buf[8 * 2 + 1];
char *p = buf;
int j,
jlimit;
jlimit = Min(l, len);
jlimit = Min(jlimit, 8); /* prevent buffer overrun */
for (j = 0; j < jlimit; j++)
p += sprintf(p, "%02x", (unsigned char) mbstr[j]);
ereport(ERROR,
(errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
errmsg("invalid byte sequence for encoding \"%s\": 0x%s",
pg_enc2name_tbl[encoding].name,
buf)));
}
/*
* report_untranslatable_char: complain about untranslatable character
*
* note: len is remaining length of string, not length of character;
* len must be greater than zero, as we always examine the first byte.
*/
void
report_untranslatable_char(int src_encoding, int dest_encoding,
const char *mbstr, int len)
{
int l = pg_encoding_mblen(src_encoding, mbstr);
char buf[8 * 2 + 1];
char *p = buf;
int j,
jlimit;
jlimit = Min(l, len);
jlimit = Min(jlimit, 8); /* prevent buffer overrun */
for (j = 0; j < jlimit; j++)
p += sprintf(p, "%02x", (unsigned char) mbstr[j]);
ereport(ERROR,
(errcode(ERRCODE_UNTRANSLATABLE_CHARACTER),
errmsg("character 0x%s of encoding \"%s\" has no equivalent in \"%s\"",
buf,
pg_enc2name_tbl[src_encoding].name,
pg_enc2name_tbl[dest_encoding].name)));
}
#endif
View Git Blame Copy Permalink