mirror of
https://github.com/postgres/postgres.git
synced 2025-07-18 17:42:25 +03:00
Back-patch addition of pg_wchar-to-multibyte conversion functionality.
Back-patch of commits72dd6291f2,f6a05fd973, and60e9c224a1. This is needed to support fixing the regex prefix extraction bug in back branches.
This commit is contained in:
Tom Lane
@ -761,6 +761,28 @@ pg_encoding_mb2wchar_with_len(int encoding,
|
||||
return (*pg_wchar_table[encoding].mb2wchar_with_len) ((const unsigned char *) from, to, len);
|
||||
}
|
||||
|
||||
/* convert a wchar string to a multibyte */
|
||||
int
|
||||
pg_wchar2mb(const pg_wchar *from, char *to)
|
||||
{
|
||||
return (*pg_wchar_table[DatabaseEncoding->encoding].wchar2mb_with_len) (from, (unsigned char *)to, pg_wchar_strlen(from));
|
||||
}
|
||||
|
||||
/* convert a wchar string to a multibyte with a limited length */
|
||||
int
|
||||
pg_wchar2mb_with_len(const pg_wchar *from, char *to, int len)
|
||||
{
|
||||
return (*pg_wchar_table[DatabaseEncoding->encoding].wchar2mb_with_len) (from, (unsigned char *)to, len);
|
||||
}
|
||||
|
||||
/* same, with any encoding */
|
||||
int
|
||||
pg_encoding_wchar2mb_with_len(int encoding,
|
||||
const pg_wchar *from, char *to, int len)
|
||||
{
|
||||
return (*pg_wchar_table[encoding].wchar2mb_with_len) (from, (unsigned char *)to, len);
|
||||
}
|
||||
|
||||
/* returns the byte length of a multibyte character */
|
||||
int
|
||||
pg_mblen(const char *mbstr)
|
||||
|
||||
@ -99,8 +99,7 @@ pg_euc2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
|
||||
*to |= *from++;
|
||||
len -= 2;
|
||||
}
|
||||
else
|
||||
/* must be ASCII */
|
||||
else /* must be ASCII */
|
||||
{
|
||||
*to = *from++;
|
||||
len--;
|
||||
@ -339,6 +338,55 @@ pg_euctw_dsplen(const unsigned char *s)
|
||||
return len;
|
||||
}
|
||||
|
||||
/*
|
||||
* Convert pg_wchar to EUC_* encoding.
|
||||
* caller must allocate enough space for "to", including a trailing zero!
|
||||
* len: length of from.
|
||||
* "from" not necessarily null terminated.
|
||||
*/
|
||||
static int
|
||||
pg_wchar2euc_with_len(const pg_wchar *from, unsigned char *to, int len)
|
||||
{
|
||||
int cnt = 0;
|
||||
|
||||
while (len > 0 && *from)
|
||||
{
|
||||
unsigned char c;
|
||||
|
||||
if ((c = (*from >> 24)))
|
||||
{
|
||||
*to++ = c;
|
||||
*to++ = (*from >> 16) & 0xff;
|
||||
*to++ = (*from >> 8) & 0xff;
|
||||
*to++ = *from & 0xff;
|
||||
cnt += 4;
|
||||
}
|
||||
else if ((c = (*from >> 16)))
|
||||
{
|
||||
*to++ = c;
|
||||
*to++ = (*from >> 8) & 0xff;
|
||||
*to++ = *from & 0xff;
|
||||
cnt += 3;
|
||||
}
|
||||
else if ((c = (*from >> 8)))
|
||||
{
|
||||
*to++ = c;
|
||||
*to++ = *from & 0xff;
|
||||
cnt += 2;
|
||||
}
|
||||
else
|
||||
{
|
||||
*to++ = *from;
|
||||
cnt++;
|
||||
}
|
||||
from++;
|
||||
len--;
|
||||
}
|
||||
*to = 0;
|
||||
return cnt;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* JOHAB
|
||||
*/
|
||||
@ -453,6 +501,31 @@ unicode_to_utf8(pg_wchar c, unsigned char *utf8string)
|
||||
return utf8string;
|
||||
}
|
||||
|
||||
/*
|
||||
* Trivial conversion from pg_wchar to UTF-8.
|
||||
* caller should allocate enough space for "to"
|
||||
* len: length of from.
|
||||
* "from" not necessarily null terminated.
|
||||
*/
|
||||
static int
|
||||
pg_wchar2utf_with_len(const pg_wchar *from, unsigned char *to, int len)
|
||||
{
|
||||
int cnt = 0;
|
||||
|
||||
while (len > 0 && *from)
|
||||
{
|
||||
int char_len;
|
||||
|
||||
unicode_to_utf8(*from, to);
|
||||
char_len = pg_utf_mblen(to);
|
||||
cnt += char_len;
|
||||
to += char_len;
|
||||
from++;
|
||||
len--;
|
||||
}
|
||||
*to = 0;
|
||||
return cnt;
|
||||
}
|
||||
|
||||
/*
|
||||
* Return the byte length of a UTF8 character pointed to by s
|
||||
@ -717,6 +790,77 @@ pg_mule2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
|
||||
return cnt;
|
||||
}
|
||||
|
||||
/*
|
||||
* convert pg_wchar to mule internal code
|
||||
* caller should allocate enough space for "to"
|
||||
* len: length of from.
|
||||
* "from" not necessarily null terminated.
|
||||
*/
|
||||
static int
|
||||
pg_wchar2mule_with_len(const pg_wchar *from, unsigned char *to, int len)
|
||||
{
|
||||
int cnt = 0;
|
||||
|
||||
while (len > 0 && *from)
|
||||
{
|
||||
unsigned char lb;
|
||||
|
||||
lb = (*from >> 16) & 0xff;
|
||||
if (IS_LC1(lb))
|
||||
{
|
||||
*to++ = lb;
|
||||
*to++ = *from & 0xff;
|
||||
cnt += 2;
|
||||
}
|
||||
else if (IS_LC2(lb))
|
||||
{
|
||||
*to++ = lb;
|
||||
*to++ = (*from >> 8) & 0xff;
|
||||
*to++ = *from & 0xff;
|
||||
cnt += 3;
|
||||
}
|
||||
else if (IS_LCPRV1_A_RANGE(lb))
|
||||
{
|
||||
*to++ = LCPRV1_A;
|
||||
*to++ = lb;
|
||||
*to++ = *from & 0xff;
|
||||
cnt += 3;
|
||||
}
|
||||
else if (IS_LCPRV1_B_RANGE(lb))
|
||||
{
|
||||
*to++ = LCPRV1_B;
|
||||
*to++ = lb;
|
||||
*to++ = *from & 0xff;
|
||||
cnt += 3;
|
||||
}
|
||||
else if (IS_LCPRV2_A_RANGE(lb))
|
||||
{
|
||||
*to++ = LCPRV2_A;
|
||||
*to++ = lb;
|
||||
*to++ = (*from >> 8) & 0xff;
|
||||
*to++ = *from & 0xff;
|
||||
cnt += 4;
|
||||
}
|
||||
else if (IS_LCPRV2_B_RANGE(lb))
|
||||
{
|
||||
*to++ = LCPRV2_B;
|
||||
*to++ = lb;
|
||||
*to++ = (*from >> 8) & 0xff;
|
||||
*to++ = *from & 0xff;
|
||||
cnt += 4;
|
||||
}
|
||||
else
|
||||
{
|
||||
*to++ = *from & 0xff;
|
||||
cnt += 1;
|
||||
}
|
||||
from++;
|
||||
len--;
|
||||
}
|
||||
*to = 0;
|
||||
return cnt;
|
||||
}
|
||||
|
||||
int
|
||||
pg_mule_mblen(const unsigned char *s)
|
||||
{
|
||||
@ -772,6 +916,28 @@ pg_latin12wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
|
||||
return cnt;
|
||||
}
|
||||
|
||||
/*
|
||||
* Trivial conversion from pg_wchar to single byte encoding. Just ignores
|
||||
* high bits.
|
||||
* caller should allocate enough space for "to"
|
||||
* len: length of from.
|
||||
* "from" not necessarily null terminated.
|
||||
*/
|
||||
static int
|
||||
pg_wchar2single_with_len(const pg_wchar *from, unsigned char *to, int len)
|
||||
{
|
||||
int cnt = 0;
|
||||
|
||||
while (len > 0 && *from)
|
||||
{
|
||||
*to++ = *from++;
|
||||
len--;
|
||||
cnt++;
|
||||
}
|
||||
*to = 0;
|
||||
return cnt;
|
||||
}
|
||||
|
||||
static int
|
||||
pg_latin1_mblen(const unsigned char *s)
|
||||
{
|
||||
@ -1339,48 +1505,48 @@ pg_utf8_islegal(const unsigned char *source, int length)
|
||||
*-------------------------------------------------------------------
|
||||
*/
|
||||
pg_wchar_tbl pg_wchar_table[] = {
|
||||
{pg_ascii2wchar_with_len, pg_ascii_mblen, pg_ascii_dsplen, pg_ascii_verifier, 1}, /* PG_SQL_ASCII */
|
||||
{pg_eucjp2wchar_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifier, 3}, /* PG_EUC_JP */
|
||||
{pg_euccn2wchar_with_len, pg_euccn_mblen, pg_euccn_dsplen, pg_euccn_verifier, 2}, /* PG_EUC_CN */
|
||||
{pg_euckr2wchar_with_len, pg_euckr_mblen, pg_euckr_dsplen, pg_euckr_verifier, 3}, /* PG_EUC_KR */
|
||||
{pg_euctw2wchar_with_len, pg_euctw_mblen, pg_euctw_dsplen, pg_euctw_verifier, 4}, /* PG_EUC_TW */
|
||||
{pg_eucjp2wchar_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifier, 3}, /* PG_EUC_JIS_2004 */
|
||||
{pg_utf2wchar_with_len, pg_utf_mblen, pg_utf_dsplen, pg_utf8_verifier, 4}, /* PG_UTF8 */
|
||||
{pg_mule2wchar_with_len, pg_mule_mblen, pg_mule_dsplen, pg_mule_verifier, 4}, /* PG_MULE_INTERNAL */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN1 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN2 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN3 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN4 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN5 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN6 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN7 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN8 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN9 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN10 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1256 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1258 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN866 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN874 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_KOI8R */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1251 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1252 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* ISO-8859-5 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* ISO-8859-6 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* ISO-8859-7 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* ISO-8859-8 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1250 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1253 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1254 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1255 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1257 */
|
||||
{pg_latin12wchar_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_KOI8U */
|
||||
{0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifier, 2}, /* PG_SJIS */
|
||||
{0, pg_big5_mblen, pg_big5_dsplen, pg_big5_verifier, 2}, /* PG_BIG5 */
|
||||
{0, pg_gbk_mblen, pg_gbk_dsplen, pg_gbk_verifier, 2}, /* PG_GBK */
|
||||
{0, pg_uhc_mblen, pg_uhc_dsplen, pg_uhc_verifier, 2}, /* PG_UHC */
|
||||
{0, pg_gb18030_mblen, pg_gb18030_dsplen, pg_gb18030_verifier, 4}, /* PG_GB18030 */
|
||||
{0, pg_johab_mblen, pg_johab_dsplen, pg_johab_verifier, 3}, /* PG_JOHAB */
|
||||
{0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifier, 2} /* PG_SHIFT_JIS_2004 */
|
||||
{pg_ascii2wchar_with_len, pg_wchar2single_with_len, pg_ascii_mblen, pg_ascii_dsplen, pg_ascii_verifier, 1}, /* PG_SQL_ASCII */
|
||||
{pg_eucjp2wchar_with_len, pg_wchar2euc_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifier, 3}, /* PG_EUC_JP */
|
||||
{pg_euccn2wchar_with_len, pg_wchar2euc_with_len, pg_euccn_mblen, pg_euccn_dsplen, pg_euccn_verifier, 2}, /* PG_EUC_CN */
|
||||
{pg_euckr2wchar_with_len, pg_wchar2euc_with_len, pg_euckr_mblen, pg_euckr_dsplen, pg_euckr_verifier, 3}, /* PG_EUC_KR */
|
||||
{pg_euctw2wchar_with_len, pg_wchar2euc_with_len, pg_euctw_mblen, pg_euctw_dsplen, pg_euctw_verifier, 4}, /* PG_EUC_TW */
|
||||
{pg_eucjp2wchar_with_len, pg_wchar2euc_with_len, pg_eucjp_mblen, pg_eucjp_dsplen, pg_eucjp_verifier, 3}, /* PG_EUC_JIS_2004 */
|
||||
{pg_utf2wchar_with_len, pg_wchar2utf_with_len, pg_utf_mblen, pg_utf_dsplen, pg_utf8_verifier, 4}, /* PG_UTF8 */
|
||||
{pg_mule2wchar_with_len, pg_wchar2mule_with_len, pg_mule_mblen, pg_mule_dsplen, pg_mule_verifier, 4}, /* PG_MULE_INTERNAL */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN1 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN2 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN3 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN4 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN5 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN6 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN7 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN8 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN9 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_LATIN10 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1256 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1258 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN866 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN874 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_KOI8R */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1251 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1252 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* ISO-8859-5 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* ISO-8859-6 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* ISO-8859-7 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* ISO-8859-8 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1250 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1253 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1254 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1255 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_WIN1257 */
|
||||
{pg_latin12wchar_with_len, pg_wchar2single_with_len, pg_latin1_mblen, pg_latin1_dsplen, pg_latin1_verifier, 1}, /* PG_KOI8U */
|
||||
{0, 0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifier, 2}, /* PG_SJIS */
|
||||
{0, 0, pg_big5_mblen, pg_big5_dsplen, pg_big5_verifier, 2}, /* PG_BIG5 */
|
||||
{0, 0, pg_gbk_mblen, pg_gbk_dsplen, pg_gbk_verifier, 2}, /* PG_GBK */
|
||||
{0, 0, pg_uhc_mblen, pg_uhc_dsplen, pg_uhc_verifier, 2}, /* PG_UHC */
|
||||
{0, 0, pg_gb18030_mblen, pg_gb18030_dsplen, pg_gb18030_verifier, 4}, /* PG_GB18030 */
|
||||
{0, 0, pg_johab_mblen, pg_johab_dsplen, pg_johab_verifier, 3}, /* PG_JOHAB */
|
||||
{0, 0, pg_sjis_mblen, pg_sjis_dsplen, pg_sjis_verifier, 2} /* PG_SHIFT_JIS_2004 */
|
||||
};
|
||||
|
||||
/* returns the byte length of a word for mule internal code */
|
||||
|
||||
Reference in New Issue
Block a user