database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-07-02 09:02:37 +03:00
Code Activity

Hi, here are the patches to enhance existing MB handling. This time

Browse Source 
I have implemented a framework of encoding translation between the
backend and the frontend. Also I have added a new variable setting
command:

SET CLIENT_ENCODING TO 'encoding';

Other features include:
	Latin1 support more 8 bit cleaness

See doc/README.mb for more details. Note that the pacthes are
against May 30 snapshot.

Tatsuo Ishii
This commit is contained in:
Bruce Momjian
1998-06-16 07:29:54 +00:00
parent 0d8e7f6381
commit cb7cbc16fa
37 changed files with 1115 additions and 341 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/backend/regex/engine.c
View File

@ -127,6 +127,9 @@ extern "C"
# elif MB == UNICODE
# define NONCHAR(c) ((c) > USHRT_MAX)
# define NNONCHAR (CODEMAX-USHRT_MAX)
# else /* assume 1 byte code such as ISO8859-1 */
# define NONCHAR(c) ((c) > UCHAR_MAX)
# define NNONCHAR (CODEMAX-UCHAR_MAX)
# endif
#else
# define NONCHAR(c) ((c) > CHAR_MAX)

2
src/backend/regex/regcomp.c
View File

@ -1344,7 +1344,7 @@ cset *cs;
for (i = 0; i < css; i++)
if (CHIN(cs, i))
return ((char) i);
return (i);
assert(never);
return (0); /* arbitrary */
}

408
src/backend/regex/utils.c
View File

@ -1,164 +1,24 @@
/*
* misc conversion functions between pg_wchar and other encodings.
* Tatsuo Ishii
* $Id: utils.c,v 1.2 1998/04/27 17:07:53 scrappy Exp $
* $Id: utils.c,v 1.3 1998/06/16 07:29:29 momjian Exp $
*/
#include <regex/pg_wchar.h>
/*
* convert EUC to pg_wchar (EUC process code)
* caller should allocate enough space for "to"
*/
static void pg_euc2wchar(const unsigned char *from, pg_wchar *to)
{
while (*from) {
if (*from == SS2) {
from++;
*to = *from++;
} else if (*from == SS3) {
from++;
*to = *from++ << 8;
*to |= 0x3f & *from++;
} else if (*from & 0x80) {
*to = *from++ << 8;
*to |= *from++;
} else {
*to = *from++;
}
to++;
}
*to = 0;
}
static void pg_eucjp2wchar(const unsigned char *from, pg_wchar *to)
{
pg_euc2wchar(from,to);
}
static void pg_euckr2wchar(const unsigned char *from, pg_wchar *to)
{
pg_euc2wchar(from,to);
}
static void pg_eucch2wchar(const unsigned char *from, pg_wchar *to)
{
while (*from) {
if (*from == SS2) {
from++;
*to = 0x3f00 & (*from++ << 8);
*to = *from++;
} else if (*from == SS3) {
from++;
*to = *from++ << 8;
*to |= 0x3f & *from++;
} else if (*from & 0x80) {
*to = *from++ << 8;
*to |= *from++;
} else {
*to = *from++;
}
to++;
}
*to = 0;
}
static void pg_euccn2wchar(const unsigned char *from, pg_wchar *to)
{
while (*from) {
if (*from == SS2) {
from++;
*to = *from++ << 16;
*to |= *from++ << 8;
*to |= *from++;
} else if (*from == SS3) {
from++;
*to = *from++ << 8;
*to |= 0x3f & *from++;
} else if (*from & 0x80) {
*to = *from++ << 8;
*to |= *from++;
} else {
*to = *from++;
}
to++;
}
*to = 0;
}
/*
* convert UTF-8 to pg_wchar (UCS-2)
* caller should allocate enough space for "to"
* conversion to pg_wchar is done by "table driven."
* to add an encoding support, define mb2wchar_with_len(), mblen()
* for the particular encoding. Note that if the encoding is only
* supported in the client, you don't need to define
* mb2wchar_with_len() function (SJIS is the case).
*/
static void pg_utf2wchar(const unsigned char *from, pg_wchar *to)
{
unsigned char c1,c2,c3;
while (*from) {
if ((*from & 0x80) == 0) {
*to = *from++;
} else if ((*from & 0xe0) == 0xc0) {
c1 = *from++ & 0x1f;
c2 = *from++ & 0x3f;
*to = c1 << 6;
*to |= c2;
} else if ((*from & 0xe0) == 0xe0) {
c1 = *from++ & 0x0f;
c2 = *from++ & 0x3f;
c3 = *from++ & 0x3f;
*to = c1 << 12;
*to |= c2 << 6;
*to |= c3;
}
to++;
}
*to = 0;
}
typedef struct {
void (*mb2wchar_with_len)(); /* convert a multi-byte string to a wchar */
int (*mblen)(); /* returns the length of a multi-byte word */
} pg_wchar_tbl;
/*
* convert mule internal code to pg_wchar.
* in this case pg_wchar consists of following 4 bytes:
*
* 0x00(unused)
* 0x00(ASCII)|leading character (one of LC1, LC12, LC2 or LC22)
* 0x00(ASCII,1 byte code)|other than 0x00(2 byte code)
* the lowest byte of the code
*
* note that Type N (variable length byte encoding) cannot be represented by
* this schema. sorry.
* caller should allocate enough space for "to"
*/
static void pg_mule2wchar(const unsigned char *from, pg_wchar *to)
{
while (*from) {
if (IS_LC1(*from)) {
*to = *from++ << 16;
*to |= *from++;
} else if (IS_LCPRV1(*from)) {
from++;
*to = *from++ << 16;
*to |= *from++;
} else if (IS_LC2(*from)) {
*to = *from++ << 16;
*to |= *from++ << 8;
*to |= *from++;
} else if (IS_LCPRV2(*from)) {
from++;
*to = *from++ << 16;
*to |= *from++ << 8;
*to |= *from++;
} else { /* assume ASCII */
*to = *from++;
}
to++;
}
*to = 0;
}
/*
* convert EUC to pg_wchar (EUC process code)
* caller should allocate enough space for "to"
* len: length of from.
* "from" not necessarily null terminated.
*/
static void pg_euc2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
static void pg_euc2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
while (*from && len > 0) {
if (*from == SS2) {
@ -184,19 +44,54 @@ static void pg_euc2wchar_with_len(const unsigned char *from, pg_wchar *to, int l
*to = 0;
}
static int pg_euc_mblen(const unsigned char *s)
{
int len;
if (*s == SS2) {
len = 2;
} else if (*s == SS3) {
len = 3;
} else if (*s & 0x80) {
len = 2;
} else {
len = 1;
}
return(len);
}
/*
* EUC_JP
*/
static void pg_eucjp2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
pg_euc2wchar_with_len(from,to,len);
}
static int pg_eucjp_mblen(const unsigned char *s)
{
return(pg_euc_mblen(s));
}
/*
* EUC_KR
*/
static void pg_euckr2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
pg_euc2wchar_with_len(from,to,len);
}
static void pg_eucch2wchar_with_len
static int pg_euckr_mblen(const unsigned char *s)
{
return(pg_euc_mblen(s));
}
/*
* EUC_CN
*/
static void pg_euccn2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
while (*from && len > 0) {
@ -224,7 +119,26 @@ static void pg_eucch2wchar_with_len
*to = 0;
}
static void pg_euccn2wchar_with_len
static int pg_euccn_mblen(const unsigned char *s)
{
int len;
if (*s == SS2) {
len = 3;
} else if (*s == SS3) {
len = 3;
} else if (*s & 0x80) {
len = 2;
} else {
len = 1;
}
return(len);
}
/*
* EUC_TW
*/
static void pg_euctw2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
while (*from && len > 0) {
@ -253,6 +167,22 @@ static void pg_euccn2wchar_with_len
*to = 0;
}
static int pg_euctw_mblen(const unsigned char *s)
{
int len;
if (*s == SS2) {
len = 4;
} else if (*s == SS3) {
len = 3;
} else if (*s & 0x80) {
len = 2;
} else {
len = 1;
}
return(len);
}
/*
* convert UTF-8 to pg_wchar (UCS-2)
* caller should allocate enough space for "to"
@ -286,6 +216,20 @@ static void pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int l
*to = 0;
}
static int pg_utf_mblen(const unsigned char *s)
{
int len = 1;
if ((*s & 0x80) == 0) {
len = 1;
} else if ((*s & 0xe0) == 0xc0) {
len = 2;
} else if ((*s & 0xe0) == 0xe0) {
len = 3;
}
return(len);
}
/*
* convert mule internal code to pg_wchar
* caller should allocate enough space for "to"
@ -324,78 +268,6 @@ static void pg_mule2wchar_with_len(const unsigned char *from, pg_wchar *to, int
*to = 0;
}
static int pg_euc_mblen(const unsigned char *s)
{
int len;
if (*s == SS2) {
len = 2;
} else if (*s == SS3) {
len = 3;
} else if (*s & 0x80) {
len = 2;
} else {
len = 1;
}
return(len);
}
static int pg_eucjp_mblen(const unsigned char *s)
{
return(pg_euc_mblen(s));
}
static int pg_euckr_mblen(const unsigned char *s)
{
return(pg_euc_mblen(s));
}
static int pg_eucch_mblen(const unsigned char *s)
{
int len;
if (*s == SS2) {
len = 3;
} else if (*s == SS3) {
len = 3;
} else if (*s & 0x80) {
len = 2;
} else {
len = 1;
}
return(len);
}
static int pg_euccn_mblen(const unsigned char *s)
{
int len;
if (*s == SS2) {
len = 4;
} else if (*s == SS3) {
len = 3;
} else if (*s & 0x80) {
len = 2;
} else {
len = 1;
}
return(len);
}
static int pg_utf_mblen(const unsigned char *s)
{
int len = 1;
if ((*s & 0x80) == 0) {
len = 1;
} else if ((*s & 0xe0) == 0xc0) {
len = 2;
} else if ((*s & 0xe0) == 0xe0) {
len = 3;
}
return(len);
}
static int pg_mule_mblen(const unsigned char *s)
{
int len;
@ -414,25 +286,71 @@ static int pg_mule_mblen(const unsigned char *s)
return(len);
}
typedef struct {
void (*mb2wchar)(); /* convert a multi-byte string to a wchar */
void (*mb2wchar_with_len)(); /* convert a multi-byte string to a wchar
with a limited length */
int (*mblen)(); /* returns the length of a multi-byte word */
} pg_wchar_tbl;
/*
* ISO8859-1
*/
static void pg_latin12wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
while (*from && len-- > 0) {
*to++ = *from++;
}
*to = 0;
}
static int pg_latin1_mblen(const unsigned char *s)
{
return(1);
}
/*
* SJIS
*/
static int pg_sjis_mblen(const unsigned char *s)
{
int len;
if (*s >= 0xa1 && *s <= 0xdf) { /* 1 byte kana? */
len = 1;
} else if (*s > 0x7f) { /* kanji? */
len = 2;
} else { /* should be ASCII */
len = 1;
}
return(len);
}
static pg_wchar_tbl pg_wchar_table[] = {
{pg_eucjp2wchar, pg_eucjp2wchar_with_len, pg_eucjp_mblen},
{pg_eucch2wchar, pg_eucch2wchar_with_len, pg_eucch_mblen},
{pg_euckr2wchar, pg_euckr2wchar_with_len, pg_euckr_mblen},
{pg_euccn2wchar, pg_euccn2wchar_with_len, pg_euccn_mblen},
{pg_utf2wchar, pg_utf2wchar_with_len, pg_utf_mblen},
{pg_mule2wchar, pg_mule2wchar_with_len, pg_mule_mblen}};
{pg_eucjp2wchar_with_len, pg_eucjp_mblen},
{pg_euccn2wchar_with_len, pg_euccn_mblen},
{pg_euckr2wchar_with_len, pg_euckr_mblen},
{pg_euctw2wchar_with_len, pg_euctw_mblen},
{pg_utf2wchar_with_len, pg_utf_mblen},
{pg_mule2wchar_with_len, pg_mule_mblen},
{pg_latin12wchar_with_len, pg_latin1_mblen},
{0, 0},
{0, 0},
{0, 0},
{0, 0},
{0, 0},
{0, 0},
{0, 0},
{0, 0},
{0, 0},
{0, pg_sjis_mblen}
};
/*
*########################################################################
*
* Public functions
*
*########################################################################
*/
/* convert a multi-byte string to a wchar */
void pg_mb2wchar(const unsigned char *from, pg_wchar *to)
{
(*pg_wchar_table[MB].mb2wchar)(from,to);
(*pg_wchar_table[MB].mb2wchar_with_len)(from,to,strlen(from));
}
/* convert a multi-byte string to a wchar with a limited length */
@ -447,6 +365,18 @@ int pg_mblen(const unsigned char *mbstr)
return((*pg_wchar_table[MB].mblen)(mbstr));
}
/* returns the byte length of a multi-byte word for an encoding */
int pg_encoding_mblen(int encoding, const unsigned char *mbstr)
{
return((*pg_wchar_table[encoding].mblen)(mbstr));
}
/* returns the byte length of a word for mule internal code */
int pg_mic_mblen(const unsigned char *mbstr)
{
return(pg_mule_mblen(mbstr));
}
/* returns the length (counted as a wchar) of a multi-byte string */
int pg_mbstrlen(const unsigned char *mbstr)
{