database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-06-30 21:42:05 +03:00
Code Activity

I really hope that I haven't missed anything in this one...

Browse Source 
From: t-ishii@sra.co.jp

Attached are patches to enhance the multi-byte support.  (patches are
against 7/18 snapshot)

* determine encoding at initdb/createdb rather than compile time

Now initdb/createdb has an option to specify the encoding. Also, I
modified the syntax of CREATE DATABASE to accept encoding option. See
README.mb for more details.

For this purpose I have added new column "encoding" to pg_database.
Also pg_attribute and pg_class are changed to catch up the
modification to pg_database.  Actually I haved added pg_database_mb.h,
pg_attribute_mb.h and pg_class_mb.h. These are used only when MB is
enabled. The reason having separate files is I couldn't find a way to
use ifdef or whatever in those files. I have to admit it looks
ugly. No way.

* support for PGCLIENTENCODING when issuing COPY command

commands/copy.c modified.

* support for SQL92 syntax "SET NAMES"

See gram.y.

* support for LATIN2-5
* add UNICODE regression test case
* new test suite for MB

New directory test/mb added.

* clean up source files

Basic idea is to have MB's own subdirectory for easier maintenance.
These are include/mb and backend/utils/mb.
This commit is contained in:
Marc G. Fournier
1998-07-24 03:32:46 +00:00
parent 6e66468f3a
commit bf00bbb0c4
82 changed files with 2161 additions and 759 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/backend/regex/Makefile
View File

@ -4,7 +4,7 @@
# Makefile for regex
#
# IDENTIFICATION
# $Header: /cvsroot/pgsql/src/backend/regex/Makefile,v 1.6 1998/04/06 00:24:39 momjian Exp $
# $Header: /cvsroot/pgsql/src/backend/regex/Makefile,v 1.7 1998/07/24 03:31:24 scrappy Exp $
#
#-------------------------------------------------------------------------
@ -16,7 +16,6 @@ CFLAGS += -DPOSIX_MISTAKE
OBJS = regcomp.o regerror.o regexec.o regfree.o
ifdef MB
OBJS += utils.o wstrcmp.o wstrncmp.o
CFLAGS += -DMB=$(MB)
endif

33
src/backend/regex/utftest.c
View File

@ -1,33 +0,0 @@
/*
* testing of utf2wchar()
* $Id: utftest.c,v 1.1 1998/03/15 07:38:37 scrappy Exp $
*/
#include <regex/regex.h>
#include <regex/utils.h>
#include <regex/regex2.h>
#include <regex/pg_wchar.h>
main()
{
/* Example 1 from RFC2044 */
char utf1[] = {0x41,0xe2,0x89,0xa2,0xce,0x91,0x2e,0};
/* Example 2 from RFC2044 */
char utf2[] = {0x48,0x69,0x20,0x4d,0x6f,0x6d,0x20,0xe2,0x98,0xba,0x21,0};
/* Example 3 from RFC2044 */
char utf3[] = {0xe6,0x97,0xa5,0xe6,0x9c,0xac,0xe8,0xaa,0x9e,0};
char *utf[] = {utf1,utf2,utf3};
pg_wchar ucs[128];
pg_wchar *p;
int i;
for (i=0;i<sizeof(utf)/sizeof(char *);i++) {
pg_utf2wchar(utf[i],ucs);
p = ucs;
while(*p) {
printf("%04x ",*p);
p++;
}
printf("\n");
}
}

404
src/backend/regex/utils.c
View File

@ -1,404 +0,0 @@
/*
* misc conversion functions between pg_wchar and other encodings.
* Tatsuo Ishii
* $Id: utils.c,v 1.4 1998/07/18 18:34:08 momjian Exp $
*/
#include <regex/pg_wchar.h>
/*
* conversion to pg_wchar is done by "table driven."
* to add an encoding support, define mb2wchar_with_len(), mblen()
* for the particular encoding. Note that if the encoding is only
* supported in the client, you don't need to define
* mb2wchar_with_len() function (SJIS is the case).
*/
typedef struct {
void (*mb2wchar_with_len)(); /* convert a multi-byte string to a wchar */
int (*mblen)(); /* returns the length of a multi-byte word */
} pg_wchar_tbl;
static void pg_euc2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
while (*from && len > 0) {
if (*from == SS2) {
from++;
len--;
*to = 0xff & *from++;
len--;
} else if (*from == SS3) {
from++;
*to = *from++ << 8;
*to |= 0x3f & *from++;
len -= 3;
} else if (*from & 0x80) {
*to = *from++ << 8;
*to |= *from++;
len -= 2;
} else {
*to = *from++;
len--;
}
to++;
}
*to = 0;
}
static int pg_euc_mblen(const unsigned char *s)
{
int len;
if (*s == SS2) {
len = 2;
} else if (*s == SS3) {
len = 3;
} else if (*s & 0x80) {
len = 2;
} else {
len = 1;
}
return(len);
}
/*
* EUC_JP
*/
static void pg_eucjp2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
pg_euc2wchar_with_len(from,to,len);
}
static int pg_eucjp_mblen(const unsigned char *s)
{
return(pg_euc_mblen(s));
}
/*
* EUC_KR
*/
static void pg_euckr2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
pg_euc2wchar_with_len(from,to,len);
}
static int pg_euckr_mblen(const unsigned char *s)
{
return(pg_euc_mblen(s));
}
/*
* EUC_CN
*/
static void pg_euccn2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
while (*from && len > 0) {
if (*from == SS2) {
from++;
len--;
*to = 0x3f00 & (*from++ << 8);
*to = *from++;
len -= 2;
} else if (*from == SS3) {
from++;
*to = *from++ << 8;
*to |= 0x3f & *from++;
len -= 3;
} else if (*from & 0x80) {
*to = *from++ << 8;
*to |= *from++;
len -= 2;
} else {
*to = *from++;
len--;
}
to++;
}
*to = 0;
}
static int pg_euccn_mblen(const unsigned char *s)
{
int len;
if (*s == SS2) {
len = 3;
} else if (*s == SS3) {
len = 3;
} else if (*s & 0x80) {
len = 2;
} else {
len = 1;
}
return(len);
}
/*
* EUC_TW
*/
static void pg_euctw2wchar_with_len
(const unsigned char *from, pg_wchar *to, int len)
{
while (*from && len > 0) {
if (*from == SS2) {
from++;
len--;
*to = *from++ << 16;
*to |= *from++ << 8;
*to |= *from++;
len -= 3;
} else if (*from == SS3) {
from++;
*to = *from++ << 8;
*to |= 0x3f & *from++;
len -= 3;
} else if (*from & 0x80) {
*to = *from++ << 8;
*to |= *from++;
len -= 2;
} else {
*to = *from++;
len--;
}
to++;
}
*to = 0;
}
static int pg_euctw_mblen(const unsigned char *s)
{
int len;
if (*s == SS2) {
len = 4;
} else if (*s == SS3) {
len = 3;
} else if (*s & 0x80) {
len = 2;
} else {
len = 1;
}
return(len);
}
/*
* convert UTF-8 to pg_wchar (UCS-2)
* caller should allocate enough space for "to"
* len: length of from.
* "from" not necessarily null terminated.
*/
static void pg_utf2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
unsigned char c1,c2,c3;
while (*from && len > 0) {
if ((*from & 0x80) == 0) {
*to = *from++;
len--;
} else if ((*from & 0xe0) == 0xc0) {
c1 = *from++ & 0x1f;
c2 = *from++ & 0x3f;
len -= 2;
*to = c1 << 6;
*to |= c2;
} else if ((*from & 0xe0) == 0xe0) {
c1 = *from++ & 0x0f;
c2 = *from++ & 0x3f;
c3 = *from++ & 0x3f;
len -= 3;
*to = c1 << 12;
*to |= c2 << 6;
*to |= c3;
}
to++;
}
*to = 0;
}
static int pg_utf_mblen(const unsigned char *s)
{
int len = 1;
if ((*s & 0x80) == 0) {
len = 1;
} else if ((*s & 0xe0) == 0xc0) {
len = 2;
} else if ((*s & 0xe0) == 0xe0) {
len = 3;
}
return(len);
}
/*
* convert mule internal code to pg_wchar
* caller should allocate enough space for "to"
* len: length of from.
* "from" not necessarily null terminated.
*/
static void pg_mule2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
while (*from && len > 0) {
if (IS_LC1(*from)) {
*to = *from++ << 16;
*to |= *from++;
len -= 2;
} else if (IS_LCPRV1(*from)) {
from++;
*to = *from++ << 16;
*to |= *from++;
len -= 3;
} else if (IS_LC2(*from)) {
*to = *from++ << 16;
*to |= *from++ << 8;
*to |= *from++;
len -= 3;
} else if (IS_LCPRV2(*from)) {
from++;
*to = *from++ << 16;
*to |= *from++ << 8;
*to |= *from++;
len -= 4;
} else { /* assume ASCII */
*to = (unsigned char)*from++;
len--;
}
to++;
}
*to = 0;
}
static int pg_mule_mblen(const unsigned char *s)
{
int len;
if (IS_LC1(*s)) {
len = 2;
} else if (IS_LCPRV1(*s)) {
len = 3;
} else if (IS_LC2(*s)) {
len = 3;
} else if (IS_LCPRV2(*s)) {
len = 4;
} else { /* assume ASCII */
len = 1;
}
return(len);
}
/*
* ISO8859-1
*/
static void pg_latin12wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
while (*from && len-- > 0) {
*to++ = *from++;
}
*to = 0;
}
static int pg_latin1_mblen(const unsigned char *s)
{
return(1);
}
/*
* SJIS
*/
static int pg_sjis_mblen(const unsigned char *s)
{
int len;
if (*s >= 0xa1 && *s <= 0xdf) { /* 1 byte kana? */
len = 1;
} else if (*s > 0x7f) { /* kanji? */
len = 2;
} else { /* should be ASCII */
len = 1;
}
return(len);
}
static pg_wchar_tbl pg_wchar_table[] = {
{pg_eucjp2wchar_with_len, pg_eucjp_mblen},
{pg_euccn2wchar_with_len, pg_euccn_mblen},
{pg_euckr2wchar_with_len, pg_euckr_mblen},
{pg_euctw2wchar_with_len, pg_euctw_mblen},
{pg_utf2wchar_with_len, pg_utf_mblen},
{pg_mule2wchar_with_len, pg_mule_mblen},
{pg_latin12wchar_with_len, pg_latin1_mblen},
{0, 0},
{0, 0},
{0, 0},
{0, 0},
{0, 0},
{0, 0},
{0, 0},
{0, 0},
{0, 0},
{0, pg_sjis_mblen}
};
/*
*########################################################################
*
* Public functions
*
*########################################################################
*/
/* convert a multi-byte string to a wchar */
void pg_mb2wchar(const unsigned char *from, pg_wchar *to)
{
(*pg_wchar_table[MULTIBYTE].mb2wchar_with_len)(from,to,strlen(from));
}
/* convert a multi-byte string to a wchar with a limited length */
void pg_mb2wchar_with_len(const unsigned char *from, pg_wchar *to, int len)
{
(*pg_wchar_table[MULTIBYTE].mb2wchar_with_len)(from,to,len);
}
/* returns the byte length of a multi-byte word */
int pg_mblen(const unsigned char *mbstr)
{
return((*pg_wchar_table[MULTIBYTE].mblen)(mbstr));
}
/* returns the byte length of a multi-byte word for an encoding */
int pg_encoding_mblen(int encoding, const unsigned char *mbstr)
{
return((*pg_wchar_table[encoding].mblen)(mbstr));
}
/* returns the byte length of a word for mule internal code */
int pg_mic_mblen(const unsigned char *mbstr)
{
return(pg_mule_mblen(mbstr));
}
/* returns the length (counted as a wchar) of a multi-byte string */
int pg_mbstrlen(const unsigned char *mbstr)
{
int len = 0;
while (*mbstr) {
mbstr += pg_mblen(mbstr);
len++;
}
return(len);
}
/* returns the length (counted as a wchar) of a multi-byte string
(not necessarily NULL terminated) */
int pg_mbstrlen_with_len(const unsigned char *mbstr, int limit)
{
int len = 0;
int l;
while (*mbstr && limit > 0) {
l = pg_mblen(mbstr);
limit -= l;
mbstr += l;
len++;
}
return(len);
}

48
src/backend/regex/wstrcmp.c
View File

@ -1,48 +0,0 @@
/*-
* Copyright (c) 1990, 1993
* The Regents of the University of California. All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* Chris Torek.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the University of
* California, Berkeley and its contributors.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <regex/pg_wchar.h>
int
pg_char_and_wchar_strcmp(s1, s2)
register const char *s1;
register const pg_wchar *s2;
{
while ((pg_wchar)*s1 == *s2++)
if (*s1++ == 0)
return (0);
return (*(const unsigned char *)s1 - *(const pg_wchar *)(s2 - 1));
}

83
src/backend/regex/wstrncmp.c
View File

@ -1,83 +0,0 @@
/*
* Copyright (c) 1989, 1993
* The Regents of the University of California. All rights reserved.
*
* This code is derived from FreeBSD 2.2.1-RELEASE software.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the University of
* California, Berkeley and its contributors.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <regex/pg_wchar.h>
int
pg_wchar_strncmp(s1, s2, n)
register const pg_wchar *s1, *s2;
register size_t n;
{
if (n == 0)
return (0);
do {
if (*s1 != *s2++)
return (*(const pg_wchar *)s1 -
*(const pg_wchar *)(s2 - 1));
if (*s1++ == 0)
break;
} while (--n != 0);
return (0);
}
int
pg_char_and_wchar_strncmp(s1, s2, n)
register const char *s1;
register const pg_wchar *s2;
register size_t n;
{
if (n == 0)
return (0);
do {
if ((pg_wchar )*s1 != *s2++)
return (*(const pg_wchar *)s1 -
*(const pg_wchar *)(s2 - 1));
if (*s1++ == 0)
break;
} while (--n != 0);
return (0);
}
size_t
pg_wchar_strlen(str)
const pg_wchar *str;
{
register const pg_wchar *s;
for (s = str; *s; ++s);
return(s - str);
}