database/postgres
1
0
Fork 0
mirror of https://github.com/postgres/postgres.git synced 2025-10-16 17:07:43 +03:00
Code Activity

Generate EUC_CN mappings from gb18030-2022.ucm

Browse Source 
In the wake of cfa6cd292, EUC_CN was the only encoding that used
gb-18030-2000.xml to generate the .map files. Since EUC_CN is a subset
of GB18030, we can easily use the same UCM file. This allows deleting
the XML file from our repository.

Author: Chao Li <lic@highgo.com>
Discussion: https://postgr.es/m/CANWCAZaNRXZ-5NuXmsaMA2mKvMZnCGHZqQusLkpE%2B8YX%2Bi5OYg%40mail.gmail.com
This commit is contained in:
John Naylor
2025-10-02 12:36:24 +07:00
parent 684a745f55
commit 48566180ef
3 changed files with 23 additions and 30929 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/backend/utils/mb/Unicode/Makefile
View File

@@ -50,7 +50,7 @@ $(eval $(call map_rule,gbk,UCS_to_most.pl,CP936.TXT,GBK))
$(eval $(call map_rule,johab,UCS_to_JOHAB.pl,JOHAB.TXT))
$(eval $(call map_rule,uhc,UCS_to_UHC.pl,windows-949-2000.xml))
$(eval $(call map_rule,euc_jp,UCS_to_EUC_JP.pl,CP932.TXT JIS0212.TXT))
$(eval $(call map_rule,euc_cn,UCS_to_EUC_CN.pl,gb-18030-2000.xml))
$(eval $(call map_rule,euc_cn,UCS_to_EUC_CN.pl,gb18030-2022.ucm))
$(eval $(call map_rule,euc_kr,UCS_to_EUC_KR.pl,KSX1001.TXT))
$(eval $(call map_rule,euc_tw,UCS_to_EUC_TW.pl,CNS11643.TXT))
$(eval $(call map_rule,sjis,UCS_to_SJIS.pl,CP932.TXT))
@@ -75,7 +75,7 @@ BIG5.TXT CNS11643.TXT:
euc-jis-2004-std.txt sjis-0213-2004-std.txt:
$(DOWNLOAD) http://x0213.org/codetable/$(@F)
gb-18030-2000.xml windows-949-2000.xml:
windows-949-2000.xml:
$(DOWNLOAD) https://raw.githubusercontent.com/unicode-org/icu-data/master/charset/data/xml/$(@F)
gb18030-2022.ucm:

32
src/backend/utils/mb/Unicode/UCS_to_EUC_CN.pl
View File

@@ -2,16 +2,17 @@
#
# Copyright (c) 2007-2025, PostgreSQL Global Development Group
#
# src/backend/utils/mb/Unicode/UCS_to_GB18030.pl
# src/backend/utils/mb/Unicode/UCS_to_EUC_CN.pl
#
# Generate UTF-8 <--> GB18030 code conversion tables from
# "gb-18030-2000.xml", obtained from
# http://source.icu-project.org/repos/icu/data/trunk/charset/data/xml/
# Generate UTF-8 <--> EUC_CN code conversion tables from
# "gb18030-2022.ucm", obtained from
# https://github.com/unicode-org/icu/blob/main/icu4c/source/data/mappings/
#
# The lines we care about in the source file look like
# <a u="009A" b="81 30 83 36"/>
# where the "u" field is the Unicode code point in hex,
# and the "b" field is the hex byte sequence for GB18030
# <UXXXX> \xYY[\xYY...] |n
# where XXXX is the Unicode code point in hex,
# and the \xYY... is the hex byte sequence for GB18030,
# and n is a flag indicating the type of mapping.
use strict;
use warnings FATAL => 'all';
@@ -22,7 +23,7 @@ my $this_script = 'src/backend/utils/mb/Unicode/UCS_to_EUC_CN.pl';
# Read the input
my $in_file = "gb-18030-2000.xml";
my $in_file = "gb18030-2022.ucm";
open(my $in, '<', $in_file) || die("cannot open $in_file");
@@ -30,9 +31,18 @@ my @mapping;
while (<$in>)
{
next if (!m/<a u="([0-9A-F]+)" b="([0-9A-F ]+)"/);
my ($u, $c) = ($1, $2);
$c =~ s/ //g;
# Mappings may have been removed by commenting out
next if /^#/;
next if !/^<U([0-9A-Fa-f]+)>\s+
((?:\\x[0-9A-Fa-f]{2})+)\s+
\|(\d+)/x;
my ($u, $c, $flag) = ($1, $2, $3);
$c =~ s/\\x//g;
# We only want round-trip mappings
next if ($flag ne '0');
my $ucs = hex($u);
my $code = hex($c);

30916
src/backend/utils/mb/Unicode/gb-18030-2000.xml
View File

File diff suppressed because it is too large Load Diff