database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-07-30 16:24:05 +03:00
Code Activity

MDEV-10743 LDML: a new syntax to reuse sort order from another 8bit simple collation

Browse Source
This commit is contained in:
Alexander Barkov
2016-09-06 12:37:11 +04:00
parent 8ae65920fa
commit e4f6fd5e12
10 changed files with 324 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
mysql-test/r/ctype_ldml.result
View File

@ -454,8 +454,12 @@ select "foo" = "foo " collate latin1_test;
The following tests check that two-byte collation IDs work
select * from information_schema.collations where id>256 and is_compiled<>'Yes' order by id;
COLLATION_NAME CHARACTER_SET_NAME ID IS_DEFAULT IS_COMPILED SORTLEN
ascii2_bin2 ascii2 319 1
ascii2_general_ci ascii2 320 Yes 1
ascii2_bin ascii2 321 1
ascii2_general_inherited_ci ascii2 322 1
ascii2_general_inherited2_ci ascii2 323 1
ascii2_badly_inherited_ci ascii2 324 1
utf8mb4_test_ci utf8mb4 326 8
utf16_test_ci utf16 327 8
utf8mb4_test_400_ci utf8mb4 328 8
@ -1185,6 +1189,7 @@ ch w ducet
DROP TABLE t1;
#
# Testing that the MY_CS_PUREASCII flag is set properly
# Comparison between ascii2 and latin1 should not give "illegal collation error"
#
CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET ascii2, b VARCHAR(10) CHARACTER SET latin1);
INSERT INTO t1 VALUES ('a','a'),('b','b');
@ -1192,13 +1197,54 @@ SELECT * FROM t1 WHERE a=b;
a b
a a
b b
ALTER TABLE t1 MODIFY a VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_bin2;
SELECT * FROM t1 WHERE a=b;
a b
a a
b b
ALTER TABLE t1 MODIFY a VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_bin;
SELECT * FROM t1 WHERE a=b;
a b
a a
b b
ALTER TABLE t1 MODIFY a VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_general_inherited_ci;
SELECT * FROM t1 WHERE a=b;
a b
a a
b b
ALTER TABLE t1 MODIFY a VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_general_inherited2_ci;
SELECT * FROM t1 WHERE a=b;
a b
a a
b b
DROP TABLE t1;
#
# Testing that in case of two binary collations
# "BINARY" in a column definition uses the collation with the least id
#
CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET ascii2 BINARY);
INSERT INTO t1 VALUES ('test');
SELECT COLLATION(a) FROM t1;
COLLATION(a)
ascii2_bin2
DROP TABLE t1;
#
# Testing mixing of two binary collations of the same character set
#
CREATE TABLE t1 (
a VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_bin,
b VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_bin2
);
INSERT INTO t1 VALUES ('a','a');
SELECT * FROM t1 WHERE a=b;
ERROR HY000: Illegal mix of collations (ascii2_bin,IMPLICIT) and (ascii2_bin2,IMPLICIT) for operation '='
DROP TABLE t1;
#
# Testing bad collation inheritance
#
CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_badly_inherited_ci);
ERROR HY000: Unknown collation: 'ascii2_badly_inherited_ci'
#
# Testing that the MY_CS_CSSORT flag is set properly
#
CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET latin1 COLLATE latin1_test);

21
mysql-test/std_data/ldml/Index.xml
View File

@ -319,8 +319,29 @@
</charset>
<charset name="ascii2">
<!--
Notes:
- ascii2 has two collations with "binary" flag.
ctype_ldml.test makes sure that
CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET ascii2 BINARY);
uses ascii2_bin2, which is the collation with the least ID.
- ascii2_general_inherited_ci inherits sort order in ascii2.xml
- ascii2_genegal_inherited2_ci inherits sort order directly in this file.
-->
<collation name="ascii2_bin2" id="319" flag="binary"/>
<collation name="ascii2_general_ci" id="320" flag="primary"/>
<collation name="ascii2_bin" id="321" flag="binary"/>
<collation name="ascii2_general_inherited_ci" id="322"/>
<collation name="ascii2_general_inherited2_ci" id="323">
<rules>
<import source="ascii2_general_ci"/>
</rules>
</collation>
<collation name="ascii2_badly_inherited_ci" id="324">
<rules>
<import source="ascii2_non_existing_ci"/>
</rules>
</collation>
</charset>
<charset name="latin1">

6
mysql-test/std_data/ldml/ascii2.xml
View File

@ -116,6 +116,12 @@
<collation name="ascii2_bin" flag="binary"/>
<collation name="ascii2_general_inherited_ci">
<rules>
<import source="ascii2_general_ci"/>
</rules>
</collation>
</charset>
</charsets>

4
mysql-test/suite/innodb/r/innodb_ctype_ldml.result
View File

@ -390,8 +390,12 @@ select "foo" = "foo " collate latin1_test;
The following tests check that two-byte collation IDs work
select * from information_schema.collations where id>256 and is_compiled<>'Yes' order by id;
COLLATION_NAME CHARACTER_SET_NAME ID IS_DEFAULT IS_COMPILED SORTLEN
ascii2_bin2 ascii2 319 1
ascii2_general_ci ascii2 320 Yes 1
ascii2_bin ascii2 321 1
ascii2_general_inherited_ci ascii2 322 1
ascii2_general_inherited2_ci ascii2 323 1
ascii2_badly_inherited_ci ascii2 324 1
utf8mb4_test_ci utf8mb4 326 8
utf16_test_ci utf16 327 8
utf8mb4_test_400_ci utf8mb4 328 8

36
mysql-test/t/ctype_ldml.test
View File

@ -413,16 +413,48 @@ DROP TABLE t1;
--echo #
--echo # Testing that the MY_CS_PUREASCII flag is set properly
--echo # Comparison between ascii2 and latin1 should not give "illegal collation error"
--echo #
CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET ascii2, b VARCHAR(10) CHARACTER SET latin1);
INSERT INTO t1 VALUES ('a','a'),('b','b');
# should not give "illegal collation" error
SELECT * FROM t1 WHERE a=b;
ALTER TABLE t1 MODIFY a VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_bin2;
SELECT * FROM t1 WHERE a=b;
ALTER TABLE t1 MODIFY a VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_bin;
# should not give "illegal collation" error
SELECT * FROM t1 WHERE a=b;
ALTER TABLE t1 MODIFY a VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_general_inherited_ci;
SELECT * FROM t1 WHERE a=b;
ALTER TABLE t1 MODIFY a VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_general_inherited2_ci;
SELECT * FROM t1 WHERE a=b;
DROP TABLE t1;
--echo #
--echo # Testing that in case of two binary collations
--echo # "BINARY" in a column definition uses the collation with the least id
--echo #
CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET ascii2 BINARY);
INSERT INTO t1 VALUES ('test');
SELECT COLLATION(a) FROM t1;
DROP TABLE t1;
--echo #
--echo # Testing mixing of two binary collations of the same character set
--echo #
CREATE TABLE t1 (
a VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_bin,
b VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_bin2
);
INSERT INTO t1 VALUES ('a','a');
--error ER_CANT_AGGREGATE_2COLLATIONS
SELECT * FROM t1 WHERE a=b;
DROP TABLE t1;
--echo #
--echo # Testing bad collation inheritance
--echo #
--error ER_UNKNOWN_COLLATION
CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET ascii2 COLLATE ascii2_badly_inherited_ci);
--echo #
--echo # Testing that the MY_CS_CSSORT flag is set properly

98
mysys/charset.c
View File

@ -197,13 +197,55 @@ err:
}
static my_bool simple_8bit_charset_data_is_full(CHARSET_INFO *cs)
{
return cs->ctype && cs->to_upper && cs->to_lower && cs->tab_to_uni;
}
/**
Inherit missing 8bit charset data from another collation.
Arrays pointed by refcs must be in the permanent memory already,
e.g. static memory, or allocated by my_once_xxx().
*/
static void
inherit_charset_data(struct charset_info_st *cs, CHARSET_INFO *refcs)
{
if (!cs->to_upper)
cs->to_upper= refcs->to_upper;
if (!cs->to_lower)
cs->to_lower= refcs->to_lower;
if (!cs->ctype)
cs->ctype= refcs->ctype;
if (!cs->tab_to_uni)
cs->tab_to_uni= refcs->tab_to_uni;
}
static my_bool simple_8bit_collation_data_is_full(CHARSET_INFO *cs)
{
return cs->sort_order || (cs->state & MY_CS_BINSORT);
}
/**
Inherit 8bit simple collation data from another collation.
refcs->sort_order must be in the permanent memory already,
e.g. static memory, or allocated by my_once_xxx().
*/
static void
inherit_collation_data(struct charset_info_st *cs, CHARSET_INFO *refcs)
{
if (!simple_8bit_collation_data_is_full(cs))
cs->sort_order= refcs->sort_order;
}
static my_bool simple_cs_is_full(CHARSET_INFO *cs)
{
return ((cs->csname && cs->tab_to_uni && cs->ctype && cs->to_upper &&
cs->to_lower) &&
(cs->number && cs->name &&
(cs->sort_order || (cs->state & MY_CS_BINSORT) )));
return cs->number && cs->csname && cs->name &&
simple_8bit_charset_data_is_full(cs) &&
(simple_8bit_collation_data_is_full(cs) || cs->tailoring);
}
@ -336,7 +378,7 @@ static int add_collation(struct charset_info_st *cs)
cs->name= NULL;
cs->state= 0;
cs->sort_order= NULL;
cs->state= 0;
cs->tailoring= NULL;
}
return MY_XML_OK;
}
@ -631,6 +673,39 @@ const char *get_charset_name(uint charset_number)
}
static CHARSET_INFO *inheritance_source_by_id(CHARSET_INFO *cs, uint refid)
{
CHARSET_INFO *refcs;
return refid && refid != cs->number &&
(refcs= all_charsets[refid]) &&
(refcs->state & MY_CS_AVAILABLE) ? refcs : NULL;
}
static CHARSET_INFO *find_collation_data_inheritance_source(CHARSET_INFO *cs)
{
const char *beg, *end;
if (cs->tailoring &&
!strncmp(cs->tailoring, "[import ", 8) &&
(end= strchr(cs->tailoring + 8, ']')) &&
(beg= cs->tailoring + 8) + MY_CS_NAME_SIZE > end)
{
char name[MY_CS_NAME_SIZE + 1];
memcpy(name, beg, end - beg);
name[end - beg]= '\0';
return inheritance_source_by_id(cs, get_collation_number(name));
}
return NULL;
}
static CHARSET_INFO *find_charset_data_inheritance_source(CHARSET_INFO *cs)
{
uint refid= get_charset_number_internal(cs->csname, MY_CS_PRIMARY);
return inheritance_source_by_id(cs, refid);
}
static CHARSET_INFO *
get_internal_charset(MY_CHARSET_LOADER *loader, uint cs_number, myf flags)
{
@ -665,6 +740,19 @@ get_internal_charset(MY_CHARSET_LOADER *loader, uint cs_number, myf flags)
{
if (!(cs->state & MY_CS_READY))
{
if (!simple_8bit_charset_data_is_full(cs))
{
CHARSET_INFO *refcs= find_charset_data_inheritance_source(cs);
if (refcs)
inherit_charset_data(cs, refcs);
}
if (!simple_8bit_collation_data_is_full(cs))
{
CHARSET_INFO *refcl= find_collation_data_inheritance_source(cs);
if (refcl)
inherit_collation_data(cs, refcl);
}
if ((cs->cset->init && cs->cset->init(cs, loader)) ||
(cs->coll->init && cs->coll->init(cs, loader)))
{

3
sql/item.cc
View File

@ -2078,6 +2078,9 @@ bool DTCollation::aggregate(const DTCollation &dt, uint flags)
set(0, DERIVATION_NONE, 0);
return 1;
}
if (collation->state & MY_CS_BINSORT &&
dt.collation->state & MY_CS_BINSORT)
return 1;
if (collation->state & MY_CS_BINSORT)
return 0;
if (dt.collation->state & MY_CS_BINSORT)

124
strings/conf_to_src.c
View File

@ -120,6 +120,9 @@ static void simple_cs_copy_data(struct charset_info_st *to, CHARSET_INFO *from)
if (from->name)
to->name= strdup(from->name);
if (from->tailoring)
to->tailoring= strdup(from->tailoring);
if (from->ctype)
to->ctype= (uchar*) mdup((char*) from->ctype, MY_CS_CTYPE_TABLE_SIZE);
if (from->to_lower)
@ -144,30 +147,60 @@ static void simple_cs_copy_data(struct charset_info_st *to, CHARSET_INFO *from)
}
static void inherit_data(struct charset_info_st *cs, CHARSET_INFO *refcs)
/*
cs->xxx arrays can be NULL in case when a collation has an entry only
in Index.xml and has no entry in csname.xml (e.g. in case of a binary
collation or a collation using <import> command).
refcs->xxx arrays can be NULL if <import> refers to a collation
which is not defined in csname.xml, e.g. an always compiled collation
such as latin1_swedish_ci.
*/
static void inherit_charset_data(struct charset_info_st *cs,
CHARSET_INFO *refcs)
{
if (refcs->ctype &&
cs->state|= (refcs->state & (MY_CS_PUREASCII|MY_CS_NONASCII));
if (refcs->ctype && cs->ctype &&
!memcmp(cs->ctype, refcs->ctype, MY_CS_CTYPE_TABLE_SIZE))
cs->ctype= NULL;
if (refcs->to_lower &&
if (refcs->to_lower && cs->to_lower &&
!memcmp(cs->to_lower, refcs->to_lower, MY_CS_TO_LOWER_TABLE_SIZE))
cs->to_lower= NULL;
if (refcs->to_upper &&
if (refcs->to_upper && cs->to_upper &&
!memcmp(cs->to_upper, refcs->to_upper, MY_CS_TO_LOWER_TABLE_SIZE))
cs->to_upper= NULL;
if (refcs->tab_to_uni &&
if (refcs->tab_to_uni && cs->tab_to_uni &&
!memcmp(cs->tab_to_uni, refcs->tab_to_uni,
MY_CS_TO_UNI_TABLE_SIZE * sizeof(uint16)))
cs->tab_to_uni= NULL;
}
static CHARSET_INFO *find_charset_data_inheritance_source(CHARSET_INFO *cs)
{
CHARSET_INFO *refcs;
uint refid= get_charset_number_internal(cs->csname, MY_CS_PRIMARY);
return refid && refid != cs->number &&
(refcs= &all_charsets[refid]) &&
(refcs->state & MY_CS_LOADED) ? refcs : NULL;
}
/**
Detect if "cs" needs further loading from csname.xml
@param cs - the character set pointer
@retval FALSE - if the current data (e.g. loaded from from Index.xml)
is not enough to dump the character set and requires
further reading from the csname.xml file.
@retval TRUE - if the current data is enough to dump,
no reading of csname.xml is needed.
*/
static my_bool simple_cs_is_full(CHARSET_INFO *cs)
{
return ((cs->csname && cs->tab_to_uni && cs->ctype && cs->to_upper &&
cs->to_lower) &&
(cs->number && cs->name &&
(cs->sort_order || (cs->state & MY_CS_BINSORT))));
(cs->sort_order || cs->tailoring || (cs->state & MY_CS_BINSORT))));
}
static int add_collation(struct charset_info_st *cs)
@ -183,6 +216,7 @@ static int add_collation(struct charset_info_st *cs)
cs->number= 0;
cs->name= NULL;
cs->tailoring= NULL;
cs->state= 0;
cs->sort_order= NULL;
cs->state= 0;
@ -255,6 +289,55 @@ void print_arrays(FILE *f, CHARSET_INFO *cs)
}
/**
Print an array member of a CHARSET_INFO.
@param f - the file to print into
@param cs0 - reference to the CHARSET_INFO to print
@param array0 - pointer to the array data (can be NULL)
@param cs1 - reference to the CHARSET_INFO that the data
can be inherited from (e.g. primary collation)
@param array1 - pointer to the array data in cs1 (can be NULL)
@param name - name of the member
If array0 is not null, then the CHARSET_INFO being dumped has its
own array (e.g. the default collation for the character set).
We print the name of this array using cs0->name and return.
If array1 is not null, then the CHARSET_INFO being dumpled reuses
the array from another collation. We print the name of the array of
the referenced collation using cs1->name and return.
Otherwise (if both array0 and array1 are NULL), we have a collation
of a character set whose primary collation is not available now,
and which does not have its own entry in csname.xml file.
For example, Index.xml has this entry:
<collation name="latin1_swedish_ci_copy">
<rules>
<import source="latin1_swedish_ci"/>
</rules>
</collation>
and latin1.xml does not have entries for latin1_swedish_ci_copy.
In such cases we print NULL as a pointer to the array.
It will be set to a not-null data during the first initialization
by the inherit_charset_data() call (see mysys/charset.c for details).
*/
static void
print_array_ref(FILE *f,
CHARSET_INFO *cs0, const void *array0,
CHARSET_INFO *cs1, const void *array1,
const char *name)
{
CHARSET_INFO *cs= array0 ? cs0 : array1 ? cs1 : NULL;
if (cs)
fprintf(f," %s_%s, /* %s */\n",
name, cs->name, name);
else
fprintf(f," NULL, /* %s */\n", name);
}
void dispcset(FILE *f,CHARSET_INFO *cs)
{
fprintf(f,"{\n");
@ -272,21 +355,23 @@ void dispcset(FILE *f,CHARSET_INFO *cs)
fprintf(f," \"%s\", /* cset name */\n",cs->csname);
fprintf(f," \"%s\", /* coll name */\n",cs->name);
fprintf(f," \"\", /* comment */\n");
fprintf(f," NULL, /* tailoring */\n");
if (cs->tailoring)
fprintf(f, " \"%s\", /* tailoring */\n", cs->tailoring);
else
fprintf(f," NULL, /* tailoring */\n");
print_array_ref(f, cs, cs->ctype, srccs, srccs->ctype, "ctype");
print_array_ref(f, cs, cs->to_lower, srccs, srccs->to_lower, "to_lower");
print_array_ref(f, cs, cs->to_upper, srccs, srccs->to_upper, "to_upper");
fprintf(f," ctype_%s, /* ctype */\n",
cs->ctype ? cs->name : srccs->name);
fprintf(f," to_lower_%s, /* lower */\n",
cs->to_lower ? cs->name : srccs->name);
fprintf(f," to_upper_%s, /* upper */\n",
cs->to_upper ? cs->name : srccs->name);
if (cs->sort_order)
fprintf(f," sort_order_%s, /* sort_order */\n",cs->name);
else
fprintf(f," NULL, /* sort_order */\n");
fprintf(f," NULL, /* uca */\n");
fprintf(f," to_uni_%s, /* to_uni */\n",
cs->tab_to_uni ? cs->name : srccs->name);
print_array_ref(f, cs, cs->tab_to_uni, srccs, srccs->tab_to_uni, "to_uni");
}
else
{
@ -403,14 +488,13 @@ main(int argc, char **argv __attribute__((unused)))
{
if (cs->state & MY_CS_LOADED)
{
uint refid= get_charset_number_internal(cs->csname, MY_CS_PRIMARY);
CHARSET_INFO *refcs= find_charset_data_inheritance_source(cs);
cs->state|= my_8bit_charset_flags_from_data(cs) |
my_8bit_collation_flags_from_data(cs);
if (refid && cs->number != refid)
if (refcs)
{
CHARSET_INFO *refcs= &all_charsets[refid];
refids[cs->number]= refid;
inherit_data(cs, refcs);
refids[cs->number]= refcs->number;
inherit_charset_data(cs, refcs);
}
fprintf(f,"#ifdef HAVE_CHARSET_%s\n",cs->csname);
print_arrays(f, cs);

4
strings/ctype-simple.c
View File

@ -1417,6 +1417,8 @@ my_cset_init_8bit(struct charset_info_st *cs, MY_CHARSET_LOADER *loader)
cs->caseup_multiply= 1;
cs->casedn_multiply= 1;
cs->pad_char= ' ';
if (!cs->to_lower || !cs->to_upper || !cs->ctype || !cs->tab_to_uni)
return TRUE;
return create_fromuni(cs, loader);
}
@ -1442,6 +1444,8 @@ static void set_max_sort_char(struct charset_info_st *cs)
static my_bool my_coll_init_simple(struct charset_info_st *cs,
MY_CHARSET_LOADER *loader __attribute__((unused)))
{
if (!cs->sort_order)
return TRUE;
cs->state|= my_8bit_collation_flags_from_data(cs);
set_max_sort_char(cs);
return FALSE;

8
strings/ctype.c
View File

@ -88,6 +88,8 @@ struct my_cs_file_section_st
#define _CS_CL_SUPPRESS_CONTRACTIONS 101
#define _CS_CL_OPTIMIZE 102
#define _CS_CL_SHIFT_AFTER_METHOD 103
#define _CS_CL_RULES_IMPORT 104
#define _CS_CL_RULES_IMPORT_SOURCE 105
/* Collation Settings */
@ -188,6 +190,8 @@ static const struct my_cs_file_section_st sec[] =
{_CS_CL_SUPPRESS_CONTRACTIONS, "charsets/charset/collation/suppress_contractions"},
{_CS_CL_OPTIMIZE, "charsets/charset/collation/optimize"},
{_CS_CL_SHIFT_AFTER_METHOD, "charsets/charset/collation/shift-after-method"},
{_CS_CL_RULES_IMPORT, "charsets/charset/collation/rules/import"},
{_CS_CL_RULES_IMPORT_SOURCE, "charsets/charset/collation/rules/import/source"},
/* Collation Settings */
{_CS_ST_SETTINGS, "charsets/charset/collation/settings"},
@ -641,6 +645,10 @@ static int cs_value(MY_XML_PARSER *st,const char *attr, size_t len)
rc= tailoring_append(st, "[version %.*s]", len, attr);
break;
case _CS_CL_RULES_IMPORT_SOURCE:
rc= tailoring_append(st, "[import %.*s]", len, attr);
break;
case _CS_CL_SUPPRESS_CONTRACTIONS:
rc= tailoring_append(st, "[suppress contractions %.*s]", len, attr);
break;