From ef451af860256fc2842d4d28ef93f06c3602428f Mon Sep 17 00:00:00 2001 From: Sergey Zefirov <72864488+mariadb-SergeyZefirov@users.noreply.github.com> Date: Fri, 10 May 2024 19:17:57 +0300 Subject: [PATCH] feat(ddl): MCOL-5744: better handling of utf8 charset aliases (#3174) Server expands ut8_XXX aliases to utf8mb3_XXX or utf8mb4_XXX depending on the UTF8_IS_UTF8MB3 setting in the OLD_MODE environment variable. Server already has the necessary code implemented in the get_utf8_flag() method of class THD. There are several uses of this flag and all we have to do to be in line with server is to use it. This patch does that for DDL as work on MCOL-5705 uncovered some problems in that area. --- dbcon/ddlpackage/ddl.y | 16 ++++++++-------- dbcon/ddlpackage/sqlparser.cpp | 5 +++-- dbcon/ddlpackage/sqlparser.h | 7 +++++-- dbcon/mysql/ha_mcs_ddl.cpp | 2 +- dbcon/mysql/is_columnstore_columns.cpp | 3 +++ .../basic/r/MCOL-5744-utf8-in-ddl.result | 19 +++++++++++++++++++ .../basic/t/MCOL-5744-utf8-in-ddl.test | 19 +++++++++++++++++++ 7 files changed, 58 insertions(+), 13 deletions(-) create mode 100644 mysql-test/columnstore/basic/r/MCOL-5744-utf8-in-ddl.result create mode 100644 mysql-test/columnstore/basic/t/MCOL-5744-utf8-in-ddl.test diff --git a/dbcon/ddlpackage/ddl.y b/dbcon/ddlpackage/ddl.y index 8aaa0307e..46d31bad8 100644 --- a/dbcon/ddlpackage/ddl.y +++ b/dbcon/ddlpackage/ddl.y @@ -56,7 +56,7 @@ int ddllex(YYSTYPE* ddllval, void* yyscanner); void ddlerror(struct pass_to_bison* x, char const *s); char* copy_string(const char *str); -void fix_column_length_and_charset(SchemaObject* elem, const CHARSET_INFO* def_cs) +void fix_column_length_and_charset(SchemaObject* elem, const CHARSET_INFO* def_cs, myf utf8_flag) { auto* column = dynamic_cast(elem); @@ -82,9 +82,9 @@ void fix_column_length_and_charset(SchemaObject* elem, const CHARSET_INFO* def_c CHARSET_INFO* cs = def_cs ? def_cs : &my_charset_latin1; if (column->fType->fCollate) - cs = get_charset_by_name(column->fType->fCollate, MYF(0)); + cs = get_charset_by_name(column->fType->fCollate, MYF(utf8_flag)); else if (column->fType->fCharset) - cs = get_charset_by_csname(column->fType->fCharset, MY_CS_PRIMARY, MYF(0)); + cs = get_charset_by_csname(column->fType->fCharset, MY_CS_PRIMARY, MYF(utf8_flag)); column->fType->fCharset = cs->cs_name.str; column->fType->fCollate = cs->coll_name.str; @@ -355,7 +355,7 @@ create_table_statement: { for (auto* elem : *$6) { - fix_column_length_and_charset(elem, x->default_table_charset); + fix_column_length_and_charset(elem, x->default_table_charset, x->utf8_flag); } $$ = new CreateTableStatement(new TableDef($4, $6, $8)); } @@ -719,17 +719,17 @@ ata_add_column: /* See the documentation for SchemaObject for an explanation of why we are using * dynamic_cast here. */ - ADD column_def { fix_column_length_and_charset($2, x->default_table_charset); $$ = new AtaAddColumn(dynamic_cast($2));} - | ADD COLUMN column_def { fix_column_length_and_charset($3, x->default_table_charset); $$ = new AtaAddColumn(dynamic_cast($3));} + ADD column_def { fix_column_length_and_charset($2, x->default_table_charset, x->utf8_flag); $$ = new AtaAddColumn(dynamic_cast($2));} + | ADD COLUMN column_def { fix_column_length_and_charset($3, x->default_table_charset, x->utf8_flag); $$ = new AtaAddColumn(dynamic_cast($3));} | ADD '(' table_element_list ')' { for (auto* elem : *$3) { - fix_column_length_and_charset(elem, x->default_table_charset); + fix_column_length_and_charset(elem, x->default_table_charset, x->utf8_flag); } $$ = new AtaAddColumns($3); } | ADD COLUMN '(' table_element_list ')' { for (auto* elem : *$4) { - fix_column_length_and_charset(elem, x->default_table_charset); + fix_column_length_and_charset(elem, x->default_table_charset, x->utf8_flag); } $$ = new AtaAddColumns($4); } diff --git a/dbcon/ddlpackage/sqlparser.cpp b/dbcon/ddlpackage/sqlparser.cpp index 4a2f0c191..8fb08905b 100644 --- a/dbcon/ddlpackage/sqlparser.cpp +++ b/dbcon/ddlpackage/sqlparser.cpp @@ -60,8 +60,9 @@ void SqlParser::setDefaultCharset(const CHARSET_INFO* default_charset) x.default_table_charset = default_charset; } -int SqlParser::Parse(const char* sqltext) +int SqlParser::Parse(const char* sqltext, myf utf8_flag) { + x.utf8_flag = utf8_flag; ddllex_init_extra(&scanData, &x.scanner); scanner_init(sqltext, x.scanner); fStatus = ddlparse(&x); @@ -129,6 +130,6 @@ int SqlFileParser::Parse(const string& sqlfile) // cout << "----------------------" << endl; // cout << sqlbuf << endl; - return SqlParser::Parse(sqlbuf); + return SqlParser::Parse(sqlbuf, MYF(0)); } } // namespace ddlpackage diff --git a/dbcon/ddlpackage/sqlparser.h b/dbcon/ddlpackage/sqlparser.h index 3dc18dfc9..460433fc6 100644 --- a/dbcon/ddlpackage/sqlparser.h +++ b/dbcon/ddlpackage/sqlparser.h @@ -29,6 +29,7 @@ #include #include "collation.h" // CHARSET_INFO #include "ddlpkg.h" +#include "mariadb_my_sys.h" // myf, MYF() #define EXPORT @@ -83,8 +84,10 @@ struct pass_to_bison std::string fDBSchema; void* scanner; const CHARSET_INFO* default_table_charset; + myf utf8_flag; - pass_to_bison(ParseTree* pt) : fParseTree(pt), scanner(nullptr), default_table_charset(nullptr){}; + pass_to_bison(ParseTree* pt) + : fParseTree(pt), scanner(NULL), default_table_charset(NULL), utf8_flag(MYF(0)){}; }; class SqlParser @@ -94,7 +97,7 @@ class SqlParser EXPORT virtual ~SqlParser(); - EXPORT int Parse(const char* sqltext); + EXPORT int Parse(const char* sqltext, myf utf8_flag); /** @brief Return the ParseTree if state is Good. Otherwise * throw a logic_error. diff --git a/dbcon/mysql/ha_mcs_ddl.cpp b/dbcon/mysql/ha_mcs_ddl.cpp index ad7e77fad..32888b8a3 100644 --- a/dbcon/mysql/ha_mcs_ddl.cpp +++ b/dbcon/mysql/ha_mcs_ddl.cpp @@ -765,7 +765,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& /*ta parser.setDefaultSchema(schema); parser.setDefaultCharset(default_table_charset); int rc = 0; - parser.Parse(ddlStatement.c_str()); + parser.Parse(ddlStatement.c_str(), thd->get_utf8_flag()); if (get_fe_conn_info_ptr() == NULL) { diff --git a/dbcon/mysql/is_columnstore_columns.cpp b/dbcon/mysql/is_columnstore_columns.cpp index 31d409479..d5bc8acaf 100644 --- a/dbcon/mysql/is_columnstore_columns.cpp +++ b/dbcon/mysql/is_columnstore_columns.cpp @@ -52,6 +52,7 @@ ST_FIELD_INFO is_columnstore_columns_fields[] = { Show::Column("NUMERIC_SCALE", Show::ULong(0), NOT_NULL), Show::Column("IS_AUTOINCREMENT", Show::STiny(0), NOT_NULL), Show::Column("COMPRESSION_TYPE", Show::Varchar(64), NOT_NULL), + Show::Column("CHARSET", Show::ULong(4), NOT_NULL), Show::CEnd()}; static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond) @@ -183,6 +184,8 @@ static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond) table->field[15]->store(compression_type.c_str(), compression_type.length(), cs); + table->field[16]->store(ct.charsetNumber); + if (schema_table_store_record(thd, table)) return 1; } diff --git a/mysql-test/columnstore/basic/r/MCOL-5744-utf8-in-ddl.result b/mysql-test/columnstore/basic/r/MCOL-5744-utf8-in-ddl.result new file mode 100644 index 000000000..dfd4742cd --- /dev/null +++ b/mysql-test/columnstore/basic/r/MCOL-5744-utf8-in-ddl.result @@ -0,0 +1,19 @@ +DROP DATABASE IF EXISTS MCOL5744; +CREATE DATABASE MCOL5744; +SET old_mode=''; +CREATE TABLE t(x text CHARACTER SET utf8 COLLATE utf8_general_ci) ENGINE=COLUMNSTORE; +SHOW CREATE TABLE t; +Table Create Table +t CREATE TABLE `t` ( + `x` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL +) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci +DROP TABLE t; +SET old_mode='UTF8_IS_UTF8MB3'; +CREATE TABLE t(x text CHARACTER SET utf8 COLLATE utf8_general_ci) ENGINE=COLUMNSTORE; +SHOW CREATE TABLE t; +Table Create Table +t CREATE TABLE `t` ( + `x` text DEFAULT NULL +) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci +DROP TABLE t; +DROP DATABASE MCOL5744; diff --git a/mysql-test/columnstore/basic/t/MCOL-5744-utf8-in-ddl.test b/mysql-test/columnstore/basic/t/MCOL-5744-utf8-in-ddl.test new file mode 100644 index 000000000..a8d0eb9a5 --- /dev/null +++ b/mysql-test/columnstore/basic/t/MCOL-5744-utf8-in-ddl.test @@ -0,0 +1,19 @@ +--disable_warnings +DROP DATABASE IF EXISTS MCOL5744; +--enable_warnings + +CREATE DATABASE MCOL5744; + +SET old_mode=''; + +CREATE TABLE t(x text CHARACTER SET utf8 COLLATE utf8_general_ci) ENGINE=COLUMNSTORE; +SHOW CREATE TABLE t; +DROP TABLE t; + +SET old_mode='UTF8_IS_UTF8MB3'; + +CREATE TABLE t(x text CHARACTER SET utf8 COLLATE utf8_general_ci) ENGINE=COLUMNSTORE; +SHOW CREATE TABLE t; +DROP TABLE t; + +DROP DATABASE MCOL5744;