feat(ddl): MCOL-5744: better handling of utf8 charset aliases (#3174)

Server expands ut8_XXX aliases to utf8mb3_XXX or utf8mb4_XXX depending
on the UTF8_IS_UTF8MB3 setting in the OLD_MODE environment variable.

Server already has the necessary code implemented in the get_utf8_flag()
method of class THD. There are several uses of this flag and all we have
to do to be in line with server is to use it.

This patch does that for DDL as work on MCOL-5705 uncovered some
problems in that area.

dbcon/ddlpackage/ddl.y

@@ -56,7 +56,7 @@ int ddllex(YYSTYPE* ddllval, void* yyscanner);
 void ddlerror(struct pass_to_bison* x, char const *s);
 char* copy_string(const char *str);
 
-void fix_column_length_and_charset(SchemaObject* elem, const CHARSET_INFO* def_cs)
+void fix_column_length_and_charset(SchemaObject* elem, const CHARSET_INFO* def_cs, myf utf8_flag)
 {
     auto* column = dynamic_cast<ColumnDef*>(elem);
 
@@ -82,9 +82,9 @@ void fix_column_length_and_charset(SchemaObject* elem, const CHARSET_INFO* def_c
         CHARSET_INFO* cs = def_cs ? def_cs : &my_charset_latin1;
 
         if (column->fType->fCollate)
-            cs = get_charset_by_name(column->fType->fCollate, MYF(0));
+            cs = get_charset_by_name(column->fType->fCollate, MYF(utf8_flag));
         else if (column->fType->fCharset)
-            cs = get_charset_by_csname(column->fType->fCharset, MY_CS_PRIMARY, MYF(0));
+            cs = get_charset_by_csname(column->fType->fCharset, MY_CS_PRIMARY, MYF(utf8_flag));
 
         column->fType->fCharset = cs->cs_name.str;
         column->fType->fCollate = cs->coll_name.str;
@@ -338,7 +338,7 @@ create_table_statement:
 	{
         for (auto* elem : *$6)
         {
-            fix_column_length_and_charset(elem, x->default_table_charset);
+            fix_column_length_and_charset(elem, x->default_table_charset, x->utf8_flag);
         }
 		$$ = new CreateTableStatement(new TableDef($4, $6, $8));
 	}
@@ -702,17 +702,17 @@ ata_add_column:
     /* See the documentation for SchemaObject for an explanation of why we are using
      * dynamic_cast here.
      */
-	ADD column_def { fix_column_length_and_charset($2, x->default_table_charset); $$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($2));}
-	| ADD COLUMN column_def { fix_column_length_and_charset($3, x->default_table_charset); $$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($3));}
+	ADD column_def { fix_column_length_and_charset($2, x->default_table_charset, x->utf8_flag); $$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($2));}
+	| ADD COLUMN column_def { fix_column_length_and_charset($3, x->default_table_charset, x->utf8_flag); $$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($3));}
 	| ADD '(' table_element_list ')' {
         for (auto* elem : *$3) {
-            fix_column_length_and_charset(elem, x->default_table_charset);
+            fix_column_length_and_charset(elem, x->default_table_charset, x->utf8_flag);
         }
         $$ = new AtaAddColumns($3);
     }
 	| ADD COLUMN '(' table_element_list ')' {
         for (auto* elem : *$4) {
-            fix_column_length_and_charset(elem, x->default_table_charset);
+            fix_column_length_and_charset(elem, x->default_table_charset, x->utf8_flag);
         }
         $$ = new AtaAddColumns($4);
     }

dbcon/ddlpackage/sqlparser.cpp

@@ -60,8 +60,9 @@ void SqlParser::setDefaultCharset(const CHARSET_INFO* default_charset)
   x.default_table_charset = default_charset;
 }
 
-int SqlParser::Parse(const char* sqltext)
+int SqlParser::Parse(const char* sqltext, myf utf8_flag)
 {
+  x.utf8_flag = utf8_flag;
   ddllex_init_extra(&scanData, &x.scanner);
   scanner_init(sqltext, x.scanner);
   fStatus = ddlparse(&x);
@@ -129,6 +130,6 @@ int SqlFileParser::Parse(const string& sqlfile)
   // cout << "----------------------" << endl;
   // cout << sqlbuf << endl;
 
-  return SqlParser::Parse(sqlbuf);
+  return SqlParser::Parse(sqlbuf, MYF(0));
 }
 }  // namespace ddlpackage

dbcon/ddlpackage/sqlparser.h

@@ -29,6 +29,7 @@
 #include <stdexcept>
 #include "collation.h"  // CHARSET_INFO
 #include "ddlpkg.h"
+#include "mariadb_my_sys.h" // myf, MYF()
 
 #define EXPORT
 
@@ -83,8 +84,14 @@ struct pass_to_bison
   std::string fDBSchema;
   void* scanner;
   const CHARSET_INFO* default_table_charset;
+  myf utf8_flag;
 
-  pass_to_bison(ParseTree* pt) : fParseTree(pt), scanner(NULL), default_table_charset(NULL){};
+  pass_to_bison(ParseTree* pt) :
+      fParseTree(pt)
+    , scanner(NULL)
+    , default_table_charset(NULL)
+    , utf8_flag(MYF(0))
+  {};
 };
 
 class SqlParser
@@ -94,7 +101,7 @@ class SqlParser
 
   EXPORT virtual ~SqlParser();
 
-  EXPORT int Parse(const char* sqltext);
+  EXPORT int Parse(const char* sqltext, myf utf8_flag);
 
   /** @brief Return the ParseTree if state is Good.  Otherwise
    *	throw a logic_error.

dbcon/mysql/ha_mcs_ddl.cpp

@@ -766,7 +766,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
   parser.setDefaultSchema(schema);
   parser.setDefaultCharset(default_table_charset);
   int rc = 0;
-  parser.Parse(ddlStatement.c_str());
+  parser.Parse(ddlStatement.c_str(), thd->get_utf8_flag());
 
   if (get_fe_conn_info_ptr() == NULL) {
     set_fe_conn_info_ptr((void*)new cal_connection_info());

dbcon/mysql/is_columnstore_columns.cpp

@@ -52,6 +52,7 @@ ST_FIELD_INFO is_columnstore_columns_fields[] = {
     Show::Column("NUMERIC_SCALE", Show::ULong(0), NOT_NULL),
     Show::Column("IS_AUTOINCREMENT", Show::STiny(0), NOT_NULL),
     Show::Column("COMPRESSION_TYPE", Show::Varchar(64), NOT_NULL),
+    Show::Column("CHARSET", Show::ULong(4), NOT_NULL),
     Show::CEnd()};
 
 static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond)
@@ -183,6 +184,8 @@ static int is_columnstore_columns_fill(THD* thd, TABLE_LIST* tables, COND* cond)
 
       table->field[15]->store(compression_type.c_str(), compression_type.length(), cs);
 
+      table->field[16]->store(ct.charsetNumber);
+
       if (schema_table_store_record(thd, table))
         return 1;
     }

mysql-test/columnstore/basic/r/MCOL-5744-utf8-in-ddl.result

@@ -0,0 +1,19 @@
+DROP DATABASE IF EXISTS MCOL5744;
+CREATE DATABASE MCOL5744;
+SET old_mode='';
+CREATE TABLE t(x text CHARACTER SET utf8 COLLATE utf8_general_ci) ENGINE=COLUMNSTORE;
+SHOW CREATE TABLE t;
+Table	Create Table
+t	CREATE TABLE `t` (
+  `x` text CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci
+DROP TABLE t;
+SET old_mode='UTF8_IS_UTF8MB3';
+CREATE TABLE t(x text CHARACTER SET utf8 COLLATE utf8_general_ci) ENGINE=COLUMNSTORE;
+SHOW CREATE TABLE t;
+Table	Create Table
+t	CREATE TABLE `t` (
+  `x` text DEFAULT NULL
+) ENGINE=Columnstore DEFAULT CHARSET=utf8mb3 COLLATE=utf8mb3_general_ci
+DROP TABLE t;
+DROP DATABASE MCOL5744;

mysql-test/columnstore/basic/t/MCOL-5744-utf8-in-ddl.test

@@ -0,0 +1,19 @@
+--disable_warnings
+DROP DATABASE IF EXISTS MCOL5744;
+--enable_warnings
+
+CREATE DATABASE MCOL5744;
+
+SET old_mode='';
+
+CREATE TABLE t(x text CHARACTER SET utf8 COLLATE utf8_general_ci) ENGINE=COLUMNSTORE;
+SHOW CREATE TABLE t;
+DROP TABLE t;
+
+SET old_mode='UTF8_IS_UTF8MB3';
+
+CREATE TABLE t(x text CHARACTER SET utf8 COLLATE utf8_general_ci) ENGINE=COLUMNSTORE;
+SHOW CREATE TABLE t;
+DROP TABLE t;
+
+DROP DATABASE MCOL5744;