MCOL-2000 Process charset definitions in the DDL

MCOL-2000 Process charset definitions in the ALTER TABLE .. ADD COLUMN

MCOL-2000 Yet another fixes for column charsets

* make respect for column (including table/db/server default) charsets
  for the TEXT(n) fields
* round TEXT(n) column length up to the next default length of TEXT-like
  subtypes, 255 (TINYTEXT), 65535 (TEXT) and so on up to 2100000000
  (LONGTEXT)

dbcon/ddlpackage/ddl.y

@@ -49,6 +49,9 @@
 #include "ddl-gram.h"
 #endif
 
+#include "my_global.h"
+#include "my_sys.h"
+
 #define scanner x->scanner
 
 using namespace std;
@@ -57,6 +60,42 @@ using namespace ddlpackage;
 int ddllex(YYSTYPE* ddllval, void* yyscanner);
 void ddlerror(struct pass_to_bison* x, char const *s);
 char* copy_string(const char *str);
+
+void fix_column_length(SchemaObject* elem, const CHARSET_INFO* def_cs) {
+    auto* column = dynamic_cast<ColumnDef*>(elem);
+    if (column == NULL || column->fType == NULL)
+    {
+        return;
+    }
+
+    if (column->fType->fType == DDL_VARCHAR ||
+         column->fType->fType == DDL_CHAR ||
+         (column->fType->fType == DDL_TEXT && column->fType->fExplicitLength))
+    {
+        unsigned mul = def_cs ? def_cs->mbmaxlen : 1;
+        if (column->fType->fCharset) {
+            const CHARSET_INFO* cs = get_charset_by_csname(column->fType->fCharset, MY_CS_PRIMARY, MYF(0));
+            if (cs)
+                mul = cs->mbmaxlen;
+        }
+        column->fType->fLength *= mul;
+    }
+
+    if (column->fType->fType == DDL_TEXT && column->fType->fExplicitLength)
+    {
+        // Rounding the resulting length of TEXT(N) field to the next default length
+        if (column->fType->fLength <= 255)
+            column->fType->fLength = 255;
+        else if (column->fType->fLength <= 65535)
+            column->fType->fLength = 65535;
+        else if (column->fType->fLength <= 16777215)
+            column->fType->fLength = 16777215;
+        else if (column->fType->fLength <= 2100000000)
+            column->fType->fLength = 2100000000;
+        // otherwise leave the decision to a caller code
+    }
+}
+
 %}
 
 %expect 17
@@ -204,6 +243,7 @@ ZEROFILL
 %type <sqlStmt>              rename_table_statement
 %type <str>                  ident
 %type <str>                  opt_quoted_literal
+%type <str>                  opt_column_charset
 %%
 stmtblock:	stmtmulti { x->fParseTree = $1; }
 		;
@@ -283,14 +323,13 @@ opt_table_options:
 	;
 
 create_table_statement:
-	CREATE TABLE opt_if_not_exists table_name '(' table_element_list ')' table_options
+	CREATE TABLE opt_if_not_exists table_name '(' table_element_list ')' opt_table_options
 	{
-		$$ = new CreateTableStatement(new TableDef($4, $6, $8));
+        for (auto* elem : *$6)
+        {
+            fix_column_length(elem, x->default_table_charset);
         }
-    |
+		$$ = new CreateTableStatement(new TableDef($4, $6, $8));
-	CREATE TABLE opt_if_not_exists table_name '(' table_element_list ')'
-	{
-		$$ = new CreateTableStatement(new TableDef($4, $6, NULL));
 	}
 	;
 
@@ -647,10 +686,20 @@ ata_add_column:
     /* See the documentation for SchemaObject for an explanation of why we are using
      * dynamic_cast here.
      */
-	ADD column_def {$$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($2));}
+	ADD column_def { fix_column_length($2, x->default_table_charset); $$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($2));}
-	| ADD COLUMN column_def {$$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($3));}
+	| ADD COLUMN column_def { fix_column_length($3, x->default_table_charset); $$ = new AtaAddColumn(dynamic_cast<ColumnDef*>($3));}
-	| ADD '(' table_element_list ')' {$$ = new AtaAddColumns($3);}
+	| ADD '(' table_element_list ')' {
-	| ADD COLUMN '(' table_element_list ')' {$$ = new AtaAddColumns($4);}
+        for (auto* elem : *$3) {
+            fix_column_length(elem, x->default_table_charset);
+        }
+        $$ = new AtaAddColumns($3);
+    }
+	| ADD COLUMN '(' table_element_list ')' {
+        for (auto* elem : *$4) {
+            fix_column_length(elem, x->default_table_charset);
+        }
+        $$ = new AtaAddColumns($4);
+    }
 	;
 
 column_name:
@@ -737,9 +786,9 @@ optional_braces:
 	;
 
 opt_column_charset:
-    /* empty */ {}
+    /* empty */ { $$ = NULL; }
     |
-    IDB_CHAR SET opt_quoted_literal {}
+    IDB_CHAR SET opt_quoted_literal { $$ = $3; }
     ;
 
 opt_column_collate:
@@ -749,12 +798,18 @@ opt_column_collate:
     ;
 
 data_type:
-	character_string_type opt_column_charset opt_column_collate
+	character_string_type opt_column_charset opt_column_collate {
+        $1->fCharset = $2;
+        $$ = $1;
+    }
 	| binary_string_type
 	| numeric_type
 	| datetime_type
 	| blob_type
-	| text_type opt_column_charset opt_column_collate
+	| text_type opt_column_charset opt_column_collate {
+        $1->fCharset = $2;
+        $$ = $1;
+    }
 	| IDB_BLOB
 	{
 		$$ = new ColumnType(DDL_BLOB);
@@ -955,6 +1010,7 @@ text_type:
 	{
 		$$ = new ColumnType(DDL_TEXT);
 		$$->fLength = atol($3);
+		$$->fExplicitLength = true;
 	}
 	| TEXT
 	{

dbcon/ddlpackage/ddlpkg.cpp

@@ -67,7 +67,9 @@ ColumnType::ColumnType(int prec, int scale) :
     fLength(0),
     fPrecision(prec),
     fScale(scale),
-    fWithTimezone(false)
+    fWithTimezone(false),
+    fCharset(NULL),
+    fExplicitLength(false)
 {
     fLength = utils::widthByPrecision(fPrecision);
 }
@@ -76,7 +78,9 @@ ColumnType::ColumnType(int type) :
     fType(type),
     fLength(0),
     fScale(0),
-    fWithTimezone(false)
+    fWithTimezone(false),
+    fCharset(NULL),
+    fExplicitLength(false)
 {
     switch ( type )
     {

dbcon/ddlpackage/ddlpkg.h

@@ -1004,7 +1004,7 @@ struct ColumnType
     EXPORT virtual int serialize(messageqcpp::ByteStream& bs);
 
     /** @brief For deserialization. */
-    ColumnType()
+    ColumnType() : fCharset(NULL), fExplicitLength(false)
     {}
 
     friend std::ostream& operator<<(std::ostream& os, const ColumnType& ac);
@@ -1044,6 +1044,12 @@ struct ColumnType
 
     uint64_t fNextvalue;
 
+    /** @brief Column charset (CHAR, VARCHAR and TEXT only) */
+    const char* fCharset;
+
+    /** @brief Is the TEXT column has explicit defined length, ie TEXT(1717) */
+    bool fExplicitLength;
+
 };
 
 

dbcon/ddlpackage/sqlparser.cpp

@@ -63,6 +63,11 @@ void SqlParser::setDefaultSchema(std::string schema)
     x.fDBSchema = schema;
 }
 
+void SqlParser::setDefaultCharset(const CHARSET_INFO* default_charset)
+{
+    x.default_table_charset = default_charset;
+}
+
 int SqlParser::Parse(const char* sqltext)
 {
     ddllex_init_extra(&scanData, &x.scanner);

dbcon/ddlpackage/sqlparser.h

@@ -27,6 +27,8 @@
  */
 
 #include <stdexcept>
+#include <my_global.h>
+#include <my_sys.h>
 #include "ddlpkg.h"
 
 #if defined(_MSC_VER) && defined(xxxDDLPKGSQLPARSER_DLLEXPORT)
@@ -86,8 +88,9 @@ struct pass_to_bison
     ParseTree* fParseTree;
     std::string fDBSchema;
     void* scanner;
+    const CHARSET_INFO* default_table_charset;
 
-    pass_to_bison(ParseTree* pt) : fParseTree(pt), scanner(NULL) {};
+    pass_to_bison(ParseTree* pt) : fParseTree(pt), scanner(NULL), default_table_charset(NULL) {};
 };
 
 class SqlParser
@@ -120,6 +123,13 @@ public:
       */
     EXPORT void setDefaultSchema(std::string schema);
 
+    /** @brief Set the default table charset. Can be overriden by column
+     *  or table options
+     *
+     *  @param default_charset the default CHARSET_INFO pointer
+     */
+     EXPORT void setDefaultCharset(const CHARSET_INFO* default_charset);
+
 protected:
     ParseTree fParseTree;
     std::string fDBSchema;

dbcon/mysql/ha_mcs_ddl.cpp

@@ -158,7 +158,7 @@ int parseCompressionComment ( std::string comment )
     {
         //Find the pattern, now get the compression type
         string compType (&(*(what[0].second)));
-        //; is the seperator between compression and autoincrement comments.
+        //; is the separator between compression and autoincrement comments.
         unsigned i = compType.find_first_of(";");
 
         if ( i <= compType.length() )
@@ -759,7 +759,8 @@ bool anyNullInTheColumn (THD* thd, string& schema, string& table, string& column
 }
 
 int ProcessDDLStatement(string& ddlStatement, string& schema, const string& table, int sessionID,
-                        string& emsg, int compressionTypeIn = 2, bool isAnyAutoincreCol = false, int64_t nextvalue = 1, std::string autoiColName = "")
+                        string& emsg, int compressionTypeIn = 2, bool isAnyAutoincreCol = false, int64_t nextvalue = 1, std::string autoiColName = "",
+                        const CHARSET_INFO* default_table_charset = NULL)
 {
     SqlParser parser;
     THD* thd = current_thd;
@@ -768,6 +769,7 @@ int ProcessDDLStatement(string& ddlStatement, string& schema, const string& tabl
 #endif
 
     parser.setDefaultSchema(schema);
+    parser.setDefaultCharset(default_table_charset);
     int rc = 0;
     IDBCompressInterface idbCompress;
     parser.Parse(ddlStatement.c_str());
@@ -2397,9 +2399,17 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea
                 oss << " NOT NULL";
 
             def_value.set(def_value_buf, sizeof(def_value_buf), system_charset_info);
-			if (get_field_default_value(thd, *field, &def_value, true)) {
+            if (get_field_default_value(thd, *field, &def_value, true))
+            {
                 oss << " DEFAULT " << def_value.c_ptr();
             }
+
+            const CHARSET_INFO* field_cs = (*field)->charset();
+            if (field_cs && (!share->table_charset || field_cs->number != share->table_charset->number))
+            {
+                oss << " CHARACTER SET " << field_cs->csname;
+            }
+
             if ((*field)->comment.length)
             {
                 String comment;
@@ -2441,13 +2451,15 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea
             oss << " MIN_ROWS=" << buff;
         }
 
-		if (share->max_rows) {
+        if (share->max_rows)
+        {
             char buff[80];
             longlong10_to_str(share->max_rows, buff, 10);
             oss << " MAX_ROWS=" << buff;
         }
 
-		if (share->comment.length) {
+        if (share->comment.length)
+        {
             String comment;
             append_unescaped(&comment, share->comment.str, share->comment.length);
             oss << " COMMENT ";
@@ -2460,7 +2472,7 @@ int ha_mcs_impl_create_(const char* name, TABLE* table_arg, HA_CREATE_INFO* crea
         tmp_restore_column_map(table_arg->read_set, old_map);
     }
 
-    rc = ProcessDDLStatement(stmt, db, tbl, tid2sid(thd->thread_id), emsg, compressiontype, isAnyAutoincreCol, startValue, columnName);
+    rc = ProcessDDLStatement(stmt, db, tbl, tid2sid(thd->thread_id), emsg, compressiontype, isAnyAutoincreCol, startValue, columnName, create_info->default_table_charset);
 
     if (rc != 0)
     {