UCS2 support in ENUM and SET, which also fixes:

Bug #5174 SHOW CREATE TABLE hangs up if the table contains half-with katakana enum values
UCS2 values are stored in HEX encoding in FRM file

mysql-test/r/ctype_ucs.result

@@ -525,3 +525,59 @@ use test;
 SET TIMESTAMP=10000;
 insert into t2 values (@v);
 drop table t2;
+set names latin1;
+create table t1 (a enum('x','y','z') character set ucs2);
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` enum('x','y','z') character set ucs2 default NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+insert into t1 values ('x');
+insert into t1 values ('y');
+insert into t1 values ('z');
+select a, hex(a) from t1 order by a;
+a	hex(a)
+x	0078
+y	0079
+z	007A
+alter table t1 change a a enum('x','y','z','d','e','<27>','<27>','<27>') character set ucs2;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` enum('x','y','z','d','e','<27>','<27>','<27>') character set ucs2 default NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+insert into t1 values ('D');
+insert into t1 values ('E ');
+insert into t1 values ('<27>');
+insert into t1 values ('<27>');
+insert into t1 values ('<27>');
+select a, hex(a) from t1 order by a;
+a	hex(a)
+x	0078
+y	0079
+z	007A
+d	0064
+e	0065
+<EFBFBD>	00E4
+<EFBFBD>	00F6
+<EFBFBD>	00FC
+drop table t1;
+create table t1 (a set ('x','y','z','<27>','<27>','<27>') character set ucs2);
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` set('x','y','z','<27>','<27>','<27>') character set ucs2 default NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+insert into t1 values ('x');
+insert into t1 values ('y');
+insert into t1 values ('z');
+insert into t1 values ('x,y');
+insert into t1 values ('x,y,z,<2C>,<2C>,<2C>');
+select a, hex(a) from t1 order by a;
+a	hex(a)
+x	0078
+y	0079
+x,y	0078002C0079
+z	007A
+x,y,z,<2C>,<2C>,<2C>	0078002C0079002C007A002C00E4002C00F6002C00FC
+drop table t1;

mysql-test/t/ctype_ucs.test

@@ -343,3 +343,34 @@ show binlog events from 79;
 --replace_result $MYSQL_TEST_DIR MYSQL_TEST_DIR
 --exec $MYSQL_BINLOG --short-form $MYSQL_TEST_DIR/var/log/master-bin.000001 
 drop table t2;
+
+
+#
+# Check that ucs2 works with ENUM and SET type
+#
+set names latin1;
+create table t1 (a enum('x','y','z') character set ucs2);
+show create table t1;
+insert into t1 values ('x');
+insert into t1 values ('y');
+insert into t1 values ('z');
+select a, hex(a) from t1 order by a;
+alter table t1 change a a enum('x','y','z','d','e','<27>','<27>','<27>') character set ucs2;
+show create table t1;
+insert into t1 values ('D');
+insert into t1 values ('E ');
+insert into t1 values ('<27>');
+insert into t1 values ('<27>');
+insert into t1 values ('<27>');
+select a, hex(a) from t1 order by a;
+drop table t1;
+
+create table t1 (a set ('x','y','z','<27>','<27>','<27>') character set ucs2);
+show create table t1;
+insert into t1 values ('x');
+insert into t1 values ('y');
+insert into t1 values ('z');
+insert into t1 values ('x,y');
+insert into t1 values ('x,y,z,<2C>,<2C>,<2C>');
+select a, hex(a) from t1 order by a;
+drop table t1;

sql/field.cc

@@ -5529,8 +5529,7 @@ int Field_enum::store(const char *from,uint length,CHARSET_INFO *cs)
   }
 
   /* Remove end space */
-  while (length > 0 && my_isspace(system_charset_info,from[length-1]))
-    length--;
+  length= field_charset->cset->lengthsp(field_charset, from, length);
   uint tmp=find_type2(typelib, from, length, field_charset);
   if (!tmp)
   {
@@ -5632,7 +5631,7 @@ String *Field_enum::val_str(String *val_buffer __attribute__((unused)),
     val_ptr->set("", 0, field_charset);
   else
     val_ptr->set((const char*) typelib->type_names[tmp-1],
-		 (uint) strlen(typelib->type_names[tmp-1]),
+		 typelib->type_lengths[tmp-1],
 		 field_charset);
   return val_ptr;
 }
@@ -5669,13 +5668,14 @@ void Field_enum::sql_type(String &res) const
   res.append("enum(");
 
   bool flag=0;
-  for (const char **pos= typelib->type_names; *pos; pos++)
+  uint *len= typelib->type_lengths;
+  for (const char **pos= typelib->type_names; *pos; pos++, len++)
   {
     uint dummy_errors;
     if (flag)
       res.append(',');
     /* convert to res.charset() == utf8, then quote */
-    enum_item.copy(*pos, strlen(*pos), charset(), res.charset(), &dummy_errors);
+    enum_item.copy(*pos, *len, charset(), res.charset(), &dummy_errors);
     append_unescaped(&res, enum_item.ptr(), enum_item.length());
     flag= 1;
   }
@@ -5760,9 +5760,9 @@ String *Field_set::val_str(String *val_buffer,
     if (tmp & 1)
     {
       if (val_buffer->length())
-	val_buffer->append(field_separator);
+	val_buffer->append(&field_separator, 1, &my_charset_latin1);
       String str(typelib->type_names[bitnr],
-		 (uint) strlen(typelib->type_names[bitnr]),
+		 typelib->type_lengths[bitnr],
 		 field_charset);
       val_buffer->append(str);
     }
@@ -5782,13 +5782,14 @@ void Field_set::sql_type(String &res) const
   res.append("set(");
 
   bool flag=0;
-  for (const char **pos= typelib->type_names; *pos; pos++)
+  uint *len= typelib->type_lengths;
+  for (const char **pos= typelib->type_names; *pos; pos++, len++)
   {
     uint dummy_errors;
     if (flag)
       res.append(',');
     /* convert to res.charset() == utf8, then quote */
-    set_item.copy(*pos, strlen(*pos), charset(), res.charset(), &dummy_errors);
+    set_item.copy(*pos, *len, charset(), res.charset(), &dummy_errors);
     append_unescaped(&res, set_item.ptr(), set_item.length());
     flag= 1;
   }

sql/item_strfunc.cc

@@ -2320,17 +2320,6 @@ String *Item_func_hex::val_str(String *str)
   return &tmp_value;
 }
 
-inline int hexchar_to_int(char c)
-{
-  if (c <= '9' && c >= '0')
-    return c-'0';
-  c|=32;
-  if (c <= 'f' && c >= 'a')
-    return c-'a'+10;
-  return -1;
-}
-
-
   /* Convert given hex string to a binary string */
 
 String *Item_func_unhex::val_str(String *str)

sql/mysql_priv.h

@@ -1200,6 +1200,23 @@ inline void setup_table_map(TABLE *table, TABLE_LIST *table_list, uint tablenr)
 }
 
 
+/*
+  SYNOPSYS
+    hexchar_to_int()
+    convert a hex digit into number
+*/
+
+inline int hexchar_to_int(char c)
+{
+  if (c <= '9' && c >= '0')
+    return c-'0';
+  c|=32;
+  if (c <= 'f' && c >= 'a')
+    return c-'a'+10;
+  return -1;
+}
+
+
 /*
   Some functions that are different in the embedded library and the normal
   server

sql/strfunc.cc

@@ -53,7 +53,21 @@ ulonglong find_set(TYPELIB *lib, const char *str, uint length, CHARSET_INFO *cs,
     {
       const char *pos= start;
       uint var_len;
+      int mblen= 1;
 
+      if (cs && cs->mbminlen > 1)
+      {
+        for ( ; pos < end; pos+= mblen)
+        {
+          my_wc_t wc;
+          if ((mblen= cs->cset->mb_wc(cs, &wc, (const uchar *) pos, 
+                                               (const uchar *) end)) < 1)
+            mblen= 1; // Not to hang on a wrong multibyte sequence
+          if (wc == (my_wc_t) field_separator)
+            break;
+        }
+      }
+      else
         for (; pos != end && *pos != field_separator; pos++) ;
       var_len= (uint) (pos - start);
       uint find= cs ? find_type2(lib, start, var_len, cs) :
@@ -66,9 +80,9 @@ ulonglong find_set(TYPELIB *lib, const char *str, uint length, CHARSET_INFO *cs,
       }
       else
         found|= ((longlong) 1 << (find - 1));
-      if (pos == end)
+      if (pos >= end)
         break;
-      start= pos + 1;
+      start= pos + mblen;
     }
   }
   return found;

sql/table.cc

@@ -485,6 +485,23 @@ int openfrm(const char *name, const char *alias, uint db_stat, uint prgflag,
         charset= outparam->table_charset;
       bzero((char*) &comment, sizeof(comment));
     }
+
+    if (interval_nr && charset->mbminlen > 1)
+    {
+      /* Unescape UCS2 intervals from HEX notation */
+      TYPELIB *interval= outparam->intervals + interval_nr - 1;
+      for (uint pos= 0; pos < interval->count; pos++)
+      {
+        char *from, *to;
+        for (from= to= (char*) interval->type_names[pos]; *from; )
+        {
+          *to++= (char) (hexchar_to_int(*from++) << 4) + 
+                         hexchar_to_int(*from++);
+        }
+        interval->type_lengths[pos] /= 2;
+      }
+    }
+    
     *field_ptr=reg_field=
       make_field(record+recpos,
 		 (uint32) field_length,

sql/unireg.cc

@@ -423,6 +423,28 @@ static bool pack_header(uchar *forminfo, enum db_type table_type,
     if (field->interval)
     {
       uint old_int_count=int_count;
+
+      if (field->charset->mbminlen > 1)
+      {
+        /* Escape UCS2 intervals using HEX notation */
+        for (uint pos= 0; pos < field->interval->count; pos++)
+        {
+          char *dst;
+          uint length= field->interval->type_lengths[pos], hex_length;
+          const char *src= field->interval->type_names[pos];
+          const char *srcend= src + length;
+          hex_length= length * 2;
+          field->interval->type_lengths[pos]= hex_length;
+          field->interval->type_names[pos]= dst= sql_alloc(hex_length + 1);
+          for ( ; src < srcend; src++)
+          {
+            *dst++= _dig_vec_upper[((uchar) *src) >> 4];
+            *dst++= _dig_vec_upper[((uchar) *src) & 15];
+          }
+          *dst= '\0';
+        }
+      }
+
       field->interval_id=get_interval_id(&int_count,create_fields,field);
       if (old_int_count != int_count)
       {