MDEV-12849 Out-of-range errors when casting hex-hybrid to SIGNED and UNSIGNED

mysql-test/r/cast.result

@@ -900,3 +900,231 @@ t1	CREATE TABLE `t1` (
   `CONCAT(CONVERT('',UNSIGNED))` varchar(1) NOT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 DROP TABLE t1;
+#
+# MDEV-12849 Out-of-range errors when casting hex-hybrid to SIGNED and UNSIGNED
+#
+CREATE PROCEDURE p1(hh TEXT)
+BEGIN
+EXECUTE IMMEDIATE
+CONCAT('CREATE OR REPLACE TABLE t1 AS SELECT CAST(0x', hh, ' AS UNSIGNED) AS c');
+DESCRIBE t1;
+SELECT c, LENGTH(c) FROM t1;
+DROP TABLE t1;
+EXECUTE IMMEDIATE
+CONCAT('CREATE OR REPLACE TABLE t1 AS SELECT CAST(0x', hh, ' AS SIGNED) AS c');
+DESCRIBE t1;
+SELECT c, LENGTH(c) FROM t1;
+DROP TABLE t1;
+SELECT '' AS `------`;
+END
+$$
+CALL p1('FF');
+Field	Type	Null	Key	Default	Extra
+c	int(3) unsigned	NO		NULL	
+c	LENGTH(c)
+255	3
+Field	Type	Null	Key	Default	Extra
+c	int(3)	NO		NULL	
+c	LENGTH(c)
+255	3
+------
+
+CALL p1('FFFF');
+Field	Type	Null	Key	Default	Extra
+c	int(5) unsigned	NO		NULL	
+c	LENGTH(c)
+65535	5
+Field	Type	Null	Key	Default	Extra
+c	int(5)	NO		NULL	
+c	LENGTH(c)
+65535	5
+------
+
+CALL p1('FFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	int(8) unsigned	NO		NULL	
+c	LENGTH(c)
+16777215	8
+Field	Type	Null	Key	Default	Extra
+c	int(8)	NO		NULL	
+c	LENGTH(c)
+16777215	8
+------
+
+CALL p1('FFFFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	int(10) unsigned	NO		NULL	
+c	LENGTH(c)
+4294967295	10
+Field	Type	Null	Key	Default	Extra
+c	int(10)	NO		NULL	
+c	LENGTH(c)
+2147483647	10
+------
+
+CALL p1('FFFFFFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(13) unsigned	NO		NULL	
+c	LENGTH(c)
+1099511627775	13
+Field	Type	Null	Key	Default	Extra
+c	bigint(13)	NO		NULL	
+c	LENGTH(c)
+1099511627775	13
+------
+
+CALL p1('FFFFFFFFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(15) unsigned	NO		NULL	
+c	LENGTH(c)
+281474976710655	15
+Field	Type	Null	Key	Default	Extra
+c	bigint(15)	NO		NULL	
+c	LENGTH(c)
+281474976710655	15
+------
+
+CALL p1('FFFFFFFFFFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(17) unsigned	NO		NULL	
+c	LENGTH(c)
+72057594037927935	17
+Field	Type	Null	Key	Default	Extra
+c	bigint(17)	NO		NULL	
+c	LENGTH(c)
+72057594037927935	17
+------
+
+CALL p1('FFFFFFFFFFFFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(20) unsigned	NO		NULL	
+c	LENGTH(c)
+18446744073709551615	20
+Field	Type	Null	Key	Default	Extra
+c	bigint(20)	NO		NULL	
+c	LENGTH(c)
+-1	2
+------
+
+CALL p1('FFFFFFFFFFFFFFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(20) unsigned	NO		NULL	
+c	LENGTH(c)
+18446744073709551615	20
+Field	Type	Null	Key	Default	Extra
+c	bigint(20)	NO		NULL	
+c	LENGTH(c)
+-1	2
+------
+
+CALL p1('FFFFFFFFFFFFFFFFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(20) unsigned	NO		NULL	
+c	LENGTH(c)
+18446744073709551615	20
+Field	Type	Null	Key	Default	Extra
+c	bigint(20)	NO		NULL	
+c	LENGTH(c)
+-1	2
+------
+
+CALL p1('8000000000000000');
+Field	Type	Null	Key	Default	Extra
+c	bigint(20) unsigned	NO		NULL	
+c	LENGTH(c)
+9223372036854775808	19
+Field	Type	Null	Key	Default	Extra
+c	bigint(20)	NO		NULL	
+c	LENGTH(c)
+-9223372036854775808	20
+------
+
+CALL p1('80000000000000FF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(20) unsigned	NO		NULL	
+c	LENGTH(c)
+9223372036854776063	19
+Field	Type	Null	Key	Default	Extra
+c	bigint(20)	NO		NULL	
+c	LENGTH(c)
+-9223372036854775553	20
+------
+
+CALL p1('800000000000FFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(20) unsigned	NO		NULL	
+c	LENGTH(c)
+9223372036854841343	19
+Field	Type	Null	Key	Default	Extra
+c	bigint(20)	NO		NULL	
+c	LENGTH(c)
+-9223372036854710273	20
+------
+
+CALL p1('8000000000FFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(20) unsigned	NO		NULL	
+c	LENGTH(c)
+9223372036871553023	19
+Field	Type	Null	Key	Default	Extra
+c	bigint(20)	NO		NULL	
+c	LENGTH(c)
+-9223372036837998593	20
+------
+
+CALL p1('80000000FFFFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(20) unsigned	NO		NULL	
+c	LENGTH(c)
+9223372041149743103	19
+Field	Type	Null	Key	Default	Extra
+c	bigint(20)	NO		NULL	
+c	LENGTH(c)
+-9223372032559808513	20
+------
+
+CALL p1('800000FFFFFFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(20) unsigned	NO		NULL	
+c	LENGTH(c)
+9223373136366403583	19
+Field	Type	Null	Key	Default	Extra
+c	bigint(20)	NO		NULL	
+c	LENGTH(c)
+-9223370937343148033	20
+------
+
+CALL p1('8000FFFFFFFFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(20) unsigned	NO		NULL	
+c	LENGTH(c)
+9223653511831486463	19
+Field	Type	Null	Key	Default	Extra
+c	bigint(20)	NO		NULL	
+c	LENGTH(c)
+-9223090561878065153	20
+------
+
+CALL p1('80FFFFFFFFFFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(20) unsigned	NO		NULL	
+c	LENGTH(c)
+9295429630892703743	19
+Field	Type	Null	Key	Default	Extra
+c	bigint(20)	NO		NULL	
+c	LENGTH(c)
+-9151314442816847873	20
+------
+
+CALL p1('8FFFFFFFFFFFFFFF');
+Field	Type	Null	Key	Default	Extra
+c	bigint(20) unsigned	NO		NULL	
+c	LENGTH(c)
+10376293541461622783	20
+Field	Type	Null	Key	Default	Extra
+c	bigint(20)	NO		NULL	
+c	LENGTH(c)
+-8070450532247928833	20
+------
+
+DROP PROCEDURE p1;

mysql-test/t/cast.test

@@ -514,3 +514,48 @@ CREATE TABLE t1 AS SELECT
   CONCAT(CONVERT('',UNSIGNED));
 SHOW CREATE TABLE t1;
 DROP TABLE t1;
+
+
+--echo #
+--echo # MDEV-12849 Out-of-range errors when casting hex-hybrid to SIGNED and UNSIGNED
+--echo #
+
+DELIMITER $$;
+CREATE PROCEDURE p1(hh TEXT)
+BEGIN
+  EXECUTE IMMEDIATE
+    CONCAT('CREATE OR REPLACE TABLE t1 AS SELECT CAST(0x', hh, ' AS UNSIGNED) AS c');
+  DESCRIBE t1;
+  SELECT c, LENGTH(c) FROM t1;
+  DROP TABLE t1;
+  EXECUTE IMMEDIATE
+    CONCAT('CREATE OR REPLACE TABLE t1 AS SELECT CAST(0x', hh, ' AS SIGNED) AS c');
+  DESCRIBE t1;
+  SELECT c, LENGTH(c) FROM t1;
+  DROP TABLE t1;
+  SELECT '' AS `------`;
+END
+$$
+DELIMITER ;$$
+CALL p1('FF');
+CALL p1('FFFF');
+CALL p1('FFFFFF');
+CALL p1('FFFFFFFF');
+CALL p1('FFFFFFFFFF');
+CALL p1('FFFFFFFFFFFF');
+CALL p1('FFFFFFFFFFFFFF');
+CALL p1('FFFFFFFFFFFFFFFF');
+CALL p1('FFFFFFFFFFFFFFFFFF');
+CALL p1('FFFFFFFFFFFFFFFFFFFF');
+
+CALL p1('8000000000000000');
+CALL p1('80000000000000FF');
+CALL p1('800000000000FFFF');
+CALL p1('8000000000FFFFFF');
+CALL p1('80000000FFFFFFFF');
+CALL p1('800000FFFFFFFFFF');
+CALL p1('8000FFFFFFFFFFFF');
+CALL p1('80FFFFFFFFFFFFFF');
+CALL p1('8FFFFFFFFFFFFFFF');
+
+DROP PROCEDURE p1;

sql/item.cc

@@ -6718,6 +6718,22 @@ void Item_hex_hybrid::print(String *str, enum_query_type query_type)
 }
 
 
+uint Item_hex_hybrid::decimal_precision() const
+{
+  switch (max_length) {// HEX                                 DEC
+  case 0:              // ----                                ---
+  case 1: return 3;    // 0xFF                                255
+  case 2: return 5;    // 0xFFFF                            65535
+  case 3: return 8;    // 0xFFFFFF                       16777215
+  case 4: return 10;   // 0xFFFFFFFF                   4294967295
+  case 5: return 13;   // 0xFFFFFFFFFF              1099511627775
+  case 6: return 15;   // 0xFFFFFFFFFFFF          281474976710655
+  case 7: return 17;   // 0xFFFFFFFFFFFFFF      72057594037927935
+  }
+  return 20;           // 0xFFFFFFFFFFFFFFFF 18446744073709551615
+}
+
+
 void Item_hex_string::print(String *str, enum_query_type query_type)
 {
   str->append("X'");

sql/item.h

@@ -3810,6 +3810,7 @@ public:
   Item_hex_hybrid(THD *thd): Item_hex_constant(thd) {}
   Item_hex_hybrid(THD *thd, const char *str, uint str_length):
     Item_hex_constant(thd, str, str_length) {}
+  uint decimal_precision() const;
   double val_real()
   { 
     DBUG_ASSERT(fixed == 1); 

sql/item_func.h

@@ -852,6 +852,19 @@ public:
     set_if_bigger(char_length, 1U + (unsigned_flag ? 0 : 1));
     fix_char_length(char_length);
   }
+  void fix_length_and_dec_string()
+  {
+    /*
+      For strings, use decimal_int_part() instead of max_char_length().
+      This is important for Item_hex_hybrid:
+        SELECT CAST(0x1FFFFFFFF AS SIGNED);
+      Length is 5, decimal_int_part() is 13.
+    */
+    uint32 char_length= MY_MIN(args[0]->decimal_int_part(),
+                               MY_INT64_NUM_DECIMAL_DIGITS);
+    set_if_bigger(char_length, 1U + (unsigned_flag ? 0 : 1));
+    fix_char_length(char_length);
+  }
   void fix_length_and_dec()
   {
     args[0]->type_handler()->Item_func_signed_fix_length_and_dec(this);

sql/sql_type.cc

@@ -4168,6 +4168,22 @@ bool Type_handler::
 }
 
 
+bool Type_handler_string_result::
+       Item_func_signed_fix_length_and_dec(Item_func_signed *item) const
+{
+  item->fix_length_and_dec_string();
+  return false;
+}
+
+
+bool Type_handler_string_result::
+       Item_func_unsigned_fix_length_and_dec(Item_func_unsigned *item) const
+{
+  item->fix_length_and_dec_string();
+  return false;
+}
+
+
 bool Type_handler::
        Item_double_typecast_fix_length_and_dec(Item_double_typecast *item) const
 {

sql/sql_type.h

@@ -1615,6 +1615,8 @@ public:
   bool Item_sum_sum_fix_length_and_dec(Item_sum_sum *) const;
   bool Item_sum_avg_fix_length_and_dec(Item_sum_avg *) const;
   bool Item_sum_variance_fix_length_and_dec(Item_sum_variance *) const;
+  bool Item_func_signed_fix_length_and_dec(Item_func_signed *item) const;
+  bool Item_func_unsigned_fix_length_and_dec(Item_func_unsigned *item) const;
   longlong Item_val_int_signed_typecast(Item *item) const;
   longlong Item_val_int_unsigned_typecast(Item *item) const;
   String *Item_func_hex_val_str_ascii(Item_func_hex *item, String *str) const;