MDEV-18150 Assertion `decimals_to_set <= 38' failed in Item_func_round::fix_length_and_dec_decimal

mysql-test/main/func_math.result

@@ -1316,5 +1316,12 @@ t2	CREATE TABLE `t2` (
 DROP TABLE t1, t2;
 SET sql_mode=DEFAULT;
 #
+# MDEV-18150 Assertion `decimals_to_set <= 38' failed in Item_func_round::fix_length_and_dec_decimal
+#
+CREATE TABLE t1 (i INT(23));
+SELECT ROUND( i, 18446744073709551594 ) AS f FROM t1;
+f
+DROP TABLE t1;
+#
 # End of 10.3 tests
 #

mysql-test/main/func_math.test

@@ -919,6 +919,14 @@ DROP TABLE t1, t2;
 SET sql_mode=DEFAULT;
 
 
+--echo #
+--echo # MDEV-18150 Assertion `decimals_to_set <= 38' failed in Item_func_round::fix_length_and_dec_decimal
+--echo #
+
+CREATE TABLE t1 (i INT(23));
+SELECT ROUND( i, 18446744073709551594 ) AS f FROM t1;
+DROP TABLE t1;
+
 --echo #
 --echo # End of 10.3 tests
 --echo #

sql/item.h

@@ -1062,20 +1062,6 @@ public:
     unsigned_flag to check the sign of the item.
   */
   inline ulonglong val_uint() { return (ulonglong) val_int(); }
-  /*
-    Adjust the result of val_int() to an unsigned number:
-    - NULL value is converted to 0. The caller can check "null_value"
-      to distinguish between 0 and NULL when necessary.
-    - Negative numbers are converted to 0.
-    - Positive numbers bigger than upper_bound are converted to upper_bound.
-    - Other numbers are returned as is.
-  */
-  ulonglong val_uint_from_val_int(ulonglong upper_bound)
-  {
-    longlong nr= val_int();
-    return (null_value || (nr < 0 && !unsigned_flag)) ? 0 :
-           (ulonglong) nr > upper_bound ? upper_bound : (ulonglong) nr;
-  }
 
   /*
     Return string representation of this item object.

sql/item_func.cc

@@ -2433,11 +2433,11 @@ void Item_func_round::fix_arg_decimal()
 {
   if (args[1]->const_item())
   {
-    uint dec= (uint) args[1]->val_uint_from_val_int(DECIMAL_MAX_SCALE);
+    Longlong_hybrid dec= args[1]->to_longlong_hybrid();
     if (args[1]->null_value)
       fix_length_and_dec_double(NOT_FIXED_DEC);
     else
-      fix_length_and_dec_decimal(dec);
+      fix_length_and_dec_decimal(dec.to_uint(DECIMAL_MAX_SCALE));
   }
   else
   {
@@ -2453,8 +2453,9 @@ void Item_func_round::fix_arg_double()
 {
   if (args[1]->const_item())
   {
-    uint dec= (uint) args[1]->val_uint_from_val_int(NOT_FIXED_DEC);
+    Longlong_hybrid dec= args[1]->to_longlong_hybrid();
-    fix_length_and_dec_double(args[1]->null_value ? NOT_FIXED_DEC : dec);
+    fix_length_and_dec_double(args[1]->null_value ? NOT_FIXED_DEC :
+                              dec.to_uint(NOT_FIXED_DEC));
   }
   else
     fix_length_and_dec_double(args[0]->decimals);
@@ -2465,17 +2466,14 @@ void Item_func_round::fix_arg_int()
 {
   if (args[1]->const_item())
   {
-    longlong val1= args[1]->val_int();
+    Longlong_hybrid val1= args[1]->to_longlong_hybrid();
-    bool val1_is_negative= val1 < 0 && !args[1]->unsigned_flag;
-    uint decimals_to_set= val1_is_negative ?
-                          0 : (uint) MY_MIN(val1, DECIMAL_MAX_SCALE);
     if (args[1]->null_value)
       fix_length_and_dec_double(NOT_FIXED_DEC);
-    else if ((!decimals_to_set && truncate) ||
+    else if ((!val1.to_uint(DECIMAL_MAX_SCALE) && truncate) ||
              args[0]->decimal_precision() < DECIMAL_LONGLONG_DIGITS)
     {
       // Length can increase in some cases: ROUND(9,-1) -> 10
-      int length_can_increase= MY_TEST(!truncate && val1_is_negative);
+      int length_can_increase= MY_TEST(!truncate && val1.neg());
       max_length= args[0]->max_length + length_can_increase;
       // Here we can keep INT_RESULT
       unsigned_flag= args[0]->unsigned_flag;
@@ -2483,7 +2481,7 @@ void Item_func_round::fix_arg_int()
       set_handler(type_handler_long_or_longlong());
     }
     else
-      fix_length_and_dec_decimal(decimals_to_set);
+      fix_length_and_dec_decimal(val1.to_uint(DECIMAL_MAX_SCALE));
   }
   else
     fix_length_and_dec_double(args[0]->decimals);

sql/sql_type_int.h

@@ -52,6 +52,22 @@ public:
       return ((ulonglong) LONGLONG_MAX) + 1;
     return m_value < 0 ? -m_value : m_value;
   }
+  /*
+    Convert to an unsigned number:
+    - Negative numbers are converted to 0.
+    - Positive numbers bigger than upper_bound are converted to upper_bound.
+    - Other numbers are returned as is.
+  */
+  ulonglong to_ulonglong(ulonglong upper_bound) const
+  {
+    return neg() ? 0 :
+           (ulonglong) m_value > upper_bound ? upper_bound :
+           (ulonglong) m_value;
+  }
+  uint to_uint(uint upper_bound) const
+  {
+    return (uint) to_ulonglong(upper_bound);
+  }
   int cmp(const Longlong_hybrid& other) const
   {
     if (m_unsigned == other.m_unsigned)