MDEV-41144 Assertion `scale <= precision' failed in decimal_bin_size

dynamic_column_decimal_read() derived intg/frac from an unbounded
var-uint and only checked scale>precision in int domain, so a crafted
value could still overflow decimal_bin_size()'s uint16 parameters,
tripping its scale<=precision assert (an out-of-bounds array read in
release builds).

Bound intg/frac against DECIMAL_MAX_POSSIBLE_PRECISION before use, and
reject the var-uint decoder's error signal, as the sibling
dynamic_column_string_read() already does. Validate the decoded
ulonglong before narrowing it to int, so a value wrapping around on
narrowing cannot bypass the bound.

mysql-test/main/dyncol.result

@@ -1987,3 +1987,23 @@ ERROR HY000: Encountered illegal format of dynamic column string
 SELECT COLUMN_JSON(CONCAT(0x040200020001000000000000004241, REPEAT('D', 512)));
 ERROR HY000: Encountered illegal format of dynamic column string
 # End of 10.6 tests
+#
+# MDEV-41144 Assertion `scale <= precision' failed in decimal_bin_size
+#
+SELECT COLUMN_JSON(0x010300070004000D007D000F000D00DFE9CE2BA3BCF60303FF290D71CD4C14D50F0000E03D0F001309890E67F466037DC899337129E8) AS ex;
+ERROR HY000: Encountered illegal format of dynamic column string
+SELECT COLUMN_CHECK(0x010300070004000D007D000F000D00DFE9CE2BA3BCF60303FF290D71CD4C14D50F0000E03D0F001309890E67F466037DC899337129E8) AS ex;
+ex
+0
+#
+# Round trip check: a decimal scale beyond the SQL-layer
+# DECIMAL_MAX_SCALE (38) must still be readable, since the dyncol
+# format itself allows up to DECIMAL_MAX_POSSIBLE_PRECISION (81).
+#
+SELECT COLUMN_JSON(COLUMN_CREATE(1, '0.123456789012345678901234567890123456789' AS decimal)) AS ex;
+ex
+{"1":0.1234567890123456789012345678901234567}
+SELECT COLUMN_CHECK(COLUMN_CREATE(1, 0.1234567890123456789012345678901234567890123456789 AS decimal)) AS ex;
+ex
+1
+# End of 10.11 tests

mysql-test/main/dyncol.test

@@ -1044,3 +1044,20 @@ SELECT COLUMN_LIST(0x040100FFFF00000300666F6F21626172);
 SELECT COLUMN_JSON(CONCAT(0x040200020001000000000000004241, REPEAT('D', 512)));
 
 --echo # End of 10.6 tests
+
+--echo #
+--echo # MDEV-41144 Assertion `scale <= precision' failed in decimal_bin_size
+--echo #
+--error ER_DYN_COL_WRONG_FORMAT
+SELECT COLUMN_JSON(0x010300070004000D007D000F000D00DFE9CE2BA3BCF60303FF290D71CD4C14D50F0000E03D0F001309890E67F466037DC899337129E8) AS ex;
+SELECT COLUMN_CHECK(0x010300070004000D007D000F000D00DFE9CE2BA3BCF60303FF290D71CD4C14D50F0000E03D0F001309890E67F466037DC899337129E8) AS ex;
+
+--echo #
+--echo # Round trip check: a decimal scale beyond the SQL-layer
+--echo # DECIMAL_MAX_SCALE (38) must still be readable, since the dyncol
+--echo # format itself allows up to DECIMAL_MAX_POSSIBLE_PRECISION (81).
+--echo #
+SELECT COLUMN_JSON(COLUMN_CREATE(1, '0.123456789012345678901234567890123456789' AS decimal)) AS ex;
+SELECT COLUMN_CHECK(COLUMN_CREATE(1, 0.1234567890123456789012345678901234567890123456789 AS decimal)) AS ex;
+
+--echo # End of 10.11 tests

mysys/ma_dyncol.c

@@ -1200,6 +1200,7 @@ dynamic_column_decimal_read(DYNAMIC_COLUMN_VALUE *store_it_here,
                             uchar *data, size_t length)
 {
   size_t intg_len, frac_len;
+  ulonglong u_intg, u_frac;
   int intg, frac, precision, scale;
 
   dynamic_column_prepare_decimal(store_it_here);
@@ -1207,18 +1208,27 @@ dynamic_column_decimal_read(DYNAMIC_COLUMN_VALUE *store_it_here,
   if (length == 0)
     return ER_DYNCOL_OK;                        /* value contains zero */
 
-  intg= (int)dynamic_column_var_uint_get(data, length, &intg_len);
+  u_intg= dynamic_column_var_uint_get(data, length, &intg_len);
   data+= intg_len;
-  frac= (int)dynamic_column_var_uint_get(data, length - intg_len, &frac_len);
+  u_frac= dynamic_column_var_uint_get(data, length - intg_len, &frac_len);
   data+= frac_len;
 
+  /* Reject before narrowing to int, so a wrap-around can't hide a bad value */
+  if (intg_len == 0 || frac_len == 0 ||
+      u_intg > DECIMAL_MAX_POSSIBLE_PRECISION ||
+      u_frac > DECIMAL_MAX_POSSIBLE_PRECISION)
+    return ER_DYNCOL_FORMAT;
+  intg= (int) u_intg;
+  frac= (int) u_frac;
+
   /* Check the size of data is correct */
   precision= intg + frac;
   scale=     frac;
-  if (scale < 0 || precision <= 0 || scale > precision ||
+  if (precision <= 0 || scale > precision ||
+      precision > DECIMAL_MAX_POSSIBLE_PRECISION ||
       (length - intg_len - frac_len) >
       (size_t) (DECIMAL_BUFF_LENGTH*sizeof(decimal_digit_t)) ||
-      decimal_bin_size(intg + frac, frac) !=
+      decimal_bin_size(precision, scale) !=
       (uint) (length - intg_len - frac_len))
     return ER_DYNCOL_FORMAT;
 

unittest/mysys/ma_dyncol-t.c

@@ -772,6 +772,51 @@ static void test_overlong_int(DYNAMIC_COLUMN_TYPE type, const char *msg)
   mariadb_dyncol_free(&str);
 }
 
+/*
+  A crafted record can encode a decimal's intg/frac digit count as a
+  var-uint wide enough that narrowing it to int wraps around into a
+  small, in-range value (2^32+1 truncates to 1 on 32-bit int). The
+  decoder must reject the value using its full width, before narrowing.
+  See MDEV-41144.
+*/
+static void test_decimal_precision_overflow()
+{
+  DYNAMIC_COLUMN str;
+  DYNAMIC_COLUMN bad;
+  DYNAMIC_COLUMN_VALUE val, res;
+  uint column_nr= 1;
+  int rc;
+  char num[]= "5";
+  char *end= num + 1;
+  /* var-uint encoding of 2^32+1, which truncates to 1 as a 32-bit int */
+  const uchar wide_intg[]= {0x81, 0x80, 0x80, 0x80, 0x10};
+
+  mariadb_dyncol_prepare_decimal(&val);
+  string2decimal(num, &val.x.decimal.value, &end);
+
+  rc= mariadb_dyncol_create_many_num(&str, 1, &column_nr, &val, 1);
+  ok(rc == ER_DYNCOL_OK, "create decimal");
+
+  /*
+    decimal "5" is stored as the last 3 bytes of the record: a 1-byte
+    intg (1), a 1-byte frac (0) and 1 byte of binary payload. Replace
+    the 1-byte intg with a 5-byte encoding of the same truncated value.
+  */
+  bad.length= str.length - 3 + sizeof(wide_intg) + 2;
+  bad.max_length= bad.length;
+  bad.alloc_increment= 0;
+  bad.str= (char *) malloc(bad.length);
+  memcpy(bad.str, str.str, str.length - 3);
+  memcpy(bad.str + str.length - 3, wide_intg, sizeof(wide_intg));
+  memcpy(bad.str + str.length - 3 + sizeof(wide_intg), str.str + str.length - 2, 2);
+
+  rc= mariadb_dyncol_get_num(&bad, column_nr, &res);
+  ok(rc == ER_DYNCOL_FORMAT, "wide intg varint rejected");
+
+  free(bad.str);
+  mariadb_dyncol_free(&str);
+}
+
 
 #define BIG_STRING_SIZE (1024*1024)
 int main(int argc __attribute__((unused)), char **argv)
@@ -780,7 +825,7 @@ int main(int argc __attribute__((unused)), char **argv)
   char *big_string= (char *)malloc(BIG_STRING_SIZE);
 
   MY_INIT(argv[0]);
-  plan(72);
+  plan(74);
 
   if (!big_string)
     exit(1);
@@ -916,6 +961,7 @@ int main(int argc __attribute__((unused)), char **argv)
   test_mdev_9773();
   test_overlong_int(DYN_COL_UINT, "uint");
   test_overlong_int(DYN_COL_INT, "sint");
+  test_decimal_precision_overflow();
 
   my_end(0);
   return exit_status();