diff --git a/mysql-test/main/ctype_ucs.result b/mysql-test/main/ctype_ucs.result index 887c01df7e9..a603d07b08a 100644 --- a/mysql-test/main/ctype_ucs.result +++ b/mysql-test/main/ctype_ucs.result @@ -6540,5 +6540,14 @@ DROP VIEW v1; DROP TABLE t1; SET NAMES utf8mb3; # +# MDEV-28386 UBSAN: runtime error: negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strntoull_8bit on SELECT ... OCT +# +CREATE TABLE t1 (c TEXT CHARACTER SET ucs2); +INSERT INTO t1 VALUES ('-9223372036854775808.5'); +SELECT OCT(c) FROM t1; +OCT(c) +1000000000000000000000 +DROP TABLE t1; +# # End of 10.5 tests # diff --git a/mysql-test/main/ctype_ucs.test b/mysql-test/main/ctype_ucs.test index 7144f0af5cc..bf967ba7619 100644 --- a/mysql-test/main/ctype_ucs.test +++ b/mysql-test/main/ctype_ucs.test @@ -1217,6 +1217,15 @@ DROP VIEW v1; DROP TABLE t1; SET NAMES utf8mb3; +--echo # +--echo # MDEV-28386 UBSAN: runtime error: negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strntoull_8bit on SELECT ... OCT +--echo # + +CREATE TABLE t1 (c TEXT CHARACTER SET ucs2); +INSERT INTO t1 VALUES ('-9223372036854775808.5'); +SELECT OCT(c) FROM t1; +DROP TABLE t1; + --echo # --echo # End of 10.5 tests diff --git a/mysql-test/main/func_str.result b/mysql-test/main/func_str.result index 52bfd3f0e3e..7aa248bffd4 100644 --- a/mysql-test/main/func_str.result +++ b/mysql-test/main/func_str.result @@ -5317,5 +5317,18 @@ SELECT SUBSTR(0,@a) FROM t; SUBSTR(0,@a) DROP TABLE t; # +# MDEV-28386 UBSAN: runtime error: negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strntoull_8bit on SELECT ... OCT +# +CREATE TABLE t1 (c BLOB); +INSERT INTO t1 VALUES ('-9223372036854775808.5'); +SELECT OCT(c) FROM t1; +OCT(c) +1000000000000000000000 +SELECT BIN(c) FROM t1; +BIN(c) +1000000000000000000000000000000000000000000000000000000000000000 +DROP TABLE t1; +DO OCT(-9223372036854775808); +# # End of 10.5 tests # diff --git a/mysql-test/main/func_str.test b/mysql-test/main/func_str.test index 3c9360a9c40..952d061c30f 100644 --- a/mysql-test/main/func_str.test +++ b/mysql-test/main/func_str.test @@ -2358,6 +2358,19 @@ CREATE TABLE t (c1 INT,c2 CHAR); SELECT SUBSTR(0,@a) FROM t; DROP TABLE t; +--echo # +--echo # MDEV-28386 UBSAN: runtime error: negation of -X cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strntoull_8bit on SELECT ... OCT +--echo # + +CREATE TABLE t1 (c BLOB); +INSERT INTO t1 VALUES ('-9223372036854775808.5'); +SELECT OCT(c) FROM t1; +SELECT BIN(c) FROM t1; +DROP TABLE t1; + +DO OCT(-9223372036854775808); + + --echo # --echo # End of 10.5 tests --echo # diff --git a/strings/ctype-simple.c b/strings/ctype-simple.c index a50c570ec6b..d40e3abdcc5 100644 --- a/strings/ctype-simple.c +++ b/strings/ctype-simple.c @@ -758,7 +758,10 @@ ulonglong my_strntoull_8bit(CHARSET_INFO *cs, return (~(ulonglong) 0); } - return (negative ? -((longlong) i) : (longlong) i); + /* Avoid undefinite behavior - negation of LONGLONG_MIN */ + return negative && (longlong) i != LONGLONG_MIN ? + -((longlong) i) : + (longlong) i; noconv: err[0]= EDOM; diff --git a/strings/ctype-ucs2.c b/strings/ctype-ucs2.c index d1ca32a8e62..fb33d8cc764 100644 --- a/strings/ctype-ucs2.c +++ b/strings/ctype-ucs2.c @@ -616,7 +616,10 @@ bs: return (~(ulonglong) 0); } - return (negative ? -((longlong) res) : (longlong) res); + /* Avoid undefinite behavior - negation of LONGLONG_MIN */ + return negative && (longlong) res != LONGLONG_MIN ? + -((longlong) res) : + (longlong) res; }