From 7c4c082349869b63ace23cd621585562684bbdf6 Mon Sep 17 00:00:00 2001
From: Alexander Barkov <bar@mariadb.com>
Date: Thu, 23 May 2024 14:18:34 +0400
Subject: [PATCH] MDEV-28387 UBSAN: runtime error: negation of
 -9223372036854775808 cannot be represented in type 'long long int'; cast to
 an unsigned type to negate this value to itself in my_strtoll10 on SELECT

Fixing the condition to raise an overflow in the ulonglong
representation of the number is greater or equal to 0x8000000000000000ULL.
Before this change the condition did not catch -9223372036854775808
(the smallest possible signed negative longlong number).
---
 mysql-test/main/func_str.result | 8 ++++++++
 mysql-test/main/func_str.test   | 9 +++++++++
 strings/my_strtoll10.c          | 2 +-
 3 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/mysql-test/main/func_str.result b/mysql-test/main/func_str.result
index c071b92318a..e5a21af03a3 100644
--- a/mysql-test/main/func_str.result
+++ b/mysql-test/main/func_str.result
@@ -5311,5 +5311,13 @@ NULL
 DROP TABLE t1;
 DROP VIEW v1;
 #
+# MDEV-28387 UBSAN: runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strtoll10 on SELECT
+#
+SET @a='-9223372036854775808';
+CREATE TABLE t (c1 INT,c2 CHAR);
+SELECT SUBSTR(0,@a) FROM t;
+SUBSTR(0,@a)
+DROP TABLE t;
+#
 # End of 10.5 tests
 #
diff --git a/mysql-test/main/func_str.test b/mysql-test/main/func_str.test
index 69161349720..49b5546abb8 100644
--- a/mysql-test/main/func_str.test
+++ b/mysql-test/main/func_str.test
@@ -2348,6 +2348,15 @@ DROP TABLE t1;
 DROP VIEW v1;
 
 
+--echo #
+--echo # MDEV-28387 UBSAN: runtime error: negation of -9223372036854775808 cannot be represented in type 'long long int'; cast to an unsigned type to negate this value to itself in my_strtoll10 on SELECT
+--echo #
+
+SET @a='-9223372036854775808';  # Quite specific value; considerably varying it will not work
+CREATE TABLE t (c1 INT,c2 CHAR);
+SELECT SUBSTR(0,@a) FROM t;
+DROP TABLE t;
+
 --echo #
 --echo # End of 10.5 tests
 --echo #
diff --git a/strings/my_strtoll10.c b/strings/my_strtoll10.c
index 183829d7074..8144be7a1da 100644
--- a/strings/my_strtoll10.c
+++ b/strings/my_strtoll10.c
@@ -241,7 +241,7 @@ end4:
   *endptr= (char*) s;
   if (negative)
   {
-   if (li > MAX_NEGATIVE_NUMBER)
+   if (li >= MAX_NEGATIVE_NUMBER)
      goto overflow;
    return -((longlong) li);
   }