From 0854d71d4ee56b9ddae609e043db4a26ecacbf1c Mon Sep 17 00:00:00 2001
From: Alexander Barkov <bar@mariadb.com>
Date: Fri, 26 Mar 2021 19:03:13 +0400
Subject: [PATCH] MCOL-4640 Narrow DECIMAL precision loss in CAST(AS SIGNED)
 and CHAR()

---
 datatypes/mcs_decimal.h         | 38 +++++++++++++++++++--------------
 mtr/basic/r/type_decimal.result |  9 ++++++++
 mtr/basic/t/type_decimal.test   | 10 +++++++++
 3 files changed, 41 insertions(+), 16 deletions(-)

diff --git a/datatypes/mcs_decimal.h b/datatypes/mcs_decimal.h
index f7f931916..94df57364 100644
--- a/datatypes/mcs_decimal.h
+++ b/datatypes/mcs_decimal.h
@@ -140,6 +140,19 @@ constexpr int128_t Decimal128Null = TSInt128::NullValue;
 constexpr int128_t Decimal128Empty = TSInt128::EmptyValue;
 
 
+template<typename T>
+T scaleDivisor(const uint32_t scale)
+{
+    if (scale < 19)
+        return (T) mcs_pow_10[scale];
+    if (scale > 39)
+    {
+        std::string msg = "scaleDivisor called with a wrong scale: " + std::to_string(scale);
+        throw std::invalid_argument(msg);
+    }
+    return (T) mcs_pow_10_128[scale - 19];
+}
+
 /**
     @brief The function to produce scale multiplier/divisor for
     wide decimals.
@@ -152,14 +165,7 @@ inline void getScaleDivisor(T& divisor, const int8_t scale)
         std::string msg = "getScaleDivisor called with negative scale: " + std::to_string(scale);
         throw std::invalid_argument(msg);
     }
-    else if (scale < 19)
-    {
-        divisor = mcs_pow_10[scale];
-    }
-    else
-    {
-        divisor = mcs_pow_10_128[scale-19];
-    }
+    divisor = scaleDivisor<T>((uint32_t) scale);
 }
 
 struct lldiv_t_128
@@ -198,14 +204,14 @@ public:
     // Divide to the scale divisor with rounding
     int64_t toSInt64Round(uint32_t scale) const
     {
-        double dscale = scale;
-        int64_t tmp = value / pow(10.0, dscale);
-        int lefto = (value - tmp * pow(10.0, dscale)) / pow(10.0, dscale - 1);
-        if (tmp >= 0 && lefto > 4)
-            return tmp + 1;
-        if (tmp < 0 && lefto < -4)
-            return tmp - 1;
-        return tmp;
+        auto divisor = scaleDivisor<int64_t>(scale);
+        int64_t intg = value / divisor;
+        int64_t frac2 = 2 * (value % divisor);
+        if (frac2 >= divisor)
+          return intg + 1;
+        if (frac2 <= -divisor)
+          return intg - 1;
+        return intg;
     }
 };
 
diff --git a/mtr/basic/r/type_decimal.result b/mtr/basic/r/type_decimal.result
index ada6a2388..f99ed6d37 100644
--- a/mtr/basic/r/type_decimal.result
+++ b/mtr/basic/r/type_decimal.result
@@ -3562,3 +3562,12 @@ SELECT HEX(CHAR(a USING latin1)) FROM t1;
 HEX(CHAR(a USING latin1))
 FFFFFFFF
 DROP TABLE t1;
+#
+# MCOL-4640 Narrow DECIMAL precision loss in CAST(AS SIGNED) and CHAR()
+#
+CREATE TABLE t1 (a DECIMAL(17,1)) ENGINE=ColumnStore;
+INSERT INTO t1 VALUES (8999999999999999.0);
+SELECT CAST(a AS SIGNED), HEX(CHAR(a USING latin1)) FROM t1;
+CAST(a AS SIGNED)	HEX(CHAR(a USING latin1))
+8999999999999999	CAFA7FFF
+DROP TABLE t1;
diff --git a/mtr/basic/t/type_decimal.test b/mtr/basic/t/type_decimal.test
index 5760f98c1..ba5c5aadb 100644
--- a/mtr/basic/t/type_decimal.test
+++ b/mtr/basic/t/type_decimal.test
@@ -207,3 +207,13 @@ CREATE TABLE t1 (a DECIMAL(30,0));
 INSERT INTO t1 VALUES (-1);
 SELECT HEX(CHAR(a USING latin1)) FROM t1;
 DROP TABLE t1;
+
+
+--echo #
+--echo # MCOL-4640 Narrow DECIMAL precision loss in CAST(AS SIGNED) and CHAR()
+--echo #
+
+CREATE TABLE t1 (a DECIMAL(17,1)) ENGINE=ColumnStore;
+INSERT INTO t1 VALUES (8999999999999999.0);
+SELECT CAST(a AS SIGNED), HEX(CHAR(a USING latin1)) FROM t1;
+DROP TABLE t1;