database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-20 13:23:00 +03:00
Code Activity

MCOL-4640 Narrow DECIMAL precision loss in CAST(AS SIGNED) and CHAR()

Browse Source
This commit is contained in:
Alexander Barkov
2021-03-26 19:03:13 +04:00
parent 77fba49e76
commit 0854d71d4e
3 changed files with 41 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
datatypes/mcs_decimal.h
View File

@ -140,6 +140,19 @@ constexpr int128_t Decimal128Null = TSInt128::NullValue;
constexpr int128_t Decimal128Empty = TSInt128::EmptyValue; constexpr int128_t Decimal128Empty = TSInt128::EmptyValue;
template<typename T>
T scaleDivisor(const uint32_t scale)
{
if (scale < 19)
return (T) mcs_pow_10[scale];
if (scale > 39)
{
std::string msg = "scaleDivisor called with a wrong scale: " + std::to_string(scale);
throw std::invalid_argument(msg);
}
return (T) mcs_pow_10_128[scale - 19];
}
/** /**
@brief The function to produce scale multiplier/divisor for @brief The function to produce scale multiplier/divisor for
wide decimals. wide decimals.
@ -152,14 +165,7 @@ inline void getScaleDivisor(T& divisor, const int8_t scale)
std::string msg = "getScaleDivisor called with negative scale: " + std::to_string(scale); std::string msg = "getScaleDivisor called with negative scale: " + std::to_string(scale);
throw std::invalid_argument(msg); throw std::invalid_argument(msg);
} }
else if (scale < 19) divisor = scaleDivisor<T>((uint32_t) scale);
{
divisor = mcs_pow_10[scale];
}
else
{
divisor = mcs_pow_10_128[scale-19];
}
} }
struct lldiv_t_128 struct lldiv_t_128
@ -198,14 +204,14 @@ public:
// Divide to the scale divisor with rounding // Divide to the scale divisor with rounding
int64_t toSInt64Round(uint32_t scale) const int64_t toSInt64Round(uint32_t scale) const
{ {
double dscale = scale; auto divisor = scaleDivisor<int64_t>(scale);
int64_t tmp = value / pow(10.0, dscale); int64_t intg = value / divisor;
int lefto = (value - tmp * pow(10.0, dscale)) / pow(10.0, dscale - 1); int64_t frac2 = 2 * (value % divisor);
if (tmp >= 0 && lefto > 4) if (frac2 >= divisor)
return tmp + 1; return intg + 1;
if (tmp < 0 && lefto < -4) if (frac2 <= -divisor)
return tmp - 1; return intg - 1;
return tmp; return intg;
} }
}; };

9
mtr/basic/r/type_decimal.result
View File

@ -3562,3 +3562,12 @@ SELECT HEX(CHAR(a USING latin1)) FROM t1;
HEX(CHAR(a USING latin1)) HEX(CHAR(a USING latin1))
FFFFFFFF FFFFFFFF
DROP TABLE t1; DROP TABLE t1;
#
# MCOL-4640 Narrow DECIMAL precision loss in CAST(AS SIGNED) and CHAR()
#
CREATE TABLE t1 (a DECIMAL(17,1)) ENGINE=ColumnStore;
INSERT INTO t1 VALUES (8999999999999999.0);
SELECT CAST(a AS SIGNED), HEX(CHAR(a USING latin1)) FROM t1;
CAST(a AS SIGNED) HEX(CHAR(a USING latin1))
8999999999999999 CAFA7FFF
DROP TABLE t1;

10
mtr/basic/t/type_decimal.test
View File

@ -207,3 +207,13 @@ CREATE TABLE t1 (a DECIMAL(30,0));
INSERT INTO t1 VALUES (-1); INSERT INTO t1 VALUES (-1);
SELECT HEX(CHAR(a USING latin1)) FROM t1; SELECT HEX(CHAR(a USING latin1)) FROM t1;
DROP TABLE t1; DROP TABLE t1;
--echo #
--echo # MCOL-4640 Narrow DECIMAL precision loss in CAST(AS SIGNED) and CHAR()
--echo #
CREATE TABLE t1 (a DECIMAL(17,1)) ENGINE=ColumnStore;
INSERT INTO t1 VALUES (8999999999999999.0);
SELECT CAST(a AS SIGNED), HEX(CHAR(a USING latin1)) FROM t1;
DROP TABLE t1;