database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-19 02:02:17 +03:00
Code Activity

MCOL-4640 Narrow DECIMAL precision loss in CAST(AS SIGNED) and CHAR()

Browse Source
This commit is contained in:
Alexander Barkov
2021-03-26 19:03:13 +04:00
parent 77fba49e76
commit 0854d71d4e
3 changed files with 41 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
datatypes/mcs_decimal.h
View File

@ -140,6 +140,19 @@ constexpr int128_t Decimal128Null = TSInt128::NullValue;
constexpr int128_t Decimal128Empty = TSInt128::EmptyValue;
template<typename T>
T scaleDivisor(const uint32_t scale)
{
if (scale < 19)
return (T) mcs_pow_10[scale];
if (scale > 39)
{
std::string msg = "scaleDivisor called with a wrong scale: " + std::to_string(scale);
throw std::invalid_argument(msg);
}
return (T) mcs_pow_10_128[scale - 19];
}
/**
@brief The function to produce scale multiplier/divisor for
wide decimals.
@ -152,14 +165,7 @@ inline void getScaleDivisor(T& divisor, const int8_t scale)
std::string msg = "getScaleDivisor called with negative scale: " + std::to_string(scale);
throw std::invalid_argument(msg);
}
else if (scale < 19)
{
divisor = mcs_pow_10[scale];
}
else
{
divisor = mcs_pow_10_128[scale-19];
}
divisor = scaleDivisor<T>((uint32_t) scale);
}
struct lldiv_t_128
@ -198,14 +204,14 @@ public:
// Divide to the scale divisor with rounding
int64_t toSInt64Round(uint32_t scale) const
{
double dscale = scale;
int64_t tmp = value / pow(10.0, dscale);
int lefto = (value - tmp * pow(10.0, dscale)) / pow(10.0, dscale - 1);
if (tmp >= 0 && lefto > 4)
return tmp + 1;
if (tmp < 0 && lefto < -4)
return tmp - 1;
return tmp;
auto divisor = scaleDivisor<int64_t>(scale);
int64_t intg = value / divisor;
int64_t frac2 = 2 * (value % divisor);
if (frac2 >= divisor)
return intg + 1;
if (frac2 <= -divisor)
return intg - 1;
return intg;
}
};

9
mtr/basic/r/type_decimal.result
View File

@ -3562,3 +3562,12 @@ SELECT HEX(CHAR(a USING latin1)) FROM t1;
HEX(CHAR(a USING latin1))
FFFFFFFF
DROP TABLE t1;
#
# MCOL-4640 Narrow DECIMAL precision loss in CAST(AS SIGNED) and CHAR()
#
CREATE TABLE t1 (a DECIMAL(17,1)) ENGINE=ColumnStore;
INSERT INTO t1 VALUES (8999999999999999.0);
SELECT CAST(a AS SIGNED), HEX(CHAR(a USING latin1)) FROM t1;
CAST(a AS SIGNED) HEX(CHAR(a USING latin1))
8999999999999999 CAFA7FFF
DROP TABLE t1;

10
mtr/basic/t/type_decimal.test
View File

@ -207,3 +207,13 @@ CREATE TABLE t1 (a DECIMAL(30,0));
INSERT INTO t1 VALUES (-1);
SELECT HEX(CHAR(a USING latin1)) FROM t1;
DROP TABLE t1;
--echo #
--echo # MCOL-4640 Narrow DECIMAL precision loss in CAST(AS SIGNED) and CHAR()
--echo #
CREATE TABLE t1 (a DECIMAL(17,1)) ENGINE=ColumnStore;
INSERT INTO t1 VALUES (8999999999999999.0);
SELECT CAST(a AS SIGNED), HEX(CHAR(a USING latin1)) FROM t1;
DROP TABLE t1;