You've already forked mariadb-columnstore-engine
mirror of
https://github.com/mariadb-corporation/mariadb-columnstore-engine.git
synced 2025-08-01 06:46:55 +03:00
MCOL-4600 CAST(decimal AS SIGNED/UNSIGNED) returns a wrong result
The "SIGNED" part of the problem was previously fixed by MCOL-4640. Fixing the "UNSIGNED" part. - Adding TDecimal64::toUInt64Round() and Decimal::decimal64ToUInt64Round() - Renaming Decimal::narrowRound() to decimal64ToSInt64Round(), for a more self-descriptive name, and for symmetry with decimal64ToUInt64Round() - Reusing TDecimal64::toSInt64Round() inside decimal64ToSInt64Round(). This change was forgotten in MCOL-4640 :( - Removing the old code in Func_cast_unsigned::getUintVal with pow(). It caused precision loss, hence the bug. Adding a call for Decimal::decimal64ToUInt64Round() instead. - Adding tests for both SIGNED and UNSIGNED casts. Additional change: - Moving the wide-decimal-to-uint64_t rounding code from Func_cast_unsigned::getUintVal() to TDecimal128::toUInt64Round() (with refactoring). Adding TDecimal::toUInt64Round() for symmetry with TDecimal::toSInt64Round(). It will be easier to reuse the code with way.
This commit is contained in:
@ -213,6 +213,12 @@ public:
|
||||
return intg - 1;
|
||||
return intg;
|
||||
}
|
||||
uint64_t toUInt64Round(uint32_t scale) const
|
||||
{
|
||||
return value < 0 ?
|
||||
0 :
|
||||
static_cast<uint64_t>(toSInt64Round(scale));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -255,6 +261,18 @@ public:
|
||||
explicit TDecimal128(const int128_t* valPtr)
|
||||
:TSInt128(valPtr)
|
||||
{ }
|
||||
uint64_t toUInt64Round(uint32_t scale) const
|
||||
{
|
||||
if (s128Value <= 0)
|
||||
return 0;
|
||||
auto divisor = scaleDivisor<uint128_t>(scale);
|
||||
uint128_t intg = s128Value / divisor;
|
||||
uint128_t frac2 = 2 * (s128Value % divisor);
|
||||
if (frac2 >= divisor)
|
||||
intg++;
|
||||
return intg > numeric_limits<uint64_t>::max() ? numeric_limits<uint64_t>::max() :
|
||||
static_cast<uint64_t>(intg);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@ -519,17 +537,13 @@ class Decimal: public TDecimal128, public TDecimal64
|
||||
return TSInt128(roundedValue);
|
||||
}
|
||||
|
||||
int64_t narrowRound() const
|
||||
int64_t decimal64ToSInt64Round() const
|
||||
{
|
||||
int64_t scaleDivisor;
|
||||
getScaleDivisor(scaleDivisor, scale);
|
||||
int64_t intg = value / scaleDivisor;
|
||||
int64_t frac2 = 2 * (value % scaleDivisor);
|
||||
if (frac2 >= scaleDivisor)
|
||||
return intg + 1;
|
||||
if (frac2 <= -scaleDivisor)
|
||||
return intg - 1;
|
||||
return intg;
|
||||
return TDecimal64::toSInt64Round((uint32_t) scale);
|
||||
}
|
||||
uint64_t decimal64ToUInt64Round() const
|
||||
{
|
||||
return TDecimal64::toUInt64Round((uint32_t) scale);
|
||||
}
|
||||
|
||||
int64_t toSInt64Round() const
|
||||
@ -538,6 +552,12 @@ class Decimal: public TDecimal128, public TDecimal64
|
||||
static_cast<int64_t>(getPosNegRoundedIntegralPart(4)) :
|
||||
TDecimal64::toSInt64Round((uint32_t) scale);
|
||||
}
|
||||
uint64_t toUInt64Round() const
|
||||
{
|
||||
return isWideDecimalTypeByPrecision(precision) ?
|
||||
TDecimal128::toUInt64Round((uint32_t) scale) :
|
||||
TDecimal64::toUInt64Round((uint32_t) scale);
|
||||
}
|
||||
|
||||
// MOD operator for an integer divisor to be used
|
||||
// for integer rhs
|
||||
|
Reference in New Issue
Block a user