database/mariadb-columnstore-engine
1
0
Fork 0
mirror of https://github.com/mariadb-corporation/mariadb-columnstore-engine.git synced 2025-07-29 08:21:15 +03:00
Code Activity

MCOL-4600 CAST(decimal AS SIGNED/UNSIGNED) returns a wrong result

Browse Source 
The "SIGNED" part of the problem was previously fixed by MCOL-4640.
Fixing the "UNSIGNED" part.

- Adding TDecimal64::toUInt64Round() and Decimal::decimal64ToUInt64Round()
- Renaming Decimal::narrowRound() to decimal64ToSInt64Round(),
   for a more self-descriptive name, and for symmetry with decimal64ToUInt64Round()
- Reusing TDecimal64::toSInt64Round() inside decimal64ToSInt64Round().
  This change was forgotten in MCOL-4640 :(
- Removing the old code in Func_cast_unsigned::getUintVal with pow().
  It caused precision loss, hence the bug. Adding a call for
  Decimal::decimal64ToUInt64Round() instead.
- Adding tests for both SIGNED and UNSIGNED casts.

Additional change:
- Moving the wide-decimal-to-uint64_t rounding code from Func_cast_unsigned::getUintVal()
  to TDecimal128::toUInt64Round() (with refactoring). Adding TDecimal::toUInt64Round()
  for symmetry with TDecimal::toSInt64Round(). It will be easier to reuse the code
  with way.
This commit is contained in:
Alexander Barkov
2021-03-29 17:30:41 +04:00
parent 2eec956977
commit 1acc631a04
5 changed files with 55 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
utils/funcexp/func_bitwise.cpp
View File

@ -100,7 +100,7 @@ datatypes::TUInt64Null DecimalToBitOperand(Row& row,
return ConvertToBitOperand<int128_t>(val);
}
return datatypes::TUInt64Null((uint64_t) d.narrowRound());
return datatypes::TUInt64Null((uint64_t) d.decimal64ToSInt64Round());
}

45
utils/funcexp/func_cast.cpp
View File

@ -331,50 +331,7 @@ uint64_t Func_cast_unsigned::getUintVal(Row& row,
case execplan::CalpontSystemCatalog::UDECIMAL:
{
IDB_Decimal d = parm[0]->data()->getDecimalVal(row, isNull);
if (parm[0]->data()->resultType().colWidth == datatypes::MAXDECIMALWIDTH)
{
if (d.s128Value < 0)
{
return 0;
}
int128_t scaleDivisor, scaleDivisor2;
datatypes::getScaleDivisor(scaleDivisor, d.scale);
scaleDivisor2 = (scaleDivisor <= 10) ? 1 : (scaleDivisor / 10);
uint128_t tmpval = d.s128Value / scaleDivisor;
int128_t lefto = (d.s128Value - tmpval * scaleDivisor) / scaleDivisor2;
if (utils::is_nonnegative(tmpval) && lefto > 4)
tmpval++;
if (tmpval > static_cast<int128_t>(UINT64_MAX))
tmpval = UINT64_MAX;
return static_cast<uint64_t>(tmpval);
}
else
{
if (d.value < 0)
{
return 0;
}
double dscale = d.scale;
uint64_t value = d.value / pow(10.0, dscale);
int lefto = (d.value - value * pow(10.0, dscale)) / pow(10.0, dscale - 1);
if ( utils::is_nonnegative(value) && lefto > 4 )
{
value++;
}
return value;
}
return d.toUInt64Round();
}
break;