database/mariadb
1
0
Fork 0
mirror of https://github.com/MariaDB/server.git synced 2025-07-29 05:21:33 +03:00
Code Activity

decimal_div bug#9501

Browse Source
This commit is contained in:
serg@serg.mylan
2005-03-31 17:46:36 +02:00
parent 8098ef79f0
commit e5a50e1927
3 changed files with 64 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
mysql-test/r/type_newdecimal.result
View File

@ -839,3 +839,6 @@ Error 1365 Division by 0
INSERT INTO Sow6_2f VALUES ('a59b');
ERROR HY000: Incorrect decimal value: 'a59b' for column 'col1' at row 1
drop table Sow6_2f;
select 10.3330000000000/12.34500000;
10.3330000000000/12.34500000
0.8370190360469825840421223160000

5
mysql-test/t/type_newdecimal.test
View File

@ -864,3 +864,8 @@ SELECT MOD(col1,0) FROM Sow6_2f;
INSERT INTO Sow6_2f VALUES ('a59b');
#-- should return SQLSTATE 22018 invalid character value for cast
drop table Sow6_2f;
#
# bug#9501
#
select 10.3330000000000/12.34500000;

61
strings/decimal.c
View File

@ -1933,16 +1933,16 @@ int decimal_mul(decimal_t *from1, decimal_t *from2, decimal_t *to)
XXX if this library is to be used with huge numbers of thousands of
digits, fast division must be implemented and alloca should be
changed to malloc (or at least fallback to malloc if alloca() fails)
but then, decimal_mod() should be rewritten too :(
but then, decimal_mul() should be rewritten too :(
*/
static int do_div_mod(decimal_t *from1, decimal_t *from2,
decimal_t *to, decimal_t *mod, int scale_incr)
{
int frac1=ROUND_UP(from1->frac)*DIG_PER_DEC1, prec1=from1->intg+frac1,
frac2=ROUND_UP(from2->frac)*DIG_PER_DEC1, prec2=from2->intg+frac2,
error, i, intg0, frac0, len1, len2, dlen1, dintg, div=(!mod);
error, i, intg0, frac0, len1, len2, dintg, div=(!mod);
dec1 *buf0, *buf1=from1->buf, *buf2=from2->buf, *tmp1,
*start2, *stop2, *stop1, *stop0, norm2, carry, *start1;
*start2, *stop2, *stop1, *stop0, norm2, carry, *start1, dcarry;
dec2 norm_factor, x, guess, y;
LINT_INIT(error);
@ -2043,7 +2043,7 @@ static int do_div_mod(decimal_t *from1, decimal_t *from2,
/* removing end zeroes */
while (*stop2 == 0 && stop2 >= start2)
stop2--;
len2= ++stop2 - start2;
len2= stop2++ - start2;
/*
calculating norm2 (normalized *start2) - we need *start2 to be large
@ -2055,38 +2055,29 @@ static int do_div_mod(decimal_t *from1, decimal_t *from2,
*/
norm_factor=DIG_BASE/(*start2+1);
norm2=(dec1)(norm_factor*start2[0]);
if (likely(len2>1))
if (likely(len2>0))
norm2+=(dec1)(norm_factor*start2[1]/DIG_BASE);
if (*start1 < *start2)
dcarry=*start1++;
else
dcarry=0;
/* main loop */
for (; buf0 < stop0; buf0++)
{
/* short-circuit, if possible */
if (unlikely(*start1 == 0))
{
start1++;
if (likely(div))
*buf0=0;
continue;
}
/* D3: make a guess */
if (*start1 >= *start2)
{
x=start1[0];
y=start1[1];
dlen1=len2-1;
}
if (unlikely(dcarry == 0 && *start1 < *start2))
guess=0;
else
{
x=((dec2)start1[0])*DIG_BASE+start1[1];
y=start1[2];
dlen1=len2;
}
/* D3: make a guess */
x=start1[0]+((dec2)dcarry)*DIG_BASE;
y=start1[1];
guess=(norm_factor*x+norm_factor*y/DIG_BASE)/norm2;
if (unlikely(guess >= DIG_BASE))
guess=DIG_BASE-1;
if (likely(len2>1))
if (likely(len2>0))
{
/* hmm, this is a suspicious trick - I removed normalization here */
if (start2[1]*guess > (x-guess*start2[0])*DIG_BASE+y)
@ -2098,7 +2089,7 @@ static int do_div_mod(decimal_t *from1, decimal_t *from2,
/* D4: multiply and subtract */
buf2=stop2;
buf1=start1+dlen1;
buf1=start1+len2;
DBUG_ASSERT(buf1 < stop1);
for (carry=0; buf2 > start2; buf1--)
{
@ -2109,32 +2100,24 @@ static int do_div_mod(decimal_t *from1, decimal_t *from2,
SUB2(*buf1, *buf1, lo, carry);
carry+=hi;
}
for (; buf1 >= start1; buf1--)
{
SUB2(*buf1, *buf1, 0, carry);
}
carry= dcarry < carry;
/* D5: check the remainder */
if (unlikely(carry))
{
DBUG_ASSERT(carry==1);
/* D6: correct the guess */
guess--;
buf2=stop2;
buf1=start1+dlen1;
buf1=start1+len2;
for (carry=0; buf2 > start2; buf1--)
{
ADD(*buf1, *buf1, *--buf2, carry);
}
for (; buf1 >= start1; buf1--)
{
SUB2(*buf1, *buf1, 0, carry);
}
DBUG_ASSERT(carry==1);
}
if (likely(div))
*buf0=(dec1)guess;
if (*start1 == 0)
dcarry= *start1;
start1++;
}
if (mod)
@ -2144,6 +2127,8 @@ static int do_div_mod(decimal_t *from1, decimal_t *from2,
intg=prec1-frac1
frac=max(frac1, frac2)=to->frac
*/
if (dcarry)
*--start1=dcarry;
buf0=to->buf;
intg0=ROUND_UP(prec1-frac1)-(start1-tmp1);
frac0=ROUND_UP(to->frac);
@ -2753,6 +2738,8 @@ int main()
test_dv("1.000000000000", "3","0.333333333333333333", 0);
test_dv("1", "1","1.000000000", 0);
test_dv("0.0123456789012345678912345", "9999999999","0.000000000001234567890246913578148141", 0);
test_dv("10.333000000", "12.34500","0.837019036046982584042122316", 0);
test_dv("10.000000000060", "2","5.000000000030000000", 0);
printf("==== decimal_mod ====\n");
test_md("234","10","4", 0);