From 65e6ecb861c22a6a80c76457a0d340fcaf7f0d28 Mon Sep 17 00:00:00 2001 From: "ramil@mysql.com" <> Date: Mon, 28 Nov 2005 14:52:38 +0400 Subject: [PATCH 1/3] Fix for bug #12956: cast make differ rounding. --- mysql-test/r/ps.result | 14 +++++++------- mysql-test/t/count_distinct3.test | 2 +- sql/item_func.cc | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mysql-test/r/ps.result b/mysql-test/r/ps.result index f1c3672083d..57272bda7d0 100644 --- a/mysql-test/r/ps.result +++ b/mysql-test/r/ps.result @@ -337,7 +337,7 @@ set @precision=10000000000; select rand(), cast(rand(10)*@precision as unsigned integer) from t1; rand() cast(rand(10)*@precision as unsigned integer) -- 6570515219 +- 6570515220 - 1282061302 - 6698761160 - 9647622201 @@ -348,23 +348,23 @@ prepare stmt from set @var=1; execute stmt using @var; rand() cast(rand(10)*@precision as unsigned integer) cast(rand(?)*@precision as unsigned integer) -- 6570515219 - +- 6570515220 - - 1282061302 - - 6698761160 - - 9647622201 - set @var=2; execute stmt using @var; rand() cast(rand(10)*@precision as unsigned integer) cast(rand(?)*@precision as unsigned integer) -- 6570515219 6555866465 -- 1282061302 1223466192 -- 6698761160 6449731873 +- 6570515220 6555866465 +- 1282061302 1223466193 +- 6698761160 6449731874 - 9647622201 8578261098 set @var=3; execute stmt using @var; rand() cast(rand(10)*@precision as unsigned integer) cast(rand(?)*@precision as unsigned integer) -- 6570515219 9057697559 +- 6570515220 9057697560 - 1282061302 3730790581 -- 6698761160 1480860534 +- 6698761160 1480860535 - 9647622201 6211931236 drop table t1; deallocate prepare stmt; diff --git a/mysql-test/t/count_distinct3.test b/mysql-test/t/count_distinct3.test index 52a4f271dac..f817b2c635d 100644 --- a/mysql-test/t/count_distinct3.test +++ b/mysql-test/t/count_distinct3.test @@ -17,7 +17,7 @@ while ($1) SET @rnd= RAND(); SET @id = CAST(@rnd * @rnd_max AS UNSIGNED); SET @id_rev= @rnd_max - @id; - SET @grp= CAST(128.0 * @rnd AS UNSIGNED); + SET @grp= CAST(127.0 * @rnd AS UNSIGNED); INSERT INTO t1 (id, grp, id_rev) VALUES (@id, @grp, @id_rev); dec $1; } diff --git a/sql/item_func.cc b/sql/item_func.cc index c20a774e57e..b4c4cf580cd 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -735,7 +735,7 @@ longlong Item_func_numhybrid::val_int() case INT_RESULT: return int_op(); case REAL_RESULT: - return (longlong)real_op(); + return (longlong) rint(real_op()); case STRING_RESULT: { int err_not_used; From b46c240b961b131aba3f92142643e46d38c842f5 Mon Sep 17 00:00:00 2001 From: "ramil@mysql.com" <> Date: Tue, 29 Nov 2005 18:06:58 +0400 Subject: [PATCH 2/3] Addition to fix for bug #12956: cast make differ rounding. - use rint() in some other val_int() methods as well. --- sql/field.cc | 4 ++-- sql/item.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/field.cc b/sql/field.cc index b70e2a92618..6de2a731030 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -3953,7 +3953,7 @@ longlong Field_float::val_int(void) else #endif memcpy_fixed((byte*) &j,ptr,sizeof(j)); - return ((longlong) j); + return (longlong) rint(j); } @@ -4241,7 +4241,7 @@ longlong Field_double::val_int(void) else #endif doubleget(j,ptr); - return ((longlong) j); + return (longlong) rint(j); } diff --git a/sql/item.h b/sql/item.h index 3c03cd23e2c..718b995bc5a 100644 --- a/sql/item.h +++ b/sql/item.h @@ -159,7 +159,7 @@ struct Hybrid_type_traits { val->real/= ulonglong2double(u); } virtual longlong val_int(Hybrid_type *val, bool unsigned_flag) const - { return (longlong) val->real; } + { return (longlong) rint(val->real); } virtual double val_real(Hybrid_type *val) const { return val->real; } virtual my_decimal *val_decimal(Hybrid_type *val, my_decimal *buf) const; virtual String *val_str(Hybrid_type *val, String *buf, uint8 decimals) const; @@ -1341,7 +1341,7 @@ public: { return LONGLONG_MAX; } - return (longlong) (value+(value > 0 ? 0.5 : -0.5)); + return (longlong) rint(value); } String *val_str(String*); my_decimal *val_decimal(my_decimal *); From 516fd4254e92c730be1258ac140ea42e896119b3 Mon Sep 17 00:00:00 2001 From: "ramil@mysql.com" <> Date: Wed, 30 Nov 2005 11:17:25 +0400 Subject: [PATCH 3/3] Addition to fix for bug #12956: cast make differ rounding. - use rint() in some other val_int() methods as well. --- sql/item.cc | 4 ++-- sql/item_func.h | 4 ++-- sql/item_sum.cc | 6 +++--- sql/item_sum.h | 8 ++++---- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/sql/item.cc b/sql/item.cc index 94dea1a7dd7..8a55049df84 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -2467,7 +2467,7 @@ longlong Item_param::val_int() { switch (state) { case REAL_VALUE: - return (longlong) (value.real + (value.real > 0 ? 0.5 : -0.5)); + return (longlong) rint(value.real); case INT_VALUE: return value.integer; case DECIMAL_VALUE: @@ -5439,7 +5439,7 @@ void Item_cache_real::store(Item *item) longlong Item_cache_real::val_int() { DBUG_ASSERT(fixed == 1); - return (longlong) (value+(value > 0 ? 0.5 : -0.5)); + return (longlong) rint(value); } diff --git a/sql/item_func.h b/sql/item_func.h index ed39cb86d3e..76647fd5cb2 100644 --- a/sql/item_func.h +++ b/sql/item_func.h @@ -199,7 +199,7 @@ public: String *val_str(String*str); my_decimal *val_decimal(my_decimal *decimal_value); longlong val_int() - { DBUG_ASSERT(fixed == 1); return (longlong) val_real(); } + { DBUG_ASSERT(fixed == 1); return (longlong) rint(val_real()); } enum Item_result result_type () const { return REAL_RESULT; } void fix_length_and_dec() { decimals= NOT_FIXED_DEC; max_length= float_length(decimals); } @@ -943,7 +943,7 @@ class Item_func_udf_float :public Item_udf_func longlong val_int() { DBUG_ASSERT(fixed == 1); - return (longlong) Item_func_udf_float::val_real(); + return (longlong) rint(Item_func_udf_float::val_real()); } my_decimal *val_decimal(my_decimal *dec_buf) { diff --git a/sql/item_sum.cc b/sql/item_sum.cc index 506d2a16108..626eb63d04f 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -452,7 +452,7 @@ longlong Item_sum_sum::val_int() &result); return result; } - return (longlong) val_real(); + return (longlong) rint(val_real()); } @@ -1285,7 +1285,7 @@ longlong Item_sum_hybrid::val_int() return sum_int; } default: - return (longlong) Item_sum_hybrid::val_real(); + return (longlong) rint(Item_sum_hybrid::val_real()); } } @@ -2001,7 +2001,7 @@ double Item_avg_field::val_real() longlong Item_avg_field::val_int() { - return (longlong) val_real(); + return (longlong) rint(val_real()); } diff --git a/sql/item_sum.h b/sql/item_sum.h index 87cc248e5e4..2c2bcedab9b 100644 --- a/sql/item_sum.h +++ b/sql/item_sum.h @@ -126,7 +126,7 @@ public: longlong val_int() { DBUG_ASSERT(fixed == 1); - return (longlong) val_real(); /* Real as default */ + return (longlong) rint(val_real()); /* Real as default */ } String *val_str(String*str); my_decimal *val_decimal(my_decimal *); @@ -392,7 +392,7 @@ public: bool add(); double val_real(); // In SPs we might force the "wrong" type with select into a declare variable - longlong val_int() { return (longlong)val_real(); } + longlong val_int() { return (longlong) rint(val_real()); } my_decimal *val_decimal(my_decimal *); String *val_str(String *str); void reset_field(); @@ -421,7 +421,7 @@ public: enum Type type() const {return FIELD_VARIANCE_ITEM; } double val_real(); longlong val_int() - { /* can't be fix_fields()ed */ return (longlong) val_real(); } + { /* can't be fix_fields()ed */ return (longlong) rint(val_real()); } String *val_str(String*); my_decimal *val_decimal(my_decimal *); bool is_null() { (void) val_int(); return null_value; } @@ -699,7 +699,7 @@ class Item_sum_udf_float :public Item_udf_sum longlong val_int() { DBUG_ASSERT(fixed == 1); - return (longlong) Item_sum_udf_float::val_real(); + return (longlong) rint(Item_sum_udf_float::val_real()); } double val_real(); String *val_str(String*str);