From b6c56f465cd7c8820768954c58c7ddebcc8a3e77 Mon Sep 17 00:00:00 2001
From: Sergey Glukhov <sergey.glukhov@oracle.com>
Date: Mon, 28 Mar 2011 12:28:30 +0400
Subject: [PATCH 1/2] Bug#11766424  59527: DECIMAL_BIN_SIZE: ASSERTION `SCALE
 >= 0 && PRECISION > 0 && SCALE <= PRE Assertion happens due to missing
 initialization of unsigned_flag for Item_func_set_user_var object. It leads
 to incorrect calculation of decimal field size. The fix is to add
 initialization of unsigned_flag.

---
 mysql-test/r/variables.result | 16 ++++++++++++++++
 mysql-test/t/variables.test   | 13 +++++++++++++
 sql/item_func.cc              |  1 +
 3 files changed, 30 insertions(+)

diff --git a/mysql-test/r/variables.result b/mysql-test/r/variables.result
index af3b76b09f3..f92e1dec4c9 100644
--- a/mysql-test/r/variables.result
+++ b/mysql-test/r/variables.result
@@ -1547,6 +1547,22 @@ Warning	1292	Truncated incorrect key_cache_block_size value: '0'
 select @@max_long_data_size;
 @@max_long_data_size
 1048576
+#
+# Bug#11766424  59527: DECIMAL_BIN_SIZE: ASSERTION `SCALE >= 0 && PRECISION > 0 && SCALE <= PRE
+#
+CREATE TABLE t1(f1 DECIMAL(1,1) UNSIGNED);
+INSERT INTO t1 VALUES (0.2),(0.1);
+SELECT 1 FROM t1 GROUP BY @a:= (SELECT ROUND(f1) FROM t1 WHERE @a=f1);
+1
+1
+DROP TABLE t1;
+CREATE TABLE t1 AS SELECT @a:= CAST(1 AS UNSIGNED) AS a;
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(1) unsigned NOT NULL DEFAULT '0'
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+DROP TABLE t1;
 SET @@global.max_binlog_cache_size=DEFAULT;
 SET @@global.max_join_size=DEFAULT;
 SET @@global.key_buffer_size=@kbs;
diff --git a/mysql-test/t/variables.test b/mysql-test/t/variables.test
index 383bdfc79a9..8f111e7cf3b 100644
--- a/mysql-test/t/variables.test
+++ b/mysql-test/t/variables.test
@@ -1298,6 +1298,19 @@ SET @@global.key_cache_block_size=0;
 #
 select @@max_long_data_size;
 
+--echo #
+--echo # Bug#11766424  59527: DECIMAL_BIN_SIZE: ASSERTION `SCALE >= 0 && PRECISION > 0 && SCALE <= PRE
+--echo #
+
+CREATE TABLE t1(f1 DECIMAL(1,1) UNSIGNED);
+INSERT INTO t1 VALUES (0.2),(0.1);
+SELECT 1 FROM t1 GROUP BY @a:= (SELECT ROUND(f1) FROM t1 WHERE @a=f1);
+DROP TABLE t1;
+
+CREATE TABLE t1 AS SELECT @a:= CAST(1 AS UNSIGNED) AS a;
+SHOW CREATE TABLE t1;
+DROP TABLE t1;
+
 # cleanup
 SET @@global.max_binlog_cache_size=DEFAULT;
 SET @@global.max_join_size=DEFAULT;
diff --git a/sql/item_func.cc b/sql/item_func.cc
index efae928a8b6..d4fd2c94e1d 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -3840,6 +3840,7 @@ Item_func_set_user_var::fix_length_and_dec()
   maybe_null=args[0]->maybe_null;
   max_length=args[0]->max_length;
   decimals=args[0]->decimals;
+  unsigned_flag= args[0]->unsigned_flag;
   collation.set(args[0]->collation.collation, DERIVATION_IMPLICIT);
 }
 

From 86c1225699b7ab403398ebfa4863aef80e6d566a Mon Sep 17 00:00:00 2001
From: Sergey Glukhov <sergey.glukhov@oracle.com>
Date: Mon, 28 Mar 2011 12:35:50 +0400
Subject: [PATCH 2/2] Bug#11764994  57900: CREATE TABLE .. SELECT ASSERTS SCALE
 >= 0 && PRECISION > 0 && SCALE <= PR Assert fails due to overflow which
 happens in Item_func_int_val::fix_num_length_and_dec() as geometry functions
 have max_length value equal to max_field_size(4294967295U). The fix is to
 skip max_length calculation for some boundary cases.

---
 mysql-test/r/func_math.result | 7 +++++++
 mysql-test/t/func_math.test   | 9 +++++++++
 sql/item_func.cc              | 7 ++++---
 3 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/mysql-test/r/func_math.result b/mysql-test/r/func_math.result
index 3a626084c9e..ad0b872145b 100644
--- a/mysql-test/r/func_math.result
+++ b/mysql-test/r/func_math.result
@@ -511,4 +511,11 @@ t1	CREATE TABLE `t1` (
   `C` varchar(23) DEFAULT NULL
 ) ENGINE=MyISAM DEFAULT CHARSET=latin1
 DROP TABLE t1;
+#
+# Bug#11764994  57900: CREATE TABLE .. SELECT ASSERTS SCALE >= 0 && PRECISION > 0 && SCALE <= PR
+#
+CREATE TABLE t1 SELECT CEIL(LINESTRINGFROMWKB(1) DIV NULL);
+DROP TABLE t1;
+CREATE TABLE t1 SELECT FLOOR(LINESTRINGFROMWKB(1) DIV NULL);
+DROP TABLE t1;
 End of 5.1 tests
diff --git a/mysql-test/t/func_math.test b/mysql-test/t/func_math.test
index c8ea11c7490..64b6a3a4ea6 100644
--- a/mysql-test/t/func_math.test
+++ b/mysql-test/t/func_math.test
@@ -324,4 +324,13 @@ CREATE TABLE t1 SELECT CAST((CASE(('')) WHEN (CONVERT(1, CHAR(1))) THEN (('' / 1
 SHOW CREATE TABLE t1;
 DROP TABLE t1;
 
+--echo #
+--echo # Bug#11764994  57900: CREATE TABLE .. SELECT ASSERTS SCALE >= 0 && PRECISION > 0 && SCALE <= PR
+--echo #
+
+CREATE TABLE t1 SELECT CEIL(LINESTRINGFROMWKB(1) DIV NULL);
+DROP TABLE t1;
+CREATE TABLE t1 SELECT FLOOR(LINESTRINGFROMWKB(1) DIV NULL);
+DROP TABLE t1;
+
 --echo End of 5.1 tests
diff --git a/sql/item_func.cc b/sql/item_func.cc
index d4fd2c94e1d..79fa37bd372 100644
--- a/sql/item_func.cc
+++ b/sql/item_func.cc
@@ -1803,9 +1803,10 @@ void Item_func_integer::fix_length_and_dec()
 
 void Item_func_int_val::fix_num_length_and_dec()
 {
-  max_length= args[0]->max_length - (args[0]->decimals ?
-                                     args[0]->decimals + 1 :
-                                     0) + 2;
+  ulonglong tmp_max_length= (ulonglong ) args[0]->max_length - 
+    (args[0]->decimals ? args[0]->decimals + 1 : 0) + 2;
+  max_length= tmp_max_length > (ulonglong) max_field_size ?
+    max_field_size : (uint32) tmp_max_length;
   uint tmp= float_length(decimals);
   set_if_smaller(max_length,tmp);
   decimals= 0;