From 672e1f20cb0d98b7d3330e367fe0c78b48a39f93 Mon Sep 17 00:00:00 2001
From: "gluh@eagle.intranet.mysql.r18.ru" <>
Date: Wed, 13 Jul 2005 14:23:09 +0500
Subject: [PATCH] Bug #9854 hex() and out of range handling   added out of
 range handling

---
 mysql-test/r/func_str.result |  3 +++
 mysql-test/t/func_str.test   |  5 +++++
 sql/item_strfunc.cc          | 16 ++++++++++++++--
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/mysql-test/r/func_str.result b/mysql-test/r/func_str.result
index fb2716fd232..18a04574596 100644
--- a/mysql-test/r/func_str.result
+++ b/mysql-test/r/func_str.result
@@ -865,3 +865,6 @@ abc	abc	c	bc	abc
 abcd	abcd	d	cd	bcd	abcd	
 abcde	abcde	e	de	cde	bcde	abcde
 drop table t1;
+select hex(29223372036854775809), hex(-29223372036854775809);
+hex(29223372036854775809)	hex(-29223372036854775809)
+FFFFFFFFFFFFFFFF	FFFFFFFFFFFFFFFF
diff --git a/mysql-test/t/func_str.test b/mysql-test/t/func_str.test
index 668e865d73a..ae3cdc361ab 100644
--- a/mysql-test/t/func_str.test
+++ b/mysql-test/t/func_str.test
@@ -606,3 +606,8 @@ insert t1 values ('ab'), ('abc'), ('abcd'), ('abcde');
 select *,substring(b,1),substring(b,-1),substring(b,-2),substring(b,-3),substring(b,-4),substring(b,-5) from t1;
 select * from (select *,substring(b,1),substring(b,-1),substring(b,-2),substring(b,-3),substring(b,-4),substring(b,-5) from t1) t;
 drop table t1;
+
+#
+# Bug #9854  hex() and out of range handling
+#
+select hex(29223372036854775809), hex(-29223372036854775809);
diff --git a/sql/item_strfunc.cc b/sql/item_strfunc.cc
index d57b3b89bf9..6a4dd0d7418 100644
--- a/sql/item_strfunc.cc
+++ b/sql/item_strfunc.cc
@@ -2348,9 +2348,21 @@ String *Item_func_hex::val_str(String *str)
   DBUG_ASSERT(fixed == 1);
   if (args[0]->result_type() != STRING_RESULT)
   {
-    /* Return hex of unsigned longlong value */
-    longlong dec= args[0]->val_int();
+    ulonglong dec;
     char ans[65],*ptr;
+    /* Return hex of unsigned longlong value */
+    if (args[0]->result_type() == REAL_RESULT)
+    {
+      double val= args[0]->val();
+      if ((val <= (double) LONGLONG_MIN) || 
+          (val >= (double) (ulonglong) ULONGLONG_MAX))
+        dec=  ~(longlong) 0;
+      else
+        dec= (ulonglong) (val + (val > 0 ? 0.5 : -0.5));
+    }
+    else
+      dec= (ulonglong) args[0]->val_int();
+
     if ((null_value= args[0]->null_value))
       return 0;
     ptr= longlong2str(dec,ans,16);