Fix for BUG#4488: first portion: sign aware '<' and '>' comparisons.

mysql-test/r/range.result

@@ -475,3 +475,80 @@ id	name	uid	id	name	uid
 1025	Y	25	1025	Y	25
 1026	Z	26	1026	Z	26
 drop table t1,t2;
+create table t1 (x bigint unsigned not null);
+insert into t1(x) values (0xfffffffffffffff0);
+insert into t1(x) values (0xfffffffffffffff1);
+select * from t1;
+x
+18446744073709551600
+18446744073709551601
+select count(*) from t1 where x>0;
+count(*)
+2
+select count(*) from t1 where x=0;
+count(*)
+0
+select count(*) from t1 where x<0;
+count(*)
+0
+select count(*) from t1 where x < -16;
+count(*)
+0
+select count(*) from t1 where x = -16;
+count(*)
+0
+select count(*) from t1 where x > -16;
+count(*)
+2
+create table t2 (x bigint not null);
+insert into t2(x) values (0xfffffffffffffff0);
+insert into t2(x) values (0xfffffffffffffff1);
+select * from t2;
+x
+-16
+-15
+select count(*) from t2 where x>0;
+count(*)
+0
+select count(*) from t2 where x=0;
+count(*)
+0
+select count(*) from t2 where x<0;
+count(*)
+2
+select count(*) from t2 where x < -16;
+count(*)
+0
+select count(*) from t2 where x = -16;
+count(*)
+1
+select count(*) from t2 where x > -16;
+count(*)
+1
+drop table t1;
+create table t1 (x bigint unsigned not null primary key) engine=innodb;
+insert into t1(x) values (0xfffffffffffffff0);
+insert into t1(x) values (0xfffffffffffffff1);
+select * from t1;
+x
+18446744073709551600
+18446744073709551601
+select count(*) from t1 where x>0;
+count(*)
+2
+select count(*) from t1 where x=0;
+count(*)
+0
+select count(*) from t1 where x<0;
+count(*)
+0
+select count(*) from t1 where x < -16;
+count(*)
+0
+select count(*) from t1 where x = -16;
+count(*)
+0
+select count(*) from t1 where x > -16;
+count(*)
+1
+drop table t1;

mysql-test/t/range.test

@@ -383,3 +383,42 @@ select * from t1, t2  where t1.uid=t2.uid AND t1.uid > 0;
 select * from t1, t2  where t1.uid=t2.uid AND t1.uid != 0;
 
 drop table t1,t2;
+
+# Fix for bug#4488 
+#
+create table t1 (x bigint unsigned not null);
+insert into t1(x) values (0xfffffffffffffff0);
+insert into t1(x) values (0xfffffffffffffff1);
+select * from t1;
+select count(*) from t1 where x>0;
+select count(*) from t1 where x=0;
+select count(*) from t1 where x<0;
+select count(*) from t1 where x < -16;
+select count(*) from t1 where x = -16;
+select count(*) from t1 where x > -16;
+
+create table t2 (x bigint not null);
+insert into t2(x) values (0xfffffffffffffff0);
+insert into t2(x) values (0xfffffffffffffff1);
+select * from t2;
+select count(*) from t2 where x>0;
+select count(*) from t2 where x=0;
+select count(*) from t2 where x<0;
+select count(*) from t2 where x < -16;
+select count(*) from t2 where x = -16;
+select count(*) from t2 where x > -16;
+
+drop table t1;
+create table t1 (x bigint unsigned not null primary key) engine=innodb;
+insert into t1(x) values (0xfffffffffffffff0);
+insert into t1(x) values (0xfffffffffffffff1);
+select * from t1;
+select count(*) from t1 where x>0;
+select count(*) from t1 where x=0;
+select count(*) from t1 where x<0;
+select count(*) from t1 where x < -16;
+select count(*) from t1 where x = -16;
+select count(*) from t1 where x > -16;
+
+drop table t1;
+

sql/item.h

@@ -862,7 +862,7 @@ public:
 };
 
 /*
-  The following class is used to optimize comparing of date columns
+  The following class is used to optimize comparing of date and bigint columns
   We need to save the original item, to be able to set the field to the
   original value in 'opt_range'.
 */
@@ -872,7 +872,9 @@ class Item_int_with_ref :public Item_int
   Item *ref;
 public:
   Item_int_with_ref(longlong i, Item *ref_arg) :Item_int(i), ref(ref_arg)
-  {}
+  {
+    unsigned_flag= ref_arg->unsigned_flag;
+  }
   int save_in_field(Field *field, bool no_conversions)
   {
     return ref->save_in_field(field, no_conversions);

sql/item_cmpfunc.cc

@@ -315,6 +315,17 @@ int Arg_comparator::set_compare_func(Item_bool_func2 *item, Item_result type)
 	func= &Arg_comparator::compare_e_binary_string;
     }
   }
+  else if (type == INT_RESULT)
+  {
+    if (func == &Arg_comparator::compare_int)
+    {
+      if ((*a)->unsigned_flag)
+        func= ((*b)->unsigned_flag)? &Arg_comparator::compare_int_unsigned : 
+                                     &Arg_comparator::compare_int_unsigned_signed;
+      else if ((*b)->unsigned_flag)
+        func= &Arg_comparator::compare_int_signed_unsigned;
+    }
+  }
   return 0;
 }
 
@@ -434,6 +445,82 @@ int Arg_comparator::compare_int()
   return -1;
 }
 
+
+/*
+  Compare values as BIGINT UNSIGNED.
+*/
+
+int Arg_comparator::compare_int_unsigned()
+{
+  ulonglong val1= (*a)->val_int();
+  if (!(*a)->null_value)
+  {
+    ulonglong val2= (*b)->val_int();
+    if (!(*b)->null_value)
+    {
+      owner->null_value= 0;
+      if (val1 < val2)	return -1;
+      if (val1 == val2)   return 0;
+      return 1;
+    }
+  }
+  owner->null_value= 1;
+  return -1;
+}
+
+
+/*
+  Compare signed (*a) with unsigned (*B)
+*/
+
+int Arg_comparator::compare_int_signed_unsigned()
+{
+  longlong sval1= (*a)->val_int();
+  if (!(*a)->null_value)
+  {
+    ulonglong uval2= (ulonglong)(*b)->val_int();
+    if (!(*b)->null_value)
+    {
+      owner->null_value= 0;
+      if (sval1 < 0 || (ulonglong)sval1 < uval2)
+        return -1;
+      if ((ulonglong)sval1 == uval2)
+        return 0;
+      return 1;
+    }
+  }
+  owner->null_value= 1;
+  return -1;
+}
+
+
+/*
+  Compare unsigned (*a) with signed (*B)
+*/
+
+int Arg_comparator::compare_int_unsigned_signed()
+{
+  ulonglong uval1= (ulonglong)(*a)->val_int();
+  if (!(*a)->null_value)
+  {
+    longlong sval2= (*b)->val_int();
+    if (!(*b)->null_value)
+    {
+      owner->null_value= 0;
+      if (sval2 < 0)
+        return 1;
+      if (uval1 < (ulonglong)sval2)
+        return -1;
+      if (uval1 == (ulonglong)sval2)
+        return 0;
+      return 1;
+    }
+  }
+  owner->null_value= 1;
+  return -1;
+}
+
+
 int Arg_comparator::compare_e_int()
 {
   longlong val1= (*a)->val_int();

sql/item_cmpfunc.h

@@ -66,6 +66,9 @@ public:
   int compare_binary_string();	 // compare args[0] & args[1]
   int compare_real();            // compare args[0] & args[1]
   int compare_int();             // compare args[0] & args[1]
+  int compare_int_signed_unsigned();
+  int compare_int_unsigned_signed();
+  int compare_int_unsigned();
   int compare_row();             // compare args[0] & args[1]
   int compare_e_string();	 // compare args[0] & args[1]
   int compare_e_binary_string(); // compare args[0] & args[1]