diff --git a/mysql-test/r/range.result b/mysql-test/r/range.result index afd91ce5fe9..07e96aee22b 100644 --- a/mysql-test/r/range.result +++ b/mysql-test/r/range.result @@ -556,11 +556,42 @@ count(*) 0 select count(*) from t1 where x > -16; count(*) -1 +2 select count(*) from t1 where x = 18446744073709551601; count(*) 1 drop table t1; +create table t1 (a bigint unsigned); +create index t1i on t1(a); +insert into t1 select 18446744073709551615; +insert into t1 select 18446744073709551614; +explain select * from t1 where a <> -1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index t1i t1i 9 NULL 2 Using where; Using index +select * from t1 where a <> -1; +a +18446744073709551614 +18446744073709551615 +explain select * from t1 where a > -1 or a < -1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index t1i t1i 9 NULL 2 Using where; Using index +select * from t1 where a > -1 or a < -1; +a +18446744073709551614 +18446744073709551615 +explain select * from t1 where a > -1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 index t1i t1i 9 NULL 2 Using where; Using index +select * from t1 where a > -1; +a +18446744073709551614 +18446744073709551615 +explain select * from t1 where a < -1; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE NULL NULL NULL NULL NULL NULL NULL Impossible WHERE noticed after reading const tables +select * from t1 where a < -1; +a +drop table t1; set names latin1; create table t1 (a char(10), b text, key (a)) character set latin1; INSERT INTO t1 (a) VALUES diff --git a/mysql-test/t/range.test b/mysql-test/t/range.test index b51a79fba77..92b7b848a8a 100644 --- a/mysql-test/t/range.test +++ b/mysql-test/t/range.test @@ -423,14 +423,30 @@ select count(*) from t1 where x=0; select count(*) from t1 where x<0; select count(*) from t1 where x < -16; select count(*) from t1 where x = -16; -# The following query returns wrong value because the range optimizer can't -# handle search on a signed value for an unsigned parameter. This will be fixed in -# 5.0 select count(*) from t1 where x > -16; select count(*) from t1 where x = 18446744073709551601; drop table t1; +# +# Bug #11185 incorrect comparison of unsigned int to signed constant +# +create table t1 (a bigint unsigned); +create index t1i on t1(a); +insert into t1 select 18446744073709551615; +insert into t1 select 18446744073709551614; + +explain select * from t1 where a <> -1; +select * from t1 where a <> -1; +explain select * from t1 where a > -1 or a < -1; +select * from t1 where a > -1 or a < -1; +explain select * from t1 where a > -1; +select * from t1 where a > -1; +explain select * from t1 where a < -1; +select * from t1 where a < -1; + +drop table t1; + # # Bug #6045: Binary Comparison regression in MySQL 4.1 # Binary searches didn't use a case insensitive index. diff --git a/sql/opt_range.cc b/sql/opt_range.cc index bd1befb436f..75cf9e6b3f0 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -960,7 +960,9 @@ get_mm_parts(PARAM *param, COND *cond_func, Field *field, if (sel_arg->type == SEL_ARG::IMPOSSIBLE) { tree->type=SEL_TREE::IMPOSSIBLE; - DBUG_RETURN(tree); + /* If this is an NE_FUNC, we still need to check GT_FUNC. */ + if (!ne_func) + DBUG_RETURN(tree); } } else @@ -979,8 +981,9 @@ get_mm_parts(PARAM *param, COND *cond_func, Field *field, SEL_TREE *tree2= get_mm_parts(param, cond_func, field, Item_func::GT_FUNC, value, cmp_type); - if (tree2) - tree= tree_or(param,tree,tree2); + if (!tree2) + DBUG_RETURN(0) + tree= tree_or(param,tree,tree2); } DBUG_RETURN(tree); } @@ -1159,6 +1162,35 @@ get_mm_leaf(PARAM *param, COND *conf_func, Field *field, KEY_PART *key_part, if (!(tree=new SEL_ARG(field,str,str2))) DBUG_RETURN(0); // out of memory + /* + Check if we are comparing an UNSIGNED integer with a negative constant. + In this case we know that: + (a) (unsigned_int [< | <=] negative_constant) == FALSE + (b) (unsigned_int [> | >=] negative_constant) == TRUE + In case (a) the condition is false for all values, and in case (b) it + is true for all values, so we can avoid unnecessary retrieval and condition + testing, and we also get correct comparison of unsinged integers with + negative integers (which otherwise fails because at query execution time + negative integers are cast to unsigned if compared with unsigned). + */ + Item_result field_result_type= field->result_type(); + Item_result value_result_type= value->result_type(); + if (field_result_type == INT_RESULT && value_result_type == INT_RESULT && + ((Field_num*)field)->unsigned_flag && !((Item_int*)value)->unsigned_flag) + { + longlong item_val= value->val_int(); + if (item_val < 0) + { + if (type == Item_func::LT_FUNC || type == Item_func::LE_FUNC) + { + tree->type= SEL_ARG::IMPOSSIBLE; + DBUG_RETURN(tree); + } + if (type == Item_func::GT_FUNC || type == Item_func::GE_FUNC) + DBUG_RETURN(0); + } + } + switch (type) { case Item_func::LT_FUNC: if (field_is_equal_to_item(field,value))