diff --git a/mysql-test/r/func_in.result b/mysql-test/r/func_in.result index e3257ce5fd0..e38e2624e19 100644 --- a/mysql-test/r/func_in.result +++ b/mysql-test/r/func_in.result @@ -326,3 +326,20 @@ deallocate prepare s; set @str=NULL; drop table t2; drop table t1; +create table t1 ( +some_id smallint(5) unsigned, +key (some_id) +); +insert into t1 values (1),(2); +select some_id from t1 where some_id not in(2,-1); +some_id +1 +select some_id from t1 where some_id not in(-4,-1,-4); +some_id +1 +2 +select some_id from t1 where some_id not in(-4,-1,3423534,2342342); +some_id +1 +2 +drop table t1; diff --git a/mysql-test/t/func_in.test b/mysql-test/t/func_in.test index 351d1fc2c92..8ddf1fbe314 100644 --- a/mysql-test/t/func_in.test +++ b/mysql-test/t/func_in.test @@ -220,3 +220,15 @@ set @str=NULL; drop table t2; drop table t1; +# BUG#19618: Crash in range optimizer for +# "unsigned_keypart NOT IN(negative_number,...)" +# (introduced in fix BUG#15872) +create table t1 ( + some_id smallint(5) unsigned, + key (some_id) +); +insert into t1 values (1),(2); +select some_id from t1 where some_id not in(2,-1); +select some_id from t1 where some_id not in(-4,-1,-4); +select some_id from t1 where some_id not in(-4,-1,3423534,2342342); +drop table t1; diff --git a/sql/opt_range.cc b/sql/opt_range.cc index af70b9aade7..ffaf3fad6c8 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -4698,17 +4698,46 @@ static SEL_TREE *get_func_mm_tree(RANGE_OPT_PARAM *param, Item_func *cond_func, if (inv) { - /* - We get here for conditions like "t.keypart NOT IN (....)". - - If the IN-list contains only constants (and func->array is an ordered - array of them), we construct the appropriate SEL_ARG tree manually, - because constructing it using the range analyzer (as - AND_i( t.keypart != c_i)) will cause lots of memory to be consumed - (see BUG#15872). - */ if (func->array && func->cmp_type != ROW_RESULT) { + /* + We get here for conditions in form "t.key NOT IN (c1, c2, ...)" + (where c{i} are constants). + Our goal is to produce a SEL_ARG graph that represents intervals: + + ($MINmem_root; + param->thd->mem_root= param->old_root; /* Create one Item_type constant object. We'll need it as get_mm_parts only accepts constant values wrapped in Item_Type @@ -4717,25 +4746,35 @@ static SEL_TREE *get_func_mm_tree(RANGE_OPT_PARAM *param, Item_func *cond_func, per-statement mem_root (while thd->mem_root is currently pointing to mem_root local to range optimizer). */ - MEM_ROOT *tmp_root= param->mem_root; - param->thd->mem_root= param->old_root; Item *value_item= func->array->create_item(); param->thd->mem_root= tmp_root; if (!value_item) break; - /* Get a SEL_TREE for "-inf < X < c_0" interval */ - func->array->value_to_item(0, value_item); - tree= get_mm_parts(param, cond_func, field, Item_func::LT_FUNC, - value_item, cmp_type); - if (!tree) + /* Get a SEL_TREE for "(-inf|NULL) < X < c_0" interval. */ + uint i=0; + do + { + func->array->value_to_item(i, value_item); + tree= get_mm_parts(param, cond_func, field, Item_func::LT_FUNC, + value_item, cmp_type); + if (!tree) + break; + i++; + } while (i < func->array->count && tree->type == SEL_TREE::IMPOSSIBLE); + + if (!tree || tree->type == SEL_TREE::IMPOSSIBLE) + { + /* We get here in cases like "t.unsigned NOT IN (-1,-2,-3) */ + tree= NULL; break; + } #define NOT_IN_IGNORE_THRESHOLD 1000 SEL_TREE *tree2; if (func->array->count < NOT_IN_IGNORE_THRESHOLD) { - for (uint i=1; i < func->array->count; i++) + for (; i < func->array->count; i++) { if (func->array->compare_elems(i, i-1)) { @@ -4743,32 +4782,44 @@ static SEL_TREE *get_func_mm_tree(RANGE_OPT_PARAM *param, Item_func *cond_func, func->array->value_to_item(i, value_item); tree2= get_mm_parts(param, cond_func, field, Item_func::LT_FUNC, value_item, cmp_type); - + if (!tree2) + { + tree= NULL; + break; + } + /* Change all intervals to be "c_{i-1} < X < c_i" */ for (uint idx= 0; idx < param->keys; idx++) { - SEL_ARG *new_interval; - if ((new_interval= tree2->keys[idx])) + SEL_ARG *new_interval, *last_val; + if (((new_interval= tree2->keys[idx])) && + ((last_val= tree->keys[idx]->last()))) { - SEL_ARG *last_val= tree->keys[idx]->last(); new_interval->min_value= last_val->max_value; new_interval->min_flag= NEAR_MIN; } } + /* + The following doesn't try to allocate memory so no need to + check for NULL. + */ tree= tree_or(param, tree, tree2); } } } else func->array->value_to_item(func->array->count - 1, value_item); - - /* - Get the SEL_TREE for the last "c_last < X < +inf" interval - (value_item cotains c_last already) - */ - tree2= get_mm_parts(param, cond_func, field, Item_func::GT_FUNC, - value_item, cmp_type); - tree= tree_or(param, tree, tree2); + + if (tree && tree->type != SEL_TREE::IMPOSSIBLE) + { + /* + Get the SEL_TREE for the last "c_last < X < +inf" interval + (value_item cotains c_last already) + */ + tree2= get_mm_parts(param, cond_func, field, Item_func::GT_FUNC, + value_item, cmp_type); + tree= tree_or(param, tree, tree2); + } } else {