BUG#19618: Crash for unsigned_col NOT IN (-1, ... )

- When manually constructing a SEL_TREE for "t.key NOT IN(...)", take into account that 
  get_mm_parts may return a tree with type SEL_TREE::IMPOSSIBLE
- Added missing OOM checks
- Added comments

mysql-test/r/func_in.result

@@ -326,3 +326,20 @@ deallocate prepare s;
 set @str=NULL;
 drop table t2;
 drop table t1;
+create table t1 (
+some_id smallint(5) unsigned,
+key (some_id)
+);
+insert into t1 values (1),(2);
+select some_id from t1 where some_id not in(2,-1);
+some_id
+1
+select some_id from t1 where some_id not in(-4,-1,-4);
+some_id
+1
+2
+select some_id from t1 where some_id not in(-4,-1,3423534,2342342);
+some_id
+1
+2
+drop table t1;

mysql-test/t/func_in.test

@@ -220,3 +220,15 @@ set @str=NULL;
 drop table t2;
 drop table t1;
 
+# BUG#19618: Crash in range optimizer for 
+#   "unsigned_keypart NOT IN(negative_number,...)" 
+#   (introduced in fix BUG#15872) 
+create table t1 (
+  some_id smallint(5) unsigned,
+  key (some_id)
+);
+insert into t1 values (1),(2);
+select some_id from t1 where some_id not in(2,-1);
+select some_id from t1 where some_id not in(-4,-1,-4);
+select some_id from t1 where some_id not in(-4,-1,3423534,2342342);
+drop table t1;

sql/opt_range.cc

@@ -3530,17 +3530,38 @@ static SEL_TREE *get_func_mm_tree(PARAM *param, Item_func *cond_func,
         if (!value_item)
           break;
         
-        /* Get a SEL_TREE for "-inf < X < c_0" interval */
+        /* 
-        func->array->value_to_item(0, value_item);
+          Get a SEL_TREE for "(-inf|NULL) < X < c_0" interval.
-        tree= get_mm_parts(param, cond_func, field, Item_func::LT_FUNC,
+          Note: for partially-covering keys the returned tree may represent
-                           value_item, cmp_type);
+          a half-closed interval (-inf < X <= c_0). In that case the for the
-        if (!tree)
+          whole NOT IN statement the (-inf < X < +inf) interval will be
+          constructed. It doesn't make sense to consider range access over
+          such intervals, but we don't eliminate them here as 1) they are 
+          handled correctly by all parts of the code, and 2) the case where
+          such intervals are constructed is rare.
+        */
+        uint i=0;
+        do 
+        {
+          func->array->value_to_item(i, value_item);
+          tree= get_mm_parts(param, cond_func, field, Item_func::LT_FUNC,
+                             value_item, cmp_type);
+          if (!tree)
+            break;
+          i++;
+        } while (i < func->array->count && tree->type == SEL_TREE::IMPOSSIBLE);
+
+        if (!tree || tree->type == SEL_TREE::IMPOSSIBLE)
+        {
+          /* We get here in cases like "t.unsigned NOT IN (-1,-2,-3) */
+          tree= NULL;
           break;
+        }
 #define NOT_IN_IGNORE_THRESHOLD 1000        
         SEL_TREE *tree2;
         if (func->array->count < NOT_IN_IGNORE_THRESHOLD)
         {
-          for (uint i=1; i < func->array->count; i++)
+          for (; i < func->array->count; i++)
           {
             if (func->array->compare_elems(i, i-1))
             {
@@ -3548,32 +3569,44 @@ static SEL_TREE *get_func_mm_tree(PARAM *param, Item_func *cond_func,
               func->array->value_to_item(i, value_item);
               tree2= get_mm_parts(param, cond_func, field, Item_func::LT_FUNC,
                                   value_item, cmp_type);
-              
+              if (!tree2)
+              {
+                tree= NULL;
+                break;
+              }
+
               /* Change all intervals to be "c_{i-1} < X < c_i" */
               for (uint idx= 0; idx < param->keys; idx++)
               {
-                SEL_ARG *new_interval;
+                SEL_ARG *new_interval, *last_val;
-                if ((new_interval=  tree2->keys[idx]))
+                if (((new_interval= tree2->keys[idx])) && 
+                    ((last_val= tree->keys[idx]->last())))
                 {
-                  SEL_ARG *last_val= tree->keys[idx]->last();
                   new_interval->min_value= last_val->max_value;
                   new_interval->min_flag= NEAR_MIN;
                 }
               }
+              /* 
+                The following doesn't try to allocate memory so no need to
+                check for NULL.
+              */
               tree= tree_or(param, tree, tree2);
             }
           }
         }
         else
           func->array->value_to_item(func->array->count - 1, value_item);
-
+        
-        /* 
+        if (tree && tree->type != SEL_TREE::IMPOSSIBLE)
-          Get the SEL_TREE for the last "c_last < X < +inf" interval 
+        {
-          (value_item cotains c_last already)
+          /* 
-        */
+            Get the SEL_TREE for the last "c_last < X < +inf" interval 
-        tree2= get_mm_parts(param, cond_func, field, Item_func::GT_FUNC,
+            (value_item cotains c_last already)
-                            value_item, cmp_type);
+          */
-        tree= tree_or(param, tree, tree2);
+          tree2= get_mm_parts(param, cond_func, field, Item_func::GT_FUNC,
+                              value_item, cmp_type);
+          tree= tree_or(param, tree, tree2);
+        }
       }
       else
       {