Merge pull request #1962 from tntnatbry/MCOL-4642

MCOL-4642 NOT IN subquery containing an isnull in the OR predicate crashes server.

dbcon/mysql/ha_in_sub.cpp

@@ -266,7 +266,6 @@ void InSub::handleFunc(gp_walk_info* gwip, Item_func* func)
             if (cond->argument_list()->elements == 1)
                 return;
 
-            // (cache=item or isnull(item)) case. remove "or isnull()"
             if (cond->argument_list()->elements == 2)
             {
                 // don't know how to deal with this. don't think it's a fatal error either.
@@ -278,35 +277,22 @@ void InSub::handleFunc(gp_walk_info* gwip, Item_func* func)
                 if (!pt->left() || !pt->right())
                     return;
 
-                SimpleFilter* sf = dynamic_cast<SimpleFilter*>(pt->left()->data());
+                SimpleFilter* lsf = dynamic_cast<SimpleFilter*>(pt->left()->data());
+                SimpleFilter* rsf = dynamic_cast<SimpleFilter*>(pt->right()->data());
 
-                //assert (sf && sf->op()->op() == execplan::OP_ISNULL);
+                if (!lsf || !rsf)
-                if (!sf || sf->op()->op() != execplan::OP_ISNULL)
                     return;
 
-                delete sf;
+                // (a=b or isnull(item))/(a=b or isnotnull(item)) case.
-                sf = dynamic_cast<SimpleFilter*>(pt->right()->data());
+                // swap the lhs and rhs operands of the OR operator.
-
+                if ((lsf->op()->op() == execplan::OP_ISNULL ||
-                //idbassert(sf && sf->op()->op() == execplan::OP_EQ);
+                     lsf->op()->op() == execplan::OP_ISNOTNULL) &&
-                if (!sf || sf->op()->op() != execplan::OP_EQ)
+                    rsf->op()->op() == execplan::OP_EQ)
-                    return;
+                {
-
+                    ParseTree* temp = pt->left();
-                // set NULLMATCH for both operand. It's really a setting for the join.
+                    pt->left(pt->right());
-                // should only set NULLMATCH when the subtype is NOT_IN. for some IN subquery
+                    pt->right(temp);
-                // with aggregation column, MySQL inefficiently convert to:
+                }
-                // (cache=item or item is null) and item is not null, which is equivalent to
-                // cache = item. Do not set NULLMATCH for this case.
-                // Because we don't know IN or NOTIN yet, set candidate bit and switch to NULLMATCH
-                // later in handleNot function.
-                if (sf->lhs()->joinInfo() & JOIN_CORRELATED)
-                    sf->lhs()->joinInfo(sf->lhs()->joinInfo() | JOIN_NULLMATCH_CANDIDATE);
-
-                if (sf->rhs()->joinInfo() & JOIN_CORRELATED)
-                    sf->rhs()->joinInfo(sf->rhs()->joinInfo() | JOIN_NULLMATCH_CANDIDATE);
-
-                pt = pt->right();
-                gwip->ptWorkStack.pop();
-                gwip->ptWorkStack.push(pt);
             }
         }
         else if (cond->functype() == Item_func::EQ_FUNC)

mysql-test/columnstore/basic/r/mcol-4642.result

@@ -0,0 +1,49 @@
+#
+# MCOL-4642 NOT IN subquery containing an isnull in the OR predicate crashes server
+#
+DROP DATABASE IF EXISTS mcol4642;
+CREATE DATABASE mcol4642;
+USE mcol4642;
+CREATE TABLE cs1 (a int);
+INSERT INTO cs1 VALUES (1), (2), (3), (4), (null);
+CREATE TABLE cs2 (b int, c int);
+INSERT INTO cs2 VALUES (1, 100), (1, 101), (2, 200),(3, 300), (3, 301), (3, 302), (null, null);
+SELECT * FROM cs1 WHERE a NOT IN (SELECT b FROM cs2 WHERE b is not null OR b is null);
+a
+SELECT * FROM cs1 WHERE a NOT IN (SELECT b FROM cs2 WHERE b is null OR b is not null);
+a
+SELECT * FROM cs1 WHERE a IN (SELECT b FROM cs2 WHERE b is not null OR b is null);
+a
+1
+2
+3
+SELECT * FROM cs1 WHERE a IN (SELECT b FROM cs2 WHERE b is null OR b is not null);
+a
+1
+2
+3
+SELECT * FROM cs1 WHERE a NOT IN (SELECT b FROM cs2 WHERE b=123 OR b is null);
+a
+SELECT * FROM cs1 WHERE a NOT IN (SELECT b FROM cs2 WHERE b is null OR b=123);
+a
+SELECT * FROM cs1 WHERE a IN (SELECT b FROM cs2 WHERE b=123 OR b is null);
+a
+SELECT * FROM cs1 WHERE a IN (SELECT b FROM cs2 WHERE b is null OR b=123);
+a
+SELECT * FROM cs1 WHERE a NOT IN (SELECT b FROM cs2 WHERE b=123 OR b is not null);
+a
+4
+SELECT * FROM cs1 WHERE a NOT IN (SELECT b FROM cs2 WHERE b is not null OR b=123);
+a
+4
+SELECT * FROM cs1 WHERE a IN (SELECT b FROM cs2 WHERE b=123 OR b is not null);
+a
+1
+2
+3
+SELECT * FROM cs1 WHERE a IN (SELECT b FROM cs2 WHERE b is not null OR b=123);
+a
+1
+2
+3
+DROP DATABASE mcol4642;

mysql-test/columnstore/basic/t/mcol-4642.test

@@ -0,0 +1,35 @@
+--source ../include/have_columnstore.inc
+--source ctype_cmp_combinations.inc
+
+--echo #
+--echo # MCOL-4642 NOT IN subquery containing an isnull in the OR predicate crashes server
+--echo #
+
+--disable_warnings
+DROP DATABASE IF EXISTS mcol4642;
+--enable_warnings
+
+CREATE DATABASE mcol4642;
+USE mcol4642;
+
+CREATE TABLE cs1 (a int);
+INSERT INTO cs1 VALUES (1), (2), (3), (4), (null);
+CREATE TABLE cs2 (b int, c int);
+INSERT INTO cs2 VALUES (1, 100), (1, 101), (2, 200),(3, 300), (3, 301), (3, 302), (null, null);
+
+SELECT * FROM cs1 WHERE a NOT IN (SELECT b FROM cs2 WHERE b is not null OR b is null);
+SELECT * FROM cs1 WHERE a NOT IN (SELECT b FROM cs2 WHERE b is null OR b is not null);
+SELECT * FROM cs1 WHERE a IN (SELECT b FROM cs2 WHERE b is not null OR b is null);
+SELECT * FROM cs1 WHERE a IN (SELECT b FROM cs2 WHERE b is null OR b is not null);
+
+SELECT * FROM cs1 WHERE a NOT IN (SELECT b FROM cs2 WHERE b=123 OR b is null);
+SELECT * FROM cs1 WHERE a NOT IN (SELECT b FROM cs2 WHERE b is null OR b=123);
+SELECT * FROM cs1 WHERE a IN (SELECT b FROM cs2 WHERE b=123 OR b is null);
+SELECT * FROM cs1 WHERE a IN (SELECT b FROM cs2 WHERE b is null OR b=123);
+
+SELECT * FROM cs1 WHERE a NOT IN (SELECT b FROM cs2 WHERE b=123 OR b is not null);
+SELECT * FROM cs1 WHERE a NOT IN (SELECT b FROM cs2 WHERE b is not null OR b=123);
+SELECT * FROM cs1 WHERE a IN (SELECT b FROM cs2 WHERE b=123 OR b is not null);
+SELECT * FROM cs1 WHERE a IN (SELECT b FROM cs2 WHERE b is not null OR b=123);
+
+DROP DATABASE mcol4642;