Fixed bug mdev-8603.

When building different range and index-merge trees the range optimizer
could build an index-merge tree with an index scan containing less ranges
then needed. This index-merge could be chosen as the best. Following this
index-merge the executioner missed some rows in the result set.
The invalid index scan was built due to an inconsistency in the code
back-ported from mysql into 5.3 that fixed mysql bug #11765831:
the code added to key_or() could change shared keys of the second
ored tree. Partially the problem was fixed in the patch for mariadb
bug #823301, but it turned out that only partially.

mysql-test/r/range_vs_index_merge.result

@@ -60,11 +60,11 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 EXPLAIN
 SELECT * FROM City
 WHERE Population > 100000 AND Name LIKE 'Aba%' OR
-Country IN ('CAN', 'ARG') AND ID < 3800 OR
+Country IN ('CAN', 'ARG') AND  ID BETWEEN 120 AND 130 OR
-Country < 'U' AND Name LIKE 'Zhu%' OR
+Country <= 'ALB' AND Name LIKE 'L%' OR
-ID BETWEEN 3800 AND 3810;
+ID BETWEEN 3807 AND 3810;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	City	index_merge	PRIMARY,Population,Country,Name	Name,Country,PRIMARY	35,3,4	NULL	132	Using sort_union(Name,Country,PRIMARY); Using where
+1	SIMPLE	City	index_merge	PRIMARY,Population,Country,Name	Name,PRIMARY,Country	35,4,3	NULL	31	Using sort_union(Name,PRIMARY,Country); Using where
 EXPLAIN 
 SELECT * FROM City
 WHERE (Population > 101000 AND Population < 115000);
@@ -1763,4 +1763,42 @@ a	b
 167	9999
 168	10000
 DROP TABLE t1;
+#
+# MDEV-8603: Wrong result OR/AND condition over index fields
+#
+CREATE TABLE t1 (
+id INT NOT NULL,
+state VARCHAR(64),
+capital VARCHAR(64),
+UNIQUE KEY (id),
+KEY state (state,id),
+KEY capital (capital, id)
+);
+INSERT INTO t1 VALUES  
+(1,'Arizona','Phoenix'), 
+(2,'Hawaii','Honolulu'),
+(3,'Georgia','Atlanta'), 
+(4,'Florida','Tallahassee'), 
+(5,'Alaska','Juneau'),
+(6,'Michigan','Lansing'),
+(7,'Pennsylvania','Harrisburg'),
+(8,'Virginia','Richmond')
+;
+EXPLAIN
+SELECT * FROM t1 FORCE KEY (state,capital) 
+WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 
+OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	range	state,capital	state	71	NULL	12	Using index condition; Using where
+SELECT * FROM t1 FORCE KEY (state,capital) 
+WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 
+OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas';
+id	state	capital
+4	Florida	Tallahassee
+3	Georgia	Atlanta
+2	Hawaii	Honolulu
+6	Michigan	Lansing
+7	Pennsylvania	Harrisburg
+8	Virginia	Richmond
+DROP TABLE t1;
 set session optimizer_switch='index_merge_sort_intersection=default';

mysql-test/r/range_vs_index_merge_innodb,innodb_plugin.rdiff

@@ -1,5 +1,5 @@
---- ./r/range_vs_index_merge_innodb.result	2012-11-21 19:35:14.000000000 +0100
+--- range_vs_index_merge_innodb.result	2017-01-17 15:00:18.039148421 -0800
-+++ ./r/range_vs_index_merge_innodb,innodb_plugin.reject	2012-11-21 20:56:00.000000000 +0100
++++ range_vs_index_merge_innodb,innodb_plugin.result	2017-01-17 14:58:45.129148312 -0800
 @@ -50,14 +50,14 @@
  WHERE (Population >= 100000 OR Name LIKE 'P%') AND Country='CAN' OR
  (Population < 100000 OR Name Like 'T%') AND Country='ARG';
@@ -278,3 +278,12 @@
  FLUSH STATUS;
  SELECT * FROM City
  WHERE ((Name > 'Ca' AND Name < 'Cf') OR (Country > 'E' AND Country < 'H'))
+@@ -1790,7 +1790,7 @@
+ WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 
+ OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas';
+ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+-1	SIMPLE	t1	range	state,capital	state	71	NULL	10	Using index condition; Using where
++1	SIMPLE	t1	range	state,capital	state	71	NULL	10	Using where
+ SELECT * FROM t1 FORCE KEY (state,capital) 
+ WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 
+ OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas';

mysql-test/r/range_vs_index_merge_innodb.result

@@ -61,11 +61,11 @@ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
 EXPLAIN
 SELECT * FROM City
 WHERE Population > 100000 AND Name LIKE 'Aba%' OR
-Country IN ('CAN', 'ARG') AND ID < 3800 OR
+Country IN ('CAN', 'ARG') AND  ID BETWEEN 120 AND 130 OR
-Country < 'U' AND Name LIKE 'Zhu%' OR
+Country <= 'ALB' AND Name LIKE 'L%' OR
-ID BETWEEN 3800 AND 3810;
+ID BETWEEN 3807 AND 3810;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	City	index_merge	PRIMARY,Population,Country,Name	Name,Country,PRIMARY	35,3,4	NULL	125	Using sort_union(Name,Country,PRIMARY); Using where
+1	SIMPLE	City	index_merge	PRIMARY,Population,Country,Name	Name,Country,PRIMARY	35,3,4	NULL	33	Using sort_union(Name,Country,PRIMARY); Using where
 EXPLAIN 
 SELECT * FROM City
 WHERE (Population > 101000 AND Population < 115000);
@@ -369,7 +369,7 @@ WHERE ((ID < 200) AND (Name LIKE 'Ha%' OR (Country > 'A' AND Country < 'ARG')))
 OR ((ID BETWEEN 100 AND 200) AND 
 (Name LIKE 'Pa%' OR (Population > 103000 AND Population < 104000)));
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-1	SIMPLE	City	range	PRIMARY,Population,Country,Name	PRIMARY	4	NULL	200	Using where
+1	SIMPLE	City	index_merge	PRIMARY,Population,Country,Name	Name,Country,PRIMARY	35,3,4	NULL	181	Using sort_union(Name,Country,PRIMARY); Using where
 SELECT * FROM City USE INDEX ()
 WHERE ((ID < 10) AND (Name LIKE 'H%' OR (Country > 'A' AND Country < 'ARG')))
 OR ((ID BETWEEN 100 AND 110) AND 
@@ -1764,5 +1764,43 @@ a	b
 167	9999
 168	10000
 DROP TABLE t1;
+#
+# MDEV-8603: Wrong result OR/AND condition over index fields
+#
+CREATE TABLE t1 (
+id INT NOT NULL,
+state VARCHAR(64),
+capital VARCHAR(64),
+UNIQUE KEY (id),
+KEY state (state,id),
+KEY capital (capital, id)
+);
+INSERT INTO t1 VALUES  
+(1,'Arizona','Phoenix'), 
+(2,'Hawaii','Honolulu'),
+(3,'Georgia','Atlanta'), 
+(4,'Florida','Tallahassee'), 
+(5,'Alaska','Juneau'),
+(6,'Michigan','Lansing'),
+(7,'Pennsylvania','Harrisburg'),
+(8,'Virginia','Richmond')
+;
+EXPLAIN
+SELECT * FROM t1 FORCE KEY (state,capital) 
+WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 
+OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	range	state,capital	state	71	NULL	10	Using index condition; Using where
+SELECT * FROM t1 FORCE KEY (state,capital) 
+WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 
+OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas';
+id	state	capital
+4	Florida	Tallahassee
+3	Georgia	Atlanta
+2	Hawaii	Honolulu
+6	Michigan	Lansing
+7	Pennsylvania	Harrisburg
+8	Virginia	Richmond
+DROP TABLE t1;
 set session optimizer_switch='index_merge_sort_intersection=default';
 SET SESSION STORAGE_ENGINE=DEFAULT;

mysql-test/t/range_vs_index_merge.test

@@ -57,9 +57,9 @@ SELECT * FROM City
 EXPLAIN
 SELECT * FROM City
   WHERE Population > 100000 AND Name LIKE 'Aba%' OR
-        Country IN ('CAN', 'ARG') AND ID < 3800 OR
+        Country IN ('CAN', 'ARG') AND  ID BETWEEN 120 AND 130 OR
-        Country < 'U' AND Name LIKE 'Zhu%' OR
+        Country <= 'ALB' AND Name LIKE 'L%' OR
-        ID BETWEEN 3800 AND 3810;
+        ID BETWEEN 3807 AND 3810;
 
 # The output of the next 3 commands tells us about selectivities
 # of the conditions utilized in 2 queries following after them  
@@ -1203,6 +1203,41 @@ SELECT * FROM t1
 
 DROP TABLE t1;
 
+
+--echo #
+--echo # MDEV-8603: Wrong result OR/AND condition over index fields
+--echo #
+
+CREATE TABLE t1 (
+  id INT NOT NULL,
+  state VARCHAR(64),
+  capital VARCHAR(64),
+  UNIQUE KEY (id),
+  KEY state (state,id),
+  KEY capital (capital, id)
+);
+ 
+INSERT INTO t1 VALUES  
+  (1,'Arizona','Phoenix'), 
+  (2,'Hawaii','Honolulu'),
+  (3,'Georgia','Atlanta'), 
+  (4,'Florida','Tallahassee'), 
+  (5,'Alaska','Juneau'),
+  (6,'Michigan','Lansing'),
+  (7,'Pennsylvania','Harrisburg'),
+  (8,'Virginia','Richmond')
+;
+ 
+EXPLAIN
+SELECT * FROM t1 FORCE KEY (state,capital) 
+WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 
+   OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas';
+SELECT * FROM t1 FORCE KEY (state,capital) 
+WHERE ( state = 'Alabama' OR state >= 'Colorado' ) AND id != 9 
+   OR ( capital >= 'Topeka' OR state = 'Kansas' ) AND state != 'Texas';
+
+DROP TABLE t1;
+
 #the following command must be the last one in the file
 set session optimizer_switch='index_merge_sort_intersection=default';
 

sql/opt_range.cc

@@ -9335,6 +9335,13 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2)
 
       if (!tmp->next_key_part)
       {
+        if (key2->use_count)
+	{
+	  SEL_ARG *key2_cpy= new SEL_ARG(*key2);
+          if (key2_cpy)
+            return 0;
+          key2= key2_cpy;
+	}
         /*
           tmp->next_key_part is empty: cut the range that is covered
           by tmp from key2. 
@@ -9366,13 +9373,6 @@ key_or(RANGE_OPT_PARAM *param, SEL_ARG *key1,SEL_ARG *key2)
             key2:               [---]
             tmp:     [---------]
           */
-          if (key2->use_count)
-	  {
-	    SEL_ARG *key2_cpy= new SEL_ARG(*key2);
-            if (key2_cpy)
-              return 0;
-            key2= key2_cpy;
-	  }
           key2->copy_max_to_min(tmp);
           continue;
         }