BUG#16166, "Can't use index_merge with FORCE INDEX": adjust the heurstics check to take into

account that "FORCE INDEX" disables full table scans, and not range/index_merge scans.
(with post-review fixes)

mysql-test/r/index_merge.result

@@ -384,3 +384,21 @@ max(A.key1 + B.key1 + A.key2 + B.key2 + A.key3 + B.key3 + A.key4 + B.key4 + A.ke
 8186
 set join_buffer_size= @save_join_buffer_size;
 drop table t0, t1, t2, t3, t4;
+CREATE TABLE t1 (
+cola char(3) not null, colb char(3) not null,  filler char(200),
+key(cola), key(colb)
+);
+INSERT INTO t1 VALUES ('foo','bar', 'ZZ'),('fuz','baz', 'ZZ');
+OPTIMIZE TABLE t1;
+Table	Op	Msg_type	Msg_text
+test.t1	optimize	status	OK
+select count(*) from t1;
+count(*)
+8704
+explain select * from t1 WHERE cola = 'foo' AND colb = 'bar';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index_merge	cola,colb	cola,colb	3,3	NULL	24	Using intersect(cola,colb); Using where
+explain select * from t1 force index(cola,colb) WHERE cola = 'foo' AND colb = 'bar';
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	index_merge	cola,colb	cola,colb	3,3	NULL	24	Using intersect(cola,colb); Using where
+drop table t1;

mysql-test/t/index_merge.test

@@ -327,3 +327,33 @@ set join_buffer_size= @save_join_buffer_size;
 
 drop table t0, t1, t2, t3, t4;
 
+# BUG#16166
+CREATE TABLE t1 (
+  cola char(3) not null, colb char(3) not null,  filler char(200),
+  key(cola), key(colb)
+);
+INSERT INTO t1 VALUES ('foo','bar', 'ZZ'),('fuz','baz', 'ZZ');
+
+--disable_query_log
+let $1=9;
+while ($1)
+{
+  eval INSERT INTO t1 SELECT * from t1 WHERE cola = 'foo';
+  dec $1;
+}
+
+let $1=13;
+while ($1)
+{
+  eval INSERT INTO t1 SELECT * from t1 WHERE cola <> 'foo';
+  dec $1;
+}
+
+--enable_query_log
+
+OPTIMIZE TABLE t1;
+select count(*) from t1;
+explain select * from t1 WHERE cola = 'foo' AND colb = 'bar';
+explain select * from t1 force index(cola,colb) WHERE cola = 'foo' AND colb = 'bar';
+drop table t1;
+

sql/sql_select.cc

@@ -3471,13 +3471,32 @@ best_access_path(JOIN      *join,
     parts of the row from any of the used index.
     This is because table scans uses index and we would not win
     anything by using a table scan.
+
+    A word for word translation of the below if-statement in psergey's
+    understanding: we check if we should use table scan if:
+    (1) The found 'ref' access produces more records than a table scan
+        (or index scan, or quick select), or 'ref' is more expensive than
+        any of them.
+    (2) This doesn't hold: the best way to perform table scan is to to perform
+        'range' access using index IDX, and the best way to perform 'ref' 
+        access is to use the same index IDX, with the same or more key parts.
+        (note: it is not clear how this rule is/should be extended to 
+        index_merge quick selects)
+    (3) See above note about InnoDB.
+    (4) NOT ("FORCE INDEX(...)" is used for table and there is 'ref' access
+             path, but there is no quick select)
+        If the condition in the above brackets holds, then the only possible
+        "table scan" access method is ALL/index (there is no quick select).
+        Since we have a 'ref' access path, and FORCE INDEX instructs us to
+        choose it over ALL/index, there is no need to consider a full table
+        scan.
   */
-  if ((records >= s->found_records || best > s->read_time) &&
+  if ((records >= s->found_records || best > s->read_time) &&            // (1)
-      !(s->quick && best_key && s->quick->index == best_key->key &&
+      !(s->quick && best_key && s->quick->index == best_key->key &&      // (2)
-        best_max_key_part >= s->table->quick_key_parts[best_key->key]) &&
+        best_max_key_part >= s->table->quick_key_parts[best_key->key]) &&// (2)
-      !((s->table->file->table_flags() & HA_TABLE_SCAN_ON_INDEX) &&
+      !((s->table->file->table_flags() & HA_TABLE_SCAN_ON_INDEX) &&      // (3)
-        ! s->table->used_keys.is_clear_all() && best_key) &&
+        ! s->table->used_keys.is_clear_all() && best_key) &&             // (3)
-      !(s->table->force_index && best_key))
+      !(s->table->force_index && best_key && !s->quick))                 // (4)
   {                                             // Check full join
     ha_rows rnd_records= s->found_records;
     /*
@@ -4460,13 +4479,15 @@ find_best(JOIN *join,table_map rest_tables,uint idx,double record_count,
 	parts of the row from any of the used index.
 	This is because table scans uses index and we would not win
 	anything by using a table scan.
+        (see comment in best_access_path() for more details on the below
+         condition)
       */
       if ((records >= s->found_records || best > s->read_time) &&
 	  !(s->quick && best_key && s->quick->index == best_key->key &&
 	    best_max_key_part >= s->table->quick_key_parts[best_key->key]) &&
 	  !((s->table->file->table_flags() & HA_TABLE_SCAN_ON_INDEX) &&
 	    ! s->table->used_keys.is_clear_all() && best_key) &&
-	  !(s->table->force_index && best_key))
+	  !(s->table->force_index && best_key & !s->quick))
       {						// Check full join
         ha_rows rnd_records= s->found_records;
         /*