MDEV-21092,MDEV-21095,MDEV-29997: Optimizer Trace for index condition pushdown, partition pruning, exists-to-in

Add Optimizer Tracing for:
        - Index Condition Pushdown
        - Partition Pruning
        - Exists-to-IN optimization

mysql-test/main/opt_trace.test

@@ -932,7 +932,64 @@ explain format=json select * from three, t1 where t1.a=three.a and t1.b<5000 and
 
 drop table three, t1;
 
+--echo #
+--echo # MDEV-21095: Index condition push down is not reflected in optimizer trace
+--echo #
+create table t10 (a int, b int, c int, key(a,b));
+insert into t10 select seq, seq, seq from seq_1_to_10000;
+explain format=json select * from t10 where a<3 and b!=5 and c<10;
+set optimizer_trace='enabled=on';
+select * from t10 where a<3 and b!=5 and c<10;
+select json_detailed(json_extract(trace, '$**.attaching_conditions_to_tables')) as out1 from information_schema.optimizer_trace;
+drop table t10;
+
+--echo #
+--echo # MDEV-21092: EXISTS to IN is not reflected in the optimizer trace
+--echo #
+set @@optimizer_switch = 'exists_to_in=on,in_to_exists=on,semijoin=on,materialization=on,partial_match_rowid_merge=on,partial_match_table_scan=on,subquery_cache=on';
+set optimizer_trace='enabled=on';
+
+create table t1 (cn_c int, cn_n char(10), cn_a int );
+create table t2 (ci_p int, ci_c int );
+create table t3 (ci_p int, ci_c int );
+
+SELECT cn_n FROM t1 WHERE (EXISTS (select 1 from t2 where ci_p > 100000 and cn_c = ci_c)
+  OR (cn_n LIKE 'L%') )
+  AND cn_a > 1000000;
+
+select json_detailed(json_extract(trace, '$.steps[*].join_optimization[0].steps[0].transformation')) from information_schema.optimizer_trace;
+
+drop table t1, t2, t3;
+
+--echo #
+--echo # MDEV-29997 Partition Pruning not included in optimizer tracing
+--echo #
+--source include/have_partition.inc
+create table t2 (a int, b int) partition by hash(a) partitions 10;
+create table t3 (a int, b int) partition by hash(a) partitions 10;
+INSERT INTO t2 SELECT seq, seq from seq_1_to_10;
+INSERT INTO t3 SELECT seq, seq from seq_1_to_10;
+
+set optimizer_trace='enabled=on';
+explain partitions select * from t2,t3 where t2.a in (2,3,4) and t3.a in (4,5);
+select json_detailed(json_extract(trace, '$**.prune_partitions')) from information_schema.optimizer_trace;
+drop table t2,t3;
+
+create table t1 (
+  a int
+) partition by range (a)
+( partition p0 values less than(10),
+  partition p1 values less than (20),
+  partition p2 values less than (25)
+);
+insert into t1 values (5),(15),(22);
+
+explain select * from t1 where a = 28;
+select json_detailed(json_extract(trace, '$**.prune_partitions')) from information_schema.optimizer_trace;
+drop table t1;
+
 set  @@optimizer_switch= @save_optimizer_switch;
 set  @@use_stat_tables= @save_use_stat_tables;
 set  @@histogram_size= @save_histogram_size;
 set  @@optimizer_use_condition_selectivity= @save_optimizer_use_condition_selectivity;
+

mysql-test/main/opt_trace_index_merge.result

@@ -257,15 +257,23 @@ explain select * from t1 where a=1 or b=1	{
             }
           },
           {
-            "attaching_conditions_to_tables": {
+            "make_join_readinfo": [
-              "attached_conditions_computation": [],
+              {
-              "attached_conditions_summary": [
+                "table": "t1",
-                {
+                "index_condition": "t1.a = 1 or t1.b = 1"
-                  "table": "t1",
+              }
-                  "attached": "t1.a = 1 or t1.b = 1"
+            ]
-                }
+          }
-              ]
+        ]
-            }
+      }
+    },
+    {
+      "attaching_conditions_to_tables": {
+        "attached_conditions_computation": [],
+        "attached_conditions_summary": [
+          {
+            "table": "t1",
+            "attached_condition": "t1.a = 1 or t1.b = 1"
           }
         ]
       }

mysql-test/main/opt_trace_index_merge_innodb.result

@@ -256,15 +256,24 @@ explain select * from t1 where pk1 != 0  and key1 = 1	{
             }
           },
           {
-            "attaching_conditions_to_tables": {
+            "make_join_readinfo": [
-              "attached_conditions_computation": [],
+              {
-              "attached_conditions_summary": [
+                "table": "t1",
-                {
+                "index_condition": null
-                  "table": "t1",
+              }
-                  "attached": "t1.pk1 <> 0"
+            ]
-                }
+          }
-              ]
+        ]
-            }
+      }
+    },
+    {
+      "attaching_conditions_to_tables": {
+        "attached_conditions_computation": [],
+        "attached_conditions_summary": [
+          {
+            "table": "t1",
+            "attached_condition": null,
+            "index_condition": "t1.pk1 <> 0"
           }
         ]
       }

mysql-test/main/opt_trace_security.result

@@ -133,15 +133,23 @@ select * from db1.t1	{
             "cost": 0.010504815
           },
           {
-            "attaching_conditions_to_tables": {
+            "make_join_readinfo": [
-              "attached_conditions_computation": [],
+              {
-              "attached_conditions_summary": [
+                "table": "t1",
-                {
+                "index_condition": null
-                  "table": "t1",
+              }
-                  "attached": null
+            ]
-                }
+          }
-              ]
+        ]
-            }
+      }
+    },
+    {
+      "attaching_conditions_to_tables": {
+        "attached_conditions_computation": [],
+        "attached_conditions_summary": [
+          {
+            "table": "t1",
+            "attached_condition": null
           }
         ]
       }
@@ -272,15 +280,23 @@ select * from db1.v1	{
             "cost": 0.010504815
           },
           {
-            "attaching_conditions_to_tables": {
+            "make_join_readinfo": [
-              "attached_conditions_computation": [],
+              {
-              "attached_conditions_summary": [
+                "table": "t1",
-                {
+                "index_condition": null
-                  "table": "t1",
+              }
-                  "attached": null
+            ]
-                }
+          }
-              ]
+        ]
-            }
+      }
+    },
+    {
+      "attaching_conditions_to_tables": {
+        "attached_conditions_computation": [],
+        "attached_conditions_summary": [
+          {
+            "table": "t1",
+            "attached_condition": null
           }
         ]
       }

sql/item.cc

@@ -42,6 +42,7 @@
                                        // RESOLVED_AGAINST_ALIAS, ...
 #include "sql_expression_cache.h"
 #include "sql_lex.h"                   // empty_clex_str
+#include "my_json_writer.h"            // for dbug_print_opt_trace()
 
 const String my_null_string("NULL", 4, default_charset_info);
 const String my_default_string("DEFAULT", 7, default_charset_info);
@@ -10887,6 +10888,29 @@ const char *dbug_print_item(Item *item)
     return "Couldn't fit into buffer";
 }
 
+
+/*
+  Return the optimizer trace collected so far for the current thread.
+*/
+
+const char *dbug_print_opt_trace()
+{
+  if (current_thd)
+  {
+    if (current_thd->opt_trace.is_started())
+    {
+      String *s= const_cast<String *>(current_thd->opt_trace
+                    .get_current_json()->output.get_string());
+      return s->c_ptr();
+    }
+    else
+      return "Trace empty";
+  }
+  else
+    return "No Thread";
+}
+
+
 const char *dbug_print_select(SELECT_LEX *sl)
 {
   char *buf= dbug_item_print_buf;

sql/item_subselect.cc

@@ -42,6 +42,7 @@
 #include "sql_parse.h"                          // check_stack_overrun
 #include "sql_cte.h"
 #include "sql_test.h"
+#include "opt_trace.h"
 
 double get_post_group_estimate(JOIN* join, double join_op_rows);
 
@@ -3342,6 +3343,14 @@ bool Item_exists_subselect::exists2in_processor(void *opt_arg)
     set possible optimization strategies
   */
   in_subs->emb_on_expr_nest= emb_on_expr_nest;
+
+  {
+    OPT_TRACE_TRANSFORM(thd, trace_wrapper, trace_transform,
+  		        in_subs->get_select_lex()->select_number,
+  			"EXISTS (SELECT)", "IN (SELECT)");
+    trace_transform.add( "upper_not", ( upper_not?true:false ) );
+  }
+
   res= check_and_do_in_subquery_rewrites(join);
   first_select->join->prepare_stage2();
 

sql/opt_range.cc

@@ -4198,6 +4198,20 @@ end:
     table->all_partitions_pruned_away= true;
     retval= TRUE;
   }
+
+  if (unlikely(thd->trace_started()))
+  {
+    String parts;
+    String_list parts_list;
+
+    make_used_partitions_str(thd->mem_root, prune_param.part_info, &parts,
+                               parts_list);
+    Json_writer_object trace_wrapper(thd);
+    Json_writer_object trace_prune(thd, "prune_partitions");
+    trace_prune.add_table_name(table);
+    trace_prune.add("used_partitions", parts.c_ptr());
+  }
+
   DBUG_RETURN(retval);
 }
 

sql/sql_select.cc

@@ -349,6 +349,8 @@ static void fix_items_after_optimize(THD *thd, SELECT_LEX *select_lex);
 static void optimize_rownum(THD *thd, SELECT_LEX_UNIT *unit, Item *cond);
 static bool process_direct_rownum_comparison(THD *thd, SELECT_LEX_UNIT *unit,
                                              Item *cond);
+void trace_attached_conditions(THD *thd, JOIN *join);
+void trace_join_readinfo(THD *thd, JOIN *join);
 
 #ifndef DBUG_OFF
 
@@ -3137,6 +3139,8 @@ int JOIN::optimize_stage2()
   if (make_join_readinfo(this, select_opts_for_readinfo, no_jbuf_after))
     DBUG_RETURN(1);
 
+  trace_join_readinfo(thd, this);
+
   /* Perform FULLTEXT search before all regular searches */
   if (!(select_options & SELECT_DESCRIBE))
     if (init_ftfuncs(thd, select_lex, MY_TEST(order)))
@@ -5112,6 +5116,8 @@ mysql_select(THD *thd, TABLE_LIST *tables, List<Item> &fields, COND *conds,
     goto err;					// 1
   }
 
+  trace_attached_conditions(thd, join);
+
   if (thd->lex->describe & DESCRIBE_EXTENDED)
   {
     join->conds_history= join->conds;
@@ -13073,10 +13079,7 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
     /*
       Step #2: Extract WHERE/ON parts
     */
-    Json_writer_object trace_wrapper(thd);
+
-    Json_writer_object trace_conditions(thd, "attaching_conditions_to_tables");
-    Json_writer_array trace_attached_comp(thd,
-                                        "attached_conditions_computation");
     uint i;
     for (i= join->top_join_tab_count - 1; i >= join->const_tables; i--)
     {
@@ -13644,23 +13647,6 @@ make_join_select(JOIN *join,SQL_SELECT *select,COND *cond)
         i++;
     }
 
-    if (unlikely(thd->trace_started()))
-    {
-      trace_attached_comp.end();
-      Json_writer_array trace_attached_summary(thd,
-                                               "attached_conditions_summary");
-      for (tab= first_depth_first_tab(join); tab;
-           tab= next_depth_first_tab(join, tab))
-      {
-        if (!tab->table)
-          continue;
-        Item *const cond = tab->select_cond;
-        Json_writer_object trace_one_table(thd);
-        trace_one_table.
-          add_table_name(tab).
-          add("attached", cond);
-      }
-    }
   }
   DBUG_RETURN(0);
 }
@@ -14778,7 +14764,7 @@ make_join_readinfo(JOIN *join, ulonglong options, uint no_jbuf_after)
   }
  
   check_join_cache_usage_for_tables(join, options, no_jbuf_after);
-  
+
   JOIN_TAB *first_tab;
   for (tab= first_tab= first_linear_tab(join,
                                         WITH_BUSH_ROOTS, WITHOUT_CONST_TABLES);
@@ -31920,6 +31906,60 @@ bool JOIN::transform_all_conds_and_on_exprs_in_join_list(
 }
 
 
+void trace_attached_conditions(THD *thd, JOIN *join)
+{
+  if (!unlikely(thd->trace_started()))
+    return;
+
+  Json_writer_object  trace_wrapper(thd);
+  Json_writer_object  trace_conditions(thd, "attaching_conditions_to_tables");
+  Json_writer_array   trace_attached_comp(thd,
+					   "attached_conditions_computation");
+  JOIN_TAB  *tab;
+
+  trace_attached_comp.end();
+  Json_writer_array  trace_attached_summary(thd,
+					       "attached_conditions_summary");
+
+  for (tab= first_depth_first_tab(join);
+       tab;
+       tab= next_depth_first_tab(join, tab))
+  {
+    if (!tab->table)
+      continue;
+
+    Item *const remaining_cond = tab->select_cond;
+    Item *const idx_cond = tab->table->file->pushed_idx_cond;
+    Json_writer_object  trace_one_table(thd);
+
+    trace_one_table.add_table_name(tab);
+    trace_one_table.add("attached_condition", remaining_cond);
+    if (idx_cond)
+      trace_one_table.add("index_condition", idx_cond);
+  }
+}
+
+
+void trace_join_readinfo(THD *thd, JOIN *join)
+{
+  if (!unlikely(thd->trace_started()))
+    return;
+
+  Json_writer_object  trace_wrapper(thd);
+  Json_writer_array  trace_conditions(thd, "make_join_readinfo");
+  JOIN_TAB  *tab;
+
+  for (tab= first_linear_tab(join, WITH_BUSH_ROOTS, WITHOUT_CONST_TABLES);
+       tab; 
+       tab= next_linear_tab(join, tab, WITH_BUSH_ROOTS))
+  {
+    Json_writer_object  trace_one_table(thd);
+    trace_one_table.add_table_name(tab);
+    trace_one_table.add("index_condition", tab->select_cond);
+  }
+}
+
+
 /**
   @} (end of group Query_Optimizer)
 */