Backport of subquery optimizations to 5.3.

There are still test failures because of:
- Wrong query results in outer join + semi join
- EXPLAIN output differences

sql/item_subselect.h

@@ -31,13 +31,15 @@ class Item_bool_func2;
 class Item_subselect :public Item_result_field
 {
   my_bool value_assigned; /* value already assigned to subselect */
-protected:
+public:
   /* thread handler, will be assigned in fix_fields only */
   THD *thd;
   /* substitution instead of subselect in case of optimization */
   Item *substitution;
+public:
   /* unit of subquery */
   st_select_lex_unit *unit;
+protected:
   /* engine that perform execution of subselect (single select or union) */
   subselect_engine *engine;
   /* old engine if engine was changed */
@@ -272,8 +274,9 @@ public:
 
 class Item_in_subselect :public Item_exists_subselect
 {
-protected:
+public:
   Item *left_expr;
+protected:
   /*
     expr & optimizer used in subselect rewriting to store Item for
     all JOIN in UNION
@@ -286,6 +289,45 @@ protected:
 public:
   /* Used to trigger on/off conditions that were pushed down to subselect */
   bool *pushed_cond_guards;
+  
+  /* Priority of this predicate in the convert-to-semi-join-nest process. */
+  int sj_convert_priority;
+  /*
+    Used by subquery optimizations to keep track about in which clause this
+    subquery predicate is located: 
+      (TABLE_LIST*) 1   - the predicate is an AND-part of the WHERE
+      join nest pointer - the predicate is an AND-part of ON expression
+                          of a join nest   
+      NULL              - for all other locations
+    See also THD::emb_on_expr_nest.
+  */
+  TABLE_LIST *emb_on_expr_nest;
+  /* 
+    Location of the subquery predicate. It is either
+     - pointer to join nest if the subquery predicate is in the ON expression
+     - (TABLE_LIST*)1 if the predicate is in the WHERE.
+  */
+  TABLE_LIST *expr_join_nest;
+  /*
+    Types of left_expr and subquery's select list allow to perform subquery
+    materialization. Currently, we set this to FALSE when it as well could
+    be TRUE. This is to be properly addressed with fix for BUG#36752.
+  */
+  bool types_allow_materialization;
+
+  /* 
+    Same as above, but they also allow to scan the materialized table. 
+  */
+  bool sjm_scan_allowed;
+
+  /* The method chosen to execute the IN predicate.  */
+  enum enum_exec_method {
+    NOT_TRANSFORMED, /* No execution method was chosen for this IN. */
+    SEMI_JOIN,   /* IN was converted to semi-join nest and should be removed. */
+    IN_TO_EXISTS, /* IN was converted to correlated EXISTS. */
+    MATERIALIZATION /* IN will be executed via subquery materialization. */
+  };
+  enum_exec_method exec_method;
 
   bool *get_cond_guard(int i)
   {
@@ -303,7 +345,7 @@ public:
   Item_in_subselect(Item * left_expr, st_select_lex *select_lex);
   Item_in_subselect()
     :Item_exists_subselect(), optimizer(0), abort_on_null(0), transformed(0),
-     pushed_cond_guards(NULL), upper_item(0)
+     pushed_cond_guards(NULL), exec_method(NOT_TRANSFORMED), upper_item(0)
   {}
 
   subs_type substype() { return IN_SUBS; }