MDEV-8786 Wrong result for SELECT FORMAT=JSON * FROM t1 WHERE a=_latin1 0xDF

mysql-test/r/explain_json.result

@@ -1070,3 +1070,23 @@ EXPLAIN
   }
 }
 drop table t1;
+#
+# MDEV-8786 Wrong result for SELECT FORMAT=JSON * FROM t1 WHERE a=_latin1 0xDF
+#
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET latin1);
+INSERT INTO t1 VALUES ('a'),('b');
+EXPLAIN FORMAT=JSON SELECT * FROM t1 WHERE a=_latin1 0xDF;
+EXPLAIN
+{
+  "query_block": {
+    "select_id": 1,
+    "table": {
+      "table_name": "t1",
+      "access_type": "ALL",
+      "rows": 2,
+      "filtered": 100,
+      "attached_condition": "(t1.a = _latin1'\xDF')"
+    }
+  }
+}
+DROP TABLE t1;

mysql-test/t/explain_json.test

@@ -279,3 +279,10 @@ explain format=json select count(distinct a1,a2,b,c) from t1 where (a2 >= 'b') a
 drop table t1;
 
 
+--echo #
+--echo # MDEV-8786 Wrong result for SELECT FORMAT=JSON * FROM t1 WHERE a=_latin1 0xDF
+--echo #
+CREATE TABLE t1 (a VARCHAR(10) CHARACTER SET latin1);
+INSERT INTO t1 VALUES ('a'),('b');
+EXPLAIN FORMAT=JSON SELECT * FROM t1 WHERE a=_latin1 0xDF;
+DROP TABLE t1;

sql/item.cc

@@ -2495,8 +2495,8 @@ void Item_ident::print(String *str, enum_query_type query_type)
       When printing EXPLAIN, don't print database name when it's the same as
       current database.
     */
-    bool skip_db= (query_type & QT_EXPLAIN) && thd->db && 
+    bool skip_db= (query_type & QT_ITEM_IDENT_SKIP_CURRENT_DATABASE) &&
-                  !strcmp(thd->db, db_name);
+                   thd->db && !strcmp(thd->db, db_name);
     if (!skip_db && 
         !(cached_table && cached_table->belong_to_view &&
           cached_table->belong_to_view->compact_view_format))
@@ -7601,7 +7601,7 @@ void Item_cache_wrapper::init_on_demand()
 
 void Item_cache_wrapper::print(String *str, enum_query_type query_type)
 {
-  if (query_type == QT_EXPLAIN)
+  if (query_type & QT_ITEM_CACHE_WRAPPER_SKIP_DETAILS)
   {
     /* Don't print the cache in EXPLAIN EXTENDED */
     orig_item->print(str, query_type);

sql/item_subselect.cc

@@ -899,7 +899,7 @@ void Item_subselect::update_used_tables()
 
 void Item_subselect::print(String *str, enum_query_type query_type)
 {
-  if (query_type == QT_EXPLAIN)
+  if (query_type & QT_ITEM_SUBSELECT_ID_ONLY)
   {
     str->append("(subquery#");
     if (unit && unit->first_select())

sql/mysqld.h

@@ -647,10 +647,26 @@ enum enum_query_type
   QT_WITHOUT_INTRODUCERS= (1 << 1),
   /// view internal representation (like QT_ORDINARY except ORDER BY clause)
   QT_VIEW_INTERNAL= (1 << 2),
-  /// This value means focus on readability, not on ability to parse back, etc.
+  /// If identifiers should not include database names for the current database
-  QT_EXPLAIN= (1 << 4)
+  QT_ITEM_IDENT_SKIP_CURRENT_DATABASE= (1 << 3),
-};
+  /// If Item_cache_wrapper should not print <expr_cache>
+  QT_ITEM_CACHE_WRAPPER_SKIP_DETAILS= (1 << 4),
+  /// If Item_subselect should print as just "(subquery#1)"
+  /// rather than display the subquery body
+  QT_ITEM_SUBSELECT_ID_ONLY= (1 << 5),
 
+  /// This value means focus on readability, not on ability to parse back, etc.
+  QT_EXPLAIN=           QT_TO_SYSTEM_CHARSET |
+                        QT_ITEM_IDENT_SKIP_CURRENT_DATABASE |
+                        QT_ITEM_CACHE_WRAPPER_SKIP_DETAILS |
+                        QT_ITEM_SUBSELECT_ID_ONLY,
+
+  /// This is used for EXPLAIN EXTENDED extra warnings
+  /// Be more detailed than QT_EXPLAIN.
+  /// Perhaps we should eventually include QT_ITEM_IDENT_SKIP_CURRENT_DATABASE
+  /// here, as it would give better readable results
+  QT_EXPLAIN_EXTENDED=  QT_TO_SYSTEM_CHARSET
+};
 
 
 /* query_id */

sql/sql_parse.cc

@@ -5812,7 +5812,7 @@ static bool execute_sqlcom_select(THD *thd, TABLE_LIST *all_tables)
               The warnings system requires input in utf8, @see
               mysqld_show_warnings().
             */
-            lex->unit.print(&str, QT_TO_SYSTEM_CHARSET);
+            lex->unit.print(&str, QT_EXPLAIN_EXTENDED);
             push_warning(thd, Sql_condition::WARN_LEVEL_NOTE,
                          ER_YES, str.c_ptr_safe());
           }