System Versioning 1.0 pre5 [closes #407]

Merge branch '10.3' into trunk

Both field_visibility and VERS_HIDDEN_FLAG exist independently.

TODO:
VERS_HIDDEN_FLAG should be replaced with SYSTEM_INVISIBLE (or COMPLETELY_INVISIBLE?).

.gitignore

@@ -494,3 +494,11 @@ UpgradeLog*.htm
 
 # Microsoft Fakes
 FakesAssemblies/
+
+compile_commands.json
+.clang-format
+.kscope/
+.vimrc
+.editorconfig
+.kateconfig
+*.kdev4

client/mysql_upgrade.c

@@ -166,8 +166,8 @@ static struct my_option my_long_options[]=
    "server with which it was built/distributed.",
    &opt_version_check, &opt_version_check, 0,
    GET_BOOL, NO_ARG, 1, 0, 0, 0, 0, 0},
-  {"write-binlog", OPT_WRITE_BINLOG, "All commands including those, "
+  {"write-binlog", OPT_WRITE_BINLOG, "All commands including those "
-   "issued by mysqlcheck, are written to the binary log.",
+   "issued by mysqlcheck are written to the binary log.",
    &opt_write_binlog, &opt_write_binlog, 0, GET_BOOL, NO_ARG,
    0, 0, 0, 0, 0, 0},
   {0, 0, 0, 0, 0, 0, GET_NO_ARG, NO_ARG, 0, 0, 0, 0, 0, 0}

client/mysqldump.c

@@ -115,10 +115,11 @@ static my_bool  verbose= 0, opt_no_create_info= 0, opt_no_data= 0, opt_no_data_m
                 opt_events= 0, opt_comments_used= 0,
                 opt_alltspcs=0, opt_notspcs= 0, opt_logging,
                 opt_drop_trigger= 0 ;
-static my_bool insert_pat_inited= 0, debug_info_flag= 0, debug_check_flag= 0;
+static my_bool insert_pat_inited= 0, debug_info_flag= 0, debug_check_flag= 0,
+               select_field_names_inited= 0;
 static ulong opt_max_allowed_packet, opt_net_buffer_length;
 static MYSQL mysql_connection,*mysql=0;
-static DYNAMIC_STRING insert_pat;
+static DYNAMIC_STRING insert_pat, select_field_names;
 static char  *opt_password=0,*current_user=0,
              *current_host=0,*path=0,*fields_terminated=0,
              *lines_terminated=0, *enclosed=0, *opt_enclosed=0, *escaped=0,
@@ -1645,6 +1646,7 @@ static void free_resources()
   dynstr_free(&extended_row);
   dynstr_free(&dynamic_where);
   dynstr_free(&insert_pat);
+  dynstr_free(&select_field_names);
   if (defaults_argv)
     free_defaults(defaults_argv);
   mysql_library_end();
@@ -2740,7 +2742,13 @@ static uint get_table_structure(char *table, char *db, char *table_type,
     else
       dynstr_set_checked(&insert_pat, "");
   }
-
+  if (!select_field_names_inited)
+  {
+    select_field_names_inited= 1;
+    init_dynamic_string_checked(&select_field_names, "", 1024, 1024);
+  }
+  else
+    dynstr_set_checked(&select_field_names, "");
   insert_option= ((delayed && opt_ignore) ? " DELAYED IGNORE " :
                   delayed ? " DELAYED " : opt_ignore ? " IGNORE " : "");
 
@@ -2976,6 +2984,19 @@ static uint get_table_structure(char *table, char *db, char *table_type,
       DBUG_RETURN(0);
     }
 
+    while ((row= mysql_fetch_row(result)))
+    {
+      if (strlen(row[SHOW_EXTRA]) && strstr(row[SHOW_EXTRA],"INVISIBLE"))
+        complete_insert= 1;
+      if (init)
+      {
+        dynstr_append_checked(&select_field_names, ", ");
+      }
+      init=1;
+      dynstr_append_checked(&select_field_names,
+              quote_name(row[SHOW_FIELDNAME], name_buff, 0));
+    }
+    init=0;
     /*
       If write_data is true, then we build up insert statements for
       the table's data. Note: in subsequent lines of code, this test
@@ -3003,19 +3024,8 @@ static uint get_table_structure(char *table, char *db, char *table_type,
       }
     }
 
-    while ((row= mysql_fetch_row(result)))
+    if (complete_insert)
-    {
+      dynstr_append_checked(&insert_pat, select_field_names.str);
-      if (complete_insert)
-      {
-        if (init)
-        {
-          dynstr_append_checked(&insert_pat, ", ");
-        }
-        init=1;
-        dynstr_append_checked(&insert_pat,
-                      quote_name(row[SHOW_FIELDNAME], name_buff, 0));
-      }
-    }
     num_fields= mysql_num_rows(result);
     mysql_free_result(result);
   }
@@ -3073,6 +3083,21 @@ static uint get_table_structure(char *table, char *db, char *table_type,
       }
     }
 
+    while ((row= mysql_fetch_row(result)))
+    {
+      if (strlen(row[SHOW_EXTRA]) && strstr(row[SHOW_EXTRA],"INVISIBLE"))
+        complete_insert= 1;
+      if (init)
+      {
+        dynstr_append_checked(&select_field_names, ", ");
+      }
+      dynstr_append_checked(&select_field_names,
+              quote_name(row[SHOW_FIELDNAME], name_buff, 0));
+      init=1;
+    }
+    init=0;
+    mysql_data_seek(result, 0);
+
     while ((row= mysql_fetch_row(result)))
     {
       ulong *lengths= mysql_fetch_lengths(result);
@@ -3713,7 +3738,9 @@ static void dump_table(char *table, char *db)
 
     /* now build the query string */
 
-    dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ * INTO OUTFILE '");
+    dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ ");
+    dynstr_append_checked(&query_string, select_field_names.str);
+    dynstr_append_checked(&query_string, " INTO OUTFILE '");
     dynstr_append_checked(&query_string, filename);
     dynstr_append_checked(&query_string, "'");
 
@@ -3762,7 +3789,9 @@ static void dump_table(char *table, char *db)
                   "\n--\n-- Dumping data for table %s\n--\n",
                   fix_for_comment(result_table));
     
-    dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ * FROM ");
+    dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ ");
+    dynstr_append_checked(&query_string, select_field_names.str);
+    dynstr_append_checked(&query_string, " FROM ");
     dynstr_append_checked(&query_string, result_table);
 
     if (where)

include/ma_dyncol.h

@@ -63,6 +63,12 @@ typedef struct st_mysql_lex_string LEX_STRING;
 /* NO and OK is the same used just to show semantics */
 #define ER_DYNCOL_NO ER_DYNCOL_OK
 
+#ifdef HAVE_CHARSET_utf8mb4
+#define DYNCOL_UTF (&my_charset_utf8mb4_general_ci)
+#else
+#define DYNCOL_UTF (&my_charset_utf8_general_ci)
+#endif
+
 enum enum_dyncol_func_result
 {
   ER_DYNCOL_OK= 0,

include/my_base.h

@@ -282,7 +282,8 @@ enum ha_base_keytype {
   This flag can be calculated -- it's based on key lengths comparison.
 */
 #define HA_KEY_HAS_PART_KEY_SEG 65536
-
+/* Internal Flag Can be calcaluted */
+#define HA_INVISIBLE_KEY 2<<18
 	/* Automatic bits in key-flag */
 
 #define HA_SPACE_PACK_USED	 4	/* Test for if SPACE_PACK used */

include/mysql_com.h

@@ -202,7 +202,7 @@ enum enum_indicator_type
 #define VERS_UPDATE_UNVERSIONED_FLAG (1 << 29) /* column that doesn't support
                                                 system versioning when table
                                                 itself supports it*/
-#define HIDDEN_FLAG (1 << 31) /* hide from SELECT * */
+#define VERS_HIDDEN_FLAG (1 << 31) /* hide from SELECT * */
 
 #define REFRESH_GRANT           (1ULL << 0)  /* Refresh grant tables */
 #define REFRESH_LOG             (1ULL << 1)  /* Start on new log file */

man/mysql_upgrade.1

@@ -691,8 +691,7 @@ it was built/distributed. Defaults to on; use \fB\-\-skip\-version\-check\fR to
 .sp
 Cause binary logging to be enabled while
 \fBmysql_upgrade\fR
-runs\&. This is the default behavior; to disable binary logging during the upgrade, use the inverse of this option (that is, start the program with
+runs\&.
-\fB\-\-skip\-write\-binlog\fR)\&.
 .RE
 .SH "COPYRIGHT"
 .br

mysql-test/include/kill_and_restart_mysqld.inc

@@ -1,15 +0,0 @@
-if (!$restart_parameters)
-{
-  let $restart_parameters = restart;
-}
-
---let $_server_id= `SELECT @@server_id`
---let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect
-
---echo # Kill and $restart_parameters
---exec echo "$restart_parameters" > $_expect_file_name
---shutdown_server 0
---source include/wait_until_disconnected.inc
---enable_reconnect
---source include/wait_until_connected_again.inc
---disable_reconnect

mysql-test/r/ctype_utf8.result

@@ -10272,6 +10272,21 @@ DROP TABLE allbytes;
 SET sql_mode = DEFAULT;
 # End of ctype_backslash.inc
 #
+# MDEV-12681 Wrong VIEW results for CHAR(0xDF USING latin1)
+#
+SET NAMES utf8;
+SELECT CHAR(0xDF USING latin1);
+CHAR(0xDF USING latin1)
+ß
+CREATE OR REPLACE VIEW v1 AS SELECT CHAR(0xDF USING latin1) AS c;
+SHOW CREATE VIEW v1;
+View	Create View	character_set_client	collation_connection
+v1	CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select char(0xdf using latin1) AS `c`	utf8	utf8_general_ci
+SELECT * FROM v1;
+c
+ß
+DROP VIEW v1;
+#
 # End of 10.0 tests
 #
 #

mysql-test/r/ctype_utf8mb4.result

@@ -3430,6 +3430,32 @@ a	b
 a 😁 b	a ? b
 DROP TABLE t1;
 #
+# MDEV-8949: COLUMN_CREATE unicode name breakage
+#
+SET NAMES utf8mb4;
+SELECT COLUMN_JSON(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1));
+COLUMN_JSON(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1))
+{"😎":1}
+SELECT COLUMN_LIST(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1));
+COLUMN_LIST(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1))
+`😎`
+SELECT COLUMN_GET(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1), _utf8mb4 0xF09F988E
+as int);
+COLUMN_GET(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1), _utf8mb4 0xF09F988E
+as int)
+1
+CREATE TABLE t1 AS SELECT
+COLUMN_LIST(COLUMN_CREATE('a',1)),
+COLUMN_JSON(COLUMN_CREATE('b',1));
+SHOW CREATE TABLE t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `COLUMN_LIST(COLUMN_CREATE('a',1))` longtext CHARACTER SET utf8mb4 DEFAULT NULL,
+  `COLUMN_JSON(COLUMN_CREATE('b',1))` longtext CHARACTER SET utf8mb4 DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+DROP TABLE t1;
+SET NAMES default;
+#
 # End of 10.0 tests
 #
 #

mysql-test/r/features.result

@@ -8,6 +8,7 @@ Feature_delay_key_write	0
 Feature_dynamic_columns	0
 Feature_fulltext	0
 Feature_gis	0
+Feature_invisible_columns	0
 Feature_locale	0
 Feature_subquery	0
 Feature_timezone	0

mysql-test/r/func_str.result

@@ -4734,6 +4734,27 @@ set global max_allowed_packet=default;
 # End of 5.6 tests
 #
 #
+# Start of 10.0 tests
+#
+#
+# MDEV-12681 Wrong VIEW results for CHAR(0xDF USING latin1)
+#
+EXPLAIN EXTENDED SELECT CHAR(0xDF USING latin1);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+Warnings:
+Note	1003	select char(0xdf using latin1) AS `CHAR(0xDF USING latin1)`
+EXPLAIN EXTENDED SELECT CHAR(0xDF USING `binary`);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+Warnings:
+Note	1003	select char(0xdf) AS `CHAR(0xDF USING ``binary``)`
+EXPLAIN EXTENDED SELECT CHAR(0xDF);
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	filtered	Extra
+1	SIMPLE	NULL	NULL	NULL	NULL	NULL	NULL	NULL	NULL	No tables used
+Warnings:
+Note	1003	select char(0xdf) AS `CHAR(0xDF)`
+#
 # Start of 10.1 tests
 #
 #
@@ -4843,6 +4864,12 @@ YQ==	61
 Yq==	62
 DROP TABLE t1;
 #
+# End of 10.1 tests
+#
+#
+# Start of 10.3 tests
+#
+#
 # MDEV-12685 Oracle-compatible function CHR()
 #
 select chr(65);
@@ -4859,12 +4886,6 @@ utf8	3	1
 drop database mysqltest1;
 use test;
 #
-# End of 10.1 tests
-#
-#
-# Start of 10.3 tests
-#
-#
 # MDEV-12592 Illegal mix of collations with the HEX function
 #
 SET NAMES utf8;

mysql-test/r/invisible_binlog.result

@@ -0,0 +1,65 @@
+include/master-slave.inc
+[connection master]
+connection master;
+create table t1(a int , b int invisible);
+insert into t1 values(1);
+insert into t1(a,b) values(2,2);
+select a,b from t1;
+a	b
+1	NULL
+2	2
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		NULL	INVISIBLE
+create table t2(a int , b int invisible default 5);
+insert into t2 values(1);
+insert into t2(a,b) values(2,2);
+select a,b from t2;
+a	b
+1	5
+2	2
+desc t2;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		5	INVISIBLE
+connection slave;
+select * from t1;
+a
+1
+2
+select a,b from t1;
+a	b
+1	NULL
+2	2
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		NULL	INVISIBLE
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL,
+  `b` int(11) INVISIBLE DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+select * from t2;
+a
+1
+2
+select a,b from t2;
+a	b
+1	5
+2	2
+desc t2;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		5	INVISIBLE
+show create table t2;
+Table	Create Table
+t2	CREATE TABLE `t2` (
+  `a` int(11) DEFAULT NULL,
+  `b` int(11) INVISIBLE DEFAULT 5
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+connection master;
+drop table t1,t2;
+include/rpl_end.inc

mysql-test/r/invisible_field.result

@@ -0,0 +1,526 @@
+FLUSH STATUS;
+create table t1(abc int primary key, xyz int invisible);
+SHOW STATUS LIKE 'Feature_invisible_columns';
+Variable_name	Value
+Feature_invisible_columns	1
+desc t1;
+Field	Type	Null	Key	Default	Extra
+abc	int(11)	NO	PRI	NULL	
+xyz	int(11)	YES		NULL	INVISIBLE
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `abc` int(11) NOT NULL,
+  `xyz` int(11) INVISIBLE DEFAULT NULL,
+  PRIMARY KEY (`abc`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+select TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,EXTRA from INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA='test' and TABLE_NAME='t1';
+TABLE_CATALOG	TABLE_SCHEMA	TABLE_NAME	COLUMN_NAME	EXTRA
+def	test	t1	abc	
+def	test	t1	xyz	INVISIBLE
+drop table t1;
+create table t1(a1 int invisible);
+ERROR 42000: A table must have at least 1 column
+create table t1(a1 blob,invisible(a1));
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '(a1))' at line 1
+create table t1(a1 int primary key invisible ,a2 int unique invisible , a3 blob,a4 int not null invisible unique);
+ERROR HY000: Invisible column `a1` must have a default value
+create table t1(abc int not null invisible);
+ERROR HY000: Invisible column `abc` must have a default value
+create table t1(a int invisible, b int);
+insert into t1 values(1);
+insert into t1(a) values(2);
+insert into t1(b) values(3);
+insert into t1(a,b) values(5,5);
+select * from t1;
+b
+1
+NULL
+3
+5
+select a,b from t1;
+a	b
+NULL	1
+2	NULL
+NULL	3
+5	5
+delete from t1;
+insert into t1 values(1),(2),(3),(4);
+select * from t1;
+b
+1
+2
+3
+4
+select a from t1;
+a
+NULL
+NULL
+NULL
+NULL
+drop table t1;
+#more complex case of invisible
+create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int);
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		NULL	INVISIBLE
+c	int(11)	NO	PRI	NULL	auto_increment, INVISIBLE
+d	blob	YES		NULL	
+e	int(11)	YES	UNI	NULL	
+f	int(11)	YES		NULL	
+insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1);
+select * from t1;
+a	d	e	f
+1	d blob	1	1
+1	d blob	11	1
+1	d blob	2	1
+1	d blob	3	1
+1	d blob	41	1
+select a,b,c,d,e,f from t1;
+a	b	c	d	e	f
+1	NULL	1	d blob	1	1
+1	NULL	2	d blob	11	1
+1	NULL	3	d blob	2	1
+1	NULL	4	d blob	3	1
+1	NULL	5	d blob	41	1
+drop table t1;
+#more complex case of invisible with sql_mode=NO_AUTO_VALUE_ON_ZERO
+set sql_mode='NO_AUTO_VALUE_ON_ZERO';
+create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int);
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		NULL	INVISIBLE
+c	int(11)	NO	PRI	NULL	auto_increment, INVISIBLE
+d	blob	YES		NULL	
+e	int(11)	YES	UNI	NULL	
+f	int(11)	YES		NULL	
+insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1);
+select * from t1;
+a	d	e	f
+1	d blob	1	1
+1	d blob	11	1
+1	d blob	2	1
+1	d blob	3	1
+1	d blob	41	1
+select a,b,c,d,e,f from t1;
+a	b	c	d	e	f
+1	NULL	1	d blob	1	1
+1	NULL	2	d blob	11	1
+1	NULL	3	d blob	2	1
+1	NULL	4	d blob	3	1
+1	NULL	5	d blob	41	1
+drop table t1;
+set sql_mode='';
+create table sdsdsd(a int , b int, invisible(a,b));
+ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '(a,b))' at line 1
+create table t1(a int,abc int as (a mod 3) virtual invisible);
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+abc	int(11)	YES		NULL	VIRTUAL GENERATED, INVISIBLE
+insert into t1 values(1,default);
+ERROR 21S01: Column count doesn't match value count at row 1
+insert into t1 values(1),(22),(233);
+select * from t1;
+a
+1
+22
+233
+select a,abc from t1;
+a	abc
+1	1
+22	1
+233	2
+drop table t1;
+create table t1(abc int primary key invisible auto_increment, a int);
+desc t1;
+Field	Type	Null	Key	Default	Extra
+abc	int(11)	NO	PRI	NULL	auto_increment, INVISIBLE
+a	int(11)	YES		NULL	
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `abc` int(11) NOT NULL INVISIBLE AUTO_INCREMENT,
+  `a` int(11) DEFAULT NULL,
+  PRIMARY KEY (`abc`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+insert into t1 values(1);
+insert into t1 values(2);
+insert into t1 values(3);
+select * from t1;
+a
+1
+2
+3
+select abc,a from t1;
+abc	a
+1	1
+2	2
+3	3
+delete  from t1;
+insert into t1 values(1),(2),(3),(4),(6);
+select abc,a from t1;
+abc	a
+4	1
+5	2
+6	3
+7	4
+8	6
+drop table t1;
+create table t1(abc int);
+alter table t1 change abc ss int invisible;
+ERROR 42000: A table must have at least 1 column
+alter table t1 add column xyz int;
+alter table t1 modify column abc  int ;
+desc t1;
+Field	Type	Null	Key	Default	Extra
+abc	int(11)	YES		NULL	
+xyz	int(11)	YES		NULL	
+insert into t1 values(22);
+ERROR 21S01: Column count doesn't match value count at row 1
+alter table t1 modify column abc  int invisible;
+desc t1;
+Field	Type	Null	Key	Default	Extra
+abc	int(11)	YES		NULL	INVISIBLE
+xyz	int(11)	YES		NULL	
+insert into t1 values(12);
+drop table t1;
+#some test on copy table structure with table data;
+#table with invisible fields and unique keys;
+create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int);
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		NULL	INVISIBLE
+c	int(11)	NO	PRI	NULL	auto_increment, INVISIBLE
+d	blob	YES		NULL	
+e	int(11)	YES	UNI	NULL	
+f	int(11)	YES		NULL	
+insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1);
+select * from t1;
+a	d	e	f
+1	d blob	1	1
+1	d blob	11	1
+1	d blob	2	1
+1	d blob	3	1
+1	d blob	41	1
+select a,b,c,d,e,f from t1;
+a	b	c	d	e	f
+1	NULL	1	d blob	1	1
+1	NULL	2	d blob	11	1
+1	NULL	3	d blob	2	1
+1	NULL	4	d blob	3	1
+1	NULL	5	d blob	41	1
+#this won't copy invisible fields and keys;
+create table t2 as select * from t1;
+desc t2;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+d	blob	YES		NULL	
+e	int(11)	YES		NULL	
+f	int(11)	YES		NULL	
+select * from t2;
+a	d	e	f
+1	d blob	1	1
+1	d blob	11	1
+1	d blob	2	1
+1	d blob	3	1
+1	d blob	41	1
+select a,b,c,d,e,f from t2;
+ERROR 42S22: Unknown column 'b' in 'field list'
+drop table t2;
+#now this will copy invisible fields
+create table t2 as select a,b,c,d,e,f from t1;
+desc t2;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		NULL	
+c	int(11)	NO		0	
+d	blob	YES		NULL	
+e	int(11)	YES		NULL	
+f	int(11)	YES		NULL	
+select * from t2;
+a	b	c	d	e	f
+1	NULL	1	d blob	1	1
+1	NULL	2	d blob	11	1
+1	NULL	3	d blob	2	1
+1	NULL	4	d blob	3	1
+1	NULL	5	d blob	41	1
+select a,b,c,d,e,f from t2;
+a	b	c	d	e	f
+1	NULL	1	d blob	1	1
+1	NULL	2	d blob	11	1
+1	NULL	3	d blob	2	1
+1	NULL	4	d blob	3	1
+1	NULL	5	d blob	41	1
+drop table t2,t1;
+#some test related to copy of data from one table to another;
+create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int);
+insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1);
+select a,b,c,d,e,f from t1;
+a	b	c	d	e	f
+1	NULL	1	d blob	1	1
+1	NULL	2	d blob	11	1
+1	NULL	3	d blob	2	1
+1	NULL	4	d blob	3	1
+1	NULL	5	d blob	41	1
+create table t2(a int , b int invisible , c int invisible , d blob , e int unique, f int);
+insert into t2 select * from t1;
+select a,b,c,d,e,f from t2;
+a	b	c	d	e	f
+1	NULL	NULL	d blob	1	1
+1	NULL	NULL	d blob	11	1
+1	NULL	NULL	d blob	2	1
+1	NULL	NULL	d blob	3	1
+1	NULL	NULL	d blob	41	1
+truncate t2;
+insert into t2 (a,b,c,d,e,f) select a,b,c,d,e,f from t1;
+select a,b,c,d,e,f from t2;
+a	b	c	d	e	f
+1	NULL	1	d blob	1	1
+1	NULL	2	d blob	11	1
+1	NULL	3	d blob	2	1
+1	NULL	4	d blob	3	1
+1	NULL	5	d blob	41	1
+truncate t2;
+drop table t1,t2;
+#some test related to creating view on table with invisible column;
+create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int);
+insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1);
+create view v as select * from t1;
+desc v;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+d	blob	YES		NULL	
+e	int(11)	YES		NULL	
+f	int(11)	YES		NULL	
+select * from v;
+a	d	e	f
+1	d blob	1	1
+1	d blob	11	1
+1	d blob	2	1
+1	d blob	3	1
+1	d blob	41	1
+#v does not have invisible column;
+select a,b,c,d,e,f from v;
+ERROR 42S22: Unknown column 'b' in 'field list'
+insert into v values(1,21,32,4);
+select * from v;
+a	d	e	f
+1	d blob	1	1
+1	d blob	11	1
+1	d blob	2	1
+1	d blob	3	1
+1	d blob	41	1
+1	21	32	4
+insert into v(a,b,c,d,e,f) values(1,12,3,4,5,6);
+ERROR 42S22: Unknown column 'b' in 'field list'
+drop view v;
+create view v as select a,b,c,d,e,f from t1;
+desc v;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		NULL	
+c	int(11)	NO		0	
+d	blob	YES		NULL	
+e	int(11)	YES		NULL	
+f	int(11)	YES		NULL	
+select * from v;
+a	b	c	d	e	f
+1	NULL	1	d blob	1	1
+1	NULL	2	d blob	11	1
+1	NULL	3	d blob	2	1
+1	NULL	4	d blob	3	1
+1	NULL	5	d blob	41	1
+1	NULL	6	21	32	4
+#v does  have invisible column but they aren't invisible anymore.
+select a,b,c,d,e,f from v;
+a	b	c	d	e	f
+1	NULL	1	d blob	1	1
+1	NULL	2	d blob	11	1
+1	NULL	3	d blob	2	1
+1	NULL	4	d blob	3	1
+1	NULL	5	d blob	41	1
+1	NULL	6	21	32	4
+insert into v values(1,26,33,4,45,66);
+select a,b,c,d,e,f from v;
+a	b	c	d	e	f
+1	NULL	1	d blob	1	1
+1	NULL	2	d blob	11	1
+1	NULL	3	d blob	2	1
+1	NULL	4	d blob	3	1
+1	NULL	5	d blob	41	1
+1	NULL	6	21	32	4
+1	26	33	4	45	66
+insert into v(a,b,c,d,e,f) values(1,32,31,41,5,6);
+select a,b,c,d,e,f from v;
+a	b	c	d	e	f
+1	NULL	1	d blob	1	1
+1	NULL	2	d blob	11	1
+1	NULL	3	d blob	2	1
+1	NULL	4	d blob	3	1
+1	NULL	5	d blob	41	1
+1	NULL	6	21	32	4
+1	26	33	4	45	66
+1	32	31	41	5	6
+drop view v;
+drop table t1;
+#now invisible column in where and some join query
+create table t1 (a int unique , b int invisible unique, c int unique  invisible);
+insert into t1(a,b,c) values(1,1,1);
+insert into t1(a,b,c) values(2,2,2);
+insert into t1(a,b,c) values(3,3,3);
+insert into t1(a,b,c) values(4,4,4);
+insert into t1(a,b,c) values(21,21,26);
+insert into t1(a,b,c) values(31,31,35);
+insert into t1(a,b,c) values(41,41,45);
+insert into t1(a,b,c) values(22,22,24);
+insert into t1(a,b,c) values(32,32,33);
+insert into t1(a,b,c) values(42,42,43);
+explain select * from t1 where b=3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	const	b	b	5	const	1	
+select * from t1 where b=3;
+a
+3
+explain select * from t1 where c=3;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	const	c	c	5	const	1	
+select * from t1 where c=3;
+a
+3
+create table t2 as select a,b,c from t1;
+desc t2;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+b	int(11)	YES		NULL	
+c	int(11)	YES		NULL	
+explain select * from t1,t2 where t1.b = t2.c and t1.c = t2.b;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t2	ALL	NULL	NULL	NULL	NULL	10	
+1	SIMPLE	t1	ALL	b,c	NULL	NULL	NULL	10	Using where; Using join buffer (flat, BNL join)
+select * from t1,t2 where t1.b = t2.c and t1.c = t2.b;
+a	a	b	c
+1	1	1	1
+2	2	2	2
+3	3	3	3
+4	4	4	4
+drop table t1,t2;
+#Unhide  invisible columns
+create table t1 (a int primary key, b int invisible, c int invisible unique);
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) NOT NULL,
+  `b` int(11) INVISIBLE DEFAULT NULL,
+  `c` int(11) INVISIBLE DEFAULT NULL,
+  PRIMARY KEY (`a`),
+  UNIQUE KEY `c` (`c`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	NO	PRI	NULL	
+b	int(11)	YES		NULL	INVISIBLE
+c	int(11)	YES	UNI	NULL	INVISIBLE
+alter table t1 modify column b int;
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	NO	PRI	NULL	
+b	int(11)	YES		NULL	
+c	int(11)	YES	UNI	NULL	INVISIBLE
+alter table t1 change column c d int;
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	NO	PRI	NULL	
+b	int(11)	YES		NULL	
+d	int(11)	YES	UNI	NULL	
+drop table t1;
+SHOW STATUS LIKE 'Feature_invisible_columns';
+Variable_name	Value
+Feature_invisible_columns	50
+#invisible is non reserved
+create table t1(a int unique , invisible int invisible, c int );
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES	UNI	NULL	
+invisible	int(11)	YES		NULL	INVISIBLE
+c	int(11)	YES		NULL	
+alter table t1 change column invisible hid int invisible;
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES	UNI	NULL	
+hid	int(11)	YES		NULL	INVISIBLE
+c	int(11)	YES		NULL	
+drop table t1;
+CREATE TABLE t1 (b int);
+INSERT t1 values(1);
+INSERT t1 values(2);
+INSERT t1 values(3);
+INSERT t1 values(4);
+INSERT t1 values(5);
+CREATE TABLE t2 (a int invisible) SELECT * FROM t1;
+select * from t2 order by b;
+b
+1
+2
+3
+4
+5
+select a,b from t2 order by b;
+a	b
+NULL	1
+NULL	2
+NULL	3
+NULL	4
+NULL	5
+CREATE TABLE t3 (b int, a int invisible) SELECT * FROM t1;
+select * from t3 order by b;
+b
+1
+2
+3
+4
+5
+select a,b from t3 order by b;
+a	b
+NULL	1
+NULL	2
+NULL	3
+NULL	4
+NULL	5
+CREATE TABLE t4 (b int invisible) SELECT * FROM t1;
+ERROR 42000: A table must have at least 1 column
+CREATE TABLE t5 (a int invisible) SELECT b as a FROM t1;
+ERROR 42000: A table must have at least 1 column
+drop table t1,t2,t3;
+create table t1 (a int , b int invisible default 3, c int , d int invisible default 6);
+CREATE PROCEDURE
+insert_t1(a int, b int)
+MODIFIES SQL DATA
+insert into t1 values(a,b);
+//
+call insert_t1(1,1);
+call insert_t1(2,2);
+select * from t1 order by a;
+a	c
+1	1
+2	2
+select a,b,c,d from t1 order by a;
+a	b	c	d
+1	3	1	6
+2	3	2	6
+DROP PROCEDURE insert_t1;
+delete from t1;
+prepare insert_1 from "insert into t1 values(@a,@c)";
+prepare insert_2 from "insert into t1(a,b,c) values(@a,@b,@c)";
+set @a=1, @c=1;
+execute insert_1;
+set @a=2,@b=2, @c=2;
+execute insert_2;
+select a,b,c,d from t1 order by a;
+a	b	c	d
+1	3	1	6
+2	2	2	6
+drop table t1;

mysql-test/r/invisible_field_debug.result

@@ -0,0 +1,285 @@
+set @old_debug= @@debug_dbug;
+set debug_dbug= "+d,test_pseudo_invisible";
+create table t1(a int);
+set debug_dbug=@old_debug;
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+insert into t1 values(1);
+select * from t1;
+a
+1
+select invisible ,a from t1;
+invisible	a
+9	1
+drop table t1;
+set debug_dbug= "+d,test_completely_invisible";
+create table t1(a int);
+set debug_dbug=@old_debug;
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+insert into t1 values(1);
+select * from t1;
+a
+1
+select invisible ,a from t1;
+ERROR 42S22: Unknown column 'invisible' in 'field list'
+set debug_dbug= "+d,test_completely_invisible";
+select invisible ,a from t1;
+invisible	a
+9	1
+set debug_dbug=@old_debug;
+drop table t1;
+set debug_dbug= "+d,test_pseudo_invisible";
+create table t1(a int);
+set debug_dbug=@old_debug;
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+insert into t1 values(1);
+select * from t1;
+a
+1
+select invisible ,a from t1;
+invisible	a
+9	1
+ALTER table t1 change invisible b int;
+ERROR 42S22: Unknown column 'invisible' in 't1'
+select * from t1;
+a
+1
+select invisible ,a from t1;
+invisible	a
+9	1
+ALTER table t1 modify invisible char;
+ERROR 42S22: Unknown column 'invisible' in 't1'
+select * from t1;
+a
+1
+select invisible ,a from t1;
+invisible	a
+9	1
+ALTER table t1 drop invisible;
+ERROR 42000: Can't DROP COLUMN `invisible`; check that it exists
+select * from t1;
+a
+1
+select invisible ,a from t1;
+invisible	a
+9	1
+ALTER table t1 add invisible int;
+ERROR 42S21: Duplicate column name 'invisible'
+select * from t1;
+a
+1
+select invisible ,a from t1;
+invisible	a
+9	1
+ALTER table t1 add invisible2 int default 2;
+select * from t1;
+a	invisible2
+1	2
+select invisible ,a from t1;
+invisible	a
+9	1
+drop table t1;
+set debug_dbug= "+d,test_completely_invisible";
+create table t1(a int);
+desc t1;
+Field	Type	Null	Key	Default	Extra
+a	int(11)	YES		NULL	
+insert into t1 values(1);
+select * from t1;
+a
+1
+select invisible ,a from t1;
+invisible	a
+9	1
+ALTER table t1 change invisible b int;
+ERROR 42S22: Unknown column 'invisible' in 't1'
+select * from t1;
+a
+1
+select invisible ,a from t1;
+invisible	a
+9	1
+ALTER table t1 modify invisible char;
+ERROR 42S22: Unknown column 'invisible' in 't1'
+select * from t1;
+a
+1
+select invisible ,a from t1;
+invisible	a
+9	1
+ALTER table t1 drop invisible;
+ERROR 42000: Can't DROP COLUMN `invisible`; check that it exists
+select * from t1;
+a
+1
+select invisible ,a from t1;
+invisible	a
+9	1
+ALTER table t1 add invisible int;
+select * from t1;
+a	invisible
+1	NULL
+select invisible1, invisible ,a from t1;
+invisible1	invisible	a
+9	NULL	1
+ALTER table t1 add hid int default 2;
+set debug_dbug= "+d,test_completely_invisible";
+select * from t1;
+a	invisible	hid
+1	NULL	2
+select invisible ,a from t1;
+invisible	a
+NULL	1
+drop table t1;
+set debug_dbug=@old_debug;
+Create table t1( a int default(99) invisible, b int);
+insert into t1 values(1);
+insert into t1 values(2);
+insert into t1 values(3);
+insert into t1 values(4);
+select * from t1 order by b;
+b
+1
+2
+3
+4
+alter table t1 add index(a);
+alter table t1 add index(a,b);
+show index from t1;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment
+t1	1	a	1	a	A	NULL	NULL	NULL	YES	BTREE		
+t1	1	a_2	1	a	A	NULL	NULL	NULL	YES	BTREE		
+t1	1	a_2	2	b	A	NULL	NULL	NULL	YES	BTREE		
+drop table t1;
+set debug_dbug= "+d,test_pseudo_invisible";
+Create table t1( a int default(99) invisible, b int);
+Create table t2( a int default(99) invisible, b int, unique(invisible));
+ERROR 42000: Key column 'invisible' doesn't exist in table
+set debug_dbug=@old_debug;
+insert into t1 values(1);
+insert into t1 values(2);
+insert into t1 values(3);
+insert into t1 values(4);
+select * from t1 order by b;
+b
+1
+2
+3
+4
+select invisible, a, b from t1 order by b;
+invisible	a	b
+9	99	1
+9	99	2
+9	99	3
+9	99	4
+alter table t1 add index(invisible);
+ERROR 42000: Key column 'invisible' doesn't exist in table
+alter table t1 add index(b,invisible);
+ERROR 42000: Key column 'invisible' doesn't exist in table
+show index from t1;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment
+drop table t1;
+set debug_dbug= "+d,test_completely_invisible";
+Create table t1( a int default(99) invisible, b int);
+Create table t2( a int default(99) invisible, b int, unique(invisible));
+ERROR 42000: Key column 'invisible' doesn't exist in table
+insert into t1 values(1);
+insert into t1 values(2);
+insert into t1 values(3);
+insert into t1 values(4);
+select * from t1 order by b;
+b
+1
+2
+3
+4
+select invisible, a, b from t1 order by b;
+invisible	a	b
+9	99	1
+9	99	2
+9	99	3
+9	99	4
+set debug_dbug=@old_debug;
+alter table t1 add index(invisible);
+ERROR 42000: Key column 'invisible' doesn't exist in table
+alter table t1 add index(b,invisible);
+ERROR 42000: Key column 'invisible' doesn't exist in table
+show index from t1;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment
+drop table t1;
+set debug_dbug= "+d,test_completely_invisible,test_invisible_index";
+Create table t1( a int default(99) , b int,c int, index(b));
+set debug_dbug=@old_debug;
+Show index from t1;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment
+t1	1	b	1	b	A	NULL	NULL	NULL	YES	BTREE		
+select * from INFORMATION_SCHEMA.STATISTICS where TABLE_SCHEMA ='test' and table_name='t1';
+TABLE_CATALOG	TABLE_SCHEMA	TABLE_NAME	NON_UNIQUE	INDEX_SCHEMA	INDEX_NAME	SEQ_IN_INDEX	COLUMN_NAME	COLLATION	CARDINALITY	SUB_PART	PACKED	NULLABLE	INDEX_TYPE	COMMENT	INDEX_COMMENT
+def	test	t1	1	test	b	1	b	A	NULL	NULL	NULL	YES	BTREE		
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT 99,
+  `b` int(11) DEFAULT NULL,
+  `c` int(11) DEFAULT NULL,
+  KEY `b` (`b`)
+) ENGINE=MyISAM DEFAULT CHARSET=latin1
+insert into t1 values(1,1,1);
+insert into t1 values(2,2,2);
+insert into t1 values(3,3,3);
+insert into t1 values(4,4,4);
+set debug_dbug= "+d,test_completely_invisible,test_invisible_index";
+select invisible, a ,b from t1 order by b;
+invisible	a	b
+9	1	1
+9	2	2
+9	3	3
+9	4	4
+explain select * from t1 where invisible =9;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ref	invisible	invisible	5	const	3	
+alter table t1 add x int default 3;
+select invisible, a ,b from t1;
+invisible	a	b
+9	1	1
+9	2	2
+9	3	3
+9	4	4
+set debug_dbug=@old_debug;
+Show index from t1;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment
+t1	1	b	1	b	A	NULL	NULL	NULL	YES	BTREE		
+create index a1 on t1(invisible);
+ERROR 42000: Key column 'invisible' doesn't exist in table
+set debug_dbug= "+d,test_completely_invisible,test_invisible_index";
+drop index invisible on t1;
+ERROR 42000: Can't DROP INDEX `invisible`; check that it exists
+explain select * from t1 where invisible =9;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ref	invisible	invisible	5	const	3	
+create index invisible on t1(c);
+explain select * from t1 where invisible =9;
+id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+1	SIMPLE	t1	ref	invisible_2	invisible_2	5	const	3	
+show indexes in t1;
+Table	Non_unique	Key_name	Seq_in_index	Column_name	Collation	Cardinality	Sub_part	Packed	Null	Index_type	Comment	Index_comment
+t1	1	b	1	b	A	NULL	NULL	NULL	YES	BTREE		
+t1	1	invisible	1	c	A	NULL	NULL	NULL	YES	BTREE		
+t1	1	invisible_2	1	invisible	A	NULL	NULL	NULL	YES	BTREE		
+drop table t1;
+set @old_debug= @@debug_dbug;

mysql-test/r/loadxml.result

@@ -134,3 +134,23 @@ LOAD XML INFILE '../../std_data/loaddata/mdev12696.xml' INTO TABLE v1 (c2);
 ERROR HY000: Column 'c2' is not updatable
 DROP VIEW v1;
 DROP TABLE t1;
+#
+# MDEV-14628 Wrong autoinc value assigned by LOAD XML in the NO_AUTO_VALUE_ON_ZERO mode
+#
+SET sql_mode=NO_AUTO_VALUE_ON_ZERO;
+CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT);
+LOAD XML INFILE '../../std_data/loaddata/mdev14628a.xml' INTO TABLE t1 ROWS IDENTIFIED BY '<row>';
+SELECT * FROM t1 ORDER BY b;
+a	b
+1	bbb1
+2	bbb2
+DROP TABLE t1;
+SET sql_mode=DEFAULT;
+SET sql_mode='';
+CREATE  TABLE t1 (id INT, g GEOMETRY NOT NULL);
+LOAD XML INFILE '../../std_data/loaddata/mdev14628b.xml' INTO TABLE t1 ROWS IDENTIFIED BY '<row>';
+ERROR 22004: Column set to default value; NULL supplied to NOT NULL column 'g' at row 1
+SELECT * FROM t1;
+id	g
+DROP TABLE t1;
+SET sql_mode=DEFAULT;

mysql-test/r/mysqldump.result

@@ -1971,7 +1971,7 @@ drop table t1, t2, t3;
 # Bug#21288 mysqldump segmentation fault when using --where
 #
 create table t1 (a int);
-mysqldump: Couldn't execute 'SELECT /*!40001 SQL_NO_CACHE */ * FROM `t1` WHERE xx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx': You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' at line 1 (1064)
+mysqldump: Couldn't execute 'SELECT /*!40001 SQL_NO_CACHE */ `a` FROM `t1` WHERE xx xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx': You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' at line 1 (1064)
 mysqldump: Got error: 1064: "You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx' at line 1" when retrieving data from server
 
 /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
@@ -5757,3 +5757,183 @@ DELIMITER ;
 /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */;
 
 DROP TABLE t1;
+#
+# Test for Invisible columns
+#
+create database d;
+use d;
+# Invisble field table
+create table t1(a int , b int invisible);
+insert into t1 values(1);
+insert into t1(a,b) values(1,2);
+# not invisible field table --complete-insert wont be used
+create table t2(a int , b int);
+insert into t2(a,b) values(1,2);
+insert into t2(a,b) values(1,2);
+# Invisble field table
+create table t3(invisible int , `a  b c & $!@#$%^&*( )` int invisible default 4, `ds=~!@ \# $% ^ & * ( ) _ - = +` int invisible default 5);
+insert into t3 values(1);
+insert into t3 values(5);
+insert into t3 values(2);
+insert into t3(`invisible`,  `a  b c & $!@#$%^&*( )`, `ds=~!@ \# $% ^ & * ( ) _ - = +` ) values(1,2,3);
+CREATE TABLE t4(ËÏÌÏÎËÁ1 INT);
+insert into t4 values(1);
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL,
+  `b` int(11) INVISIBLE DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `t1` (`a`, `b`) VALUES (1,NULL),(1,2);
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `t2` (
+  `a` int(11) DEFAULT NULL,
+  `b` int(11) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `t2` VALUES (1,2),(1,2);
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `t3` (
+  `invisible` int(11) DEFAULT NULL,
+  `a  b c & $!@#$%^&*( )` int(11) INVISIBLE DEFAULT 4,
+  `ds=~!@ \# $% ^ & * ( ) _ - = +` int(11) INVISIBLE DEFAULT 5
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `t3` (`invisible`, `a  b c & $!@#$%^&*( )`, `ds=~!@ \# $% ^ & * ( ) _ - = +`) VALUES (1,4,5),(5,4,5),(2,4,5),(1,2,3);
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `t4` (
+  `ËÏÌÏÎËÁ1` int(11) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `t4` VALUES (1);
+#Check side effect on --complete insert
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL,
+  `b` int(11) INVISIBLE DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `t1` (`a`, `b`) VALUES (1,NULL),(1,2);
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `t2` (
+  `a` int(11) DEFAULT NULL,
+  `b` int(11) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `t2` (`a`, `b`) VALUES (1,2),(1,2);
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `t3` (
+  `invisible` int(11) DEFAULT NULL,
+  `a  b c & $!@#$%^&*( )` int(11) INVISIBLE DEFAULT 4,
+  `ds=~!@ \# $% ^ & * ( ) _ - = +` int(11) INVISIBLE DEFAULT 5
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `t3` (`invisible`, `a  b c & $!@#$%^&*( )`, `ds=~!@ \# $% ^ & * ( ) _ - = +`) VALUES (1,4,5),(5,4,5),(2,4,5),(1,2,3);
+/*!40101 SET @saved_cs_client     = @@character_set_client */;
+/*!40101 SET character_set_client = utf8 */;
+CREATE TABLE `t4` (
+  `ËÏÌÏÎËÁ1` int(11) DEFAULT NULL
+) ENGINE=MyISAM DEFAULT CHARSET=latin1;
+/*!40101 SET character_set_client = @saved_cs_client */;
+INSERT INTO `t4` (`ËÏÌÏÎËÁ1`) VALUES (1);
+#Check xml
+<?xml version="1.0"?>
+<mysqldump xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+<database name="d">
+	<table_structure name="t1">
+		<field Field="a" Type="int(11)" Null="YES" Key="" Default="NULL" Extra="" Comment="" />
+		<field Field="b" Type="int(11)" Null="YES" Key="" Default="NULL" Extra="INVISIBLE" Comment="" />
+	</table_structure>
+	<table_data name="t1">
+	<row>
+		<field name="a">1</field>
+		<field name="b" xsi:nil="true" />
+	</row>
+	<row>
+		<field name="a">1</field>
+		<field name="b">2</field>
+	</row>
+	</table_data>
+	<table_structure name="t2">
+		<field Field="a" Type="int(11)" Null="YES" Key="" Default="NULL" Extra="" Comment="" />
+		<field Field="b" Type="int(11)" Null="YES" Key="" Default="NULL" Extra="" Comment="" />
+	</table_structure>
+	<table_data name="t2">
+	<row>
+		<field name="a">1</field>
+		<field name="b">2</field>
+	</row>
+	<row>
+		<field name="a">1</field>
+		<field name="b">2</field>
+	</row>
+	</table_data>
+	<table_structure name="t3">
+		<field Field="invisible" Type="int(11)" Null="YES" Key="" Default="NULL" Extra="" Comment="" />
+		<field Field="a  b c &amp; $!@#$%^&amp;*( )" Type="int(11)" Null="YES" Key="" Default="4" Extra="INVISIBLE" Comment="" />
+		<field Field="ds=~!@ \# $% ^ &amp; * ( ) _ - = +" Type="int(11)" Null="YES" Key="" Default="5" Extra="INVISIBLE" Comment="" />
+	</table_structure>
+	<table_data name="t3">
+	<row>
+		<field name="invisible">1</field>
+		<field name="a  b c &amp; $!@#$%^&amp;*( )">4</field>
+		<field name="ds=~!@ \# $% ^ &amp; * ( ) _ - = +">5</field>
+	</row>
+	<row>
+		<field name="invisible">5</field>
+		<field name="a  b c &amp; $!@#$%^&amp;*( )">4</field>
+		<field name="ds=~!@ \# $% ^ &amp; * ( ) _ - = +">5</field>
+	</row>
+	<row>
+		<field name="invisible">2</field>
+		<field name="a  b c &amp; $!@#$%^&amp;*( )">4</field>
+		<field name="ds=~!@ \# $% ^ &amp; * ( ) _ - = +">5</field>
+	</row>
+	<row>
+		<field name="invisible">1</field>
+		<field name="a  b c &amp; $!@#$%^&amp;*( )">2</field>
+		<field name="ds=~!@ \# $% ^ &amp; * ( ) _ - = +">3</field>
+	</row>
+	</table_data>
+	<table_structure name="t4">
+		<field Field="ËÏÌÏÎËÁ1" Type="int(11)" Null="YES" Key="" Default="NULL" Extra="" Comment="" />
+	</table_structure>
+	<table_data name="t4">
+	<row>
+		<field name="ËÏÌÏÎËÁ1">1</field>
+	</row>
+	</table_data>
+</database>
+</mysqldump>
+DROP table t1,t2,t3;
+select * from t1;
+a
+1
+1
+select a,b from t1;
+a	b
+1	NULL
+1	2
+select * from t2;
+a	b
+1	2
+1	2
+select * from t3;
+invisible
+1
+5
+2
+1
+desc t3;
+Field	Type	Null	Key	Default	Extra
+invisible	int(11)	YES		NULL	
+a  b c & $!@#$%^&*( )	int(11)	YES		4	INVISIBLE
+ds=~!@ \# $% ^ & * ( ) _ - = +	int(11)	YES		5	INVISIBLE
+drop database d;

mysql-test/r/show_check.result

@@ -1206,26 +1206,6 @@ DROP TABLE t1;
 DROP VIEW v1;
 DROP PROCEDURE p1;
 DROP FUNCTION f1;
-set names koi8r;
-DROP DATABASE IF EXISTS mysqltest1;
-CREATE DATABASE mysqltest1;
-use mysqltest1;
-CREATE TABLE t1(<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>1 INT);
-
----> Dumping mysqltest1 to outfile1
-
-
-DROP DATABASE mysqltest1;
-
-
----> Restoring mysqltest1...
-SHOW CREATE TABLE mysqltest1.t1;
-Table	Create Table
-t1	CREATE TABLE `t1` (
-  `<60><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>1` int(11) DEFAULT NULL
-) ENGINE=MyISAM DEFAULT CHARSET=latin1
-DROP DATABASE mysqltest1;
-use test;
 flush status;
 show variables like "log_queries_not_using_indexes";
 Variable_name	Value
@@ -1273,8 +1253,8 @@ PRIMARY KEY  (Codigo)
 ) ENGINE=MyISAM AUTO_INCREMENT=11 DEFAULT CHARSET=utf8;
 show create table t1;
 Catalog	Database	Table	Table_alias	Column	Column_alias	Type	Length	Max length	Is_null	Flags	Decimals	Charsetnr
-def					Table	253	64	2	N	1	39	7
+def					Table	253	192	2	N	1	39	33
-def					Create Table	253	1024	458	N	1	39	7
+def					Create Table	253	3072	458	N	1	39	33
 Table	Create Table
 t1	CREATE TABLE `t1` (
   `Codigo` int(10) unsigned NOT NULL AUTO_INCREMENT,

mysql-test/std_data/loaddata/mdev14628a.xml

@@ -0,0 +1,4 @@
+<list>
+  <row a="1" b="bbb1"/>
+  <row b="bbb2"/>
+</list>

mysql-test/std_data/loaddata/mdev14628b.xml

@@ -0,0 +1,3 @@
+<list>
+  <row id="1"/>
+</list>

mysql-test/suite/innodb/r/recovery_shutdown.result

@@ -0,0 +1,64 @@
+#
+# MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup
+# while rolling back recovered incomplete transactions
+#
+CREATE TABLE t (a INT) ENGINE=InnoDB;
+BEGIN;
+COMMIT;
+connect  con$c,localhost,root,,;
+CREATE TABLE t8 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t8 (a) SELECT NULL FROM t;
+UPDATE t8 SET a=a+100, b=a;
+DELETE FROM t8;
+connect  con$c,localhost,root,,;
+CREATE TABLE t7 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t7 (a) SELECT NULL FROM t;
+UPDATE t7 SET a=a+100, b=a;
+DELETE FROM t7;
+connect  con$c,localhost,root,,;
+CREATE TABLE t6 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t6 (a) SELECT NULL FROM t;
+UPDATE t6 SET a=a+100, b=a;
+DELETE FROM t6;
+connect  con$c,localhost,root,,;
+CREATE TABLE t5 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t5 (a) SELECT NULL FROM t;
+UPDATE t5 SET a=a+100, b=a;
+DELETE FROM t5;
+connect  con$c,localhost,root,,;
+CREATE TABLE t4 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t4 (a) SELECT NULL FROM t;
+UPDATE t4 SET a=a+100, b=a;
+DELETE FROM t4;
+connect  con$c,localhost,root,,;
+CREATE TABLE t3 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t3 (a) SELECT NULL FROM t;
+UPDATE t3 SET a=a+100, b=a;
+DELETE FROM t3;
+connect  con$c,localhost,root,,;
+CREATE TABLE t2 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t2 (a) SELECT NULL FROM t;
+UPDATE t2 SET a=a+100, b=a;
+DELETE FROM t2;
+connect  con$c,localhost,root,,;
+CREATE TABLE t1 (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+INSERT INTO t1 (a) SELECT NULL FROM t;
+UPDATE t1 SET a=a+100, b=a;
+DELETE FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+connection default;
+SET GLOBAL innodb_flush_log_at_trx_commit=1;
+CREATE TABLE u(a SERIAL) ENGINE=INNODB;
+DROP TABLE t,u;

mysql-test/suite/innodb/r/update_time.result

@@ -43,7 +43,6 @@ XA PREPARE 'xatrx';
 CONNECT  con1,localhost,root,,;
 call mtr.add_suppression("Found 1 prepared XA transactions");
 FLUSH TABLES;
-# Kill and restart
 SELECT update_time FROM information_schema.tables WHERE table_name = 't';
 update_time
 NULL

mysql-test/suite/innodb/t/recovery_shutdown.test

@@ -0,0 +1,59 @@
+--source include/have_innodb.inc
+--source include/not_embedded.inc
+
+--echo #
+--echo # MDEV-13797 InnoDB may hang if shutdown is initiated soon after startup
+--echo # while rolling back recovered incomplete transactions
+--echo #
+
+CREATE TABLE t (a INT) ENGINE=InnoDB;
+let $size = 100;
+let $trx = 8;
+let $c = $size;
+BEGIN;
+--disable_query_log
+while ($c) {
+INSERT INTO t VALUES();
+dec $c;
+}
+--enable_query_log
+COMMIT;
+
+let $c = $trx;
+while ($c)
+{
+connect (con$c,localhost,root,,);
+eval CREATE TABLE t$c (a SERIAL, b INT UNIQUE, c INT UNIQUE) ENGINE=InnoDB;
+BEGIN;
+eval INSERT INTO t$c (a) SELECT NULL FROM t;
+eval UPDATE t$c SET a=a+$size, b=a;
+eval DELETE FROM t$c;
+dec $c;
+}
+
+INSERT INTO t1(a) SELECT NULL FROM t;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+INSERT INTO t1(a) SELECT NULL FROM t1;
+
+--connection default
+SET GLOBAL innodb_flush_log_at_trx_commit=1;
+CREATE TABLE u(a SERIAL) ENGINE=INNODB;
+
+--let $shutdown_timeout=0
+--source include/restart_mysqld.inc
+--let $shutdown_timeout=60
+--source include/restart_mysqld.inc
+
+--disable_query_log
+let $c = $trx;
+while ($c)
+{
+disconnect con$c;
+eval DROP TABLE t$c;
+dec $c;
+}
+--enable_query_log
+
+DROP TABLE t,u;

mysql-test/suite/innodb/t/table_definition_cache_debug.opt

@@ -1 +1 @@
---innodb-open-files=13
+--innodb-open-files=20

mysql-test/suite/innodb/t/update_time.test

@@ -67,7 +67,8 @@ CONNECT (con1,localhost,root,,);
 call mtr.add_suppression("Found 1 prepared XA transactions");
 FLUSH TABLES;
 
---source include/kill_and_restart_mysqld.inc
+--let $shutdown_timeout=0
+--source include/restart_mysqld.inc
 
 SELECT update_time FROM information_schema.tables WHERE table_name = 't';
 

mysql-test/suite/innodb_gis/r/kill_server.result

@@ -36,6 +36,5 @@ call mtr.add_suppression("InnoDB: A copy of page \[page id: space=[0-9]+, page n
 START TRANSACTION;
 CALL insert_t1(5000);
 COMMIT;
-# Kill and restart
 drop procedure insert_t1;
 drop table t1;

mysql-test/suite/innodb_gis/r/rtree_compress2.result

@@ -20,6 +20,5 @@ call mtr.add_suppression("InnoDB: A copy of page \[page id: space=[0-9]+, page n
 START TRANSACTION;
 CALL insert_t1(5000);
 COMMIT;
-# Kill and restart
 drop procedure insert_t1;
 drop table t1;

mysql-test/suite/innodb_gis/r/rtree_crash.result

@@ -1,41 +0,0 @@
-create table t1 (c1 int, c2 geometry not null, spatial index (c2))engine=innodb;
-create procedure insert_t1(IN total int)
-begin
-declare i int default 1;
-while (i <= total) DO
-insert into t1 values (i, Point(i, i));
-set i = i + 1;
-end while;
-end|
-CALL insert_t1(5000);
-select count(*) from t1;
-count(*)
-5000
-check table t1;
-Table	Op	Msg_type	Msg_text
-test.t1	check	status	OK
-truncate table t1;
-CALL insert_t1(10000);
-select count(*) from t1;
-count(*)
-10000
-drop index c2 on t1;
-create spatial index idx on t1(c2);
-show create table t1;
-Table	Create Table
-t1	CREATE TABLE `t1` (
-  `c1` int(11) DEFAULT NULL,
-  `c2` geometry NOT NULL,
-  SPATIAL KEY `idx` (`c2`)
-) ENGINE=InnoDB DEFAULT CHARSET=latin1
-check table t1;
-Table	Op	Msg_type	Msg_text
-test.t1	check	status	OK
-truncate table t1;
-call mtr.add_suppression("InnoDB: page [0-9]+ in the doublewrite buffer is not within space bounds.*");
-START TRANSACTION;
-CALL insert_t1(5000);
-COMMIT;
-# Kill and restart
-drop procedure insert_t1;
-drop table t1;

mysql-test/suite/innodb_gis/r/rtree_recovery.result

@@ -18,7 +18,6 @@ end while;
 end|
 CALL insert_t1(367);
 COMMIT;
-# Kill and restart
 check table t1;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK
@@ -29,7 +28,6 @@ CALL update_t1(367);
 SET @poly1 = ST_GeomFromText('POLYGON((10000 10000, 10000 10350, 10350 10350, 10350 10000, 10000 10000))');
 delete from t1 where ST_Contains(@poly1, c2);
 COMMIT;
-# Kill and restart
 check table t1;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK

mysql-test/suite/innodb_gis/r/types.result

@@ -53,7 +53,6 @@ COMMIT;
 INSERT INTO t_wl6455 VALUES(11, POINT(11,11));
 BEGIN;
 INSERT INTO t_wl6455 VALUES(1, POINT(1,1));
-# Kill and restart
 CHECK TABLE t_wl6455;
 Table	Op	Msg_type	Msg_text
 test.t_wl6455	check	status	OK

mysql-test/suite/innodb_gis/t/kill_server.test

@@ -58,7 +58,8 @@ CALL insert_t1(5000);
 
 COMMIT;
 
---source include/kill_and_restart_mysqld.inc
+--let $shutdown_timeout=0
+--source include/restart_mysqld.inc
 
 # Clean up.
 drop procedure insert_t1;

mysql-test/suite/innodb_gis/t/rtree_compress2.test

@@ -51,7 +51,8 @@ CALL insert_t1(5000);
 
 COMMIT;
 
---source include/kill_and_restart_mysqld.inc
+--let $shutdown_timeout=0
+--source include/restart_mysqld.inc
 
 # Clean up.
 drop procedure insert_t1;

mysql-test/suite/innodb_gis/t/rtree_recovery.test

@@ -44,7 +44,8 @@ delimiter ;|
 CALL insert_t1(367);
 COMMIT;
 
---source include/kill_and_restart_mysqld.inc
+--let $shutdown_timeout=0
+--source include/restart_mysqld.inc
 
 # Check table.
 check table t1;
@@ -63,7 +64,7 @@ SET @poly1 = ST_GeomFromText('POLYGON((10000 10000, 10000 10350, 10350 10350, 10
 delete from t1 where ST_Contains(@poly1, c2);
 COMMIT;
 
---source include/kill_and_restart_mysqld.inc
+--source include/restart_mysqld.inc
 
 # Check table.
 check table t1;

mysql-test/suite/innodb_gis/t/types.test

@@ -73,7 +73,8 @@ INSERT INTO t_wl6455 VALUES(11, POINT(11,11));
 BEGIN;
 INSERT INTO t_wl6455 VALUES(1, POINT(1,1));
 
---source include/kill_and_restart_mysqld.inc
+--let $shutdown_timeout=0
+--source include/restart_mysqld.inc
 
 CHECK TABLE t_wl6455;
 SELECT ST_AsText(g) FROM t_wl6455;

mysql-test/suite/storage_engine/type_bit_indexes.result

@@ -69,7 +69,7 @@ INSERT INTO t1 (a,b,c,d) VALUES
 (1,0xFFFF,0xFFFFFFFF,0xFFFFFFFFFFFFFFFF);
 EXPLAIN SELECT HEX(b+c) FROM t1 WHERE c > 1 OR HEX(b) < 0xFFFFFF;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-#	#	#	#	#	NULL	#	#	#	#
+#	#	#	#	#	b_c	#	#	#	#
 SELECT HEX(b+c) FROM t1 WHERE c > 1 OR HEX(b) < 0xFFFFFF;
 HEX(b+c)
 10
@@ -98,7 +98,7 @@ INSERT INTO t1 (a,b,c,d) VALUES
 (1,0xFFFF,0xFFFFFFFF,0xFFFFFFFFFFFFFFFF);
 EXPLAIN SELECT DISTINCT a+0 FROM t1 ORDER BY a;
 id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
-#	#	#	#	#	NULL	#	#	#	#
+#	#	#	#	#	a	#	#	#	#
 SELECT DISTINCT a+0 FROM t1 ORDER BY a;
 a+0
 0

mysql-test/suite/versioning/r/optimized.result

@@ -17,59 +17,59 @@ a	b	b+0
 1	NULL	NULL
 3	NULL	NULL
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select * from t for system_time as of timestamp now(6);
 a	b
 1	NULL
 3	NULL
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select count(*) from t group by b for system_time as of timestamp now(6);
 count(*)
 2
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select * from t for system_time as of timestamp now(6) order by b asc;
 a	b
 1	NULL
 3	NULL
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select * from t for system_time as of timestamp now(6) order by b desc;
 a	b
 1	NULL
 3	NULL
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select * from t group by a having a=2 for system_time as of timestamp now(6);
 a	b
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select * from t group by b having b=2 for system_time as of timestamp now(6);
 a	b
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select a from t where b=2 for system_time as of timestamp now(6);
 a
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select a from t where b=NULL for system_time as of timestamp now(6);
 a
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select a from t where b is NULL for system_time as of timestamp now(6);
 a
 1
 3
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select count(*), b from t group by b having b=NULL for system_time as of timestamp now(6);
 count(*)	b
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select a, b from t;
 a	b
 1	2
@@ -78,29 +78,29 @@ select count(*) from t for system_time as of timestamp now(6) group by b;
 count(*)
 2
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select * from t for system_time as of timestamp now(6) group by b having b=2;
 a	b
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select a from t for system_time as of timestamp now(6) where b=2;
 a
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select a from t for system_time as of timestamp now(6) where b=NULL;
 a
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select a from t for system_time as of timestamp now(6) where b is NULL;
 a
 1
 3
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select count(*), b from t for system_time as of timestamp now(6) group by b having b=NULL;
 count(*)	b
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 create or replace table t (
 a int,
 b int not null without system versioning
@@ -111,12 +111,12 @@ a	b
 1	NULL
 3	NULL
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 select * from t for system_time as of timestamp now(6) where b is NULL;
 a	b
 1	NULL
 3	NULL
 Warnings:
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
-Warning	4111	Attempt to read unversioned field `b` in historical query
+Warning	4112	Attempt to read unversioned field `b` in historical query
 drop table t;

mysql-test/suite/versioning/r/partition.result

@@ -78,7 +78,7 @@ ERROR HY000: Wrong partitions consistency for `t1`: must have at least one 'VERS
 alter table t1 add partition (
 partition p1 versioning);
 Warnings:
-Warning	4114	Maybe missing parameters: no rotation condition for multiple `VERSIONING` partitions.
+Warning	4115	Maybe missing parameters: no rotation condition for multiple `VERSIONING` partitions.
 show create table t1;
 Table	Create Table
 t1	CREATE TABLE `t1` (
@@ -215,7 +215,7 @@ x
 2
 delete from t1;
 Warnings:
-Note	4115	Switching from partition `p0` to `p1`
+Note	4116	Switching from partition `p0` to `p1`
 select * from t1 partition (p0) for system_time all;
 x
 1
@@ -225,7 +225,7 @@ x
 insert into t1 values (3);
 delete from t1;
 Warnings:
-Warning	4113	Using full partition `p1`, need more VERSIONING partitions!
+Warning	4114	Using full partition `p1`, need more VERSIONING partitions!
 select * from t1 partition (p1) for system_time all;
 x
 2
@@ -258,7 +258,7 @@ x
 insert into t1 values (4);
 delete from t1;
 Warnings:
-Note	4115	Switching from partition `p0` to `p1`
+Note	4116	Switching from partition `p0` to `p1`
 select * from t1 partition (p1) for system_time all;
 x
 4
@@ -280,8 +280,8 @@ x
 2
 delete from t1;
 Warnings:
-Note	4115	Switching from partition `p0` to `p1`
+Note	4116	Switching from partition `p0` to `p1`
-Warning	4113	Using full partition `p1`, need more VERSIONING partitions!
+Warning	4114	Using full partition `p1`, need more VERSIONING partitions!
 select * from t1 partition (p0sp0) for system_time all;
 x
 1

mysql-test/t/ctype_utf8.test

@@ -1878,6 +1878,18 @@ SELECT _utf8 0x7E, _utf8 X'7E', _utf8 B'01111110';
 let $ctype_unescape_combinations=selected;
 --source include/ctype_unescape.inc
 
+--echo #
+--echo # MDEV-12681 Wrong VIEW results for CHAR(0xDF USING latin1)
+--echo #
+
+SET NAMES utf8;
+SELECT CHAR(0xDF USING latin1);
+CREATE OR REPLACE VIEW v1 AS SELECT CHAR(0xDF USING latin1) AS c;
+SHOW CREATE VIEW v1;
+SELECT * FROM v1;
+DROP VIEW v1;
+
+
 --echo #
 --echo # End of 10.0 tests
 --echo #

mysql-test/t/ctype_utf8mb4.test

@@ -1922,6 +1922,24 @@ INSERT IGNORE INTO t1 SELECT 'a 😁 b', 'a 😁 b';
 SELECT * FROM t1;
 DROP TABLE t1;
 
+--echo #
+--echo # MDEV-8949: COLUMN_CREATE unicode name breakage
+--echo #
+
+SET NAMES utf8mb4;
+SELECT COLUMN_JSON(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1));
+SELECT COLUMN_LIST(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1));
+SELECT COLUMN_GET(COLUMN_CREATE(_utf8mb4 0xF09F988E, 1), _utf8mb4 0xF09F988E
+as int);
+
+CREATE TABLE t1 AS SELECT
+  COLUMN_LIST(COLUMN_CREATE('a',1)),
+  COLUMN_JSON(COLUMN_CREATE('b',1));
+SHOW CREATE TABLE t1;
+DROP TABLE t1;
+
+SET NAMES default;
+
 --echo #
 --echo # End of 10.0 tests
 --echo #

mysql-test/t/func_str.test

@@ -1809,6 +1809,18 @@ set global max_allowed_packet=default;
 --echo # End of 5.6 tests
 --echo #
 
+--echo #
+--echo # Start of 10.0 tests
+--echo #
+
+--echo #
+--echo # MDEV-12681 Wrong VIEW results for CHAR(0xDF USING latin1)
+--echo #
+
+EXPLAIN EXTENDED SELECT CHAR(0xDF USING latin1);
+EXPLAIN EXTENDED SELECT CHAR(0xDF USING `binary`);
+EXPLAIN EXTENDED SELECT CHAR(0xDF);
+
 --echo #
 --echo # Start of 10.1 tests
 --echo #
@@ -1853,6 +1865,15 @@ SELECT f1,HEX(f2) FROM t1                  WHERE f1='YQ==' AND (f2= from_base64(
 SELECT f1,HEX(f2) FROM t1                  WHERE f1='YQ==' AND (f2= from_base64("Yq==") OR f2= from_base64("YQ=="));
 DROP TABLE t1;
 
+--echo #
+--echo # End of 10.1 tests
+--echo #
+
+
+--echo #
+--echo # Start of 10.3 tests
+--echo #
+
 --echo #
 --echo # MDEV-12685 Oracle-compatible function CHR()
 --echo #
@@ -1864,15 +1885,6 @@ select charset(chr(14844588)), length(chr(14844588)),char_length(chr(14844588));
 drop database mysqltest1;
 use test;
 
---echo #
---echo # End of 10.1 tests
---echo #
-
-
---echo #
---echo # Start of 10.3 tests
---echo #
-
 --echo #
 --echo # MDEV-12592 Illegal mix of collations with the HEX function
 --echo #

mysql-test/t/invisible_binlog.test

@@ -0,0 +1,32 @@
+--source include/master-slave.inc
+
+--connection master
+create table t1(a int , b int invisible);
+insert into t1 values(1);
+insert into t1(a,b) values(2,2);
+select a,b from t1;
+desc t1;
+
+create table t2(a int , b int invisible default 5);
+insert into t2 values(1);
+insert into t2(a,b) values(2,2);
+select a,b from t2;
+desc t2;
+
+
+--sync_slave_with_master
+select * from t1;
+select a,b from t1;
+desc t1;
+show create table t1;
+
+select * from t2;
+select a,b from t2;
+desc t2;
+show create table t2;
+
+
+--connection master
+drop table t1,t2;
+
+--source include/rpl_end.inc

mysql-test/t/invisible_field.test

@@ -0,0 +1,216 @@
+FLUSH STATUS;
+create table t1(abc int primary key, xyz int invisible);
+SHOW STATUS LIKE 'Feature_invisible_columns';
+desc t1;
+show create table t1;
+select TABLE_CATALOG,TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,EXTRA from INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA='test' and TABLE_NAME='t1';
+drop table t1;
+--error ER_TABLE_MUST_HAVE_COLUMNS
+create table t1(a1 int invisible);
+--error ER_PARSE_ERROR
+create table t1(a1 blob,invisible(a1));
+--error ER_INVISIBLE_NOT_NULL_WITHOUT_DEFAULT
+create table t1(a1 int primary key invisible ,a2 int unique invisible , a3 blob,a4 int not null invisible unique);
+--error ER_INVISIBLE_NOT_NULL_WITHOUT_DEFAULT
+create table t1(abc int not null invisible);
+create table t1(a int invisible, b int);
+#should automatically add null
+insert into t1 values(1);
+insert into t1(a) values(2);
+insert into t1(b) values(3);
+insert into t1(a,b) values(5,5);
+select * from t1;
+select a,b from t1;
+delete from t1;
+insert into t1 values(1),(2),(3),(4);
+select * from t1;
+select a from t1;
+drop table t1;
+
+--echo #more complex case of invisible
+create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int);
+desc t1;
+insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1);
+select * from t1;
+select a,b,c,d,e,f from t1;
+drop table t1;
+
+--echo #more complex case of invisible with sql_mode=NO_AUTO_VALUE_ON_ZERO
+set sql_mode='NO_AUTO_VALUE_ON_ZERO';
+create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int);
+desc t1;
+insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1);
+select * from t1;
+select a,b,c,d,e,f from t1;
+drop table t1;
+set sql_mode='';
+
+--error ER_PARSE_ERROR
+create table sdsdsd(a int , b int, invisible(a,b));
+create table t1(a int,abc int as (a mod 3) virtual invisible);
+desc t1;
+--error ER_WRONG_VALUE_COUNT_ON_ROW
+insert into t1 values(1,default);
+insert into t1 values(1),(22),(233);
+select * from t1;
+select a,abc from t1;
+drop table t1;
+create table t1(abc int primary key invisible auto_increment, a int);
+desc t1;
+show create table t1;
+insert into t1 values(1);
+insert into t1 values(2);
+insert into t1 values(3);
+select * from t1;
+select abc,a from t1;
+delete  from t1;
+insert into t1 values(1),(2),(3),(4),(6);
+select abc,a from t1;
+drop table t1;
+create table t1(abc int);
+--error ER_TABLE_MUST_HAVE_COLUMNS
+alter table t1 change abc ss int invisible;
+alter table t1 add column xyz int;
+alter table t1 modify column abc  int ;
+desc t1;
+--error ER_WRONG_VALUE_COUNT_ON_ROW
+insert into t1 values(22);
+alter table t1 modify column abc  int invisible;
+desc t1;
+insert into t1 values(12);
+drop table t1;
+--echo #some test on copy table structure with table data;
+--echo #table with invisible fields and unique keys;
+create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int);
+desc t1;
+insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1);
+select * from t1;
+select a,b,c,d,e,f from t1;
+--echo #this won't copy invisible fields and keys;
+create table t2 as select * from t1;
+desc t2;
+select * from t2;
+--error ER_BAD_FIELD_ERROR
+select a,b,c,d,e,f from t2;
+drop table t2;
+--echo #now this will copy invisible fields
+create table t2 as select a,b,c,d,e,f from t1;
+desc t2;
+select * from t2;
+select a,b,c,d,e,f from t2;
+drop table t2,t1;
+--echo #some test related to copy of data from one table to another;
+create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int);
+insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1);
+select a,b,c,d,e,f from t1;
+create table t2(a int , b int invisible , c int invisible , d blob , e int unique, f int);
+insert into t2 select * from t1;
+select a,b,c,d,e,f from t2;
+truncate t2;
+insert into t2 (a,b,c,d,e,f) select a,b,c,d,e,f from t1;
+select a,b,c,d,e,f from t2;
+truncate t2;
+drop table t1,t2;
+--echo #some test related to creating view on table with invisible column;
+create table t1(a int , b int invisible , c int invisible auto_increment unique, d blob , e int unique, f int);
+insert into t1 values(1,'d blob',1,1),(1,'d blob',11,1),(1,'d blob',2,1),(1,'d blob',3,1),(1,'d blob',41,1);
+create view v as select * from t1;
+desc v;
+select * from v;
+--echo #v does not have invisible column;
+--error ER_BAD_FIELD_ERROR
+select a,b,c,d,e,f from v;
+insert into v values(1,21,32,4);
+select * from v;
+--error ER_BAD_FIELD_ERROR
+insert into v(a,b,c,d,e,f) values(1,12,3,4,5,6);
+drop view v;
+create view v as select a,b,c,d,e,f from t1;
+desc v;
+select * from v;
+--echo #v does  have invisible column but they aren't invisible anymore.
+select a,b,c,d,e,f from v;
+insert into v values(1,26,33,4,45,66);
+select a,b,c,d,e,f from v;
+insert into v(a,b,c,d,e,f) values(1,32,31,41,5,6);
+select a,b,c,d,e,f from v;
+drop view v;
+drop table t1;
+--echo #now invisible column in where and some join query
+create table t1 (a int unique , b int invisible unique, c int unique  invisible);
+insert into t1(a,b,c) values(1,1,1);
+insert into t1(a,b,c) values(2,2,2);
+insert into t1(a,b,c) values(3,3,3);
+insert into t1(a,b,c) values(4,4,4);
+insert into t1(a,b,c) values(21,21,26);
+insert into t1(a,b,c) values(31,31,35);
+insert into t1(a,b,c) values(41,41,45);
+insert into t1(a,b,c) values(22,22,24);
+insert into t1(a,b,c) values(32,32,33);
+insert into t1(a,b,c) values(42,42,43);
+explain select * from t1 where b=3;
+select * from t1 where b=3;
+explain select * from t1 where c=3;
+select * from t1 where c=3;
+create table t2 as select a,b,c from t1;
+desc t2;
+explain select * from t1,t2 where t1.b = t2.c and t1.c = t2.b;
+select * from t1,t2 where t1.b = t2.c and t1.c = t2.b;
+drop table t1,t2;
+--echo #Unhide  invisible columns
+create table t1 (a int primary key, b int invisible, c int invisible unique);
+show create table t1;
+desc t1;
+alter table t1 modify column b int;
+desc t1;
+alter table t1 change column c d int;
+desc t1;
+drop table t1;
+SHOW STATUS LIKE 'Feature_invisible_columns';
+--echo #invisible is non reserved
+create table t1(a int unique , invisible int invisible, c int );
+desc t1;
+alter table t1 change column invisible hid int invisible;
+desc t1;
+drop table t1;
+##Internal temp table
+CREATE TABLE t1 (b int);
+INSERT t1 values(1);
+INSERT t1 values(2);
+INSERT t1 values(3);
+INSERT t1 values(4);
+INSERT t1 values(5);
+CREATE TABLE t2 (a int invisible) SELECT * FROM t1;
+select * from t2 order by b;
+select a,b from t2 order by b;
+CREATE TABLE t3 (b int, a int invisible) SELECT * FROM t1;
+select * from t3 order by b;
+select a,b from t3 order by b;
+--error ER_TABLE_MUST_HAVE_COLUMNS
+CREATE TABLE t4 (b int invisible) SELECT * FROM t1;
+--error ER_TABLE_MUST_HAVE_COLUMNS
+CREATE TABLE t5 (a int invisible) SELECT b as a FROM t1;
+drop table t1,t2,t3;
+
+create table t1 (a int , b int invisible default 3, c int , d int invisible default 6);
+DELIMITER //;
+CREATE PROCEDURE
+insert_t1(a int, b int)
+MODIFIES SQL DATA
+insert into t1 values(a,b);
+//
+DELIMITER ;//
+call insert_t1(1,1);
+call insert_t1(2,2);
+select * from t1 order by a;
+select a,b,c,d from t1 order by a;
+DROP PROCEDURE insert_t1;
+delete from t1;
+prepare insert_1 from "insert into t1 values(@a,@c)";
+prepare insert_2 from "insert into t1(a,b,c) values(@a,@b,@c)";
+set @a=1, @c=1;
+execute insert_1;
+set @a=2,@b=2, @c=2;
+execute insert_2;
+select a,b,c,d from t1 order by a;
+drop table t1;

mysql-test/t/invisible_field_debug.test

@@ -0,0 +1,202 @@
+--source include/have_debug.inc
+##TEST for invisible coloumn level 2
+set @old_debug= @@debug_dbug;
+set debug_dbug= "+d,test_pseudo_invisible";
+create table t1(a int);
+set debug_dbug=@old_debug;
+
+desc t1;
+show create table t1;
+insert into t1 values(1);
+select * from t1;
+select invisible ,a from t1;
+drop table t1;
+
+##TEST for invisible coloumn level 3
+
+set debug_dbug= "+d,test_completely_invisible";
+create table t1(a int);
+set debug_dbug=@old_debug;
+
+desc t1;
+show create table t1;
+insert into t1 values(1);
+select * from t1;
+
+--error ER_BAD_FIELD_ERROR
+select invisible ,a from t1;
+
+set debug_dbug= "+d,test_completely_invisible";
+select invisible ,a from t1;
+set debug_dbug=@old_debug;
+
+drop table t1;
+
+##TEST for Alter table for invisibleness level 2
+
+set debug_dbug= "+d,test_pseudo_invisible";
+create table t1(a int);
+set debug_dbug=@old_debug;
+
+desc t1;
+insert into t1 values(1);
+select * from t1;
+select invisible ,a from t1;
+
+## Alter should not be possible
+
+--error ER_BAD_FIELD_ERROR
+ALTER table t1 change invisible b int;
+select * from t1;
+select invisible ,a from t1;
+
+--error ER_BAD_FIELD_ERROR
+ALTER table t1 modify invisible char;
+select * from t1;
+select invisible ,a from t1;
+
+--error ER_CANT_DROP_FIELD_OR_KEY
+ALTER table t1 drop invisible;
+select * from t1;
+select invisible ,a from t1;
+
+--error ER_DUP_FIELDNAME
+ALTER table t1 add invisible int;
+select * from t1;
+select invisible ,a from t1;
+
+ALTER table t1 add invisible2 int default 2;
+select * from t1;
+select invisible ,a from t1;
+
+drop table t1;
+
+##TEST for Alter table for invisibleness level 3
+
+set debug_dbug= "+d,test_completely_invisible";
+create table t1(a int);
+
+desc t1;
+insert into t1 values(1);
+select * from t1;
+select invisible ,a from t1;
+
+## Alter should not be possible
+
+--error ER_BAD_FIELD_ERROR
+ALTER table t1 change invisible b int;
+select * from t1;
+select invisible ,a from t1;
+
+--error ER_BAD_FIELD_ERROR
+ALTER table t1 modify invisible char;
+select * from t1;
+select invisible ,a from t1;
+
+--error ER_CANT_DROP_FIELD_OR_KEY
+ALTER table t1 drop invisible;
+select * from t1;
+select invisible ,a from t1;
+
+ALTER table t1 add invisible int;
+select * from t1;
+select invisible1, invisible ,a from t1;
+
+#set debug_dbug=@old_debug;
+ALTER table t1 add hid int default 2;
+set debug_dbug= "+d,test_completely_invisible";
+select * from t1;
+select invisible ,a from t1;
+
+drop table t1;
+set debug_dbug=@old_debug;
+
+## Test Index on USER_DEFINED_INVISIBLE
+
+Create table t1( a int default(99) invisible, b int);
+insert into t1 values(1);
+insert into t1 values(2);
+insert into t1 values(3);
+insert into t1 values(4);
+select * from t1 order by b;
+alter table t1 add index(a);
+alter table t1 add index(a,b);
+show index from t1;
+drop table t1;
+
+## Test Index on PSEUDO_invisible_INVISIBLE
+
+set debug_dbug= "+d,test_pseudo_invisible";
+Create table t1( a int default(99) invisible, b int);
+
+--error ER_KEY_COLUMN_DOES_NOT_EXITS
+Create table t2( a int default(99) invisible, b int, unique(invisible));
+
+set debug_dbug=@old_debug;
+insert into t1 values(1);
+insert into t1 values(2);
+insert into t1 values(3);
+insert into t1 values(4);
+select * from t1 order by b;
+select invisible, a, b from t1 order by b;
+--error ER_KEY_COLUMN_DOES_NOT_EXITS
+alter table t1 add index(invisible);
+--error ER_KEY_COLUMN_DOES_NOT_EXITS
+alter table t1 add index(b,invisible);
+show index from t1;
+drop table t1;
+
+## Test Index on COMPLETELY_INVISIBLE
+
+set debug_dbug= "+d,test_completely_invisible";
+Create table t1( a int default(99) invisible, b int);
+
+--error ER_KEY_COLUMN_DOES_NOT_EXITS
+Create table t2( a int default(99) invisible, b int, unique(invisible));
+insert into t1 values(1);
+insert into t1 values(2);
+insert into t1 values(3);
+insert into t1 values(4);
+select * from t1 order by b;
+select invisible, a, b from t1 order by b;
+set debug_dbug=@old_debug;
+
+--error ER_KEY_COLUMN_DOES_NOT_EXITS
+alter table t1 add index(invisible);
+--error ER_KEY_COLUMN_DOES_NOT_EXITS
+alter table t1 add index(b,invisible);
+show index from t1;
+drop table t1;
+## Sytem Generated index on invisible column
+set debug_dbug= "+d,test_completely_invisible,test_invisible_index";
+## index name will be invisible
+Create table t1( a int default(99) , b int,c int, index(b));
+set debug_dbug=@old_debug;
+Show index from t1;
+select * from INFORMATION_SCHEMA.STATISTICS where TABLE_SCHEMA ='test' and table_name='t1';
+show create table t1;
+insert into t1 values(1,1,1);
+insert into t1 values(2,2,2);
+insert into t1 values(3,3,3);
+insert into t1 values(4,4,4);
+set debug_dbug= "+d,test_completely_invisible,test_invisible_index";
+select invisible, a ,b from t1 order by b;
+explain select * from t1 where invisible =9;
+alter table t1 add x int default 3;
+select invisible, a ,b from t1;
+set debug_dbug=@old_debug;
+Show index from t1;
+## Sytem Generated Index modification
+--error ER_KEY_COLUMN_DOES_NOT_EXITS
+create index a1 on t1(invisible);
+set debug_dbug= "+d,test_completely_invisible,test_invisible_index";
+## index does not exist for user
+--error ER_CANT_DROP_FIELD_OR_KEY
+drop index invisible on t1;
+explain select * from t1 where invisible =9;
+## index name will be changed
+create index invisible on t1(c);
+explain select * from t1 where invisible =9;
+show indexes in t1;
+drop table t1;
+set @old_debug= @@debug_dbug;

mysql-test/t/loadxml.test

@@ -143,3 +143,22 @@ LOAD XML INFILE '../../std_data/loaddata/mdev12696.xml' INTO TABLE v1 (c1);
 LOAD XML INFILE '../../std_data/loaddata/mdev12696.xml' INTO TABLE v1 (c2);
 DROP VIEW v1;
 DROP TABLE t1;
+
+--echo #
+--echo # MDEV-14628 Wrong autoinc value assigned by LOAD XML in the NO_AUTO_VALUE_ON_ZERO mode
+--echo #
+
+SET sql_mode=NO_AUTO_VALUE_ON_ZERO;
+CREATE TABLE t1 (a INT AUTO_INCREMENT PRIMARY KEY, b TEXT);
+LOAD XML INFILE '../../std_data/loaddata/mdev14628a.xml' INTO TABLE t1 ROWS IDENTIFIED BY '<row>';
+SELECT * FROM t1 ORDER BY b;
+DROP TABLE t1;
+SET sql_mode=DEFAULT;
+
+SET sql_mode='';
+CREATE  TABLE t1 (id INT, g GEOMETRY NOT NULL);
+--error ER_WARN_NULL_TO_NOTNULL
+LOAD XML INFILE '../../std_data/loaddata/mdev14628b.xml' INTO TABLE t1 ROWS IDENTIFIED BY '<row>';
+SELECT * FROM t1;
+DROP TABLE t1;
+SET sql_mode=DEFAULT;

mysql-test/t/mysqldump.test

@@ -2646,3 +2646,46 @@ CREATE TRIGGER tt1_t1 BEFORE INSERT ON t1 FOR EACH ROW
 INSERT INTO t1 (a) VALUES (1),(2),(3);
 --exec $MYSQL_DUMP --triggers --no-data --no-create-info --add-drop-trigger --skip-comments --databases test
 DROP TABLE t1;
+--echo #
+--echo # Test for Invisible columns
+--echo #
+create database d;
+use d;
+
+--echo # Invisble field table
+create table t1(a int , b int invisible);
+insert into t1 values(1);
+insert into t1(a,b) values(1,2);
+
+--echo # not invisible field table --complete-insert wont be used
+create table t2(a int , b int);
+insert into t2(a,b) values(1,2);
+insert into t2(a,b) values(1,2);
+
+--echo # Invisble field table
+create table t3(invisible int , `a  b c & $!@#$%^&*( )` int invisible default 4, `ds=~!@ \# $% ^ & * ( ) _ - = +` int invisible default 5);
+insert into t3 values(1);
+insert into t3 values(5);
+insert into t3 values(2);
+insert into t3(`invisible`,  `a  b c & $!@#$%^&*( )`, `ds=~!@ \# $% ^ & * ( ) _ - = +` ) values(1,2,3);
+CREATE TABLE t4(ËÏÌÏÎËÁ1 INT);
+insert into t4 values(1);
+--exec $MYSQL_DUMP --compact d
+
+--echo #Check side effect on --complete insert
+--exec $MYSQL_DUMP --compact --complete-insert d
+--echo #Check xml
+--exec $MYSQL_DUMP --skip-create-options --skip-comments -X d
+
+#import data
+--exec $MYSQL_DUMP --skip-comments d > $MYSQLTEST_VARDIR/tmp/invisible_dump.sql
+DROP table t1,t2,t3;
+
+--exec $MYSQL d < $MYSQLTEST_VARDIR/tmp/invisible_dump.sql
+
+select * from t1;
+select a,b from t1;
+select * from t2;
+select * from t3;
+desc t3;
+drop database d;

mysql-test/t/show_check.test

@@ -890,62 +890,62 @@ DROP FUNCTION f1;
 # Part 2: check that table with non-latin1 characters are dumped/restored
 # correctly.
 #
-
+# See MDEV-14651 for details
 # Ensure that all needed objects are dropped.
 
-set names koi8r;
+#set names koi8r;
 
---disable_warnings
+#--disable_warnings
-DROP DATABASE IF EXISTS mysqltest1;
+#DROP DATABASE IF EXISTS mysqltest1;
---enable_warnings
+#--enable_warnings
 
 # Create objects.
 
-CREATE DATABASE mysqltest1;
+#CREATE DATABASE mysqltest1;
 
-use mysqltest1;
+#use mysqltest1;
 
-CREATE TABLE t1(<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>1 INT);
+#CREATE TABLE t1(<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>1 INT);
 
 # Check:
 #   - Dump mysqltest1;
 
---let $outfile1=$MYSQLTEST_VARDIR/tmp/show_check.mysqltest1.sql
+#--let $outfile1=$MYSQLTEST_VARDIR/tmp/show_check.mysqltest1.sql
 
---source include/count_sessions.inc
+#--source include/count_sessions.inc
---echo
+#--echo
---echo ---> Dumping mysqltest1 to outfile1
+#--echo ---> Dumping mysqltest1 to outfile1
---exec $MYSQL_DUMP --default-character-set=latin1 --character-sets-dir=$MYSQL_SHAREDIR/charsets --databases mysqltest1 > $outfile1
+#--exec $MYSQL_DUMP --default-character-set=latin1 --character-sets-dir=$MYSQL_SHAREDIR/charsets --databases mysqltest1 > $outfile1
 # Take care that the additional session caused by MYSQL_DUMP has disappeared.
---source include/wait_until_count_sessions.inc
+#--source include/wait_until_count_sessions.inc
 
 #   - Clean mysqltest1;
 
---echo
+#--echo
---echo
+#--echo
 
-DROP DATABASE mysqltest1;
+#DROP DATABASE mysqltest1;
 
 #   - Restore mysqltest1;
 
---echo
+#--echo
---echo
+#--echo
 
---source include/count_sessions.inc
+#--source include/count_sessions.inc
---echo ---> Restoring mysqltest1...
+#--echo ---> Restoring mysqltest1...
---exec $MYSQL test < $outfile1
+#--exec $MYSQL test < $outfile1
---remove_file $outfile1
+#--remove_file $outfile1
 # Take care that the additional session caused by MYSQL has disappeared.
---source include/wait_until_count_sessions.inc
+#--source include/wait_until_count_sessions.inc
 
 #   - Check definition of the table.
 
-SHOW CREATE TABLE mysqltest1.t1;
+#SHOW CREATE TABLE mysqltest1.t1;
 
 # Cleanup.
 
-DROP DATABASE mysqltest1;
+#DROP DATABASE mysqltest1;
-use test;
+#use test;
 
 #
 # Bug#28808 log_queries_not_using_indexes variable dynamic change is ignored

mysys/ma_dyncol.c

@@ -4183,8 +4183,7 @@ mariadb_dyncol_json_internal(DYNAMIC_COLUMN *str, DYNAMIC_STRING *json,
     }
     else
     {
-      if ((rc= mariadb_dyncol_val_str(json, &val,
+      if ((rc= mariadb_dyncol_val_str(json, &val, DYNCOL_UTF, '"')) < 0)
-                                      &my_charset_utf8_general_ci, '"')) < 0)
         goto err;
     }
   }

mysys/thr_alarm.c

@@ -274,7 +274,7 @@ void thr_end_alarm(thr_alarm_t *alarmed)
 /*
   Come here when some alarm in queue is due.
   Mark all alarms with are finnished in list.
-  Shedule alarms to be sent again after 1-10 sec (many alarms at once)
+  Schedule alarms to be sent again after 1-10 sec (many alarms at once)
   If alarm_aborted is set then all alarms are given and resent
   every second.
 */
@@ -426,7 +426,7 @@ void end_thr_alarm(my_bool free_structures)
   if (alarm_aborted != 1)			/* If memory not freed */
   {
     mysql_mutex_lock(&LOCK_alarm);
-    DBUG_PRINT("info",("Resheduling %d waiting alarms",alarm_queue.elements));
+    DBUG_PRINT("info",("Rescheduling %d waiting alarms",alarm_queue.elements));
     alarm_aborted= -1;				/* mark aborted */
     if (alarm_queue.elements || (alarm_thread_running && free_structures))
     {

sql/field.cc

@@ -1253,6 +1253,20 @@ warn:
 }
 
 
+bool Field::load_data_set_null(THD *thd)
+{
+  reset();
+  set_null();
+  if (!maybe_null())
+  {
+    if (this != table->next_number_field)
+      set_warning(Sql_condition::WARN_LEVEL_WARN, ER_WARN_NULL_TO_NOTNULL, 1);
+  }
+  set_has_explicit_value(); // Do not auto-update this field
+  return false;
+}
+
+
 /**
   Numeric fields base class constructor.
 */
@@ -1589,7 +1603,8 @@ String *Field::val_int_as_str(String *val_buffer, bool unsigned_val)
 Field::Field(uchar *ptr_arg,uint32 length_arg,uchar *null_ptr_arg,
 	     uchar null_bit_arg,
 	     utype unireg_check_arg, const LEX_CSTRING *field_name_arg)
-  :ptr(ptr_arg), null_ptr(null_ptr_arg), table(0), orig_table(0),
+  :ptr(ptr_arg), field_visibility(NOT_INVISIBLE),
+  null_ptr(null_ptr_arg), table(0), orig_table(0),
   table_name(0), field_name(*field_name_arg), option_list(0),
   option_struct(0), key_start(0), part_of_key(0),
   part_of_key_not_clustered(0), part_of_sortkey(0),
@@ -2223,8 +2238,9 @@ Field *Field::make_new_field(MEM_ROOT *root, TABLE *new_table,
   tmp->flags&= (NOT_NULL_FLAG | BLOB_FLAG | UNSIGNED_FLAG |
                 ZEROFILL_FLAG | BINARY_FLAG | ENUM_FLAG | SET_FLAG |
                 VERS_SYS_START_FLAG | VERS_SYS_END_FLAG |
-                VERS_UPDATE_UNVERSIONED_FLAG | HIDDEN_FLAG);
+                VERS_UPDATE_UNVERSIONED_FLAG | VERS_HIDDEN_FLAG);
   tmp->reset_fields();
+  tmp->field_visibility= NOT_INVISIBLE;
   return tmp;
 }
 
@@ -5216,6 +5232,27 @@ int Field_timestamp::set_time()
   return 0;
 }
 
+
+bool Field_timestamp::load_data_set_null(THD *thd)
+{
+  if (!maybe_null())
+  {
+    /*
+      Timestamp fields that are NOT NULL are autoupdated if there is no
+      corresponding value in the data file.
+    */
+    set_time();
+  }
+  else
+  {
+    reset();
+    set_null();
+  }
+  set_has_explicit_value(); // Do not auto-update this field
+  return false;
+}
+
+
 #ifdef NOT_USED
 static void store_native(ulonglong num, uchar *to, uint bytes)
 {
@@ -8819,6 +8856,22 @@ bool Field_geom::can_optimize_range(const Item_bool_func *cond,
   return item->cmp_type() == STRING_RESULT;
 }
 
+
+bool Field_geom::load_data_set_null(THD *thd)
+{
+  Field_blob::reset();
+  if (!maybe_null())
+  {
+    my_error(ER_WARN_NULL_TO_NOTNULL, MYF(0), field_name.str,
+             thd->get_stmt_da()->current_row_for_warning());
+    return true;
+  }
+  set_null();
+  set_has_explicit_value(); // Do not auto-update this field
+  return false;
+}
+
+
 #endif /*HAVE_SPATIAL*/
 
 /****************************************************************************
@@ -10697,6 +10750,7 @@ Column_definition::Column_definition(THD *thd, Field *old_field,
   pack_flag= 0;
   compression_method_ptr= 0;
   versioning= VERSIONING_NOT_SET;
+  field_visibility= old_field->field_visibility;
 
   if (orig_field)
   {
@@ -10834,6 +10888,7 @@ Column_definition::redefine_stage1_common(const Column_definition *dup_field,
   flags=        dup_field->flags;
   interval=     dup_field->interval;
   vcol_info=    dup_field->vcol_info;
+  field_visibility= dup_field->field_visibility;
 }
 
 

sql/field.h

@@ -682,6 +682,8 @@ public:
   { DBUG_ASSERT(0); return false; }
 
   uchar		*ptr;			// Position to field in record
+
+  field_visible_type field_visibility;
   /**
      Byte where the @c NULL bit is stored inside a record. If this Field is a
      @c NOT @c NULL field, this member is @c NULL.
@@ -1176,6 +1178,8 @@ public:
     { if (null_ptr) null_ptr[row_offset]&= (uchar) ~null_bit; }
   inline bool maybe_null(void) const
   { return null_ptr != 0 || table->maybe_null; }
+  // Set to NULL on LOAD DATA or LOAD XML
+  virtual bool load_data_set_null(THD *thd);
 
   /* @return true if this field is NULL-able (even if temporarily) */
   inline bool real_maybe_null(void) const { return null_ptr != 0; }
@@ -2538,6 +2542,7 @@ public:
   {
     return get_equal_const_item_datetime(thd, ctx, const_item);
   }
+  bool load_data_set_null(THD *thd);
   uint size_of() const { return sizeof(*this); }
 };
 
@@ -3736,6 +3741,7 @@ public:
     but the underlying blob must still be reset.
    */
   int reset(void) { return Field_blob::reset() || !maybe_null(); }
+  bool load_data_set_null(THD *thd);
 
   geometry_type get_geometry_type() { return geom_type; };
   static geometry_type geometry_type_merge(geometry_type, geometry_type);
@@ -4136,6 +4142,7 @@ public:
     max number of characters. 
   */
   ulonglong length;
+  field_visible_type field_visibility;
   /*
     The value of `length' as set by parser: is the number of characters
     for most of the types, or of bytes for BLOBs or numeric types.
@@ -4168,7 +4175,7 @@ public:
    :Type_handler_hybrid_field_type(&type_handler_null),
     compression_method_ptr(0),
     comment(null_clex_str),
-    on_update(NULL), length(0), decimals(0),
+    on_update(NULL), length(0),field_visibility(NOT_INVISIBLE), decimals(0),
     flags(0), pack_length(0), key_length(0), unireg_check(Field::NONE),
     interval(0), charset(&my_charset_bin),
     srid(0), geom_type(Field::GEOM_GEOMETRY),
@@ -4647,6 +4654,7 @@ bool check_expression(Virtual_column_info *vcol, LEX_CSTRING *name,
 #define f_no_default(x)		((x) & FIELDFLAG_NO_DEFAULT)
 #define f_bit_as_char(x)        ((x) & FIELDFLAG_TREAT_BIT_AS_CHAR)
 #define f_is_hex_escape(x)      ((x) & FIELDFLAG_HEX_ESCAPE)
+#define f_visibility(x)         (static_cast<field_visible_type> ((x) & MAX_BITS_INVISIBLE))
 
 inline
 ulonglong TABLE::vers_end_id() const

sql/handler.cc

@@ -6768,7 +6768,7 @@ static Create_field *vers_init_sys_field(THD *thd, const char *field_name,
   f->field_name.str= field_name;
   f->field_name.length= strlen(field_name);
   f->charset= system_charset_info;
-  f->flags= flags | HIDDEN_FLAG;
+  f->flags= flags | VERS_HIDDEN_FLAG;
   if (integer_fields)
   {
     f->set_handler(&type_handler_longlong);
@@ -7124,13 +7124,13 @@ bool Vers_parse_info::fix_alter_info(THD *thd, Alter_info *alter_info,
       return true;
     }
 
-    if (!(share->vers_start_field()->flags & HIDDEN_FLAG))
+    if (!(share->vers_start_field()->flags & VERS_HIDDEN_FLAG))
     {
       my_error(ER_VERS_SYS_FIELD_NOT_HIDDEN, MYF(0),
                share->vers_start_field()->field_name.str);
       return true;
     }
-    if (!(share->vers_end_field()->flags & HIDDEN_FLAG))
+    if (!(share->vers_end_field()->flags & VERS_HIDDEN_FLAG))
     {
       my_error(ER_VERS_SYS_FIELD_NOT_HIDDEN, MYF(0),
                share->vers_end_field()->field_name.str);
@@ -7259,7 +7259,7 @@ bool Vers_parse_info::fix_alter_info(THD *thd, Alter_info *alter_info,
         }
         else
           continue;
-        if (f->flags & HIDDEN_FLAG)
+        if (f->flags & VERS_HIDDEN_FLAG)
         {
           my_error(ER_CANT_DROP_FIELD_OR_KEY, MYF(0), d->type_name(), name);
           return true;

sql/item_cmpfunc.cc

@@ -6962,20 +6962,20 @@ longlong Item_func_dyncol_exists::val_int()
       null_value= 1;
       return 1;
     }
-    if (my_charset_same(nm->charset(), &my_charset_utf8_general_ci))
+    if (my_charset_same(nm->charset(), DYNCOL_UTF))
     {
       buf.str= (char *) nm->ptr();
       buf.length= nm->length();
     }
     else
     {
-      uint strlen= nm->length() * my_charset_utf8_general_ci.mbmaxlen + 1;
+      uint strlen= nm->length() * DYNCOL_UTF->mbmaxlen + 1;
       uint dummy_errors;
       buf.str= (char *) current_thd->alloc(strlen);
       if (buf.str)
       {
         buf.length=
-          copy_and_convert(buf.str, strlen, &my_charset_utf8_general_ci,
+          copy_and_convert(buf.str, strlen, DYNCOL_UTF,
                            nm->ptr(), nm->length(), nm->charset(),
                            &dummy_errors);
       }

sql/item_strfunc.cc

@@ -2918,6 +2918,20 @@ String *Item_func_make_set::val_str(String *str)
 }
 
 
+void Item_func_char::print(String *str, enum_query_type query_type)
+{
+  str->append(Item_func_char::func_name());
+  str->append('(');
+  print_args(str, 0, query_type);
+  if (collation.collation != &my_charset_bin)
+  {
+    str->append(C_STRING_WITH_LEN(" using "));
+    str->append(collation.collation->csname);
+  }
+  str->append(')');
+}
+
+
 String *Item_func_char::val_str(String *str)
 {
   DBUG_ASSERT(fixed == 1);
@@ -4517,20 +4531,19 @@ bool Item_func_dyncol_create::prepare_arguments(THD *thd, bool force_names_arg)
       if (res)
       {
         // guaranty UTF-8 string for names
-        if (my_charset_same(res->charset(), &my_charset_utf8_general_ci))
+        if (my_charset_same(res->charset(), DYNCOL_UTF))
         {
           keys_str[i].length= res->length();
           keys_str[i].str= thd->strmake(res->ptr(), res->length());
         }
         else
         {
-          uint strlen= res->length() * my_charset_utf8_general_ci.mbmaxlen + 1;
+          uint strlen= res->length() * DYNCOL_UTF->mbmaxlen + 1;
           uint dummy_errors;
-          char *str= (char *) thd->alloc(strlen);
+          if (char *str= (char *) thd->alloc(strlen))
-          if (str)
           {
             keys_str[i].length=
-              copy_and_convert(str, strlen, &my_charset_utf8_general_ci,
+              copy_and_convert(str, strlen, DYNCOL_UTF,
                                res->ptr(), res->length(), res->charset(),
                                &dummy_errors);
               keys_str[i].str= str;
@@ -4750,9 +4763,10 @@ String *Item_func_dyncol_json::val_str(String *str)
     char *ptr;
     size_t length, alloc_length;
     dynstr_reassociate(&json, &ptr, &length, &alloc_length);
-    str->reset(ptr, length, alloc_length, &my_charset_utf8_general_ci);
+    str->reset(ptr, length, alloc_length, DYNCOL_UTF);
     null_value= FALSE;
   }
+  str->set_charset(DYNCOL_UTF);
   return str;
 
 null:
@@ -4852,20 +4866,20 @@ bool Item_dyncol_get::get_dyn_value(THD *thd, DYNAMIC_COLUMN_VALUE *val,
       return 1;
     }
 
-    if (my_charset_same(nm->charset(), &my_charset_utf8_general_ci))
+    if (my_charset_same(nm->charset(), DYNCOL_UTF))
     {
       buf.str= (char *) nm->ptr();
       buf.length= nm->length();
     }
     else
     {
-      uint strlen= nm->length() * my_charset_utf8_general_ci.mbmaxlen + 1;
+      uint strlen= nm->length() * DYNCOL_UTF->mbmaxlen + 1;
       uint dummy_errors;
       buf.str= (char *) thd->alloc(strlen);
       if (buf.str)
       {
         buf.length=
-          copy_and_convert(buf.str, strlen, &my_charset_utf8_general_ci,
+          copy_and_convert(buf.str, strlen, DYNCOL_UTF,
                            nm->ptr(), nm->length(), nm->charset(),
                            &dummy_errors);
       }
@@ -5289,7 +5303,6 @@ String *Item_func_dyncol_list::val_str(String *str)
     goto null;
 
   str->length(0);
-  str->set_charset(&my_charset_utf8_general_ci);
   for (i= 0; i < count; i++)
   {
     append_identifier(current_thd, str, names[i].str, names[i].length);
@@ -5299,6 +5312,7 @@ String *Item_func_dyncol_list::val_str(String *str)
   null_value= FALSE;
   if (names)
     my_free(names);
+  str->set_charset(DYNCOL_UTF);
   return str;
 
 null:

sql/item_strfunc.h

@@ -978,6 +978,7 @@ public:
     max_length= arg_count * 4;
   }
   const char *func_name() const { return "char"; }
+  void print(String *str, enum_query_type query_type);
   Item *get_copy(THD *thd)
   { return get_item_copy<Item_func_char>(thd, this); }
 };
@@ -1585,14 +1586,14 @@ public:
 class Item_func_dyncol_json: public Item_str_func
 {
 public:
-  Item_func_dyncol_json(THD *thd, Item *str): Item_str_func(thd, str) {}
+  Item_func_dyncol_json(THD *thd, Item *str): Item_str_func(thd, str)
+    {collation.set(DYNCOL_UTF);}
   const char *func_name() const{ return "column_json"; }
   String *val_str(String *);
   void fix_length_and_dec()
   {
     max_length= MAX_BLOB_WIDTH;
     maybe_null= 1;
-    collation.set(&my_charset_bin);
     decimals= 0;
   }
   Item *get_copy(THD *thd)
@@ -1645,7 +1646,8 @@ public:
 class Item_func_dyncol_list: public Item_str_func
 {
 public:
-  Item_func_dyncol_list(THD *thd, Item *str): Item_str_func(thd, str) {};
+  Item_func_dyncol_list(THD *thd, Item *str): Item_str_func(thd, str)
+    {collation.set(DYNCOL_UTF);}
   void fix_length_and_dec() { maybe_null= 1; max_length= MAX_BLOB_WIDTH; };
   const char *func_name() const{ return "column_list"; }
   String *val_str(String *);

sql/lex.h

@@ -273,6 +273,7 @@ static SYMBOL symbols[] = {
   { "HAVING",		SYM(HAVING)},
   { "HELP",		SYM(HELP_SYM)},
   { "HIGH_PRIORITY",	SYM(HIGH_PRIORITY)},
+  { "INVISIBLE",        SYM(INVISIBLE_SYM)},
   { "HOST",		SYM(HOST_SYM)},
   { "HOSTS",		SYM(HOSTS_SYM)},
   { "HOUR",		SYM(HOUR_SYM)},

sql/mysqld.cc

@@ -8531,6 +8531,7 @@ SHOW_VAR status_vars[]= {
   {"Feature_dynamic_columns",  (char*) offsetof(STATUS_VAR, feature_dynamic_columns), SHOW_LONG_STATUS},
   {"Feature_fulltext",         (char*) offsetof(STATUS_VAR, feature_fulltext), SHOW_LONG_STATUS},
   {"Feature_gis",              (char*) offsetof(STATUS_VAR, feature_gis), SHOW_LONG_STATUS},
+  {"Feature_invisible_columns",   (char*) offsetof(STATUS_VAR, feature_invisible_columns), SHOW_LONG_STATUS},
   {"Feature_locale",           (char*) offsetof(STATUS_VAR, feature_locale), SHOW_LONG_STATUS},
   {"Feature_subquery",         (char*) offsetof(STATUS_VAR, feature_subquery), SHOW_LONG_STATUS},
   {"Feature_timezone",         (char*) offsetof(STATUS_VAR, feature_timezone), SHOW_LONG_STATUS},

sql/share/errmsg-utf8.txt

@@ -7810,6 +7810,8 @@ ER_INVALID_AGGREGATE_FUNCTION
         eng "Aggregate specific instruction(FETCH GROUP NEXT ROW) missing from the aggregate function"
 ER_INVALID_VALUE_TO_LIMIT
         eng "Limit only accepts integer values"
+ER_INVISIBLE_NOT_NULL_WITHOUT_DEFAULT
+        eng "Invisible column %`s must have a default value"
 
 # MariaDB error numbers related to System Versioning
 

sql/sql_base.cc

@@ -5528,7 +5528,11 @@ find_field_in_table(THD *thd, TABLE *table, const char *name, uint length,
 
   if (field_ptr && *field_ptr)
   {
-    *cached_field_index_ptr= (uint)(field_ptr - table->field);
+    if ((*field_ptr)->field_visibility == COMPLETELY_INVISIBLE &&
+        DBUG_EVALUATE_IF("test_completely_invisible", 0, 1))
+      DBUG_RETURN((Field*)0);
+
+    *cached_field_index_ptr= field_ptr - table->field;
     field= *field_ptr;
   }
   else
@@ -7613,6 +7617,14 @@ insert_fields(THD *thd, Name_resolution_context *context, const char *db_name,
 
     for (; !field_iterator.end_of_fields(); field_iterator.next())
     {
+      /*
+        field() is always NULL for views (see, e.g. Field_iterator_view or
+        Field_iterator_natural_join).
+        But view fields can never be invisible.
+      */
+      if ((field= field_iterator.field()) &&
+          field->field_visibility != NOT_INVISIBLE)
+        continue;
       Item *item;
 
       if (!(item= field_iterator.create_item(thd)))
@@ -7637,12 +7649,12 @@ insert_fields(THD *thd, Name_resolution_context *context, const char *db_name,
               (sql_command == SQLCOM_CREATE_VIEW ||
               slex->nest_level > 0 ||
               vers_hide == VERS_HIDE_FULL ||
-              ((fl & HIDDEN_FLAG) && (
+              ((fl & VERS_HIDDEN_FLAG) && (
                 vers_hide == VERS_HIDE_IMPLICIT ||
                   (vers_hide == VERS_HIDE_AUTO && (
                     vers_type == FOR_SYSTEM_TIME_UNSPECIFIED ||
                     vers_type == FOR_SYSTEM_TIME_AS_OF))))) :
-            (fl & HIDDEN_FLAG))
+            (fl & VERS_HIDDEN_FLAG))
         {
           if (sql_command != SQLCOM_CREATE_TABLE ||
             !(create_options & HA_VERSIONED_TABLE))
@@ -7658,7 +7670,7 @@ insert_fields(THD *thd, Name_resolution_context *context, const char *db_name,
         {
           Item_field *f= (Item_field *)i;
           DBUG_ASSERT(f->field);
-          if (f->field->flags & HIDDEN_FLAG)
+          if (f->field->flags & VERS_HIDDEN_FLAG)
             continue;
         }
       }
@@ -8273,7 +8285,6 @@ fill_record(THD *thd, TABLE *table, Field **ptr, List<Item> &values,
                         ? table->next_number_field->field_index
                         : ~0U;
   DBUG_ENTER("fill_record");
-
   if (!*ptr)
   {
     /* No fields to update, quite strange!*/
@@ -8302,7 +8313,10 @@ fill_record(THD *thd, TABLE *table, Field **ptr, List<Item> &values,
       goto err;
     }
 
-    value=v++;
+    if (field->field_visibility != NOT_INVISIBLE)
+      continue;
+    else
+      value=v++;
     if (field->field_index == autoinc_index)
       table->auto_increment_field_not_null= TRUE;
     if (field->vcol_info)

sql/sql_class.cc

@@ -138,7 +138,7 @@ Key::Key(const Key &rhs, MEM_ROOT *mem_root)
   columns(rhs.columns, mem_root),
   name(rhs.name),
   option_list(rhs.option_list),
-  generated(rhs.generated)
+  generated(rhs.generated), invisible(false)
 {
   list_copy_and_replace_each_value(columns, mem_root);
 }

sql/sql_class.h

@@ -305,22 +305,25 @@ public:
   LEX_CSTRING name;
   engine_option_value *option_list;
   bool generated;
+  bool invisible;
 
   Key(enum Keytype type_par, const LEX_CSTRING *name_arg,
       ha_key_alg algorithm_arg, bool generated_arg, DDL_options_st ddl_options)
     :DDL_options(ddl_options),
      type(type_par), key_create_info(default_key_create_info),
-    name(*name_arg), option_list(NULL), generated(generated_arg)
+    name(*name_arg), option_list(NULL), generated(generated_arg),
+    invisible(false)
   {
     key_create_info.algorithm= algorithm_arg;
-  } 
+  }
   Key(enum Keytype type_par, const LEX_CSTRING *name_arg,
       KEY_CREATE_INFO *key_info_arg,
       bool generated_arg, List<Key_part_spec> *cols,
       engine_option_value *create_opt, DDL_options_st ddl_options)
     :DDL_options(ddl_options),
      type(type_par), key_create_info(*key_info_arg), columns(*cols),
-    name(*name_arg), option_list(create_opt), generated(generated_arg)
+    name(*name_arg), option_list(create_opt), generated(generated_arg),
+    invisible(false)
   {}
   Key(const Key &rhs, MEM_ROOT *mem_root);
   virtual ~Key() {}
@@ -798,6 +801,7 @@ typedef struct system_status_var
   ulong feature_dynamic_columns;    /* +1 when creating a dynamic column */
   ulong feature_fulltext;	    /* +1 when MATCH is used */
   ulong feature_gis;                /* +1 opening a table with GIS features */
+  ulong feature_invisible_columns;     /* +1 opening a table with invisible column */
   ulong feature_locale;		    /* +1 when LOCALE is set */
   ulong feature_subquery;	    /* +1 when subqueries are used */
   ulong feature_timezone;	    /* +1 when XPATH is used */

sql/sql_insert.cc

@@ -997,7 +997,8 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list,
           No field list, all fields are set explicitly:
           INSERT INTO t1 VALUES (values)
         */
-        if (thd->lex->used_tables)		      // Column used in values()
+        if (thd->lex->used_tables || // Column used in values()
+                table->s->visible_fields != table->s->fields)
 	  restore_record(table,s->default_values);	// Get empty record
         else
         {

sql/sql_load.cc

@@ -1015,7 +1015,6 @@ continue_loop:;
 }
 
 
-
 static int
 read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list,
                List<Item> &fields_vars, List<Item> &set_fields,
@@ -1094,28 +1093,9 @@ read_sep_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list,
         }
         else
         {
-          Field *field= real_item->field;
+          DBUG_ASSERT(real_item->field->table == table);
-          if (field->reset())
+          if (real_item->field->load_data_set_null(thd))
-          {
-            my_error(ER_WARN_NULL_TO_NOTNULL, MYF(0), field->field_name.str,
-                     thd->get_stmt_da()->current_row_for_warning());
             DBUG_RETURN(1);
-          }
-          field->set_null();
-          if (!field->maybe_null())
-          {
-            /*
-              Timestamp fields that are NOT NULL are autoupdated if there is no
-              corresponding value in the data file.
-            */
-            if (field->type() == MYSQL_TYPE_TIMESTAMP)
-              field->set_time();
-            else if (field != table->next_number_field)
-              field->set_warning(Sql_condition::WARN_LEVEL_WARN,
-                                 ER_WARN_NULL_TO_NOTNULL, 1);
-          }
-          /* Do not auto-update this field. */
-          field->set_has_explicit_value();
 	}
 
 	continue;
@@ -1262,6 +1242,7 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list,
   
   for ( ; ; it.rewind())
   {
+    bool err;
     if (thd->killed)
     {
       thd->send_kill_message();
@@ -1313,21 +1294,9 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list,
         }
         else
         {
-          Field *field= real_item->field;
+          DBUG_ASSERT(real_item->field->table == table);
-          field->reset();
+          if (real_item->field->load_data_set_null(thd))
-          field->set_null();
+            DBUG_RETURN(1);
-          if (field == table->next_number_field)
-            table->auto_increment_field_not_null= TRUE;
-          if (!field->maybe_null())
-          {
-            if (field->type() == FIELD_TYPE_TIMESTAMP)
-              field->set_time();
-            else if (field != table->next_number_field)
-              field->set_warning(Sql_condition::WARN_LEVEL_WARN,
-                                 ER_WARN_NULL_TO_NOTNULL, 1);
-          }
-          /* Do not auto-update this field. */
-          field->set_has_explicit_value();
         }
         continue;
       }
@@ -1361,39 +1330,8 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list,
       skip_lines--;
       continue;
     }
-    
+
-    if (item)
+    DBUG_ASSERT(!item);
-    {
-      /* Have not read any field, thus input file is simply ended */
-      if (item == fields_vars.head())
-        break;
-      
-      for ( ; item; item= it++)
-      {
-        Item_field *real_item= item->field_for_view_update();
-        if (item->type() == Item::STRING_ITEM)
-          ((Item_user_var_as_out_param *)item)->set_null_value(cs);
-        else if (!real_item)
-        {
-          my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), item->name.str);
-          DBUG_RETURN(1);
-        }
-        else
-        {
-          /*
-            QQ: We probably should not throw warning for each field.
-            But how about intention to always have the same number
-            of warnings in THD::cuted_fields (and get rid of cuted_fields
-            in the end ?)
-          */
-          thd->cuted_fields++;
-          push_warning_printf(thd, Sql_condition::WARN_LEVEL_WARN,
-                              ER_WARN_TOO_FEW_RECORDS,
-                              ER_THD(thd, ER_WARN_TOO_FEW_RECORDS),
-                              thd->get_stmt_da()->current_row_for_warning());
-        }
-      }
-    }
 
     if (thd->killed ||
         fill_record_n_invoke_before_triggers(thd, table, set_fields, set_values,
@@ -1410,7 +1348,9 @@ read_xml_field(THD *thd, COPY_INFO &info, TABLE_LIST *table_list,
       DBUG_RETURN(-1);
     }
     
-    if (write_record(thd, table, &info))
+    err= write_record(thd, table, &info);
+    table->auto_increment_field_not_null= false;
+    if (err)
       DBUG_RETURN(1);
     
     /*

sql/sql_select.cc

@@ -16836,7 +16836,7 @@ Field *create_tmp_field_from_field(THD *thd, Field *org_field,
       new_field->field_name= *name;
     new_field->flags|= (org_field->flags & (
       NO_DEFAULT_VALUE_FLAG |
-      HIDDEN_FLAG |
+      VERS_HIDDEN_FLAG |
       VERS_SYS_START_FLAG |
       VERS_SYS_END_FLAG |
       VERS_UPDATE_UNVERSIONED_FLAG));
@@ -17670,8 +17670,8 @@ create_tmp_table(THD *thd, TMP_TABLE_PARAM *param, List<Item> &fields,
 
   if (sys_trx_start && sys_trx_end)
   {
-    sys_trx_start->flags|= VERS_SYS_START_FLAG | HIDDEN_FLAG;
+    sys_trx_start->flags|= VERS_SYS_START_FLAG | VERS_HIDDEN_FLAG;
-    sys_trx_end->flags|= VERS_SYS_END_FLAG | HIDDEN_FLAG;
+    sys_trx_end->flags|= VERS_SYS_END_FLAG | VERS_HIDDEN_FLAG;
     share->versioned= true;
     share->field= table->field;
     share->row_start_field= sys_trx_start->field_index;

sql/sql_show.cc

@@ -2183,17 +2183,22 @@ int show_create_table(THD *thd, TABLE_LIST *table_list, String *packet,
   */
   old_map= tmp_use_all_columns(table, table->read_set);
 
+  bool not_the_first_field= false;
   for (ptr=table->field ; (field= *ptr); ptr++)
   {
+
     uint flags = field->flags;
 
     if (vers_hide == VERS_HIDE_FULL &&
         (flags & (VERS_SYS_START_FLAG | VERS_SYS_END_FLAG)))
       continue;
 
-    if (ptr != table->field)
+    if (field->field_visibility > USER_DEFINED_INVISIBLE)
+       continue;
+    if (not_the_first_field)
       packet->append(STRING_WITH_LEN(",\n"));
 
+    not_the_first_field= true;
     packet->append(STRING_WITH_LEN("  "));
     append_identifier(thd,packet,field->field_name.str,
                       field->field_name.length);
@@ -2254,6 +2259,10 @@ int show_create_table(THD *thd, TABLE_LIST *table_list, String *packet,
         packet->append(STRING_WITH_LEN(" NULL"));
       }
 
+      if (field->field_visibility == USER_DEFINED_INVISIBLE)
+      {
+        packet->append(STRING_WITH_LEN(" INVISIBLE"));
+      }
       def_value.set(def_value_buf, sizeof(def_value_buf), system_charset_info);
       if (get_field_default_value(thd, field, &def_value, 1))
       {
@@ -2300,6 +2309,8 @@ int show_create_table(THD *thd, TABLE_LIST *table_list, String *packet,
 
   for (uint i=0 ; i < share->keys ; i++,key_info++)
   {
+    if (key_info->flags & HA_INVISIBLE_KEY)
+      continue;
     KEY_PART_INFO *key_part= key_info->key_part;
     bool found_primary=0;
     packet->append(STRING_WITH_LEN(",\n  "));
@@ -2853,6 +2864,7 @@ void mysqld_list_processes(THD *thd,const char *user, bool verbose)
       {
         thd_info->proc_info= "Busy";
         thd_info->progress= 0.0;
+        thd_info->db= "";
       }
 
       thd_info->state_info= thread_state_info(tmp);
@@ -5835,6 +5847,8 @@ static int get_schema_column_record(THD *thd, TABLE_LIST *tables,
 
   for (; (field= *ptr) ; ptr++)
   {
+    if(field->field_visibility > USER_DEFINED_INVISIBLE)
+      continue;
     uchar *pos;
     char tmp[MAX_FIELD_WIDTH];
     String type(tmp,sizeof(tmp), system_charset_info);
@@ -5892,11 +5906,11 @@ static int get_schema_column_record(THD *thd, TABLE_LIST *tables,
     table->field[16]->store((const char*) pos,
                             strlen((const char*) pos), cs);
 
+    StringBuffer<256> buf;
     if (field->unireg_check == Field::NEXT_NUMBER)
-      table->field[17]->store(STRING_WITH_LEN("auto_increment"), cs);
+        buf.set(STRING_WITH_LEN("auto_increment"),cs);
     if (print_on_update_clause(field, &type, true))
-      table->field[17]->store(type.ptr(), type.length(), cs);
+        buf.set(type.ptr(), type.length(),cs);
-
     if (field->vcol_info)
     {
       String gen_s(tmp,sizeof(tmp), system_charset_info);
@@ -5907,13 +5921,20 @@ static int get_schema_column_record(THD *thd, TABLE_LIST *tables,
       table->field[20]->store(STRING_WITH_LEN("ALWAYS"), cs);
 
       if (field->vcol_info->stored_in_db)
-        table->field[17]->store(STRING_WITH_LEN("STORED GENERATED"), cs);
+        buf.set(STRING_WITH_LEN("STORED GENERATED"), cs);
       else
-        table->field[17]->store(STRING_WITH_LEN("VIRTUAL GENERATED"), cs);
+        buf.set(STRING_WITH_LEN("VIRTUAL GENERATED"), cs);
     }
     else
       table->field[20]->store(STRING_WITH_LEN("NEVER"), cs);
-
+    /*Invisible can coexist with auto_increment and virtual */
+    if (field->field_visibility == USER_DEFINED_INVISIBLE)
+    {
+      if (buf.length())
+        buf.append(STRING_WITH_LEN(", "));
+      buf.append(STRING_WITH_LEN("INVISIBLE"),cs);
+    }
+    table->field[17]->store(buf.ptr(), buf.length(), cs);
     table->field[19]->store(field->comment.str, field->comment.length, cs);
     if (schema_table_store_record(thd, table))
       DBUG_RETURN(1);
@@ -6493,6 +6514,9 @@ static int get_schema_stat_record(THD *thd, TABLE_LIST *tables,
     }
     for (uint i=0 ; i < show_table->s->keys ; i++,key_info++)
     {
+      if ((key_info->flags & HA_INVISIBLE_KEY) &&
+          DBUG_EVALUATE_IF("test_invisible_index", 0, 1))
+        continue;
       KEY_PART_INFO *key_part= key_info->key_part;
       LEX_CSTRING *str;
       LEX_CSTRING unknown= {STRING_WITH_LEN("?unknown field?") };

sql/sql_table.cc

@@ -65,12 +65,16 @@
 
 const char *primary_key_name="PRIMARY";
 
-static bool check_if_keyname_exists(const char *name,KEY *start, KEY *end);
+static int check_if_keyname_exists(const char *name,KEY *start, KEY *end);
 static char *make_unique_key_name(THD *thd, const char *field_name, KEY *start,
                                   KEY *end);
 static void make_unique_constraint_name(THD *thd, LEX_CSTRING *name,
                                         List<Virtual_column_info> *vcol,
                                         uint *nr);
+static const
+char * make_unique_invisible_field_name(THD *thd, const char *field_name,
+                        List<Create_field> *fields);
+
 static int copy_data_between_tables(THD *thd, TABLE *from,TABLE *to,
                                     List<Create_field> &create, bool ignore,
 				    uint order_num, ORDER *order,
@@ -3304,8 +3308,70 @@ bool Column_definition::prepare_stage1_check_typelib_default()
   }
   return false;
 }
+/*
+   This function adds a invisible field to field_list
+   SYNOPSIS
+    mysql_add_invisible_field()
+      thd                      Thread Object
+      field_list               list of all table fields
+      field_name               name/prefix of invisible field
+                               ( Prefix in the case when it is
+                                *COMPLETELY_INVISIBLE*
+                               and given name is duplicate)
+      type_handler             field data type
+      field_visibility
+      default value
+    RETURN VALUE
+      Create_field pointer
+*/
+int mysql_add_invisible_field(THD *thd, List<Create_field> * field_list,
+        const char *field_name, Type_handler *type_handler,
+        field_visible_type field_visibility, Item* default_value)
+{
+  Create_field *fld= new(thd->mem_root)Create_field();
+  const char *new_name= NULL;
+  /* Get unique field name if field_visibility == COMPLETELY_INVISIBLE */
+  if (field_visibility == COMPLETELY_INVISIBLE)
+  {
+    if ((new_name= make_unique_invisible_field_name(thd, field_name,
+                                                     field_list)))
+    {
+      fld->field_name.str= new_name;
+      fld->field_name.length= strlen(new_name);
+    }
+    else
+      return 1;  //Should not happen
+  }
+  else
+  {
+    fld->field_name.str= thd->strmake(field_name, strlen(field_name));
+    fld->field_name.length= strlen(field_name);
+  }
+  fld->set_handler(type_handler);
+  fld->field_visibility= field_visibility;
+  if (default_value)
+  {
+    Virtual_column_info *v= new (thd->mem_root) Virtual_column_info();
+    v->expr= default_value;
+    v->utf8= 0;
+    fld->default_value= v;
+  }
+  field_list->push_front(fld, thd->mem_root);
+  return 0;
+}
 
-
+Key *
+mysql_add_invisible_index(THD *thd, List<Key> *key_list,
+        LEX_CSTRING* field_name, enum Key::Keytype type)
+{
+  Key *key= NULL;
+  key= new (thd->mem_root) Key(type, &null_clex_str, HA_KEY_ALG_UNDEF,
+         false, DDL_options(DDL_options::OPT_NONE));
+  key->columns.push_back(new(thd->mem_root) Key_part_spec(field_name, 0),
+          thd->mem_root);
+  key_list->push_back(key, thd->mem_root);
+  return key;
+}
 /*
   Preparation for table creation
 
@@ -3353,6 +3419,23 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info,
   bool tmp_table= create_table_mode == C_ALTER_TABLE;
   DBUG_ENTER("mysql_prepare_create_table");
 
+  DBUG_EXECUTE_IF("test_pseudo_invisible",{
+          mysql_add_invisible_field(thd, &alter_info->create_list,
+                      "invisible", &type_handler_long, SYSTEM_INVISIBLE,
+                      new (thd->mem_root)Item_int(thd, 9));
+          });
+  DBUG_EXECUTE_IF("test_completely_invisible",{
+          mysql_add_invisible_field(thd, &alter_info->create_list,
+                      "invisible", &type_handler_long, COMPLETELY_INVISIBLE,
+                      new (thd->mem_root)Item_int(thd, 9));
+          });
+  DBUG_EXECUTE_IF("test_invisible_index",{
+          LEX_CSTRING temp;
+          temp.str= "invisible";
+          temp.length= strlen("invisible");
+          mysql_add_invisible_index(thd, &alter_info->key_list
+                  , &temp, Key::MULTIPLE);
+          });
   LEX_CSTRING* connect_string = &create_info->connect_string;
   if (connect_string->length != 0 &&
       connect_string->length > CONNECT_STRING_MAXLEN &&
@@ -3482,7 +3565,6 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info,
   while ((sql_field=it++))
   {
     DBUG_ASSERT(sql_field->charset != 0);
-
     if (sql_field->prepare_stage2(file, file->ha_table_flags()))
       DBUG_RETURN(TRUE);
     if (sql_field->real_field_type() == MYSQL_TYPE_VARCHAR)
@@ -3502,8 +3584,18 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info,
     */
     if (sql_field->stored_in_db())
       record_offset+= sql_field->pack_length;
+    if (sql_field->field_visibility == USER_DEFINED_INVISIBLE &&
+        sql_field->flags & NOT_NULL_FLAG &&
+        sql_field->flags & NO_DEFAULT_VALUE_FLAG)
+    {
+      my_error(ER_INVISIBLE_NOT_NULL_WITHOUT_DEFAULT, MYF(0),
+                          sql_field->field_name.str);
+      DBUG_RETURN(TRUE);
+    }
   }
-  /* Update virtual fields' offset*/
+  /* Update virtual fields' offset and give error if
+     All fields are invisible */
+  bool is_all_invisible= true;
   it.rewind();
   while ((sql_field=it++))
   {
@@ -3512,6 +3604,13 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info,
       sql_field->offset= record_offset;
       record_offset+= sql_field->pack_length;
     }
+    if (sql_field->field_visibility == NOT_INVISIBLE)
+      is_all_invisible= false;
+  }
+  if (is_all_invisible)
+  {
+    my_error(ER_TABLE_MUST_HAVE_COLUMNS, MYF(0));
+    DBUG_RETURN(TRUE);
   }
   if (auto_increment > 1)
   {
@@ -3762,11 +3861,21 @@ mysql_prepare_create_table(THD *thd, HA_CREATE_INFO *create_info,
                             &column->field_name,
                             &sql_field->field_name))
 	field++;
+      /*
+         Either field is not present or field visibility is >
+         USER_DEFINED_INVISIBLE
+      */
       if (!sql_field)
       {
 	my_error(ER_KEY_COLUMN_DOES_NOT_EXITS, MYF(0), column->field_name.str);
 	DBUG_RETURN(TRUE);
       }
+      if (sql_field->field_visibility > USER_DEFINED_INVISIBLE &&
+          !key->invisible && DBUG_EVALUATE_IF("test_invisible_index", 0, 1))
+      {
+        my_error(ER_KEY_COLUMN_DOES_NOT_EXITS, MYF(0), column->field_name.str);
+        DBUG_RETURN(TRUE);
+      }
       while ((dup_column= cols2++) != column)
       {
         if (!lex_string_cmp(system_charset_info,
@@ -5146,17 +5255,36 @@ err:
 
 /*
 ** Give the key name after the first field with an optional '_#' after
+   @returns
+    0        if keyname does not exists
+    [1..)    index + 1 of duplicate key name
 **/
 
-static bool
+static int
 check_if_keyname_exists(const char *name, KEY *start, KEY *end)
 {
-  for (KEY *key=start ; key != end ; key++)
+  uint i= 1;
+  for (KEY *key=start; key != end ; key++, i++)
     if (!my_strcasecmp(system_charset_info, name, key->name.str))
-      return 1;
+      return i;
   return 0;
 }
 
+/**
+ Returns 1 if field name exists otherwise 0
+*/
+static bool
+check_if_field_name_exists(const char *name, List<Create_field> * fields)
+{
+  Create_field *fld;
+  List_iterator<Create_field>it(*fields);
+  while ((fld = it++))
+  {
+    if (!my_strcasecmp(system_charset_info, fld->field_name.str, name))
+      return 1;
+  }
+  return 0;
+}
 
 static char *
 make_unique_key_name(THD *thd, const char *field_name,KEY *start,KEY *end)
@@ -5214,6 +5342,33 @@ static void make_unique_constraint_name(THD *thd, LEX_CSTRING *name,
   }
 }
 
+/**
+  COMPLETELY_INVISIBLE are internally created. They are completely invisible
+  to Alter command (Opposite of SYSTEM_INVISIBLE which throws an
+  error when same name column is added by Alter). So in the case of when
+  user added a same column name as of COMPLETELY_INVISIBLE , we change
+  COMPLETELY_INVISIBLE column name.
+*/
+static const
+char * make_unique_invisible_field_name(THD *thd, const char *field_name,
+                        List<Create_field> *fields)
+{
+  if (!check_if_field_name_exists(field_name, fields))
+    return field_name;
+  char buff[MAX_FIELD_NAME], *buff_end;
+  buff_end= strmake_buf(buff, field_name);
+  if (buff_end - buff < 5)
+    return NULL; // Should not happen
+
+  for (uint i=1 ; i < 10000; i++)
+  {
+    char *real_end= int10_to_str(i, buff_end, 10);
+    if (check_if_field_name_exists(buff, fields))
+      continue;
+    return (const char *)thd->strmake(buff, real_end - buff);
+  }
+  return NULL; //Should not happen
+}
 
 /****************************************************************************
 ** Alter a table definition
@@ -7732,6 +7887,8 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
   bitmap_clear_all(&table->tmp_set);
   for (f_ptr=table->field ; (field= *f_ptr) ; f_ptr++)
   {
+    if (field->field_visibility == COMPLETELY_INVISIBLE)
+        continue;
     Alter_drop *drop;
     if (field->type() == MYSQL_TYPE_VARCHAR)
       create_info->varchar= TRUE;
@@ -7743,7 +7900,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
           !my_strcasecmp(system_charset_info,field->field_name.str, drop->name))
         break;
     }
-    if (drop)
+    if (drop && field->field_visibility < SYSTEM_INVISIBLE)
     {
       /* Reset auto_increment value if it was dropped */
       if (MTYP_TYPENR(field->unireg_check) == Field::NEXT_NUMBER &&
@@ -7768,7 +7925,7 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
                           &def->change))
 	break;
     }
-    if (def)
+    if (def && field->field_visibility < SYSTEM_INVISIBLE)
     {						// Field is changed
       def->field=field;
       /*
@@ -7823,12 +7980,12 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
   def_it.rewind();
   while ((def=def_it++))			// Add new columns
   {
+    Create_field *find;
     if (def->change.str && ! def->field)
     {
       /*
         Check if there is modify for newly added field.
       */
-      Create_field *find;
       find_it.rewind();
       while((find=find_it++))
       {
@@ -7868,7 +8025,6 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
       new_create_list.push_back(def, thd->mem_root);
     else
     {
-      Create_field *find;
       if (def->change.str)
       {
         find_it.rewind();
@@ -7955,9 +8111,11 @@ mysql_prepare_alter_table(THD *thd, TABLE *table,
     Collect all keys which isn't in drop list. Add only those
     for which some fields exists.
   */
- 
+
   for (uint i=0 ; i < table->s->keys ; i++,key_info++)
   {
+    if (key_info->flags & HA_INVISIBLE_KEY)
+      continue;
     const char *key_name= key_info->name.str;
     Alter_drop *drop;
     drop_it.rewind();

sql/sql_yacc.yy

@@ -1160,6 +1160,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  HEX_NUM
 %token  HEX_STRING
 %token  HIGH_PRIORITY
+%token  INVISIBLE_SYM
 %token  HOST_SYM
 %token  HOSTS_SYM
 %token  HOUR_MICROSECOND_SYM
@@ -6648,6 +6649,10 @@ vcol_attribute:
             lex->alter_info.flags|= Alter_info::ALTER_ADD_INDEX;
           }
         | COMMENT_SYM TEXT_STRING_sys { Lex->last_field->comment= $2; }
+        | INVISIBLE_SYM
+          {
+              Lex->last_field->field_visibility= USER_DEFINED_INVISIBLE;
+          }
         ;
 
 parse_vcol_expr:
@@ -7032,7 +7037,7 @@ attribute:
           }
         | AUTO_INC { Lex->last_field->flags|= AUTO_INCREMENT_FLAG | NOT_NULL_FLAG; }
         | SERIAL_SYM DEFAULT VALUE_SYM
-          { 
+          {
             LEX *lex=Lex;
             lex->last_field->flags|= AUTO_INCREMENT_FLAG | NOT_NULL_FLAG | UNIQUE_KEY_FLAG;
             lex->alter_info.flags|= Alter_info::ALTER_ADD_INDEX;
@@ -7068,19 +7073,7 @@ asrow_attribute:
             lex->last_field->flags|= PRI_KEY_FLAG | NOT_NULL_FLAG;
             lex->alter_info.flags|= Alter_info::ALTER_ADD_INDEX;
           }
-        | UNIQUE_SYM
+        | vcol_attribute
-          {
-            LEX *lex=Lex;
-            lex->last_field->flags|= UNIQUE_KEY_FLAG;
-            lex->alter_info.flags|= Alter_info::ALTER_ADD_INDEX;
-          }
-        | UNIQUE_SYM KEY_SYM
-          {
-            LEX *lex=Lex;
-            lex->last_field->flags|= UNIQUE_KEY_FLAG;
-            lex->alter_info.flags|= Alter_info::ALTER_ADD_INDEX;
-          }
-        | COMMENT_SYM TEXT_STRING_sys { Lex->last_field->comment= $2; }
         ;
 
 serial_attribute:
@@ -15534,6 +15527,7 @@ keyword_sp_not_data_type:
         | GOTO_SYM                 {}
         | HASH_SYM                 {}
         | HARD_SYM                 {}
+        | INVISIBLE_SYM            {}
         | HOSTS_SYM                {}
         | HOUR_SYM                 {}
         | ID_SYM                   {}

sql/sql_yacc_ora.yy

@@ -538,6 +538,7 @@ bool my_yyoverflow(short **a, YYSTYPE **b, ulong *yystacksize);
 %token  HEX_NUM
 %token  HEX_STRING
 %token  HIGH_PRIORITY
+%token  INVISIBLE_SYM
 %token  HOST_SYM
 %token  HOSTS_SYM
 %token  HOUR_MICROSECOND_SYM
@@ -6132,6 +6133,10 @@ vcol_attribute:
             lex->alter_info.flags|= Alter_info::ALTER_ADD_INDEX;
           }
         | COMMENT_SYM TEXT_STRING_sys { Lex->last_field->comment= $2; }
+        | INVISIBLE_SYM
+          {
+            Lex->last_field->field_visibility= USER_DEFINED_INVISIBLE;
+          }
         ;
 
 parse_vcol_expr:
@@ -15053,6 +15058,7 @@ keyword_sp_not_data_type:
         | GLOBAL_SYM               {}
         | HASH_SYM                 {}
         | HARD_SYM                 {}
+        | INVISIBLE_SYM            {}
         | HOSTS_SYM                {}
         | HOUR_SYM                 {}
         | ID_SYM                   {}

sql/table.cc

@@ -43,6 +43,7 @@
 #include "rpl_filter.h"
 #include "sql_cte.h"
 #include "ha_sequence.h"
+#include "sql_show.h"
 
 /* For MySQL 5.7 virtual fields */
 #define MYSQL57_GENERATED_FIELD 128
@@ -1175,7 +1176,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
   const uchar *frm_image_end = frm_image + frm_length;
   uchar *record, *null_flags, *null_pos, *mysql57_vcol_null_pos= 0;
   const uchar *disk_buff, *strpos;
-  ulong pos, record_offset; 
+  ulong pos, record_offset;
   ulong rec_buff_length;
   handler *handler_file= 0;
   KEY	*keyinfo;
@@ -1298,12 +1299,6 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
           goto err;
         system_period = extra2;
         break;
-      case EXTRA2_FIELD_FLAGS:
-        if (extra2_field_flags)
-          goto err;
-        extra2_field_flags= extra2;
-        extra2_field_flags_length= length;
-        break;
       case EXTRA2_VTMD:
         if (vtmd_used)
           goto err;
@@ -1315,6 +1310,12 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
         }
         vtmd_used= true;
         break;
+      case EXTRA2_FIELD_FLAGS:
+        if (extra2_field_flags)
+          goto err;
+        extra2_field_flags= extra2;
+        extra2_field_flags_length= length;
+        break;
       default:
         /* abort frm parsing if it's an unknown but important extra2 value */
         if (type >= EXTRA2_ENGINE_IMPORTANT)
@@ -1631,7 +1632,6 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
   memcpy(record, frm_image + record_offset, share->reclength);
 
   disk_buff= frm_image + pos + FRM_FORMINFO_SIZE;
-
   share->fields= uint2korr(forminfo+258);
   if (extra2_field_flags && extra2_field_flags_length != share->fields)
     goto err;
@@ -1645,6 +1645,7 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
   vcol_screen_length= uint2korr(forminfo+286);
   share->virtual_fields= share->default_expressions=
     share->field_check_constraints= share->default_fields= 0;
+  share->visible_fields= 0;
   share->stored_fields= share->fields;
   if (forminfo[46] != (uchar)255)
   {
@@ -2052,9 +2053,15 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
       uchar flags= *extra2_field_flags++;
       if (flags & VERS_OPTIMIZED_UPDATE)
         reg_field->flags|= VERS_UPDATE_UNVERSIONED_FLAG;
-      if (flags & HIDDEN)
+      if (flags & VERS_HIDDEN)
-        reg_field->flags|= HIDDEN_FLAG;
+        reg_field->flags|= VERS_HIDDEN_FLAG;
+
+      reg_field->field_visibility= f_visibility(flags);
     }
+    if (reg_field->field_visibility == USER_DEFINED_INVISIBLE)
+      status_var_increment(thd->status_var.feature_invisible_columns);
+    if (reg_field->field_visibility == NOT_INVISIBLE)
+      share->visible_fields++;
     if (field_type == MYSQL_TYPE_BIT && !f_bit_as_char(pack_flag))
     {
       null_bits_are_used= 1;
@@ -2306,6 +2313,8 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write,
 
         field= key_part->field= share->field[key_part->fieldnr-1];
         key_part->type= field->key_type();
+        if (field->field_visibility > USER_DEFINED_INVISIBLE)
+          keyinfo->flags |= HA_INVISIBLE_KEY;
         if (field->null_ptr)
         {
           key_part->null_offset=(uint) ((uchar*) field->null_ptr -
@@ -5239,7 +5248,6 @@ int TABLE::verify_constraints(bool ignore_failure)
   return(VIEW_CHECK_OK);
 }
 
-
 /*
   Find table in underlying tables by mask and check that only this
   table belong to given mask

sql/table.h

@@ -335,6 +335,17 @@ enum enum_vcol_update_mode
   VCOL_UPDATE_FOR_REPLACE
 };
 
+/* Field visibility enums */
+
+enum field_visible_type{
+  NOT_INVISIBLE= 0,
+  USER_DEFINED_INVISIBLE,
+  /* automatically added by the server. Can be queried explicitly
+  in SELECT, otherwise invisible from anything" */
+  SYSTEM_INVISIBLE,
+  COMPLETELY_INVISIBLE,
+  MAX_BITS_INVISIBLE= 3
+};
 
 /**
   Category of table found in the table share.
@@ -673,6 +684,7 @@ struct TABLE_SHARE
   uint blob_fields;                     /* number of blob fields */
   uint varchar_fields;                  /* number of varchar fields */
   uint default_fields;                  /* number of default fields */
+  uint visible_fields;                  /* number of visible fields */
 
   uint default_expressions;
   uint table_check_constraints, field_check_constraints;
@@ -1559,8 +1571,8 @@ public:
   uint vers_user_fields() const
   {
     return s->versioned ?
-      s->fields - VERSIONING_FIELDS :
+      s->visible_fields - VERSIONING_FIELDS :
-      s->fields;
+      s->visible_fields;
   }
 };
 

sql/unireg.cc

@@ -87,10 +87,31 @@ static uchar *extra2_write(uchar *pos, enum extra2_frm_value_type type,
   return extra2_write(pos, type, reinterpret_cast<LEX_CSTRING *>(str));
 }
 
+static uchar *extra2_write_field_properties(uchar *pos,
+                   List<Create_field> &create_fields)
+{
+  List_iterator<Create_field> it(create_fields);
+  *pos++= EXTRA2_FIELD_FLAGS;
+  /*
+   always first 2  for field visibility
+  */
+  pos= extra2_write_len(pos, create_fields.elements);
+  while (Create_field *cf= it++)
+  {
+    uchar flags= cf->field_visibility;
+    if (cf->flags & VERS_UPDATE_UNVERSIONED_FLAG)
+      flags|= VERS_OPTIMIZED_UPDATE;
+    if (cf->flags & VERS_HIDDEN_FLAG)
+      flags|= VERS_HIDDEN;
+    *pos++= flags;
+  }
+  return pos;
+}
+
 static const bool ROW_START = true;
 static const bool ROW_END = false;
 
-inline
+static inline
 uint16
 vers_get_field(HA_CREATE_INFO *create_info, List<Create_field> &create_fields, bool row_start)
 {
@@ -116,12 +137,15 @@ vers_get_field(HA_CREATE_INFO *create_info, List<Create_field> &create_fields, b
   return 0;
 }
 
+static inline
 bool has_extra2_field_flags(List<Create_field> &create_fields)
 {
   List_iterator<Create_field> it(create_fields);
   while (Create_field *f= it++)
   {
-    if (f->flags & (VERS_UPDATE_UNVERSIONED_FLAG | HIDDEN_FLAG))
+    if (f->field_visibility != NOT_INVISIBLE)
+      return true;
+    if (f->flags & (VERS_UPDATE_UNVERSIONED_FLAG | VERS_HIDDEN_FLAG))
       return true;
   }
   return false;
@@ -272,8 +296,8 @@ LEX_CUSTRING build_frm_image(THD *thd, const char *table,
   bool has_extra2_field_flags_= has_extra2_field_flags(create_fields);
   if (has_extra2_field_flags_)
   {
-    extra2_size+=
+    extra2_size+= 1 + (create_fields.elements > 255 ? 3 : 1) +
-        1 + (create_fields.elements <= 255 ? 1 : 3) + create_fields.elements;
+        create_fields.elements;
   }
 
   key_buff_length= uint4korr(fileinfo+47);
@@ -349,20 +373,7 @@ LEX_CUSTRING build_frm_image(THD *thd, const char *table,
   }
 
   if (has_extra2_field_flags_)
-  {
+    pos= extra2_write_field_properties(pos, create_fields);
-    *pos++= EXTRA2_FIELD_FLAGS;
-    pos= extra2_write_len(pos, create_fields.elements);
-    List_iterator<Create_field> it(create_fields);
-    while (Create_field *field= it++)
-    {
-      uchar flags= 0;
-      if (field->flags & VERS_UPDATE_UNVERSIONED_FLAG)
-        flags|= VERS_OPTIMIZED_UPDATE;
-      if (field->flags & HIDDEN_FLAG)
-        flags|= HIDDEN;
-      *pos++= flags;
-    }
-  }
 
   int4store(pos, filepos); // end of the extra2 segment
   pos+= 4;

sql/unireg.h

@@ -173,17 +173,17 @@ enum extra2_frm_value_type {
   EXTRA2_DEFAULT_PART_ENGINE=1,
   EXTRA2_GIS=2,
   EXTRA2_PERIOD_FOR_SYSTEM_TIME=4,
-  EXTRA2_FIELD_FLAGS=8,
+  EXTRA2_VTMD=8,
-  EXTRA2_VTMD=16,
 
 #define EXTRA2_ENGINE_IMPORTANT 128
 
   EXTRA2_ENGINE_TABLEOPTS=128,
+  EXTRA2_FIELD_FLAGS=129
 };
 
 enum extra2_field_flags {
-  VERS_OPTIMIZED_UPDATE=1,
+  VERS_OPTIMIZED_UPDATE= 1 << MAX_BITS_INVISIBLE,
-  HIDDEN=2,
+  VERS_HIDDEN= 1 << (MAX_BITS_INVISIBLE + 1),
 };
 
 int rea_create_table(THD *thd, LEX_CUSTRING *frm,

storage/innobase/ha/ha0ha.cc

@@ -489,62 +489,4 @@ ha_validate(
 	return(ok);
 }
 #endif /* defined UNIV_AHI_DEBUG || defined UNIV_DEBUG */
-
-/*************************************************************//**
-Prints info of a hash table. */
-void
-ha_print_info(
-/*==========*/
-	FILE*		file,	/*!< in: file where to print */
-	hash_table_t*	table)	/*!< in: hash table */
-{
-#ifdef UNIV_DEBUG
-/* Some of the code here is disabled for performance reasons in production
-builds, see http://bugs.mysql.com/36941 */
-#define PRINT_USED_CELLS
-#endif /* UNIV_DEBUG */
-
-#ifdef PRINT_USED_CELLS
-	hash_cell_t*	cell;
-	ulint		cells	= 0;
-	ulint		i;
-#endif /* PRINT_USED_CELLS */
-	ulint		n_bufs;
-
-	ut_ad(table);
-	ut_ad(table->magic_n == HASH_TABLE_MAGIC_N);
-#ifdef PRINT_USED_CELLS
-	for (i = 0; i < hash_get_n_cells(table); i++) {
-
-		cell = hash_get_nth_cell(table, i);
-
-		if (cell->node) {
-
-			cells++;
-		}
-	}
-#endif /* PRINT_USED_CELLS */
-
-	fprintf(file, "Hash table size %lu",
-		(ulong) hash_get_n_cells(table));
-
-#ifdef PRINT_USED_CELLS
-	fprintf(file, ", used cells %lu", (ulong) cells);
-#endif /* PRINT_USED_CELLS */
-
-	if (table->heaps == NULL && table->heap != NULL) {
-
-		/* This calculation is intended for the adaptive hash
-		index: how many buffer frames we have reserved? */
-
-		n_bufs = UT_LIST_GET_LEN(table->heap->base) - 1;
-
-		if (table->heap->free_block) {
-			n_bufs++;
-		}
-
-		fprintf(file, ", node heap has %lu buffer(s)\n",
-			(ulong) n_bufs);
-	}
-}
 #endif /* BTR_CUR_HASH_ADAPT */

storage/innobase/handler/ha_innodb.cc

@@ -3783,8 +3783,7 @@ innobase_init(
 	/* Currently, Galera does not support VATS lock schedule algorithm. */
 	if (innodb_lock_schedule_algorithm == INNODB_LOCK_SCHEDULE_ALGORITHM_VATS
 	    && global_system_variables.wsrep_on) {
-		ib::info() << "In Galera environment Variance-Aware-Transaction-Sheduling Algorithm"
+		ib::info() << "For Galera, using innodb_lock_schedule_algorithm=fcfs";
-			   " is not supported. Falling back to First-Come-First-Served order. ";
 		innodb_lock_schedule_algorithm = INNODB_LOCK_SCHEDULE_ALGORITHM_FCFS;
 	}
 #endif /* WITH_WSREP */

storage/innobase/include/ha0ha.h

@@ -198,13 +198,6 @@ ha_validate(
 	ulint		start_index,	/*!< in: start index */
 	ulint		end_index);	/*!< in: end index */
 #endif /* defined UNIV_AHI_DEBUG || defined UNIV_DEBUG */
-/*************************************************************//**
-Prints info of a hash table. */
-void
-ha_print_info(
-/*==========*/
-	FILE*		file,	/*!< in: file where to print */
-	hash_table_t*	table);	/*!< in: hash table */
 
 /** The hash table external chain node */
 struct ha_node_t {

storage/innobase/include/que0que.h

@@ -380,9 +380,6 @@ struct que_thr_t{
 	UT_LIST_NODE_T(que_thr_t)
 			thrs;		/*!< list of thread nodes of the fork
 					node */
-	UT_LIST_NODE_T(que_thr_t)
-			trx_thrs;	/*!< lists of threads in wait list of
-					the trx */
 	UT_LIST_NODE_T(que_thr_t)
 			queue;		/*!< list of runnable thread nodes in
 					the server task queue */

storage/innobase/include/trx0roll.h

@@ -33,7 +33,8 @@ Created 3/26/1996 Heikki Tuuri
 #include "mtr0mtr.h"
 #include "trx0sys.h"
 
-extern bool	trx_rollback_or_clean_is_active;
+extern bool		trx_rollback_or_clean_is_active;
+extern const trx_t*	trx_roll_crash_recv_trx;
 
 /*******************************************************************//**
 Determines if this transaction is rolling back an incomplete transaction
@@ -62,6 +63,10 @@ trx_undo_rec_t*
 trx_roll_pop_top_rec_of_trx(trx_t* trx, roll_ptr_t* roll_ptr, mem_heap_t* heap)
 	MY_ATTRIBUTE((nonnull, warn_unused_result));
 
+/** Report progress when rolling back a row of a recovered transaction.
+@return	whether the rollback should be aborted due to pending shutdown */
+bool
+trx_roll_must_shutdown();
 /*******************************************************************//**
 Rollback or clean up any incomplete transactions which were
 encountered in crash recovery.  If the transaction already was

storage/innobase/row/row0quiesce.cc

@@ -520,7 +520,7 @@ row_quiesce_table_start(
 	ut_ad(fil_space_get(table->space) != NULL);
 	ib::info() << "Sync to disk of " << table->name << " started.";
 
-	if (trx_purge_state() != PURGE_STATE_DISABLED) {
+	if (srv_undo_sources) {
 		trx_purge_stop();
 	}
 
@@ -603,7 +603,7 @@ row_quiesce_table_complete(
 		ib::info() << "Deleting the meta-data file '" << cfg_name << "'";
 	}
 
-	if (trx_purge_state() != PURGE_STATE_DISABLED) {
+	if (srv_undo_sources) {
 		trx_purge_run();
 	}
 

storage/innobase/row/row0umod.cc

@@ -1192,7 +1192,16 @@ close_table:
 	if (!row_undo_search_clust_to_pcur(node)) {
 		/* As long as this rolling-back transaction exists,
 		the PRIMARY KEY value pointed to by the undo log
-		record must exist. But, it is possible that the record
+		record should exist.
+
+		However, if InnoDB is killed during a rollback, or
+		shut down during the rollback of recovered
+		transactions, then after restart we may try to roll
+		back some of the same undo log records again, because
+		trx_roll_try_truncate() is not being invoked after
+		every undo log record.
+
+		It is also possible that the record
 		was not modified yet (the DB_ROLL_PTR does not match
 		node->roll_ptr) and thus there is nothing to roll back.
 
@@ -1200,8 +1209,11 @@ close_table:
 		record after successfully acquiring an exclusive lock
 		on the the clustered index record. That lock will not
 		be released before the transaction is committed or
-		fully rolled back. */
+		fully rolled back. (Exception: if the server was
-		ut_ad(node->pcur.btr_cur.low_match == node->ref->n_fields);
+		killed, restarted, and shut down again before the
+		rollback of the recovered transaction was completed,
+		it is possible that the transaction was partially
+		rolled back and locks released.) */
 		goto close_table;
 	}
 

storage/innobase/row/row0undo.cc

@@ -343,6 +343,13 @@ row_undo_step(
 
 	ut_ad(que_node_get_type(node) == QUE_NODE_UNDO);
 
+	if (UNIV_UNLIKELY(trx == trx_roll_crash_recv_trx)
+	    && trx_roll_must_shutdown()) {
+		/* Shutdown has been initiated. */
+		trx->error_state = DB_INTERRUPTED;
+		return(NULL);
+	}
+
 	err = row_undo(node, thr);
 
 	trx->error_state = err;

storage/innobase/srv/srv0srv.cc

@@ -1301,9 +1301,28 @@ srv_printf_innodb_monitor(
 
 #ifdef BTR_CUR_HASH_ADAPT
 	for (ulint i = 0; i < btr_ahi_parts; ++i) {
-		rw_lock_s_lock(btr_search_latches[i]);
+		const hash_table_t* table = btr_search_sys->hash_tables[i];
-		ha_print_info(file, btr_search_sys->hash_tables[i]);
+
-		rw_lock_s_unlock(btr_search_latches[i]);
+		ut_ad(table->magic_n == HASH_TABLE_MAGIC_N);
+		/* this is only used for buf_pool->page_hash */
+		ut_ad(!table->heaps);
+		/* this is used for the adaptive hash index */
+		ut_ad(table->heap);
+
+		const mem_heap_t* heap = table->heap;
+		/* The heap may change during the following call,
+		so the data displayed may be garbage. We intentionally
+		avoid acquiring btr_search_latches[] so that the
+		diagnostic output will not stop here even in case another
+		thread hangs while holding btr_search_latches[].
+
+		This should be safe from crashes, because
+		table->heap will be pointing to the same object
+		for the full lifetime of the server. Even during
+		btr_search_disable() the heap will stay valid. */
+		fprintf(file, "Hash table size " ULINTPF
+			", node heap has " ULINTPF " buffer(s)\n",
+			table->n_cells, heap->base.count - !heap->free_block);
 	}
 
 	fprintf(file,
@@ -2831,6 +2850,9 @@ DECLARE_THREAD(srv_purge_coordinator_thread)(
 
 	purge_sys->running = false;
 
+	/* Ensure that the wait in trx_purge_stop() will terminate. */
+	os_event_set(purge_sys->event);
+
 	rw_lock_x_unlock(&purge_sys->latch);
 
 #ifdef UNIV_DEBUG_THREAD_CREATION

storage/innobase/srv/srv0start.cc

@@ -2618,8 +2618,6 @@ files_checked:
 		srv_start_state_set(SRV_START_STATE_MASTER);
 	}
 
-	srv_is_being_started = false;
-
 	if (!srv_read_only_mode && srv_operation == SRV_OPERATION_NORMAL
 	    && srv_force_recovery < SRV_FORCE_NO_BACKGROUND) {
 
@@ -2647,6 +2645,8 @@ files_checked:
 		purge_sys->state = PURGE_STATE_DISABLED;
 	}
 
+	srv_is_being_started = false;
+
 	if (!srv_read_only_mode) {
 		/* wake main loop of page cleaner up */
 		os_event_set(buf_flush_event);

storage/innobase/trx/trx0purge.cc

@@ -1724,52 +1724,48 @@ void
 trx_purge_stop(void)
 /*================*/
 {
-	ut_a(srv_n_purge_threads > 0);
-
 	rw_lock_x_lock(&purge_sys->latch);
 
-	const int64_t		sig_count = os_event_reset(purge_sys->event);
+	switch (purge_sys->state) {
-	const purge_state_t	state = purge_sys->state;
+	case PURGE_STATE_INIT:
-
+	case PURGE_STATE_DISABLED:
-	ut_a(state == PURGE_STATE_RUN || state == PURGE_STATE_STOP);
+		ut_error;
-
+	case PURGE_STATE_EXIT:
-	++purge_sys->n_stop;
+		/* Shutdown must have been initiated during
-
+		FLUSH TABLES FOR EXPORT. */
-	if (state == PURGE_STATE_RUN) {
+		ut_ad(!srv_undo_sources);
+unlock:
+		rw_lock_x_unlock(&purge_sys->latch);
+		break;
+	case PURGE_STATE_STOP:
+		ut_ad(srv_n_purge_threads > 0);
+		++purge_sys->n_stop;
+		purge_sys->state = PURGE_STATE_STOP;
+		if (!purge_sys->running) {
+			goto unlock;
+		}
+		ib::info() << "Waiting for purge to stop";
+		do {
+			rw_lock_x_unlock(&purge_sys->latch);
+			os_thread_sleep(10000);
+			rw_lock_x_lock(&purge_sys->latch);
+		} while (purge_sys->running);
+		goto unlock;
+	case PURGE_STATE_RUN:
+		ut_ad(srv_n_purge_threads > 0);
+		++purge_sys->n_stop;
 		ib::info() << "Stopping purge";
 
 		/* We need to wakeup the purge thread in case it is suspended,
 		so that it can acknowledge the state change. */
 
+		const int64_t sig_count = os_event_reset(purge_sys->event);
+		purge_sys->state = PURGE_STATE_STOP;
 		srv_purge_wakeup();
-	}
-
-	purge_sys->state = PURGE_STATE_STOP;
-
-	if (state != PURGE_STATE_STOP) {
 		rw_lock_x_unlock(&purge_sys->latch);
 		/* Wait for purge coordinator to signal that it
 		is suspended. */
 		os_event_wait_low(purge_sys->event, sig_count);
-	} else {
-		bool	once = true;
-
-		/* Wait for purge to signal that it has actually stopped. */
-		while (purge_sys->running) {
-
-			if (once) {
-				ib::info() << "Waiting for purge to stop";
-				once = false;
-			}
-
-			rw_lock_x_unlock(&purge_sys->latch);
-
-			os_thread_sleep(10000);
-
-			rw_lock_x_lock(&purge_sys->latch);
-		}
-
-		rw_lock_x_unlock(&purge_sys->latch);
 	}
 
 	MONITOR_INC_VALUE(MONITOR_PURGE_STOP_COUNT, 1);
@@ -1784,8 +1780,12 @@ trx_purge_run(void)
 	rw_lock_x_lock(&purge_sys->latch);
 
 	switch (purge_sys->state) {
-	case PURGE_STATE_INIT:
 	case PURGE_STATE_EXIT:
+		/* Shutdown must have been initiated during
+		FLUSH TABLES FOR EXPORT. */
+		ut_ad(!srv_undo_sources);
+		break;
+	case PURGE_STATE_INIT:
 	case PURGE_STATE_DISABLED:
 		ut_error;
 

storage/innobase/trx/trx0roll.cc

@@ -24,8 +24,10 @@ Transaction rollback
 Created 3/26/1996 Heikki Tuuri
 *******************************************************/
 
-#include "ha_prototypes.h"
+#include "my_config.h"
+#include <my_systemd.h>
 
+#include "ha_prototypes.h"
 #include "trx0roll.h"
 
 #include <mysql/service_wsrep.h>
@@ -56,14 +58,7 @@ static const ulint TRX_ROLL_TRUNC_THRESHOLD = 1;
 bool			trx_rollback_or_clean_is_active;
 
 /** In crash recovery, the current trx to be rolled back; NULL otherwise */
-static const trx_t*	trx_roll_crash_recv_trx	= NULL;
+const trx_t*		trx_roll_crash_recv_trx;
-
-/** In crash recovery we set this to the undo n:o of the current trx to be
-rolled back. Then we can print how many % the rollback has progressed. */
-static undo_no_t	trx_roll_max_undo_no;
-
-/** Auxiliary variable which tells the previous progress % we printed */
-static ulint		trx_roll_progress_printed_pct;
 
 /****************************************************************//**
 Finishes a transaction rollback. */
@@ -643,8 +638,6 @@ trx_rollback_active(
 	que_thr_t*	thr;
 	roll_node_t*	roll_node;
 	dict_table_t*	table;
-	int64_t		rows_to_undo;
-	const char*	unit		= "";
 	ibool		dictionary_locked = FALSE;
 
 	heap = mem_heap_create(512);
@@ -663,28 +656,8 @@ trx_rollback_active(
 
 	ut_a(thr == que_fork_start_command(fork));
 
-	trx_sys_mutex_enter();
-
 	trx_roll_crash_recv_trx	= trx;
 
-	trx_roll_max_undo_no = trx->undo_no;
-
-	trx_roll_progress_printed_pct = 0;
-
-	rows_to_undo = trx_roll_max_undo_no;
-
-	trx_sys_mutex_exit();
-
-	if (rows_to_undo > 1000000000) {
-		rows_to_undo = rows_to_undo / 1000000;
-		unit = "M";
-	}
-
-	const trx_id_t	trx_id = trx_get_id_for_print(trx);
-
-	ib::info() << "Rolling back trx with id " << trx_id << ", "
-		<< rows_to_undo << unit << " rows to undo";
-
 	if (trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) {
 		row_mysql_lock_data_dictionary(trx);
 		dictionary_locked = TRUE;
@@ -695,6 +668,17 @@ trx_rollback_active(
 
 	que_run_threads(roll_node->undo_thr);
 
+	if (trx->error_state != DB_SUCCESS) {
+		ut_ad(trx->error_state == DB_INTERRUPTED);
+		ut_ad(!srv_is_being_started);
+		ut_ad(!srv_undo_sources);
+		ut_ad(srv_fast_shutdown);
+		ut_ad(!dictionary_locked);
+		que_graph_free(static_cast<que_t*>(
+				       roll_node->undo_thr->common.parent));
+		goto func_exit;
+	}
+
 	trx_rollback_finish(thr_get_trx(roll_node->undo_thr));
 
 	/* Free the memory reserved by the undo graph */
@@ -726,11 +710,13 @@ trx_rollback_active(
 		}
 	}
 
+func_exit:
 	if (dictionary_locked) {
 		row_mysql_unlock_data_dictionary(trx);
 	}
 
-	ib::info() << "Rollback of trx with id " << trx_id << " completed";
+	ib::info() << "Rollback of trx with id " << ib::hex(trx->id)
+		<< " completed";
 
 	mem_heap_free(heap);
 
@@ -748,7 +734,7 @@ ibool
 trx_rollback_resurrected(
 /*=====================*/
 	trx_t*	trx,	/*!< in: transaction to rollback or clean */
-	ibool	all)	/*!< in: FALSE=roll back dictionary transactions;
+	ibool*	all)	/*!< in/out: FALSE=roll back dictionary transactions;
 			TRUE=roll back all non-PREPARED transactions */
 {
 	ut_ad(trx_sys_mutex_own());
@@ -759,40 +745,102 @@ trx_rollback_resurrected(
 	to accidentally clean up a non-recovered transaction here. */
 
 	trx_mutex_enter(trx);
-	bool		is_recovered	= trx->is_recovered;
+	if (!trx->is_recovered) {
-	trx_state_t	state		= trx->state;
+func_exit:
-	trx_mutex_exit(trx);
+		trx_mutex_exit(trx);
-
-	if (!is_recovered) {
 		return(FALSE);
 	}
 
-	switch (state) {
+	switch (trx->state) {
 	case TRX_STATE_COMMITTED_IN_MEMORY:
+		trx_mutex_exit(trx);
 		trx_sys_mutex_exit();
-		ib::info() << "Cleaning up trx with id "
+		ib::info() << "Cleaning up trx with id " << ib::hex(trx->id);
-			<< trx_get_id_for_print(trx);
 
 		trx_cleanup_at_db_startup(trx);
 		trx_free_resurrected(trx);
 		return(TRUE);
 	case TRX_STATE_ACTIVE:
-		if (all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) {
+		if (!srv_is_being_started
+		    && !srv_undo_sources && srv_fast_shutdown) {
+fake_prepared:
+			trx->state = TRX_STATE_PREPARED;
+			trx_sys->n_prepared_trx++;
+			trx_sys->n_prepared_recovered_trx++;
+			*all = FALSE;
+			goto func_exit;
+		}
+		trx_mutex_exit(trx);
+
+		if (*all || trx_get_dict_operation(trx) != TRX_DICT_OP_NONE) {
 			trx_sys_mutex_exit();
 			trx_rollback_active(trx);
+			if (trx->error_state != DB_SUCCESS) {
+				ut_ad(trx->error_state == DB_INTERRUPTED);
+				trx->error_state = DB_SUCCESS;
+				ut_ad(!srv_undo_sources);
+				ut_ad(srv_fast_shutdown);
+				mutex_enter(&trx_sys->mutex);
+				trx_mutex_enter(trx);
+				goto fake_prepared;
+			}
 			trx_free_for_background(trx);
 			return(TRUE);
 		}
 		return(FALSE);
 	case TRX_STATE_PREPARED:
-		return(FALSE);
+		goto func_exit;
 	case TRX_STATE_NOT_STARTED:
 	case TRX_STATE_FORCED_ROLLBACK:
 		break;
 	}
 
 	ut_error;
-	return(FALSE);
+	goto func_exit;
+}
+
+/** Report progress when rolling back a row of a recovered transaction.
+@return	whether the rollback should be aborted due to pending shutdown */
+bool
+trx_roll_must_shutdown()
+{
+	const trx_t* trx = trx_roll_crash_recv_trx;
+	ut_ad(trx);
+	ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE));
+	ut_ad(trx->in_rollback);
+
+	if (trx_get_dict_operation(trx) == TRX_DICT_OP_NONE
+	    && !srv_is_being_started
+	    && !srv_undo_sources && srv_fast_shutdown) {
+		return true;
+	}
+
+	ib_time_t time = ut_time();
+	mutex_enter(&trx_sys->mutex);
+	mutex_enter(&recv_sys->mutex);
+
+	if (recv_sys->report(time)) {
+		ulint n_trx = 0, n_rows = 0;
+		for (const trx_t* t = UT_LIST_GET_FIRST(trx_sys->rw_trx_list);
+		     t != NULL;
+		     t = UT_LIST_GET_NEXT(trx_list, t)) {
+
+			assert_trx_in_rw_list(t);
+			if (t->is_recovered
+			    && trx_state_eq(t, TRX_STATE_ACTIVE)) {
+				n_trx++;
+				n_rows += t->undo_no;
+			}
+		}
+		ib::info() << "To roll back: " << n_trx << " transactions, "
+			   << n_rows << " rows";
+		sd_notifyf(0, "STATUS=To roll back: " ULINTPF " transactions, "
+			   ULINTPF " rows", n_trx, n_rows);
+	}
+
+	mutex_exit(&recv_sys->mutex);
+	mutex_exit(&trx_sys->mutex);
+	return false;
 }
 
 /*******************************************************************//**
@@ -837,17 +885,11 @@ trx_rollback_or_clean_recovered(
 
 			assert_trx_in_rw_list(trx);
 
-			if (srv_shutdown_state != SRV_SHUTDOWN_NONE
-			    && srv_fast_shutdown != 0) {
-				all = FALSE;
-				break;
-			}
-
 			/* If this function does a cleanup or rollback
 			then it will release the trx_sys->mutex, therefore
 			we need to reacquire it before retrying the loop. */
 
-			if (trx_rollback_resurrected(trx, all)) {
+			if (trx_rollback_resurrected(trx, &all)) {
 
 				trx_sys_mutex_enter();
 
@@ -1057,27 +1099,6 @@ trx_roll_pop_top_rec_of_trx(trx_t* trx, roll_ptr_t* roll_ptr, mem_heap_t* heap)
 	ut_ad(trx_roll_check_undo_rec_ordering(
 		undo_no, undo->rseg->space, trx));
 
-	/* We print rollback progress info if we are in a crash recovery
-	and the transaction has at least 1000 row operations to undo. */
-
-	if (trx == trx_roll_crash_recv_trx && trx_roll_max_undo_no > 1000) {
-
-		ulint	progress_pct = 100 - (ulint)
-			((undo_no * 100) / trx_roll_max_undo_no);
-		if (progress_pct != trx_roll_progress_printed_pct) {
-			if (trx_roll_progress_printed_pct == 0) {
-				fprintf(stderr,
-					"\nInnoDB: Progress in percents:"
-					" %lu", (ulong) progress_pct);
-			} else {
-				fprintf(stderr,
-					" %lu", (ulong) progress_pct);
-			}
-			fflush(stderr);
-			trx_roll_progress_printed_pct = progress_pct;
-		}
-	}
-
 	trx->undo_no = undo_no;
 	trx->undo_rseg_space = undo->rseg->space;
 	mutex_exit(&trx->undo_mutex);

storage/innobase/trx/trx0undo.cc

@@ -1602,10 +1602,14 @@ trx_undo_free_prepared(
 			/* fall through */
 		case TRX_UNDO_ACTIVE:
 			/* lock_trx_release_locks() assigns
-			trx->is_recovered=false */
+			trx->is_recovered=false and
+			trx->state = TRX_STATE_COMMITTED_IN_MEMORY,
+			also for transactions that we faked
+			to TRX_STATE_PREPARED in trx_rollback_resurrected(). */
 			ut_a(!srv_was_started
 			     || srv_read_only_mode
-			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO);
+			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO
+			     || srv_fast_shutdown);
 			break;
 		default:
 			ut_error;
@@ -1628,10 +1632,14 @@ trx_undo_free_prepared(
 			/* fall through */
 		case TRX_UNDO_ACTIVE:
 			/* lock_trx_release_locks() assigns
-			trx->is_recovered=false */
+			trx->is_recovered=false and
+			trx->state = TRX_STATE_COMMITTED_IN_MEMORY,
+			also for transactions that we faked
+			to TRX_STATE_PREPARED in trx_rollback_resurrected(). */
 			ut_a(!srv_was_started
 			     || srv_read_only_mode
-			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO);
+			     || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO
+			     || srv_fast_shutdown);
 			break;
 		default:
 			ut_error;

storage/maria/ma_loghandler.c

@@ -6342,7 +6342,6 @@ my_bool translog_write_record(LSN *lsn,
                                       short_trid, &parts, trn, hook_arg);
       break;
     case LOGRECTYPE_NOT_ALLOWED:
-      DBUG_ASSERT(0);
     default:
       DBUG_ASSERT(0);
       rc= 1;

storage/rocksdb/mysql-test/storage_engine/disabled.def

@@ -14,6 +14,7 @@ lock_concurrent      : MDEV-13148 - LOCK TABLE on RocksDB table fails with a bog
 optimize_table       : MDEV-13148 - LOCK TABLE on RocksDB table fails with a bogus error message
 repair_table         : MDEV-13148 - LOCK TABLE on RocksDB table fails with a bogus error message
 select_high_prio     : Not supported
+show_engine          : SHOW ENGINE produces different number of lines depending on previous tests
 show_table_status    : MDEV-13152 - Indeterministic row number in SHOW TABLE STATUS on RocksDB table
 tbl_opt_data_dir     : Not supported
 tbl_opt_index_dir    : Not supported

storage/rocksdb/mysql-test/storage_engine/type_bit_indexes.rdiff

@@ -0,0 +1,20 @@
+--- suite/storage_engine/type_bit_indexes.result	2017-12-12 20:34:34.000000000 +0200
++++ suite/storage_engine/type_bit_indexes.reject	2017-12-12 20:35:24.539330056 +0200
+@@ -69,7 +69,7 @@
+ (1,0xFFFF,0xFFFFFFFF,0xFFFFFFFFFFFFFFFF);
+ EXPLAIN SELECT HEX(b+c) FROM t1 WHERE c > 1 OR HEX(b) < 0xFFFFFF;
+ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+-#	#	#	#	#	b_c	#	#	#	#
++#	#	#	#	#	NULL	#	#	#	#
+ SELECT HEX(b+c) FROM t1 WHERE c > 1 OR HEX(b) < 0xFFFFFF;
+ HEX(b+c)
+ 10
+@@ -98,7 +98,7 @@
+ (1,0xFFFF,0xFFFFFFFF,0xFFFFFFFFFFFFFFFF);
+ EXPLAIN SELECT DISTINCT a+0 FROM t1 ORDER BY a;
+ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+-#	#	#	#	#	a	#	#	#	#
++#	#	#	#	#	NULL	#	#	#	#
+ SELECT DISTINCT a+0 FROM t1 ORDER BY a;
+ a+0
+ 0

storage/rocksdb/mysql-test/storage_engine/type_enum_indexes.rdiff

@@ -0,0 +1,11 @@
+--- suite/storage_engine/type_enum_indexes.result	2017-03-12 04:38:50.000000000 +0200
++++ suite/storage_engine/type_enum_indexes.reject	2017-12-12 20:36:47.455331726 +0200
+@@ -30,7 +30,7 @@
+ t1	0	a_b	2	b	#	#	NULL	NULL	#	#		
+ EXPLAIN SELECT a FROM t1 WHERE b > 'test2' ORDER BY a;
+ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+-#	#	#	#	#	a_b	#	#	#	#
++#	#	#	#	#	NULL	#	#	#	#
+ SELECT a FROM t1 WHERE b > 'test2' ORDER BY a;
+ a
+ Africa

storage/rocksdb/mysql-test/storage_engine/type_set_indexes.rdiff

@@ -0,0 +1,20 @@
+--- suite/storage_engine/type_set_indexes.result	2017-03-12 04:38:50.000000000 +0200
++++ suite/storage_engine/type_set_indexes.reject	2017-12-12 20:37:16.187332305 +0200
+@@ -97,7 +97,7 @@
+ Warning	1265	Data truncated for column 'b' at row 7
+ EXPLAIN SELECT a FROM t1 WHERE FIND_IN_SET('Europe',a) > 0;
+ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+-#	#	#	#	#	a	#	#	#	#
++#	#	#	#	#	NULL	#	#	#	#
+ SELECT a FROM t1 WHERE FIND_IN_SET('Europe',a) > 0;
+ a
+ Africa,Europe,Asia
+@@ -124,7 +124,7 @@
+ Warning	1265	Data truncated for column 'b' at row 7
+ EXPLAIN SELECT DISTINCT a, b FROM t1 ORDER BY b DESC, a;
+ id	select_type	table	type	possible_keys	key	key_len	ref	rows	Extra
+-#	#	#	#	#	b_a	#	#	#	#
++#	#	#	#	#	NULL	#	#	#	#
+ SELECT DISTINCT a, b FROM t1 ORDER BY b DESC, a;
+ a	b
+ 	test1,test3

storage/tokudb/mysql-test/rpl/r/rpl_tokudb_row_log.result

@@ -226,7 +226,6 @@ master-bin.000001	#	Xid	#	#	COMMIT /* XID */
 master-bin.000001	#	Rotate	#	#	master-bin.000002;pos=POS
 include/show_binlog_events.inc
 Log_name	Pos	Event_type	Server_id	End_log_pos	Info
-master-bin.000002	#	Binlog_checkpoint	#	#	master-bin.000002
 master-bin.000002	#	Gtid	#	#	GTID #-#-#
 master-bin.000002	#	Query	#	#	use `test`; create table t3 (a int)ENGINE=TokuDB
 master-bin.000002	#	Gtid	#	#	GTID #-#-#
@@ -268,7 +267,6 @@ slave-bin.000001	#	Query	#	#	use `test`; create table t3 (a int)ENGINE=TokuDB
 slave-bin.000001	#	Rotate	#	#	slave-bin.000002;pos=POS
 include/show_binlog_events.inc
 Log_name	Pos	Event_type	Server_id	End_log_pos	Info
-slave-bin.000002	#	Binlog_checkpoint	#	#	slave-bin.000002
 slave-bin.000002	#	Gtid	#	#	GTID #-#-#
 slave-bin.000002	#	Query	#	#	use `test`; create table t2 (n int)ENGINE=TokuDB
 slave-bin.000002	#	Gtid	#	#	BEGIN GTID #-#-#

storage/tokudb/mysql-test/rpl/r/rpl_tokudb_stm_log.result

@@ -222,7 +222,6 @@ master-bin.000001	#	Xid	#	#	COMMIT /* XID */
 master-bin.000001	#	Rotate	#	#	master-bin.000002;pos=POS
 include/show_binlog_events.inc
 Log_name	Pos	Event_type	Server_id	End_log_pos	Info
-master-bin.000002	#	Binlog_checkpoint	#	#	master-bin.000002
 master-bin.000002	#	Gtid	#	#	GTID #-#-#
 master-bin.000002	#	Query	#	#	use `test`; create table t3 (a int)ENGINE=TokuDB
 master-bin.000002	#	Gtid	#	#	GTID #-#-#
@@ -260,7 +259,6 @@ slave-bin.000001	#	Query	#	#	use `test`; create table t3 (a int)ENGINE=TokuDB
 slave-bin.000001	#	Rotate	#	#	slave-bin.000002;pos=POS
 include/show_binlog_events.inc
 Log_name	Pos	Event_type	Server_id	End_log_pos	Info
-slave-bin.000002	#	Binlog_checkpoint	#	#	slave-bin.000002
 slave-bin.000002	#	Gtid	#	#	GTID #-#-#
 slave-bin.000002	#	Query	#	#	use `test`; create table t2 (n int)ENGINE=TokuDB
 slave-bin.000002	#	Gtid	#	#	BEGIN GTID #-#-#